diff --git a/.azure-pipelines/jobs/package.yml b/.azure-pipelines/jobs/package.yml deleted file mode 100644 index bdb0254a1ba..00000000000 --- a/.azure-pipelines/jobs/package.yml +++ /dev/null @@ -1,36 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Package - dependsOn: - - Test_Primary - - Test_Secondary - pool: - vmImage: ${{ parameters.vmImage }} - - steps: - - task: UsePythonVersion@0 - displayName: Use Python 3 latest - inputs: - versionSpec: '3' - - - bash: | - git config --global user.email "distutils-sig@python.org" - git config --global user.name "pip" - displayName: Setup Git credentials - - - bash: pip install nox - displayName: Install dependencies - - - bash: nox -s prepare-release -- 99.9 - displayName: Prepare dummy release - - - bash: nox -s build-release -- 99.9 - displayName: Generate distributions for the dummy release - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: dist' - inputs: - pathtoPublish: dist - artifactName: dist diff --git a/.azure-pipelines/jobs/test-windows.yml b/.azure-pipelines/jobs/test-windows.yml deleted file mode 100644 index 99cd8a836bd..00000000000 --- a/.azure-pipelines/jobs/test-windows.yml +++ /dev/null @@ -1,53 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Tests / - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.6": # lowest Python version - python.version: '3.6' - python.architecture: x64 - "3.8": # current - python.version: '3.8' - python.architecture: x64 - maxParallel: 6 - - steps: - - template: ../steps/run-tests-windows.yml - parameters: - runIntegrationTests: true - -- job: Test_Secondary - displayName: Tests / - # Don't run integration tests for these runs - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - dependsOn: Test_Primary - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.7": - python.version: '3.7' - python.architecture: x64 - # This is for Windows, so test x86 builds - "3.6-x86": - python.version: '3.6' - python.architecture: x86 - "3.7-x86": - python.version: '3.7' - python.architecture: x86 - "3.8-x86": - python.version: '3.8' - python.architecture: x86 - maxParallel: 6 - - steps: - - template: ../steps/run-tests-windows.yml - parameters: - runIntegrationTests: false diff --git a/.azure-pipelines/jobs/test.yml b/.azure-pipelines/jobs/test.yml deleted file mode 100644 index a3a0ef80b6d..00000000000 --- a/.azure-pipelines/jobs/test.yml +++ /dev/null @@ -1,38 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Tests / - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.6": # lowest Python version - python.version: '3.6' - python.architecture: x64 - "3.8": - python.version: '3.8' - python.architecture: x64 - maxParallel: 2 - - steps: - - template: ../steps/run-tests.yml - -- job: Test_Secondary - displayName: Tests / - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - dependsOn: Test_Primary - - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - "3.7": - python.version: '3.7' - python.architecture: x64 - maxParallel: 4 - - steps: - - template: ../steps/run-tests.yml diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml deleted file mode 100644 index e5598074344..00000000000 --- a/.azure-pipelines/linux.yml +++ /dev/null @@ -1,11 +0,0 @@ -variables: - CI: true - -jobs: -- template: jobs/test.yml - parameters: - vmImage: ubuntu-16.04 - -- template: jobs/package.yml - parameters: - vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml deleted file mode 100644 index 39282a3cc80..00000000000 --- a/.azure-pipelines/steps/run-tests-windows.yml +++ /dev/null @@ -1,54 +0,0 @@ -parameters: - runIntegrationTests: - -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- task: PowerShell@2 - inputs: - filePath: .azure-pipelines/scripts/New-RAMDisk.ps1 - arguments: "-Drive R -Size 1GB" - displayName: Setup RAMDisk - -- powershell: | - mkdir R:\Temp - $acl = Get-Acl "R:\Temp" - $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( - "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" - ) - $acl.AddAccessRule($rule) - Set-Acl "R:\Temp" $acl - displayName: Set RAMDisk Permissions - -- bash: pip install --upgrade 'virtualenv<20' setuptools tox - displayName: Install Tox - -- script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml - env: - TEMP: "R:\\Temp" - displayName: Tox run unit tests - -- ${{ if eq(parameters.runIntegrationTests, 'true') }}: - - powershell: | - # Fix Git SSL errors - pip install certifi tox - python -m certifi > cacert.txt - $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt) - - # Shorten paths to get under MAX_PATH or else integration tests will fail - # https://bugs.python.org/issue18199 - $env:TEMP = "R:\Temp" - - tox -e py -- -m integration -n auto --durations=5 --junit-xml=junit/integration-test.xml - displayName: Tox run integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: junit/*.xml - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml deleted file mode 100644 index 5b9a9c50c89..00000000000 --- a/.azure-pipelines/steps/run-tests.yml +++ /dev/null @@ -1,25 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - -- bash: pip install --upgrade 'virtualenv<20' setuptools tox - displayName: Install Tox - -- script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml - displayName: Tox run unit tests - -# Run integration tests in two groups so we will fail faster if there is a failure in the first group -- script: tox -e py -- -m integration -n auto --durations=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml - displayName: Tox run Group 0 integration tests - -- script: tox -e py -- -m integration -n auto --durations=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml - displayName: Tox run Group 1 integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: junit/*.xml - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml deleted file mode 100644 index f56b8f50486..00000000000 --- a/.azure-pipelines/windows.yml +++ /dev/null @@ -1,11 +0,0 @@ -variables: - CI: true - -jobs: -- template: jobs/test-windows.yml - parameters: - vmImage: vs2017-win2016 - -- template: jobs/package.yml - parameters: - vmImage: vs2017-win2016 diff --git a/.gitattributes b/.gitattributes index 7b547a58cc2..6a0fc6943c1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ # Patches must have Unix-style line endings, even on Windows -tools/automation/vendoring/patches/* eol=lf +tools/vendoring/patches/* eol=lf # The CA Bundle should always use Unix-style line endings, even on Windows src/pip/_vendor/certifi/*.pem eol=lf diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 508153d8d25..00000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,3 +0,0 @@ -* pip version: -* Python version: -* Operating system: diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index 157be28b678..00000000000 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve ---- - - - -**Environment** - -* pip version: -* Python version: -* OS: - - - -**Description** - - -**Expected behavior** - - -**How to Reproduce** - - -1. Get package from '...' -2. Then run '...' -3. An error occurs. - -**Output** - -``` -Paste the output of the steps above, including the commands themselves and -pip's output/traceback etc. -``` diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000000..c691a0058fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,79 @@ +name: Bug report +description: Something is not working correctly. +labels: "S: needs triage, type: bug" + +body: + - type: markdown + attributes: + value: >- + Hi there! + + We'd appreciate it if you could search on pip's existing issues prior to filing + a bug report. + + We get a lot of duplicate tickets and have limited maintainer capacity to triage + them. Thanks! + + - type: textarea + attributes: + label: Description + description: >- + A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + attributes: + label: Expected behavior + description: >- + A clear and concise description of what you expected to happen. + + - type: input + attributes: + label: pip version + validations: + required: true + - type: input + attributes: + label: Python version + validations: + required: true + - type: input + attributes: + label: OS + validations: + required: true + + - type: textarea + attributes: + label: How to Reproduce + description: Please provide steps to reproduce this bug. + placeholder: | + 1. Get package from '...' + 2. Then run '...' + 3. An error occurs. + validations: + required: true + + - type: textarea + attributes: + label: Output + description: >- + Provide the output of the steps above, including the commands + themselves and pip's output/traceback etc. If you're familiar with + Markdown, this block will have triple backticks added automatically + around it -- you don't have to add them. + + If you want to present output from multiple commands, please present + that as a shell session (commands you run get prefixed with `$ `). + Please also ensure that the "How to reproduce" section contains matching + instructions for reproducing this. + render: sh-session + + - type: checkboxes + attributes: + label: Code of Conduct + options: + - label: >- + I agree to follow the [PSF Code of Conduct](https://www.python.org/psf/conduct/). + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 8e5c268c114..416ae07000f 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,11 +1,11 @@ -# Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser -blank_issues_enabled: true # default +# Documentation for this file can be found at: +# https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository + +blank_issues_enabled: false contact_links: -- name: 💬 Discourse - url: https://discuss.python.org/c/packaging - about: | - Please ask typical Q&A here: general ideas for Python packaging, - questions about structuring projects and so on -- name: '💬 IRC: #pypa @ Freenode' - url: https://webchat.freenode.net/#pypa - about: Chat with devs + - name: "💬 IRC: #pypa" + url: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa + about: Chat with devs + - name: "(maintainers only) Blank issue" + url: https://github.com/pypa/pip/issues/new + about: For maintainers only. diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md deleted file mode 100644 index a0addbdb7ff..00000000000 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project - ---- - -**What's the problem this feature will solve?** - - -**Describe the solution you'd like** - - - - -**Alternative Solutions** - - -**Additional context** - diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 00000000000..bc87da0a008 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,59 @@ +name: Feature request +description: Suggest an idea for this project +labels: "S: needs triage, type: feature request" + +body: + - type: markdown + attributes: + value: >- + Hi there! + + We'd appreciate it if you could search on pip's existing issues prior to filing + a feature request. + + We get a lot of duplicate tickets and have limited maintainer capacity to triage + them. Thanks! + + - type: textarea + attributes: + label: What's the problem this feature will solve? + description: >- + What are you trying to do, that you are unable to achieve with pip as it + currently stands? + validations: + required: true + + - type: textarea + attributes: + label: Describe the solution you'd like + description: >- + Clear and concise description of what you want to happen. Please use examples + of real world use cases that this would help with, and how it solves the + problem described above. + validations: + required: true + + - type: textarea + attributes: + label: Alternative Solutions + description: >- + Have you tried to workaround the problem using pip or other tools? Or a + different approach to solving this issue? Please elaborate here. + validations: + required: true + + - type: textarea + attributes: + label: Additional context + description: >- + Add any other context, links, etc. relevant to the feature request. + validations: + required: true + + - type: checkboxes + attributes: + label: Code of Conduct + options: + - label: >- + I agree to follow the [PSF Code of Conduct](https://www.python.org/psf/conduct/). + required: true diff --git a/.github/ISSUE_TEMPLATE/resolver-failure.md b/.github/ISSUE_TEMPLATE/resolver-failure.md deleted file mode 100644 index b5215cef94d..00000000000 --- a/.github/ISSUE_TEMPLATE/resolver-failure.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Dependency resolver failures / errors -about: Report when the pip dependency resolver fails -labels: ["K: UX", "K: crash", "C: new resolver", "C: dependency resolution"] ---- - - - -**What did you want to do?** - - -**Output** - -``` -Paste what pip outputted in a code block. https://github.github.com/gfm/#fenced-code-blocks -``` - -**Additional information** - - diff --git a/.github/ISSUE_TEMPLATE/~good-first-issue.md b/.github/ISSUE_TEMPLATE/~good-first-issue.md deleted file mode 100644 index 885198b63a7..00000000000 --- a/.github/ISSUE_TEMPLATE/~good-first-issue.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -name: (Maintainers Only) Good First Issue -about: For maintainers, to create an issue that is good for new contributors -labels: ["good first issue"] - ---- - - - - - - ---- - -**Good First Issue**: This issue is a good starting point for first time contributors -- the process of fixing this should be a good introduction to pip's development workflow. If you've already contributed to pip, work on [another issue without this label](https://github.com/pypa/pip/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+-label%3A%22good+first+issue%22) instead. If there is not a corresponding pull request for this issue, it is up for grabs. For directions for getting set up, see our [Getting Started Guide](https://pip.pypa.io/en/latest/development/getting-started/). If you are working on this issue and have questions, feel free to ask them here, [`#pypa-dev` on Freenode](https://webchat.freenode.net/?channels=%23pypa-dev), or the [distutils-sig mailing list](https://mail.python.org/mailman3/lists/distutils-sig.python.org/). diff --git a/.github/ISSUE_TEMPLATE/~good-first-issue.yml b/.github/ISSUE_TEMPLATE/~good-first-issue.yml new file mode 100644 index 00000000000..81e206a35f9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/~good-first-issue.yml @@ -0,0 +1,38 @@ +name: Good first issue +description: If you're a pip maintainer, use this to create a "good first issue" for new contributors. +labels: "good first issue" + +body: + - type: textarea + attributes: + label: Description + description: >- + A clear and concise description of what the task is. + validations: + required: true + + - type: textarea + attributes: + label: What needs to be done + description: >- + Describe what the contributor would need to do, describing the change. + See https://github.com/pypa/pip/issues/7661 for example. + validations: + required: true + + - type: textarea + attributes: + label: Guidance for potential contributors + description: >- + Usually, you don't have to modify the content here. + value: >- + This issue is a good starting point for first time contributors -- the + process of fixing this should be a good introduction to pip's + development workflow. If there is not a corresponding pull request for + this issue, it is up for grabs. For directions for getting set up, see our + [Getting Started Guide](https://pip.pypa.io/en/latest/development/getting-started/). + If you are working on this issue and have questions, feel free to ask + them here. If you've contributed code to pip before, we encourage you to + pick up an issue without this label. + validations: + required: true diff --git a/.github/lock.yml b/.github/lock.yml deleted file mode 100644 index dd12e3da8bc..00000000000 --- a/.github/lock.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Number of days of inactivity before a closed issue or pull request is locked -daysUntilLock: 30 -# Issues and pull requests with these labels will not be locked. -exemptLabels: [] -# Label to add before locking, such as `outdated`. Set to `false` to disable -lockLabel: "S: auto-locked" -# Comment to post before locking. Set to `false` to disable -lockComment: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..85b575c3e7a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,220 @@ +name: CI + +on: + push: + branches: [main] + tags: + # Tags for all potential release numbers till 2030. + - "2[0-9].[0-3]" # 20.0 -> 29.3 + - "2[0-9].[0-3].[0-9]+" # 20.0.0 -> 29.3.[0-9]+ + pull_request: + schedule: + - cron: 0 0 * * MON # Run every Monday at 00:00 UTC + +env: + # The "FORCE_COLOR" variable, when set to 1, + # tells Nox to colorize itself. + FORCE_COLOR: "1" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + docs: + name: docs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install nox + - run: nox -s docs + + determine-changes: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.filter.outputs.tests }} + vendoring: ${{ steps.filter.outputs.vendoring }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + vendoring: + # Anything that's touching "vendored code" + - "src/pip/_vendor/**" + - "pyproject.toml" + tests: + # Anything that's touching code-related stuff + - ".github/workflows/ci.yml" + - "src/**" + - "tests/**" + if: github.event_name == 'pull_request' + + pre-commit: + name: pre-commit + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 + with: + extra_args: --all-files --hook-stage=manual + + packaging: + name: packaging + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Set up git credentials + run: | + git config --global user.email "pypa-dev@googlegroups.com" + git config --global user.name "pip" + + - run: pip install nox + - run: nox -s prepare-release -- 99.9 + - run: nox -s build-release -- 99.9 + + vendoring: + name: vendoring + runs-on: ubuntu-latest + + needs: [determine-changes] + if: >- + needs.determine-changes.outputs.vendoring == 'true' || + github.event_name != 'pull_request' + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + + - run: pip install nox + - run: nox -s vendoring + - run: git diff --exit-code + + tests-unix: + name: tests / ${{ matrix.python }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest + + needs: [pre-commit, packaging, determine-changes] + if: >- + needs.determine-changes.outputs.tests == 'true' || + github.event_name != 'pull_request' + + strategy: + fail-fast: true + matrix: + os: [Ubuntu, MacOS] + python: + - 3.7 + - 3.8 + - 3.9 + - "3.10" + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Install Ubuntu dependencies + if: matrix.os == 'Ubuntu' + run: sudo apt-get install bzr + + - name: Install MacOS dependencies + if: matrix.os == 'MacOS' + run: brew install bzr + + - run: pip install nox 'virtualenv<20' 'setuptools != 60.6.0' + + # Main check + - name: Run unit tests + run: >- + nox -s test-${{ matrix.python }} -- + -m unit + --verbose --numprocesses auto --showlocals + - name: Run integration tests + run: >- + nox -s test-${{ matrix.python }} -- + -m integration + --verbose --numprocesses auto --showlocals + --durations=5 + + tests-windows: + name: tests / ${{ matrix.python }} / ${{ matrix.os }} / ${{ matrix.group }} + runs-on: ${{ matrix.os }}-latest + + needs: [pre-commit, packaging, determine-changes] + if: >- + needs.determine-changes.outputs.tests == 'true' || + github.event_name != 'pull_request' + + strategy: + fail-fast: true + matrix: + os: [Windows] + python: + - 3.7 + # Commented out, since Windows tests are expensively slow. + # - 3.8 + # - 3.9 + - "3.10" + group: [1, 2] + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + # We use a RAMDisk on Windows, since filesystem IO is a big slowdown + # for our tests. + - name: Create a RAMDisk + run: ./tools/ci/New-RAMDisk.ps1 -Drive R -Size 1GB + + - name: Setup RAMDisk permissions + run: | + mkdir R:\Temp + $acl = Get-Acl "R:\Temp" + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" + ) + $acl.AddAccessRule($rule) + Set-Acl "R:\Temp" $acl + + - run: pip install nox 'virtualenv<20' + env: + TEMP: "R:\\Temp" + + # Main check + - name: Run unit tests + if: matrix.group == 1 + run: >- + nox -s test-${{ matrix.python }} -- + -m unit + --verbose --numprocesses auto --showlocals + env: + TEMP: "R:\\Temp" + + - name: Run integration tests (group 1) + if: matrix.group == 1 + run: >- + nox -s test-${{ matrix.python }} -- + -m integration -k "not test_install" + --verbose --numprocesses auto --showlocals + env: + TEMP: "R:\\Temp" + + - name: Run integration tests (group 2) + if: matrix.group == 2 + run: >- + nox -s test-${{ matrix.python }} -- + -m integration -k "test_install" + --verbose --numprocesses auto --showlocals + env: + TEMP: "R:\\Temp" diff --git a/.github/workflows/label-merge-conflicts.yml b/.github/workflows/label-merge-conflicts.yml new file mode 100644 index 00000000000..1de897ca1c4 --- /dev/null +++ b/.github/workflows/label-merge-conflicts.yml @@ -0,0 +1,19 @@ +name: Autolabel merge conflicts + +permissions: + issues: write + pull-requests: write + +on: + push: + branches: [main] + +jobs: + label-merge-conflicts: + if: github.repository_owner == 'pypa' + runs-on: ubuntu-latest + steps: + - uses: pradyunsg/auto-label-merge-conflicts@v3 + with: + CONFLICT_LABEL_NAME: "needs rebase or merge" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index 71459d660e8..00000000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Linting - -on: - push: - pull_request: - schedule: - # Run every Friday at 18:02 UTC - - cron: 2 18 * * 5 - -jobs: - lint: - name: ${{ matrix.os }} - runs-on: ${{ matrix.os }}-latest - env: - TOXENV: lint,docs,vendoring - - strategy: - matrix: - os: - - Ubuntu - - Windows - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - # Setup Caching - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Set PY (for pre-commit cache) - run: echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV - - name: pre-commit cache - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|2020-02-14|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - # Get the latest tox - - name: Install tox - run: python -m pip install tox - - # Main check - - run: python -m tox diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml new file mode 100644 index 00000000000..990440dd6c8 --- /dev/null +++ b/.github/workflows/lock-threads.yml @@ -0,0 +1,23 @@ +name: 'Lock Closed Threads' + +on: + schedule: + - cron: '0 7 * * *' # 7am UTC, daily + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +concurrency: + group: lock + +jobs: + action: + if: github.repository_owner == 'pypa' + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v3 + with: + issue-inactive-days: '30' + pr-inactive-days: '15' diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index de226389d26..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: MacOS - -on: - push: - pull_request: - schedule: - # Run every Friday at 18:02 UTC - - cron: 2 18 * * 5 - -jobs: - dev-tools: - name: Quality Check - runs-on: macos-latest - - steps: - # Caches - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - name: Set PY (for pre-commit cache) - run: echo "PY=$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV - - name: pre-commit cache - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|2020-02-14|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - # Setup - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Install tox - run: python -m pip install tox - - # Main check - - run: python -m tox -e "lint,docs" - - packaging: - name: Packaging - runs-on: macos-latest - - steps: - # Caches - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - # Setup - - name: Set up git credentials - run: | - git config --global user.email "pypa-dev@googlegroups.com" - git config --global user.name "pip" - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install tox and nox - run: python -m pip install tox nox - - # Main check - - name: Check vendored packages - run: python -m tox -e "vendoring" - - - name: Prepare dummy release - run: nox -s prepare-release -- 99.9 - - - name: Generate distributions for the dummy release - run: nox -s build-release -- 99.9 - - tests: - name: Tests / ${{ matrix.python }} - runs-on: macos-latest - - needs: dev-tools - - strategy: - fail-fast: false - matrix: - python: [3.6, 3.7, 3.8, 3.9] - - steps: - # Caches - - name: pip cache - uses: actions/cache@v1 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('tools/requirements/tests.txt') }}-${{ hashFiles('tools/requirements/docs.txt') }}-${{ hashFiles('tox.ini') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - # Setup - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Install tox - run: python -m pip install tox 'virtualenv<20' - - # Main check - - name: Run unit tests - run: >- - python -m tox -e py -- - -m unit - --verbose - --numprocesses auto - - - name: Run integration tests - run: >- - python -m tox -e py -- - -m integration - --verbose - --numprocesses auto - --durations=5 diff --git a/.github/workflows/news-file.yml b/.github/workflows/news-file.yml new file mode 100644 index 00000000000..2c7680d0b68 --- /dev/null +++ b/.github/workflows/news-file.yml @@ -0,0 +1,25 @@ +name: Check + +on: + pull_request: + types: [labeled, unlabeled, opened, reopened, synchronize] + +jobs: + check-news-entry: + name: news entry + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + # `towncrier check` runs `git diff --name-only origin/main...`, which + # needs a non-shallow clone. + fetch-depth: 0 + + - name: Check news entry + if: "!contains(github.event.pull_request.labels.*.name, 'trivial')" + run: | + if ! pipx run towncrier check --compare-with origin/${{ github.base_ref }}; then + echo "Please see https://pip.pypa.io/dev/news-entry-failure for guidance." + false + fi diff --git a/.gitignore b/.gitignore index dc6244855fe..79b8ab84b06 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ tests/data/common_wheels/ *~ .*.sw? .env/ +.venv/ # For IntelliJ IDEs (basically PyCharm) .idea/ @@ -48,3 +49,6 @@ tests/data/common_wheels/ # Mac .DS_Store + +# Profiling related artifacts +*.prof diff --git a/.mailmap b/.mailmap index 29f9ec039b6..d0c64300fd2 100644 --- a/.mailmap +++ b/.mailmap @@ -19,8 +19,11 @@ Dongweiming Endoh Takanao Erik M. Bray +Ee Durbin Gabriel de Perthuis Hsiaoming Yang +Hugo van Kemenade Hugo +Hugo van Kemenade hugovk Igor Kuzmitshov Ilya Baryshev Jakub Stasiak @@ -31,6 +34,7 @@ Ludovic Gasc Markus Hametner Masklinn Matthew Iversen +Ofek Lev Pi Delport Pradyun Gedam diff --git a/.pre-commit-config-slow.yaml b/.pre-commit-config-slow.yaml deleted file mode 100644 index 2179c665769..00000000000 --- a/.pre-commit-config-slow.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Slow pre-commit checks we don't want to run locally with each commit. - -repos: -- repo: https://github.com/mgedmin/check-manifest - rev: '0.43' - hooks: - - id: check-manifest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 636fdfd3d4c..a2a147be0bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: 'src/pip/_vendor/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v3.4.0 hooks: - id: check-builtin-literals - id: check-added-large-files @@ -17,66 +17,44 @@ repos: exclude: .patch - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.7b0 hooks: - id: black - exclude: | - (?x) - ^docs/| - ^src/pip/_internal/cli| - ^src/pip/_internal/commands| - ^src/pip/_internal/distributions| - ^src/pip/_internal/index| - ^src/pip/_internal/models| - ^src/pip/_internal/network| - ^src/pip/_internal/operations| - ^src/pip/_internal/req| - ^src/pip/_internal/resolution| - ^src/pip/_internal/utils| - ^src/pip/_internal/vcs| - ^src/pip/_internal/\w+\.py$| - ^src/pip/__main__.py$| - ^tools/| - # Tests - ^tests/conftest.py| - ^tests/yaml| - ^tests/lib| - ^tests/data| - ^tests/unit| - ^tests/functional/(?!test_install)| - ^tests/functional/test_install| - # Files in the root of the repository - ^setup.py| - ^noxfile.py| - # A blank ignore, to avoid merge conflicts later. - ^$ - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: [ 'flake8-bugbear==20.1.4', 'flake8-logging-format==0.6.0', + 'flake8-implicit-str-concat==0.2.0', ] exclude: tests/data - repo: https://github.com/PyCQA/isort - rev: 5.5.3 + rev: 5.7.0 hooks: - id: isort files: \.py$ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + rev: v0.910 hooks: - id: mypy - exclude: docs|tests - args: ["--pretty"] + exclude: tests/data + args: ["--pretty", "--show-error-codes"] + additional_dependencies: [ + 'keyring==23.0.1', + 'nox==2021.6.12', + 'pytest==6.2.5', + 'types-docutils==0.1.8', + 'types-setuptools==57.0.2', + 'types-six==0.1.9', + ] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.6.0 + rev: v1.7.0 hooks: - id: python-no-log-warn - id: python-no-eval @@ -93,3 +71,9 @@ repos: entry: NEWS fragment files must be named *.(process|removal|feature|bugfix|vendor|doc|trivial).rst exclude: ^news/(.gitignore|.*\.(process|removal|feature|bugfix|vendor|doc|trivial).rst) files: ^news/ + +- repo: https://github.com/mgedmin/check-manifest + rev: '0.46' + hooks: + - id: check-manifest + stages: [manual] diff --git a/.readthedocs.yml b/.readthedocs.yml index c123a1939fb..7d62011a6e3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,6 @@ sphinx: configuration: docs/html/conf.py python: - version: 3.7 + version: 3.8 install: - - requirements: tools/requirements/docs.txt + - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6610b6eb019..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -language: python -cache: pip -dist: xenial -python: 3.9 -addons: - apt: - packages: - - bzr - -stages: -- primary -- secondary - -jobs: - include: - # Basic Checks - - stage: primary - env: TOXENV=docs - - env: TOXENV=lint - - env: TOXENV=vendoring - - # Complete checking for ensuring compatibility - # PyPy - - stage: secondary - env: GROUP=1 - python: pypy3.6-7.3.1 - - env: GROUP=2 - python: pypy3.6-7.3.1 - -before_install: tools/travis/setup.sh -install: travis_retry tools/travis/install.sh -script: tools/travis/run.sh diff --git a/AUTHORS.txt b/AUTHORS.txt index 0360f988f2b..bcca65bf521 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -18,10 +18,12 @@ Alan Yee Albert Tugushev Albert-Guan albertg +Alberto Sottile Aleks Bunin Alethea Flowers Alex Gaynor Alex Grönholm +Alex Hedges Alex Loosley Alex Morega Alex Stachowiak @@ -37,6 +39,7 @@ Andre Aguiar Andreas Lutro Andrei Geacar Andrew Gaul +Andrey Bienkowski Andrey Bulgakov Andrés Delfino Andy Freeland @@ -60,6 +63,7 @@ AraHaan Arindam Choudhury Armin Ronacher Artem +Arun Babu Neelicattu Ashley Manton Ashwin Ramaswami atse @@ -71,8 +75,10 @@ Barney Gale barneygale Bartek Ogryczak Bastian Venthur +Ben Bodenmiller Ben Darnell Ben Hoyt +Ben Mares Ben Rosser Bence Nagy Benjamin Peterson @@ -85,6 +91,7 @@ Bernardo B. Marques Bernhard M. Wiedemann Bertil Hatt Bhavam Vidyarthi +Blazej Michalik Bogdan Opanchuk BorisZZZ Brad Erickson @@ -94,13 +101,16 @@ Brandt Bucher Brett Randall Brian Cristante Brian Rosner +briantracy BrownTruck Bruno Oliveira Bruno Renié +Bruno S Bstrdsmkr Buck Golemon burrows Bussonnier Matthias +bwoodsend c22 Caleb Martinez Calvin Smith @@ -115,6 +125,7 @@ Chris Brinker Chris Hunt Chris Jerdonek Chris McDonough +Chris Pawley Chris Wolfe Christian Clauss Christian Heimes @@ -139,10 +150,14 @@ Cristina Cristina Muñoz Curtis Doty cytolentino +Daan De Meyer +Damian Damian Quiroga +Damian Shaw Dan Black Dan Savilonis Dan Sully +Dane Hillard daniel Daniel Collins Daniel Hahler @@ -154,6 +169,7 @@ Daniele Esposti Daniele Procida Danny Hermes Danny McClanahan +Darren Kavanagh Dav Clark Dave Abrahams Dave Jones @@ -161,7 +177,9 @@ David Aguilar David Black David Bordeynik David Caro +David D Lowe David Evans +David Hewitt David Linke David Poggi David Pursehouse @@ -169,13 +187,20 @@ David Tucker David Wales Davidovich Deepak Sharma +Deepyaman Datta +Denise Yu derwolfe Desetude Devesh Kumar Singh Diego Caraballo +Diego Ramirez DiegoCaraballo +Dimitri Merejkowsky +Dirk Stolle Dmitry Gladkov +Dmitry Volodin Domen Kožar +Dominic Davis-Foster Donald Stufft Dongweiming Douglas Thor @@ -183,6 +208,7 @@ DrFeathers Dustin Ingram Dwayne Bailey Ed Morley +Ee Durbin Eitan Adler ekristina elainechan @@ -195,13 +221,12 @@ Emmanuel Arias Endoh Takanao enoch Erdinc Mutlu +Eric Cousineau Eric Gillingham Eric Hanchrow Eric Hopper Erik M. Bray Erik Rose -Ernest W Durbin III -Ernest W. Durbin III Erwin Janssen Eugene Vereshchagin everdimension @@ -227,21 +252,25 @@ gizmoguy1 gkdoc Gopinath M GOTO Hayato +gousaiyang gpiks +Greg Roodt Greg Ward Guilherme Espada gutsytechster Guy Rozendorn +Guy Tuval gzpan123 Hanjun Kim Hari Charan Harsh Vardhan +harupy +Harutaka Kawamura +Henry Schreiner Herbert Pfennig Hsiaoming Yang -Hugo Hugo Lopes Tavares Hugo van Kemenade -hugovk Hynek Schlawack Ian Bicking Ian Cordasco @@ -251,18 +280,22 @@ Ian Wienand Igor Kuzmitshov Igor Sobreira Ilan Schnell +Illia Volochii Ilya Baryshev -INADA Naoki +Inada Naoki Ionel Cristian Mărieș Ionel Maries Cristian Ivan Pozdeev Jacob Kim +Jacob Walls jakirkham Jakub Stasiak Jakub Vysoky Jakub Wilk James Cleveland +James Curtin James Firth +James Gerity James Polley Jan Pokorný Jannis Leidel @@ -276,9 +309,12 @@ Jelmer Vernooij jenix21 Jeremy Stanley Jeremy Zafran +Jesse Rittner Jiashuo Li +Jim Fisher Jim Garrison Jivan Amara +Joe Michelini John Paton John T. Wodder II John-Scott Atlakson @@ -290,6 +326,7 @@ Jonas Nockert Jonathan Herbert Joost Molenaar Jorge Niedbalski +Joseph Bylund Joseph Long Josh Bronson Josh Hansen @@ -315,6 +352,7 @@ Kevin Frommelt Kevin R Patterson Kexuan Sun Kit Randel +Klaas van Schelven KOLANICH kpinc Krishna Oza @@ -323,8 +361,10 @@ Kyle Persohn lakshmanaram Laszlo Kiss-Kollar Laurent Bristiel +Laurent LAPORTE Laurie O Laurie Opperman +layday Leon Sasson Lev Givon Lincoln de Sousa @@ -332,6 +372,7 @@ Lipis Loren Carvalho Lucas Cimon Ludovic Gasc +Lukas Juhrich Luke Macken Luo Jiebin luojiebin @@ -344,11 +385,15 @@ Mariatta Mark Kohler Mark Williams Markus Hametner +Martey Dodoo +Martin Häcker +Martin Pavlasek Masaki Masklinn Matej Stuchlik Mathew Jennings Mathieu Bridon +Matt Bacchi Matt Good Matt Maker Matt Robenolt @@ -360,18 +405,22 @@ Matthew Trumbell Matthew Willson Matthias Bussonnier mattip +Maurits van Rees +Max W Chase Maxim Kurnikov Maxime Rouyrre mayeut mbaluna mdebi memoselyk +meowmeowcat Michael Michael Aquilina Michael E. Karpeles Michael Klich Michael Williamson michaelpacer +Michał Górny Mickaël Schoentgen Miguel Araujo Perez Mihir Singh @@ -383,25 +432,33 @@ Miro Hrončok Monica Baluna montefra Monty Taylor +Nadav Wexler Nate Coraor +Nate Prewitt +Nathan Houghton Nathaniel J. Smith Nehal J Wani Neil Botelho Nguyễn Gia Phong +Nicholas Serra Nick Coghlan Nick Stenning Nick Timkovich Nicolas Bock Nicole Harris Nikhil Benesch +Nikita Chepanov Nikolay Korolev +Nipunn Koorapati Nitesh Sharma +Niyas Sait Noah Noah Gorny Nowell Strite NtaleGrey nvdv -Ofekmeister +OBITORASU +Ofek Lev ofrinevo Oliver Jeeves Oliver Mannion @@ -427,9 +484,12 @@ Paul Nasrat Paul Oswald Paul van der Linden Paulus Schoutsen +Pavel Safronov Pavithra Eswaramoorthy Pawel Jasinski +Paweł Szramowski Pekka Klärck +Peter Gessler Peter Lisák Peter Waller petr-tik @@ -454,7 +514,9 @@ Preet Thakkar Preston Holmes Przemek Wrzos Pulkit Goyal +q0w Qiangning Hong +Quentin Lee Quentin Pradet R. David Murray Rafael Caricio @@ -466,6 +528,7 @@ Remi Rampin Rene Dudfield Riccardo Magliocchetti Richard Jones +Richard Si Ricky Ng-Adam RobberPhex Robert Collins @@ -504,6 +567,7 @@ Simon Cross Simon Pichugin sinoroc sinscary +snook92 socketubs Sorin Sbarnea Srinivas Nyayapati @@ -525,7 +589,9 @@ Sumana Harihareswara Surbhi Sharma Sviatoslav Sydorenko Swat009 +Sylvain Takayuki SHIMIZUKAWA +Taneli Hukkinen tbeswick Thijs Triemstra Thomas Fenzl @@ -553,6 +619,7 @@ toonarmycaptain Toshio Kuratomi toxinu Travis Swicegood +Tushar Sadhwani Tzu-ping Chung Valentin Haenel Victor Stinner @@ -573,7 +640,9 @@ William ML Leslie William T Olson Wilson Mo wim glenn +Winson Luk Wolfgang Maier +XAMES3 Xavier Fernandez xoviat xtreak @@ -585,6 +654,8 @@ Yu Jian Yuan Jing Vincent Yan Zearin Zhiping Deng +ziebam Zvezdan Petkovic Łukasz Langa Семён Марьясин +‮rekcäH nitraM‮ diff --git a/LICENSE.txt b/LICENSE.txt index 75eb0fd80b0..8e7b65eaf62 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +Copyright (c) 2008-present The pip developers (see AUTHORS.txt file) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/MANIFEST.in b/MANIFEST.in index 2cf636ce3f7..07699ddd6e6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,31 +6,30 @@ include pyproject.toml include src/pip/_vendor/README.rst include src/pip/_vendor/vendor.txt +include src/pip/_vendor/pyparsing/diagram/template.jinja2 recursive-include src/pip/_vendor *LICENSE* recursive-include src/pip/_vendor *COPYING* include docs/docutils.conf +include docs/requirements.txt exclude .coveragerc exclude .mailmap exclude .appveyor.yml -exclude .travis.yml exclude .readthedocs.yml exclude .pre-commit-config.yaml -exclude .pre-commit-config-slow.yaml exclude tox.ini exclude noxfile.py recursive-include src/pip/_vendor *.pem recursive-include src/pip/_vendor py.typed -recursive-include docs *.css *.rst *.py +recursive-include docs *.css *.py *.rst *.md exclude src/pip/_vendor/six exclude src/pip/_vendor/six/moves recursive-exclude src/pip/_vendor *.pyi prune .github -prune .azure-pipelines prune docs/build prune news prune tasks diff --git a/NEWS.rst b/NEWS.rst index a082cddf314..5f274c0ef7f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,576 @@ .. towncrier release notes start +22.0.4 (2022-03-06) +=================== + +Deprecations and Removals +------------------------- + +- Drop the doctype check, that presented a warning for index pages that use non-compliant HTML 5. (`#10903 `_) + +Vendored Libraries +------------------ + +- Downgrade distlib to 0.3.3. + + +22.0.3 (2022-02-03) +=================== + +Features +-------- + +- Print the exception via ``rich.traceback``, when running with ``--debug``. (`#10791 `_) + +Bug Fixes +--------- + +- Only calculate topological installation order, for packages that are going to be installed/upgraded. + + This fixes an `AssertionError` that occured when determining installation order, for a very specific combination of upgrading-already-installed-package + change of dependencies + fetching some packages from a package index. This combination was especially common in Read the Docs' builds. (`#10851 `_) +- Use ``html.parser`` by default, instead of falling back to ``html5lib`` when ``--use-deprecated=html5lib`` is not passed. (`#10869 `_) + +Improved Documentation +---------------------- + +- Clarify that using per-requirement overrides disables the usage of wheels. (`#9674 `_) + + +22.0.2 (2022-01-30) +=================== + +Deprecations and Removals +------------------------- + +- Instead of failing on index pages that use non-compliant HTML 5, print a deprecation warning and fall back to ``html5lib``-based parsing for now. This simplifies the migration for non-compliant index pages, by letting such indexes function with a warning. (`#10847 `_) + + +22.0.1 (2022-01-30) +=================== + +Bug Fixes +--------- + +- Accept lowercase ```` on index pages. (`#10844 `_) +- Properly handle links parsed by html5lib, when using ``--use-deprecated=html5lib``. (`#10846 `_) + + +22.0 (2022-01-29) +================= + +Process +------- + +- Completely replace :pypi:`tox` in our development workflow, with :pypi:`nox`. + +Deprecations and Removals +------------------------- + +- Deprecate alternative progress bar styles, leaving only ``on`` and ``off`` as available choices. (`#10462 `_) +- Drop support for Python 3.6. (`#10641 `_) +- Disable location mismatch warnings on Python versions prior to 3.10. + + These warnings were helping identify potential issues as part of the sysconfig -> distutils transition, and we no longer need to rely on reports from older Python versions for information on the transition. (`#10840 `_) + +Features +-------- + +- Changed ``PackageFinder`` to parse HTML documents using the stdlib :class:`html.parser.HTMLParser` class instead of the ``html5lib`` package. + + For now, the deprecated ``html5lib`` code remains and can be used with the ``--use-deprecated=html5lib`` command line option. However, it will be removed in a future pip release. (`#10291 `_) +- Utilise ``rich`` for presenting pip's default download progress bar. (`#10462 `_) +- Present a better error message when an invalid wheel file is encountered, providing more context where the invalid wheel file is. (`#10535 `_) +- Documents the ``--require-virtualenv`` flag for ``pip install``. (`#10588 `_) +- ``pip install `` autocompletes paths. (`#10646 `_) +- Allow Python distributors to opt-out from or opt-in to the ``sysconfig`` installation scheme backend by setting ``sysconfig._PIP_USE_SYSCONFIG`` to ``True`` or ``False``. (`#10647 `_) +- Make it possible to deselect tests requiring cryptography package on systems where it cannot be installed. (`#10686 `_) +- Start using Rich for presenting error messages in a consistent format. (`#10703 `_) +- Improve presentation of errors from subprocesses. (`#10705 `_) +- Forward pip's verbosity configuration to VCS tools to control their output accordingly. (`#8819 `_) + +Bug Fixes +--------- + +- Optimize installation order calculation to improve performance when installing requirements that form a complex dependency graph with a large amount of edges. (`#10557 `_) +- When a package is requested by the user for upgrade, correctly identify that the extra-ed variant of that same package depended by another user-requested package is requesting the same package, and upgrade it accordingly. (`#10613 `_) +- Prevent pip from installing yanked releases unless explicitly pinned via the ``==`` or ``===`` operators. (`#10617 `_) +- Stop backtracking on build failures, by instead surfacing them to the user and aborting immediately. This behaviour provides more immediate feedback when a package cannot be built due to missing build dependencies or platform incompatibility. (`#10655 `_) +- Silence ``Value for does not match`` warning caused by an erroneous patch in Slackware-distributed Python 3.9. (`#10668 `_) +- Fix an issue where pip did not consider dependencies with and without extras to be equal (`#9644 `_) + +Vendored Libraries +------------------ + +- Upgrade CacheControl to 0.12.10 +- Upgrade certifi to 2021.10.8 +- Upgrade distlib to 0.3.4 +- Upgrade idna to 3.3 +- Upgrade msgpack to 1.0.3 +- Upgrade packaging to 21.3 +- Upgrade platformdirs to 2.4.1 +- Add pygments 2.11.2 as a vendored dependency. +- Tree-trim unused portions of vendored pygments, to reduce the distribution size. +- Upgrade pyparsing to 3.0.7 +- Upgrade Requests to 2.27.1 +- Upgrade resolvelib to 0.8.1 +- Add rich 11.0.0 as a vendored dependency. +- Tree-trim unused portions of vendored rich, to reduce the distribution size. +- Add typing_extensions 4.0.1 as a vendored dependency. +- Upgrade urllib3 to 1.26.8 + + +21.3.1 (2021-10-22) +=================== + + +Bug Fixes +--------- + + +- Always refuse installing or building projects that have no ``pyproject.toml`` nor + ``setup.py``. (`#10531 `_) +- Tweak running-as-root detection, to check ``os.getuid`` if it exists, on Unix-y and non-Linux/non-MacOS machines. (`#10565 `_) +- When installing projects with a ``pyproject.toml`` in editable mode, and the build + backend does not support :pep:`660`, prepare metadata using + ``prepare_metadata_for_build_wheel`` instead of ``setup.py egg_info``. Also, refuse + installing projects that only have a ``setup.cfg`` and no ``setup.py`` nor + ``pyproject.toml``. These restore the pre-21.3 behaviour. (`#10573 `_) +- Restore compatibility of where configuration files are loaded from on MacOS (back to ``Library/Application Support/pip``, instead of ``Preferences/pip``). (`#10585 `_) + +Vendored Libraries +------------------ + + +- Upgrade pep517 to 0.12.0 + + +21.3 (2021-10-11) +================= + +Deprecations and Removals +------------------------- + +- Improve deprecation warning regarding the copying of source trees when installing from a local directory. (`#10128 `_) +- Suppress location mismatch warnings when pip is invoked from a Python source + tree, so ``ensurepip`` does not emit warnings on CPython ``make install``. (`#10270 `_) +- On Python 3.10 or later, the installation scheme backend has been changed to use + ``sysconfig``. This is to anticipate the deprecation of ``distutils`` in Python + 3.10, and its scheduled removal in 3.12. For compatibility considerations, pip + installations running on Python 3.9 or lower will continue to use ``distutils``. (`#10358 `_) +- Remove the ``--build-dir`` option and aliases, one last time. (`#10485 `_) +- In-tree builds are now the default. ``--use-feature=in-tree-build`` is now + ignored. ``--use-deprecated=out-of-tree-build`` may be used temporarily to ease + the transition. (`#10495 `_) +- Un-deprecate source distribution re-installation behaviour. (`#8711 `_) + +Features +-------- + +- Replace vendored appdirs with platformdirs. (`#10202 `_) +- Support `PEP 610 `_ to detect + editable installs in ``pip freeze`` and ``pip list``. The ``pip list`` column output + has a new ``Editable project location`` column, and the JSON output has a new + ``editable_project_location`` field. (`#10249 `_) +- ``pip freeze`` will now always fallback to reporting the editable project + location when it encounters a VCS error while analyzing an editable + requirement. Before, it sometimes reported the requirement as non-editable. (`#10410 `_) +- ``pip show`` now sorts ``Requires`` and ``Required-By`` alphabetically. (`#10422 `_) +- Do not raise error when there are no files to remove with ``pip cache purge/remove``. + Instead log a warning and continue (to log that we removed 0 files). (`#10459 `_) +- When backtracking during dependency resolution, prefer the dependencies which are involved in the most recent conflict. This can significantly reduce the amount of backtracking required. (`#10479 `_) +- Cache requirement objects, to improve performance reducing reparses of requirement strings. (`#10550 `_) +- Support editable installs for projects that have a ``pyproject.toml`` and use a + build backend that supports :pep:`660`. (`#8212 `_) +- When a revision is specified in a Git URL, use git's partial clone feature to speed up source retrieval. (`#9086 `_) +- Add a ``--debug`` flag, to enable a mode that doesn't log errors and propagates them to the top level instead. This is primarily to aid with debugging pip's crashes. (`#9349 `_) +- If a host is explicitly specified as trusted by the user (via the --trusted-host option), cache HTTP responses from it in addition to HTTPS ones. (`#9498 `_) + +Bug Fixes +--------- + +- Present a better error message, when a ``file:`` URL is not found. (`#10263 `_) +- Fix the auth credential cache to allow for the case in which + the index url contains the username, but the password comes + from an external source, such as keyring. (`#10269 `_) +- Fix double unescape of HTML ``data-requires-python`` and ``data-yanked`` attributes. (`#10378 `_) +- New resolver: Fixes depth ordering of packages during resolution, e.g. a dependency 2 levels deep will be ordered before a dependency 3 levels deep. (`#10482 `_) +- Correctly indent metadata preparation messages in pip output. (`#10524 `_) + +Vendored Libraries +------------------ + +- Remove appdirs as a vendored dependency. +- Upgrade distlib to 0.3.3 +- Upgrade distro to 1.6.0 +- Patch pkg_resources to use platformdirs rather than appdirs. +- Add platformdirs as a vendored dependency. +- Upgrade progress to 1.6 +- Upgrade resolvelib to 0.8.0 +- Upgrade urllib3 to 1.26.7 + +Improved Documentation +---------------------- + +- Update links of setuptools as setuptools moved these documents. The Simple Repository link now points to PyPUG as that is the canonical place of packaging specification, and setuptools's ``easy_install`` is deprecated. (`#10430 `_) +- Create a "Build System Interface" reference section, for documenting how pip interacts with build systems. (`#10497 `_) + + +21.2.4 (2021-08-12) +=================== + +Bug Fixes +--------- + +- Fix 3.6.0 compatibility in link comparison logic. (`#10280 `_) + + +21.2.3 (2021-08-06) +=================== + +Bug Fixes +--------- + +- Modify the ``sysconfig.get_preferred_scheme`` function check to be + compatible with CPython 3.10’s alphareleases. (`#10252 `_) + + +21.2.2 (2021-07-31) +=================== + +Bug Fixes +--------- + +- New resolver: When a package is specified with extras in constraints, and with + extras in non-constraint requirements, the resolver now correctly identifies the + constraint's existence and avoids backtracking. (`#10233 `_) + + +21.2.1 (2021-07-25) +=================== + +Process +------- + +- The source distribution re-installation feature removal has been delayed to 21.3. + + +21.2 (2021-07-24) +================= + +Process +------- + +- ``pip freeze``, ``pip list``, and ``pip show`` no longer normalize underscore + (``_``) in distribution names to dash (``-``). This is a side effect of the + migration to ``importlib.metadata``, since the underscore-dash normalization + behavior is non-standard and specific to setuptools. This should not affect + other parts of pip (for example, when feeding the ``pip freeze`` result back + into ``pip install``) since pip internally performs standard PEP 503 + normalization independently to setuptools. + +Deprecations and Removals +------------------------- + +- Git version parsing is now done with regular expression to prepare for the + pending upstream removal of non-PEP-440 version parsing logic. (`#10117 `_) +- Re-enable the "Value for ... does not match" location warnings to field a new + round of feedback for the ``distutils``-``sysconfig`` transition. (`#10151 `_) +- Remove deprecated ``--find-links`` option in ``pip freeze`` (`#9069 `_) + +Features +-------- + +- New resolver: Loosen URL comparison logic when checking for direct URL reference + equivalency. The logic includes the following notable characteristics: + + * The authentication part of the URL is explicitly ignored. + * Most of the fragment part, including ``egg=``, is explicitly ignored. Only + ``subdirectory=`` and hash values (e.g. ``sha256=``) are kept. + * The query part of the URL is parsed to allow ordering differences. (`#10002 `_) +- Support TOML v1.0.0 syntax in ``pyproject.toml``. (`#10034 `_) +- Added a warning message for errors caused due to Long Paths being disabled on Windows. (`#10045 `_) +- Change the encoding of log file from default text encoding to UTF-8. (`#10071 `_) +- Log the resolved commit SHA when installing a package from a Git repository. (`#10149 `_) +- Add a warning when passing an invalid requirement to ``pip uninstall``. (`#4958 `_) +- Add new subcommand ``pip index`` used to interact with indexes, and implement + ``pip index version`` to list available versions of a package. (`#7975 `_) +- When pip is asked to uninstall a project without the dist-info/RECORD file + it will no longer traceback with FileNotFoundError, + but it will provide a better error message instead, such as:: + + ERROR: Cannot uninstall foobar 0.1, RECORD file not found. You might be able to recover from this via: 'pip install --force-reinstall --no-deps foobar==0.1'. + + When dist-info/INSTALLER is present and contains some useful information, the info is included in the error message instead:: + + ERROR: Cannot uninstall foobar 0.1, RECORD file not found. Hint: The package was installed by rpm. + + (`#8954 `_) +- Add an additional level of verbosity. ``--verbose`` (and the shorthand ``-v``) now + contains significantly less output, and users that need complete full debug-level output + should pass it twice (``--verbose --verbose`` or ``-vv``). (`#9450 `_) +- New resolver: The order of dependencies resolution has been tweaked to traverse + the dependency graph in a more breadth-first approach. (`#9455 `_) +- Make "yes" the default choice in ``pip uninstall``'s prompt. (`#9686 `_) +- Add a special error message when users forget the ``-r`` flag when installing. (`#9915 `_) +- New resolver: A distribution's ``Requires-Python`` metadata is now checked + before its Python dependencies. This makes the resolver fail quicker when + there's an interpreter version conflict. (`#9925 `_) +- Suppress "not on PATH" warning when ``--prefix`` is given. (`#9931 `_) +- Include ``rustc`` version in pip's ``User-Agent``, when the system has ``rustc``. (`#9987 `_) + +Bug Fixes +--------- + +- Update vendored six to 1.16.0 and urllib3 to 1.26.5 (`#10043 `_) +- Correctly allow PEP 517 projects to be detected without warnings in ``pip freeze``. (`#10080 `_) +- Strip leading slash from a ``file://`` URL built from an path with the Windows + drive notation. This fixes bugs where the ``file://`` URL cannot be correctly + used as requirement, constraint, or index URLs on Windows. (`#10115 `_) +- New resolver: URL comparison logic now treats ``file://localhost/`` and + ``file:///`` as equivalent to conform to RFC 8089. (`#10162 `_) +- Prefer credentials from the URL over the previously-obtained credentials from URLs of the same domain, so it is possible to use different credentials on the same index server for different ``--extra-index-url`` options. (`#3931 `_) +- Fix extraction of files with utf-8 encoded paths from tars. (`#7667 `_) +- Skip distutils configuration parsing on encoding errors. (`#8931 `_) +- New resolver: Detect an unnamed requirement is user-specified (by building its + metadata for the project name) so it can be correctly ordered in the resolver. (`#9204 `_) +- Fix :ref:`pip freeze` to output packages :ref:`installed from git ` + in the correct ``git+protocol://git.example.com/MyProject#egg=MyProject`` format + rather than the old and no longer supported ``git+git@`` format. (`#9822 `_) +- Fix warnings about install scheme selection for Python framework builds + distributed by Apple's Command Line Tools. (`#9844 `_) +- Relax interpreter detection to quelch a location mismatch warning where PyPy + is deliberately breaking backwards compatibility. (`#9845 `_) + +Vendored Libraries +------------------ + +- Upgrade certifi to 2021.05.30. +- Upgrade idna to 3.2. +- Upgrade packaging to 21.0 +- Upgrade requests to 2.26.0. +- Upgrade resolvelib to 0.7.1. +- Upgrade urllib3 to 1.26.6. + + +21.1.3 (2021-06-26) +=================== + +Bug Fixes +--------- + +- Remove unused optional ``tornado`` import in vendored ``tenacity`` to prevent old versions of Tornado from breaking pip. (`#10020 `_) +- Require ``setup.cfg``-only projects to be built via PEP 517, by requiring an explicit dependency on setuptools declared in pyproject.toml. (`#10031 `_) + + +21.1.2 (2021-05-23) +=================== + +Bug Fixes +--------- + +- New resolver: Correctly exclude an already installed package if its version is + known to be incompatible to stop the dependency resolution process with a clear + error message. (`#9841 `_) +- Allow ZIP to archive files with timestamps earlier than 1980. (`#9910 `_) +- Emit clearer error message when a project root does not contain either + ``pyproject.toml``, ``setup.py`` or ``setup.cfg``. (`#9944 `_) +- Fix detection of existing standalone pip instance for PEP 517 builds. (`#9953 `_) + + +21.1.1 (2021-04-30) +=================== + +Deprecations and Removals +------------------------- + +- Temporarily set the new "Value for ... does not match" location warnings level + to *DEBUG*, to hide them from casual users. This prepares pip 21.1 for CPython + inclusion, while pip maintainers digest the first intake of location mismatch + issues for the ``distutils``-``sysconfig`` transition. (`#9912 `_) + +Bug Fixes +--------- + +- This change fixes a bug on Python <=3.6.1 with a Typing feature added in 3.6.2 (`#9831 `_) +- Fix compatibility between distutils and sysconfig when the project name is unknown outside of a virtual environment. (`#9838 `_) +- Fix Python 3.6 compatibility when a PEP 517 build requirement itself needs to be + built in an isolated environment. (`#9878 `_) + + +21.1 (2021-04-24) +================= + +Process +------- + +- Start installation scheme migration from ``distutils`` to ``sysconfig``. A + warning is implemented to detect differences between the two implementations to + encourage user reports, so we can avoid breakages before they happen. + +Features +-------- + +- Add the ability for the new resolver to process URL constraints. (`#8253 `_) +- Add a feature ``--use-feature=in-tree-build`` to build local projects in-place + when installing. This is expected to become the default behavior in pip 21.3; + see `Installing from local packages `_ + for more information. (`#9091 `_) +- Bring back the "(from versions: ...)" message, that was shown on resolution failures. (`#9139 `_) +- Add support for editable installs for project with only setup.cfg files. (`#9547 `_) +- Improve performance when picking the best file from indexes during ``pip install``. (`#9748 `_) +- Warn instead of erroring out when doing a PEP 517 build in presence of + ``--build-option``. Warn when doing a PEP 517 build in presence of + ``--global-option``. (`#9774 `_) + +Bug Fixes +--------- + +- Fixed ``--target`` to work with ``--editable`` installs. (`#4390 `_) +- Add a warning, discouraging the usage of pip as root, outside a virtual environment. (`#6409 `_) +- Ignore ``.dist-info`` directories if the stem is not a valid Python distribution + name, so they don't show up in e.g. ``pip freeze``. (`#7269 `_) +- Only query the keyring for URLs that actually trigger error 401. + This prevents an unnecessary keyring unlock prompt on every pip install + invocation (even with default index URL which is not password protected). (`#8090 `_) +- Prevent packages already-installed alongside with pip to be injected into an + isolated build environment during build-time dependency population. (`#8214 `_) +- Fix ``pip freeze`` permission denied error in order to display an understandable error message and offer solutions. (`#8418 `_) +- Correctly uninstall script files (from setuptools' ``scripts`` argument), when installed with ``--user``. (`#8733 `_) +- New resolver: When a requirement is requested both via a direct URL + (``req @ URL``) and via version specifier with extras (``req[extra]``), the + resolver will now be able to use the URL to correctly resolve the requirement + with extras. (`#8785 `_) +- New resolver: Show relevant entries from user-supplied constraint files in the + error message to improve debuggability. (`#9300 `_) +- Avoid parsing version to make the version check more robust against lousily + debundled downstream distributions. (`#9348 `_) +- ``--user`` is no longer suggested incorrectly when pip fails with a permission + error in a virtual environment. (`#9409 `_) +- Fix incorrect reporting on ``Requires-Python`` conflicts. (`#9541 `_) +- Make wheel compatibility tag preferences more important than the build tag (`#9565 `_) +- Fix pip to work with warnings converted to errors. (`#9779 `_) +- **SECURITY**: Stop splitting on unicode separators in git references, + which could be maliciously used to install a different revision on the + repository. (`#9827 `_) + +Vendored Libraries +------------------ + +- Update urllib3 to 1.26.4 to fix CVE-2021-28363 +- Remove contextlib2. +- Upgrade idna to 3.1 +- Upgrade pep517 to 0.10.0 +- Upgrade vendored resolvelib to 0.7.0. +- Upgrade tenacity to 7.0.0 + +Improved Documentation +---------------------- + +- Update "setuptools extras" link to match upstream. (`#4822829F-6A45-4202-87BA-A80482DF6D4E `_) +- Improve SSL Certificate Verification docs and ``--cert`` help text. (`#6720 `_) +- Add a section in the documentation to suggest solutions to the ``pip freeze`` permission denied issue. (`#8418 `_) +- Add warning about ``--extra-index-url`` and dependency confusion (`#9647 `_) +- Describe ``--upgrade-strategy`` and direct requirements explicitly; add a brief + example. (`#9692 `_) + + +21.0.1 (2021-01-30) +=================== + +Bug Fixes +--------- + +- commands: debug: Use packaging.version.parse to compare between versions. (`#9461 `_) +- New resolver: Download and prepare a distribution only at the last possible + moment to avoid unnecessary network access when the same version is already + installed locally. (`#9516 `_) + +Vendored Libraries +------------------ + +- Upgrade packaging to 20.9 + + +21.0 (2021-01-23) +================= + +Deprecations and Removals +------------------------- + +- Drop support for Python 2. (`#6148 `_) +- Remove support for legacy wheel cache entries that were created with pip + versions older than 20.0. (`#7502 `_) +- Remove support for VCS pseudo URLs editable requirements. It was emitting + deprecation warning since version 20.0. (`#7554 `_) +- Modernise the codebase after Python 2. (`#8802 `_) +- Drop support for Python 3.5. (`#9189 `_) +- Remove the VCS export feature that was used only with editable VCS + requirements and had correctness issues. (`#9338 `_) + +Features +-------- + +- Add ``--ignore-requires-python`` support to pip download. (`#1884 `_) +- New resolver: Error message shown when a wheel contains inconsistent metadata + is made more helpful by including both values from the file name and internal + metadata. (`#9186 `_) + +Bug Fixes +--------- + +- Fix a regression that made ``pip wheel`` do a VCS export instead of a VCS clone + for editable requirements. This broke VCS requirements that need the VCS + information to build correctly. (`#9273 `_) +- Fix ``pip download`` of editable VCS requirements that need VCS information + to build correctly. (`#9337 `_) + +Vendored Libraries +------------------ + +- Upgrade msgpack to 1.0.2. +- Upgrade requests to 2.25.1. + +Improved Documentation +---------------------- + +- Render the unreleased pip version change notes on the news page in docs. (`#9172 `_) +- Fix broken email link in docs feedback banners. (`#9343 `_) + + +20.3.4 (2021-01-23) +=================== + +Features +-------- + +- ``pip wheel`` now verifies the built wheel contains valid metadata, and can be + installed by a subsequent ``pip install``. This can be disabled with + ``--no-verify``. (`#9206 `_) +- Improve presentation of XMLRPC errors in pip search. (`#9315 `_) + +Bug Fixes +--------- + +- Fixed hanging VCS subprocess calls when the VCS outputs a large amount of data + on stderr. Restored logging of VCS errors that was inadvertently removed in pip + 20.2. (`#8876 `_) +- Fix error when an existing incompatibility is unable to be applied to a backtracked state. (`#9180 `_) +- New resolver: Discard a faulty distribution, instead of quitting outright. + This implementation is taken from 20.2.2, with a fix that always makes the + resolver iterate through candidates from indexes lazily, to avoid downloading + candidates we do not need. (`#9203 `_) +- New resolver: Discard a source distribution if it fails to generate metadata, + instead of quitting outright. This implementation is taken from 20.2.2, with a + fix that always makes the resolver iterate through candidates from indexes + lazily, to avoid downloading candidates we do not need. (`#9246 `_) + +Vendored Libraries +------------------ + +- Upgrade resolvelib to 0.5.4. + + 20.3.3 (2020-12-15) =================== @@ -142,7 +712,7 @@ Features - When installing a git URL that refers to a commit that is not available locally after git clone, attempt to fetch it from the remote. (`#8815 `_) - Include http subdirectory in ``pip cache info`` and ``pip cache purge`` commands. (`#8892 `_) -- Cache package listings on index packages so they are guarenteed to stay stable +- Cache package listings on index packages so they are guaranteed to stay stable during a pip command session. This also improves performance when a index page is accessed multiple times during the command session. (`#8905 `_) - New resolver: Tweak resolution logic to improve user experience when @@ -214,7 +784,7 @@ Features and considered good enough. (`#8023 `_) - Improve error message friendliness when an environment has packages with corrupted metadata. (`#8676 `_) -- Cache package listings on index packages so they are guarenteed to stay stable +- Cache package listings on index packages so they are guaranteed to stay stable during a pip command session. This also improves performance when a index page is accessed multiple times during the command session. (`#8905 `_) - New resolver: Tweak resolution logic to improve user experience when diff --git a/README.rst b/README.rst index a15de466b7f..6810315526b 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ We release updates regularly, with a new version every 3 months. Find more detai In pip 20.3, we've `made a big improvement to the heart of pip`_; `learn more`_. We want your input, so `sign up for our user experience research studies`_ to help us do it right. -**Note**: pip 21.0, in January 2021, will remove Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3. +**Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3. If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms: @@ -44,7 +44,7 @@ rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _package installer: https://packaging.python.org/guides/tool-recommendations/ .. _Python Package Index: https://pypi.org -.. _Installation: https://pip.pypa.io/en/stable/installing.html +.. _Installation: https://pip.pypa.io/en/stable/installation/ .. _Usage: https://pip.pypa.io/en/stable/ .. _Release notes: https://pip.pypa.io/en/stable/news.html .. _Release process: https://pip.pypa.io/en/latest/development/release-process/ @@ -57,6 +57,6 @@ rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _Issue tracking: https://github.com/pypa/pip/issues .. _Discourse channel: https://discuss.python.org/c/packaging .. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/ -.. _User IRC: https://webchat.freenode.net/?channels=%23pypa -.. _Development IRC: https://webchat.freenode.net/?channels=%23pypa-dev +.. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa +.. _Development IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev .. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/docs/docs_feedback_sphinxext.py b/docs/docs_feedback_sphinxext.py deleted file mode 100644 index a8ab94e5cbd..00000000000 --- a/docs/docs_feedback_sphinxext.py +++ /dev/null @@ -1,164 +0,0 @@ -"""A sphinx extension for collecting per doc feedback.""" - -from __future__ import annotations - -from itertools import chain -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import Dict, List, Union - - from sphinx.application import Sphinx - - -DEFAULT_DOC_LINES_THRESHOLD = 250 -RST_INDENT = 4 -EMAIL_INDENT = 6 - - -def _modify_rst_document_source_on_read( - app: Sphinx, - docname: str, - source: List[str], -) -> None: - """Add info block to top and bottom of each document source. - - This function modifies RST source in-place by adding an admonition - block at the top and the bottom of each document right after it's - been read from disk preserving :orphan: at top, if present. - """ - admonition_type = app.config.docs_feedback_admonition_type - big_doc_lines = app.config.docs_feedback_big_doc_lines - escaped_email = app.config.docs_feedback_email.replace(' ', r'\ ') - excluded_documents = set(app.config.docs_feedback_excluded_documents) - questions_list = app.config.docs_feedback_questions_list - - valid_admonitions = { - 'attention', 'caution', 'danger', 'error', 'hint', - 'important', 'note', 'tip', 'warning', 'admonition', - } - - if admonition_type not in valid_admonitions: - raise ValueError( - 'Expected `docs_feedback_admonition_type` to be one of ' - f'{valid_admonitions} but got {admonition_type}.' - ) - - if not questions_list: - raise ValueError( - 'Expected `docs_feedback_questions_list` to list questions ' - 'but got none.' - ) - - if docname in excluded_documents: - # NOTE: Completely ignore any document - # NOTE: listed in 'docs_feedback_excluded_documents'. - return - - is_doc_big = source[0].count('\n') >= big_doc_lines - - questions_list_rst = '\n'.join( - f'{" " * RST_INDENT}{number!s}. {question}' - for number, question in enumerate(questions_list, 1) - ) - questions_list_urlencoded = ( - '\n'.join( - f'\n{" " * RST_INDENT}{number!s}. {question} ' - for number, question in enumerate( - chain( - (f'Document: {docname}. Page URL: https://', ), - questions_list, - ), - ) - ). - rstrip('\r\n\t '). - replace('\r', '%0D'). - replace('\n', '%0A'). - replace(' ', '%20') - ) - - admonition_msg = rf""" - **Did this article help?** - - We are currently doing research to improve pip's documentation - and would love your feedback. - Please `email us`_ and let us know{{let_us_know_ending}} - -{{questions_list_rst}} - - .. _email us: - mailto:{escaped_email}\ - ?subject=[Doc:\ {docname}]\ Pip\ docs\ feedback\ \ - (URL\:\ https\://)\ - &body={questions_list_urlencoded} - """ - let_us_know_ending = ':' - - info_block_bottom = ( - f'.. {admonition_type}::\n\t\t{admonition_msg.format_map(locals())}\n' - ) - - questions_list_rst = '' - let_us_know_ending = ( - ' why you came to this page and what on it helped ' - 'you and what did not. ' - '(:issue:`Read more about this research <8517>`)' - ) - info_block_top = '' if is_doc_big else ( - f'.. {admonition_type}::\n\t\t{admonition_msg.format_map(locals())}\n' - ) - - orphan_mark = ':orphan:' - is_orphan = orphan_mark in source[0] - if is_orphan: - source[0] = source[0].replace(orphan_mark, '') - else: - orphan_mark = '' - - source[0] = '\n\n'.join(( - orphan_mark, info_block_top, source[0], info_block_bottom, - )) - - -def setup(app: Sphinx) -> Dict[str, Union[bool, str]]: - """Initialize the Sphinx extension. - - This function adds a callback for modifying the document sources - in-place on read. - - It also declares the extension settings changable via :file:`conf.py`. - """ - rebuild_trigger = 'html' # rebuild full html on settings change - app.add_config_value( - 'docs_feedback_admonition_type', - default='important', - rebuild=rebuild_trigger, - ) - app.add_config_value( - 'docs_feedback_big_doc_lines', - default=DEFAULT_DOC_LINES_THRESHOLD, - rebuild=rebuild_trigger, - ) - app.add_config_value( - 'docs_feedback_email', - default='Docs UX Team ', - rebuild=rebuild_trigger, - ) - app.add_config_value( - 'docs_feedback_excluded_documents', - default=set(), - rebuild=rebuild_trigger, - ) - app.add_config_value( - 'docs_feedback_questions_list', - default=(), - rebuild=rebuild_trigger, - ) - - app.connect('source-read', _modify_rst_document_source_on_read) - - return { - 'parallel_read_safe': True, - 'parallel_write_safe': True, - 'version': 'builtin', - } diff --git a/docs/html/cli/index.md b/docs/html/cli/index.md new file mode 100644 index 00000000000..a3497c308c2 --- /dev/null +++ b/docs/html/cli/index.md @@ -0,0 +1,48 @@ +# Commands + +The general options that apply to all the commands listed below can be +found [under the `pip` page in this section](pip). + +```{toctree} +:maxdepth: 1 +:hidden: + +pip +``` + +```{toctree} +:maxdepth: 1 +:caption: Environment Management and Introspection + +pip_install +pip_uninstall +pip_list +pip_show +pip_freeze +pip_check +``` + +```{toctree} +:maxdepth: 1 +:caption: Handling Distribution Files + +pip_download +pip_wheel +pip_hash +``` + +```{toctree} +:maxdepth: 1 +:caption: Package Index information + +pip_search +``` + +```{toctree} +:maxdepth: 1 +:caption: Managing pip itself + +pip_cache +pip_config +pip_debug +``` diff --git a/docs/html/cli/pip.rst b/docs/html/cli/pip.rst new file mode 100644 index 00000000000..2c42174eab5 --- /dev/null +++ b/docs/html/cli/pip.rst @@ -0,0 +1,88 @@ +=== +pip +=== + + +Usage +***** + +.. tab:: Unix/macOS + + .. code-block:: shell + + python -m pip [options] + +.. tab:: Windows + + .. code-block:: shell + + py -m pip [options] + +Description +*********** + + +.. _`Logging`: + + +Logging +======= + +Console logging +~~~~~~~~~~~~~~~ + +pip offers :ref:`-v, --verbose <--verbose>` and :ref:`-q, --quiet <--quiet>` +to control the console log level. By default, some messages (error and warnings) +are colored in the terminal. If you want to suppress the colored output use +:ref:`--no-color <--no-color>`. + + +.. _`FileLogging`: + +File logging +~~~~~~~~~~~~ + +pip offers the :ref:`--log <--log>` option for specifying a file where a maximum +verbosity log will be kept. This option is empty by default. This log appends +to previous logging. + +Like all pip options, ``--log`` can also be set as an environment variable, or +placed into the pip config file. See the :doc:`../topics/configuration` section. + +.. _`exists-action`: + +--exists-action option +====================== + +This option specifies default behavior when path already exists. +Possible cases: downloading files or checking out repositories for installation, +creating archives. If ``--exists-action`` is not defined, pip will prompt +when decision is needed. + +*(s)witch* + Only relevant to VCS checkout. Attempt to switch the checkout + to the appropriate URL and/or revision. +*(i)gnore* + Abort current operation (e.g. don't copy file, don't create archive, + don't modify a checkout). +*(w)ipe* + Delete the file or VCS checkout before trying to create, download, or checkout a new one. +*(b)ackup* + Rename the file or checkout to ``{name}{'.bak' * n}``, where n is some number + of ``.bak`` extensions, such that the file didn't exist at some point. + So the most recent backup will be the one with the largest number after ``.bak``. +*(a)bort* + Abort pip and return non-zero exit status. + + +Build System Interface +====================== + +This is now covered in :doc:`../reference/build-system/index`. + +.. _`General Options`: + +General Options +*************** + +.. pip-general-options:: diff --git a/docs/html/cli/pip_cache.rst b/docs/html/cli/pip_cache.rst new file mode 100644 index 00000000000..0a23c510d6f --- /dev/null +++ b/docs/html/cli/pip_cache.rst @@ -0,0 +1,27 @@ + +.. _`pip cache`: + +pip cache +--------- + + +Usage +***** + +.. tab:: Unix/macOS + + .. pip-command-usage:: cache "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: cache "py -m pip" + +Description +*********** + +.. pip-command-description:: cache + +Options +******* + +.. pip-command-options:: cache diff --git a/docs/html/cli/pip_check.rst b/docs/html/cli/pip_check.rst new file mode 100644 index 00000000000..268cf9a143c --- /dev/null +++ b/docs/html/cli/pip_check.rst @@ -0,0 +1,87 @@ +.. _`pip check`: + +========= +pip check +========= + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: check "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: check "py -m pip" + + +Description +=========== + +.. pip-command-description:: check + + +Examples +======== + +#. If all dependencies are compatible: + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip check + No broken requirements found. + $ echo $? + 0 + + .. tab:: Windows + + .. code-block:: console + + C:\> py -m pip check + No broken requirements found. + C:\> echo %errorlevel% + 0 + +#. If a package is missing: + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip check + pyramid 1.5.2 requires WebOb, which is not installed. + $ echo $? + 1 + + .. tab:: Windows + + .. code-block:: console + + C:\> py -m pip check + pyramid 1.5.2 requires WebOb, which is not installed. + C:\> echo %errorlevel% + 1 + +#. If a package has the wrong version: + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip check + pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. + $ echo $? + 1 + + .. tab:: Windows + + .. code-block:: console + + C:\> py -m pip check + pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. + C:\> echo %errorlevel% + 1 diff --git a/docs/html/cli/pip_config.rst b/docs/html/cli/pip_config.rst new file mode 100644 index 00000000000..8b2f846304f --- /dev/null +++ b/docs/html/cli/pip_config.rst @@ -0,0 +1,30 @@ + +.. _`pip config`: + +========== +pip config +========== + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: config "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: config "py -m pip" + + +Description +=========== + +.. pip-command-description:: config + + +Options +======= + +.. pip-command-options:: config diff --git a/docs/html/cli/pip_debug.rst b/docs/html/cli/pip_debug.rst new file mode 100644 index 00000000000..4023533c905 --- /dev/null +++ b/docs/html/cli/pip_debug.rst @@ -0,0 +1,35 @@ +.. _`pip debug`: + +========= +pip debug +========= + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: debug "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: debug "py -m pip" + + +.. warning:: + + This command is only meant for debugging. + Its options and outputs are provisional and may change without notice. + + +Description +=========== + +.. pip-command-description:: debug + + +Options +======= + +.. pip-command-options:: debug diff --git a/docs/html/cli/pip_download.rst b/docs/html/cli/pip_download.rst new file mode 100644 index 00000000000..4f15314d765 --- /dev/null +++ b/docs/html/cli/pip_download.rst @@ -0,0 +1,226 @@ + +.. _`pip download`: + +============ +pip download +============ + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: download "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: download "py -m pip" + + +Description +=========== + +.. pip-command-description:: download + +Overview +-------- + +``pip download`` does the same resolution and downloading as ``pip install``, +but instead of installing the dependencies, it collects the downloaded +distributions into the directory provided (defaulting to the current +directory). This directory can later be passed as the value to ``pip install +--find-links`` to facilitate offline or locked down package installation. + +``pip download`` with the ``--platform``, ``--python-version``, +``--implementation``, and ``--abi`` options provides the ability to fetch +dependencies for an interpreter and system other than the ones that pip is +running on. ``--only-binary=:all:`` or ``--no-deps`` is required when using any +of these options. It is important to note that these options all default to the +current system/interpreter, and not to the most restrictive constraints (e.g. +platform any, abi none, etc). To avoid fetching dependencies that happen to +match the constraint of the current interpreter (but not your target one), it +is recommended to specify all of these options if you are specifying one of +them. Generic dependencies (e.g. universal wheels, or dependencies with no +platform, abi, or implementation constraints) will still match an over- +constrained download requirement. + + + +Options +======= + +.. pip-command-options:: download + +.. pip-index-options:: download + + +Examples +======== + +#. Download a package and all of its dependencies + + .. tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download SomePackage + python -m pip download -d . SomePackage # equivalent to above + python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + + .. tab:: Windows + + .. code-block:: shell + + py -m pip download SomePackage + py -m pip download -d . SomePackage # equivalent to above + py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + + +#. Download a package and all of its dependencies with OSX specific interpreter constraints. + This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, + this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, + etc. + It will also match deps with platform ``any``. Also force the interpreter version to ``27`` + (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). + + .. tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage + + .. tab:: Windows + + .. code-block:: shell + + py -m pip download ^ + --only-binary=:all: ^ + --platform macosx-10_10_x86_64 ^ + --python-version 27 ^ + --implementation cp ^ + SomePackage + +#. Download a package and its dependencies with linux specific constraints. + Force the interpreter to be any minor version of py3k, and only accept + ``cp34m`` or ``none`` as the abi. + + .. tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage + + .. tab:: Windows + + .. code-block:: shell + + py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 3 ^ + --implementation cp ^ + --abi cp34m ^ + SomePackage + +#. Force platform, implementation, and abi agnostic deps. + + .. tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage + + .. tab:: Windows + + .. code-block:: shell + + py -m pip download ^ + --only-binary=:all: ^ + --platform any ^ + --python-version 3 ^ + --implementation py ^ + --abi none ^ + SomePackage + +#. Even when overconstrained, this will still correctly fetch the pip universal wheel. + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 33 \ + --implementation cp \ + --abi cp34m \ + pip>=8 + + .. code-block:: console + + $ ls pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl + + .. tab:: Windows + + .. code-block:: console + + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 33 ^ + --implementation cp ^ + --abi cp34m ^ + pip>=8 + + .. code-block:: console + + C:\> dir pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl + +#. Download a package supporting one of several ABIs and platforms. + This is useful when fetching wheels for a well-defined interpreter, whose + supported ABIs and platforms are known and fixed, different than the one pip is + running under. + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip download \ + --only-binary=:all: \ + --platform manylinux1_x86_64 --platform linux_x86_64 --platform any \ + --python-version 36 \ + --implementation cp \ + --abi cp36m --abi cp36 --abi abi3 --abi none \ + SomePackage + + .. tab:: Windows + + .. code-block:: console + + C:> py -m pip download ^ + --only-binary=:all: ^ + --platform manylinux1_x86_64 --platform linux_x86_64 --platform any ^ + --python-version 36 ^ + --implementation cp ^ + --abi cp36m --abi cp36 --abi abi3 --abi none ^ + SomePackage diff --git a/docs/html/cli/pip_freeze.rst b/docs/html/cli/pip_freeze.rst new file mode 100644 index 00000000000..3533db7930c --- /dev/null +++ b/docs/html/cli/pip_freeze.rst @@ -0,0 +1,92 @@ + +.. _`pip freeze`: + +========== +pip freeze +========== + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: freeze "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: freeze "py -m pip" + + +Description +=========== + +.. pip-command-description:: freeze + + +Options +======= + +.. pip-command-options:: freeze + + +Examples +======== + +#. Generate output suitable for a requirements file. + + .. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip freeze + docutils==0.11 + Jinja2==2.7.2 + MarkupSafe==0.19 + Pygments==1.6 + Sphinx==1.2.2 + + .. tab:: Windows + + .. code-block:: console + + C:\> py -m pip freeze + docutils==0.11 + Jinja2==2.7.2 + MarkupSafe==0.19 + Pygments==1.6 + Sphinx==1.2.2 + +#. Generate a requirements file and then install from it in another environment. + + .. tab:: Unix/macOS + + .. code-block:: shell + + env1/bin/python -m pip freeze > requirements.txt + env2/bin/python -m pip install -r requirements.txt + + .. tab:: Windows + + .. code-block:: shell + + env1\bin\python -m pip freeze > requirements.txt + env2\bin\python -m pip install -r requirements.txt + + +Fixing "Permission denied:" errors +================================== + +The purpose of this section of documentation is to provide practical +suggestions to users seeing a `"Permission denied" error `__ on ``pip freeze``. + +This error occurs, for instance, when the command is installed only for another +user, and the current user doesn't have the permission to execute the other +user's command. + +To solve that issue, you can try one of the followings: + +- Install the command for yourself (e.g. in your home directory). +- Ask the system admin to allow this command for all users. +- Check and correct the PATH variable of your own environment. +- Check the `ACL (Access-Control List) `_ for this command. diff --git a/docs/html/cli/pip_hash.rst b/docs/html/cli/pip_hash.rst new file mode 100644 index 00000000000..7df0d5a4f13 --- /dev/null +++ b/docs/html/cli/pip_hash.rst @@ -0,0 +1,72 @@ +.. _`pip hash`: + +======== +pip hash +======== + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: hash "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: hash "py -m pip" + + +Description +=========== + +.. pip-command-description:: hash + +Overview +-------- + +``pip hash`` is a convenient way to get a hash digest for use with +:ref:`hash-checking mode`, especially for packages with multiple archives. The +error message from ``pip install --require-hashes ...`` will give you one +hash, but, if there are multiple archives (like source and binary ones), you +will need to manually download and compute a hash for the others. Otherwise, a +spurious hash mismatch could occur when :ref:`pip install` is passed a +different set of options, like :ref:`--no-binary `. + + +Options +======= + +.. pip-command-options:: hash + + +Example +======= + +Compute the hash of a downloaded archive: + +.. tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + $ python -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + +.. tab:: Windows + + .. code-block:: console + + C:\> py -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + C:\> py -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst new file mode 100644 index 00000000000..cfff4f7e270 --- /dev/null +++ b/docs/html/cli/pip_install.rst @@ -0,0 +1,550 @@ +.. _`pip install`: + +=========== +pip install +=========== + + + +Usage +===== + +.. tab:: Unix/macOS + + .. pip-command-usage:: install "python -m pip" + +.. tab:: Windows + + .. pip-command-usage:: install "py -m pip" + + + +Description +=========== + +.. pip-command-description:: install + +Overview +-------- + +pip install has several stages: + +1. Identify the base requirements. The user supplied arguments are processed + here. +2. Resolve dependencies. What will be installed is determined here. +3. Build wheels. All the dependencies that can be are built into wheels. +4. Install the packages (and uninstall anything being upgraded/replaced). + +Note that ``pip install`` prefers to leave the installed version as-is +unless ``--upgrade`` is specified. + +Argument Handling +----------------- + +When looking at the items to be installed, pip checks what type of item +each is, in the following order: + +1. Project or archive URL. +2. Local directory (which must contain a ``setup.py``, or pip will report + an error). +3. Local file (a sdist or wheel format archive, following the naming + conventions for those formats). +4. A requirement, as specified in :pep:`440`. + +Each item identified is added to the set of requirements to be satisfied by +the install. + +Working Out the Name and Version +-------------------------------- + +For each candidate item, pip needs to know the project name and version. For +wheels (identified by the ``.whl`` file extension) this can be obtained from +the filename, as per the Wheel spec. For local directories, or explicitly +specified sdist files, the ``setup.py egg_info`` command is used to determine +the project metadata. For sdists located via an index, the filename is parsed +for the name and project version (this is in theory slightly less reliable +than using the ``egg_info`` command, but avoids downloading and processing +unnecessary numbers of files). + +Any URL may use the ``#egg=name`` syntax (see :doc:`../topics/vcs-support`) to +explicitly state the project name. + +Satisfying Requirements +----------------------- + +Once pip has the set of requirements to satisfy, it chooses which version of +each requirement to install using the simple rule that the latest version that +satisfies the given constraints will be installed (but see :ref:`here
`
+for an exception regarding pre-release versions). Where more than one source of
+the chosen version is available, it is assumed that any source is acceptable
+(as otherwise the versions would differ).
+
+Installation Order
+------------------
+
+.. note::
+
+   This section is only about installation order of runtime dependencies, and
+   does not apply to build dependencies (those are specified using PEP 518).
+
+As of v6.1.0, pip installs dependencies before their dependents, i.e. in
+"topological order."  This is the only commitment pip currently makes related
+to order.  While it may be coincidentally true that pip will install things in
+the order of the install arguments or in the order of the items in a
+requirements file, this is not a promise.
+
+In the event of a dependency cycle (aka "circular dependency"), the current
+implementation (which might possibly change later) has it such that the first
+encountered member of the cycle is installed last.
+
+For instance, if quux depends on foo which depends on bar which depends on baz,
+which depends on foo:
+
+.. tab:: Unix/macOS
+
+   .. code-block:: console
+
+      $ python -m pip install quux
+      ...
+      Installing collected packages baz, bar, foo, quux
+
+      $ python -m pip install bar
+      ...
+      Installing collected packages foo, baz, bar
+
+.. tab:: Windows
+
+   .. code-block:: console
+
+      C:\> py -m pip install quux
+      ...
+      Installing collected packages baz, bar, foo, quux
+
+      C:\> py -m pip install bar
+      ...
+      Installing collected packages foo, baz, bar
+
+
+Prior to v6.1.0, pip made no commitments about install order.
+
+The decision to install topologically is based on the principle that
+installations should proceed in a way that leaves the environment usable at each
+step. This has two main practical benefits:
+
+1. Concurrent use of the environment during the install is more likely to work.
+2. A failed install is less likely to leave a broken environment.  Although pip
+   would like to support failure rollbacks eventually, in the mean time, this is
+   an improvement.
+
+Although the new install order is not intended to replace (and does not replace)
+the use of ``setup_requires`` to declare build dependencies, it may help certain
+projects install from sdist (that might previously fail) that fit the following
+profile:
+
+1. They have build dependencies that are also declared as install dependencies
+   using ``install_requires``.
+2. ``python setup.py egg_info`` works without their build dependencies being
+   installed.
+3. For whatever reason, they don't or won't declare their build dependencies using
+   ``setup_requires``.
+
+
+Requirements File Format
+------------------------
+
+This section has been moved to :doc:`../reference/requirements-file-format`.
+
+Requirement Specifiers
+----------------------
+
+This section has been moved to :doc:`../reference/requirement-specifiers`.
+
+Per-requirement Overrides
+-------------------------
+
+This is now covered in :doc:`../reference/requirements-file-format`.
+
+.. _`Pre Release Versions`:
+
+Pre-release Versions
+--------------------
+
+Starting with v1.4, pip will only install stable versions as specified by
+`pre-releases`_ by default. If a version cannot be parsed as a compliant :pep:`440`
+version then it is assumed to be a pre-release.
+
+If a Requirement specifier includes a pre-release or development version
+(e.g. ``>=0.0.dev0``) then pip will allow pre-release and development versions
+for that requirement. This does not include the != flag.
+
+The ``pip install`` command also supports a :ref:`--pre ` flag
+that enables installation of pre-releases and development releases.
+
+
+.. _pre-releases: https://www.python.org/dev/peps/pep-0440/#handling-of-pre-releases
+
+
+.. _`VCS Support`:
+
+VCS Support
+-----------
+
+This is now covered in :doc:`../topics/vcs-support`.
+
+Finding Packages
+----------------
+
+pip searches for packages on `PyPI`_ using the
+`HTTP simple interface `_,
+which is documented `here `_
+and `there `_.
+
+pip offers a number of package index options for modifying how packages are
+found.
+
+pip looks for packages in a number of places: on PyPI (if not disabled via
+``--no-index``), in the local filesystem, and in any additional repositories
+specified via ``--find-links`` or ``--index-url``. There is no ordering in
+the locations that are searched. Rather they are all checked, and the "best"
+match for the requirements (in terms of version number - see :pep:`440` for
+details) is selected.
+
+See the :ref:`pip install Examples`.
+
+
+.. _`SSL Certificate Verification`:
+
+SSL Certificate Verification
+----------------------------
+
+Starting with v1.3, pip provides SSL certificate verification over HTTP, to
+prevent man-in-the-middle attacks against PyPI downloads. This does not use
+the system certificate store but instead uses a bundled CA certificate
+store. The default bundled CA certificate store certificate store may be
+overridden by using ``--cert`` option or by using ``PIP_CERT``,
+``REQUESTS_CA_BUNDLE``, or ``CURL_CA_BUNDLE`` environment variables.
+
+
+.. _`Caching`:
+
+Caching
+-------
+
+This is now covered in :doc:`../topics/caching`.
+
+.. _`Wheel cache`:
+
+Wheel Cache
+^^^^^^^^^^^
+
+This is now covered in :doc:`../topics/caching`.
+
+Hash checking mode
+------------------
+
+This is now covered in :doc:`../topics/secure-installs`.
+
+Local Project Installs
+----------------------
+
+This is now covered in :doc:`../topics/local-project-installs`.
+
+Editable installs
+-----------------
+
+This is now covered in :doc:`../topics/local-project-installs`.
+
+Build System Interface
+----------------------
+
+This is now covered in :doc:`../reference/build-system/index`.
+
+
+.. _`pip install Options`:
+
+
+Options
+=======
+
+.. pip-command-options:: install
+
+.. pip-index-options:: install
+
+
+.. _`pip install Examples`:
+
+
+Examples
+========
+
+#. Install ``SomePackage`` and its dependencies from `PyPI`_ using :ref:`Requirement Specifiers`
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install SomePackage            # latest version
+         python -m pip install SomePackage==1.0.4     # specific version
+         python -m pip install 'SomePackage>=1.0.4'   # minimum version
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install SomePackage            # latest version
+         py -m pip install SomePackage==1.0.4     # specific version
+         py -m pip install 'SomePackage>=1.0.4'   # minimum version
+
+
+#. Install a list of requirements specified in a file.  See the :ref:`Requirements files `.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install -r requirements.txt
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install -r requirements.txt
+
+
+#. Upgrade an already installed ``SomePackage`` to the latest from PyPI.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install --upgrade SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install --upgrade SomePackage
+
+    .. note::
+
+      This will guarantee an update to ``SomePackage`` as it is a direct
+      requirement, and possibly upgrade dependencies if their installed
+      versions do not meet the minimum requirements of ``SomePackage``.
+      Any non-requisite updates of its dependencies (indirect requirements)
+      will be affected by the ``--upgrade-strategy`` command.
+
+#. Install a local project in "editable" mode. See the section on :ref:`Editable Installs `.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install -e .                # project in current directory
+         python -m pip install -e path/to/project  # project in another directory
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install -e .                 # project in current directory
+         py -m pip install -e path/to/project   # project in another directory
+
+
+#. Install a project from VCS
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1
+
+
+#. Install a project from VCS in "editable" mode. See the sections on :doc:`../topics/vcs-support` and :ref:`Editable Installs `.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage          # from git
+         python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage            # from mercurial
+         python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage         # from svn
+         python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage  # from 'feature' branch
+         python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage          # from git
+         py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage            # from mercurial
+         py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage         # from svn
+         py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage  # from 'feature' branch
+         py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory
+
+#. Install a package with `extras`_.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install SomePackage[PDF]
+         python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@main#subdirectory=subdir_path"
+         python -m pip install .[PDF]  # project in current directory
+         python -m pip install SomePackage[PDF]==3.0
+         python -m pip install SomePackage[PDF,EPUB]  # multiple extras
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install SomePackage[PDF]
+         py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@main#subdirectory=subdir_path"
+         py -m pip install .[PDF]  # project in current directory
+         py -m pip install SomePackage[PDF]==3.0
+         py -m pip install SomePackage[PDF,EPUB]  # multiple extras
+
+#. Install a particular source archive file.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install ./downloads/SomePackage-1.0.4.tar.gz
+         python -m pip install http://my.package.repo/SomePackage-1.0.4.zip
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install ./downloads/SomePackage-1.0.4.tar.gz
+         py -m pip install http://my.package.repo/SomePackage-1.0.4.zip
+
+#. Install a particular source archive file following :pep:`440` direct references.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl
+         python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl"
+         python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl
+         py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl"
+         py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz
+
+#. Install from alternative package repositories.
+
+   Install from a different index, and not `PyPI`_
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install --index-url http://my.package.repo/simple/ SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install --index-url http://my.package.repo/simple/ SomePackage
+
+   Install from a local flat directory containing archives (and don't scan indexes):
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install --no-index --find-links=file:///local/dir/ SomePackage
+         python -m pip install --no-index --find-links=/local/dir/ SomePackage
+         python -m pip install --no-index --find-links=relative/dir/ SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install --no-index --find-links=file:///local/dir/ SomePackage
+         py -m pip install --no-index --find-links=/local/dir/ SomePackage
+         py -m pip install --no-index --find-links=relative/dir/ SomePackage
+
+   Search an additional index during install, in addition to `PyPI`_
+
+   .. warning::
+
+       Using this option to search for packages which are not in the main
+       repository (such as private packages) is unsafe, per a security
+       vulnerability called
+       `dependency confusion `_:
+       an attacker can claim the package on the public repository in a way that
+       will ensure it gets chosen over the private package.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install --extra-index-url http://my.package.repo/simple SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install --extra-index-url http://my.package.repo/simple SomePackage
+
+
+#. Find pre-release and development versions, in addition to stable versions.  By default, pip only finds stable versions.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install --pre SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install --pre SomePackage
+
+
+#. Install packages from source.
+
+   Do not use any binary packages
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install SomePackage1 SomePackage2 --no-binary :all:
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install SomePackage1 SomePackage2 --no-binary :all:
+
+   Specify ``SomePackage1`` to be installed from source:
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1
+
+.. _extras: https://www.python.org/dev/peps/pep-0508/#extras
+.. _PyPI: https://pypi.org/
diff --git a/docs/html/cli/pip_list.rst b/docs/html/cli/pip_list.rst
new file mode 100644
index 00000000000..739435981c3
--- /dev/null
+++ b/docs/html/cli/pip_list.rst
@@ -0,0 +1,231 @@
+.. _`pip list`:
+
+========
+pip list
+========
+
+
+
+Usage
+=====
+
+.. tab:: Unix/macOS
+
+   .. pip-command-usage:: list "python -m pip"
+
+.. tab:: Windows
+
+   .. pip-command-usage:: list "py -m pip"
+
+
+Description
+===========
+
+.. pip-command-description:: list
+
+
+Options
+=======
+
+.. pip-command-options:: list
+
+.. pip-index-options:: list
+
+
+Examples
+========
+
+#. List installed packages (with the default column formatting).
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list
+         Package Version
+         ------- -------
+         docopt  0.6.2
+         idlex   1.13
+         jedi    0.9.0
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list
+         Package Version
+         ------- -------
+         docopt  0.6.2
+         idlex   1.13
+         jedi    0.9.0
+
+#. List outdated packages with column formatting.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list --outdated --format columns
+         Package    Version Latest Type
+         ---------- ------- ------ -----
+         retry      0.8.1   0.9.1  wheel
+         setuptools 20.6.7  21.0.0 wheel
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list --outdated --format columns
+         Package    Version Latest Type
+         ---------- ------- ------ -----
+         retry      0.8.1   0.9.1  wheel
+         setuptools 20.6.7  21.0.0 wheel
+
+#. List packages that are not dependencies of other packages. Can be combined with
+   other options.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list --outdated --not-required
+         Package  Version Latest Type
+         -------- ------- ------ -----
+         docutils 0.14    0.17.1 wheel
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list --outdated --not-required
+         Package  Version Latest Type
+         -------- ------- ------ -----
+         docutils 0.14    0.17.1 wheel
+
+#. Use json formatting
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list --format=json
+         [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ...
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list --format=json
+         [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ...
+
+#. Use freeze formatting
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list --format=freeze
+         colorama==0.3.7
+         docopt==0.6.2
+         idlex==1.13
+         jedi==0.9.0
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list --format=freeze
+         colorama==0.3.7
+         docopt==0.6.2
+         idlex==1.13
+         jedi==0.9.0
+
+#. List packages installed in editable mode
+
+When some packages are installed in editable mode, ``pip list`` outputs an
+additional column that shows the directory where the editable project is
+located (i.e. the directory that contains the ``pyproject.toml`` or
+``setup.py`` file).
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list
+         Package          Version  Editable project location
+         ---------------- -------- -------------------------------------
+         pip              21.2.4
+         pip-test-package 0.1.1    /home/you/.venv/src/pip-test-package
+         setuptools       57.4.0
+         wheel            0.36.2
+
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list
+         Package          Version  Editable project location
+         ---------------- -------- ----------------------------------------
+         pip              21.2.4
+         pip-test-package 0.1.1    C:\Users\You\.venv\src\pip-test-package
+         setuptools       57.4.0
+         wheel            0.36.2
+
+The json format outputs an additional ``editable_project_location`` field.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip list --format=json | python -m json.tool
+         [
+           {
+             "name": "pip",
+             "version": "21.2.4",
+           },
+           {
+             "name": "pip-test-package",
+             "version": "0.1.1",
+             "editable_project_location": "/home/you/.venv/src/pip-test-package"
+           },
+           {
+             "name": "setuptools",
+             "version": "57.4.0"
+           },
+           {
+             "name": "wheel",
+             "version": "0.36.2"
+           }
+         ]
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip list --format=json | py -m json.tool
+         [
+           {
+             "name": "pip",
+             "version": "21.2.4",
+           },
+           {
+             "name": "pip-test-package",
+             "version": "0.1.1",
+             "editable_project_location": "C:\Users\You\.venv\src\pip-test-package"
+           },
+           {
+             "name": "setuptools",
+             "version": "57.4.0"
+           },
+           {
+             "name": "wheel",
+             "version": "0.36.2"
+           }
+         ]
+
+.. note::
+
+   Contrary to the ``freeze``  command, ``pip list --format=freeze`` will not
+   report editable install information, but the version of the package at the
+   time it was installed.
diff --git a/docs/html/cli/pip_search.rst b/docs/html/cli/pip_search.rst
new file mode 100644
index 00000000000..9905a1bafac
--- /dev/null
+++ b/docs/html/cli/pip_search.rst
@@ -0,0 +1,52 @@
+.. _`pip search`:
+
+==========
+pip search
+==========
+
+
+
+Usage
+=====
+
+.. tab:: Unix/macOS
+
+   .. pip-command-usage:: search "python -m pip"
+
+.. tab:: Windows
+
+   .. pip-command-usage:: search "py -m pip"
+
+
+Description
+===========
+
+.. pip-command-description:: search
+
+
+Options
+=======
+
+.. pip-command-options:: search
+
+
+Examples
+========
+
+#. Search for "peppercorn"
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip search peppercorn
+         pepperedform    - Helpers for using peppercorn with formprocess.
+         peppercorn      - A library for converting a token stream into [...]
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip search peppercorn
+         pepperedform    - Helpers for using peppercorn with formprocess.
+         peppercorn      - A library for converting a token stream into [...]
diff --git a/docs/html/cli/pip_show.rst b/docs/html/cli/pip_show.rst
new file mode 100644
index 00000000000..b603f786fd9
--- /dev/null
+++ b/docs/html/cli/pip_show.rst
@@ -0,0 +1,154 @@
+.. _`pip show`:
+
+========
+pip show
+========
+
+
+
+Usage
+=====
+
+.. tab:: Unix/macOS
+
+   .. pip-command-usage:: show "python -m pip"
+
+.. tab:: Windows
+
+   .. pip-command-usage:: show "py -m pip"
+
+
+Description
+===========
+
+.. pip-command-description:: show
+
+
+Options
+=======
+
+.. pip-command-options:: show
+
+
+Examples
+========
+
+#. Show information about a package:
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip show sphinx
+         Name: Sphinx
+         Version: 1.4.5
+         Summary: Python documentation generator
+         Home-page: http://sphinx-doc.org/
+         Author: Georg Brandl
+         Author-email: georg@python.org
+         License: BSD
+         Location: /my/env/lib/python2.7/site-packages
+         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip show sphinx
+         Name: Sphinx
+         Version: 1.4.5
+         Summary: Python documentation generator
+         Home-page: http://sphinx-doc.org/
+         Author: Georg Brandl
+         Author-email: georg@python.org
+         License: BSD
+         Location: /my/env/lib/python2.7/site-packages
+         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
+
+#. Show all information about a package
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip show --verbose sphinx
+         Name: Sphinx
+         Version: 1.4.5
+         Summary: Python documentation generator
+         Home-page: http://sphinx-doc.org/
+         Author: Georg Brandl
+         Author-email: georg@python.org
+         License: BSD
+         Location: /my/env/lib/python2.7/site-packages
+         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
+         Metadata-Version: 2.0
+         Installer:
+         Classifiers:
+            Development Status :: 5 - Production/Stable
+            Environment :: Console
+            Environment :: Web Environment
+            Intended Audience :: Developers
+            Intended Audience :: Education
+            License :: OSI Approved :: BSD License
+            Operating System :: OS Independent
+            Programming Language :: Python
+            Programming Language :: Python :: 2
+            Programming Language :: Python :: 3
+            Framework :: Sphinx
+            Framework :: Sphinx :: Extension
+            Framework :: Sphinx :: Theme
+            Topic :: Documentation
+            Topic :: Documentation :: Sphinx
+            Topic :: Text Processing
+            Topic :: Utilities
+         Entry-points:
+            [console_scripts]
+            sphinx-apidoc = sphinx.apidoc:main
+            sphinx-autogen = sphinx.ext.autosummary.generate:main
+            sphinx-build = sphinx:main
+            sphinx-quickstart = sphinx.quickstart:main
+            [distutils.commands]
+            build_sphinx = sphinx.setup_command:BuildDoc
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip show --verbose sphinx
+         Name: Sphinx
+         Version: 1.4.5
+         Summary: Python documentation generator
+         Home-page: http://sphinx-doc.org/
+         Author: Georg Brandl
+         Author-email: georg@python.org
+         License: BSD
+         Location: /my/env/lib/python2.7/site-packages
+         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
+         Metadata-Version: 2.0
+         Installer:
+         Classifiers:
+            Development Status :: 5 - Production/Stable
+            Environment :: Console
+            Environment :: Web Environment
+            Intended Audience :: Developers
+            Intended Audience :: Education
+            License :: OSI Approved :: BSD License
+            Operating System :: OS Independent
+            Programming Language :: Python
+            Programming Language :: Python :: 2
+            Programming Language :: Python :: 3
+            Framework :: Sphinx
+            Framework :: Sphinx :: Extension
+            Framework :: Sphinx :: Theme
+            Topic :: Documentation
+            Topic :: Documentation :: Sphinx
+            Topic :: Text Processing
+            Topic :: Utilities
+         Entry-points:
+            [console_scripts]
+            sphinx-apidoc = sphinx.apidoc:main
+            sphinx-autogen = sphinx.ext.autosummary.generate:main
+            sphinx-build = sphinx:main
+            sphinx-quickstart = sphinx.quickstart:main
+            [distutils.commands]
+            build_sphinx = sphinx.setup_command:BuildDoc
diff --git a/docs/html/cli/pip_uninstall.rst b/docs/html/cli/pip_uninstall.rst
new file mode 100644
index 00000000000..0dd52619d80
--- /dev/null
+++ b/docs/html/cli/pip_uninstall.rst
@@ -0,0 +1,58 @@
+.. _`pip uninstall`:
+
+=============
+pip uninstall
+=============
+
+
+
+Usage
+=====
+
+.. tab:: Unix/macOS
+
+   .. pip-command-usage:: uninstall "python -m pip"
+
+.. tab:: Windows
+
+   .. pip-command-usage:: uninstall "py -m pip"
+
+
+Description
+===========
+
+.. pip-command-description:: uninstall
+
+
+Options
+=======
+
+.. pip-command-options:: uninstall
+
+
+Examples
+========
+
+#. Uninstall a package.
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: console
+
+         $ python -m pip uninstall simplejson
+         Uninstalling simplejson:
+            /home/me/env/lib/python3.9/site-packages/simplejson
+            /home/me/env/lib/python3.9/site-packages/simplejson-2.2.1-py3.9.egg-info
+         Proceed (Y/n)? y
+            Successfully uninstalled simplejson
+
+   .. tab:: Windows
+
+      .. code-block:: console
+
+         C:\> py -m pip uninstall simplejson
+         Uninstalling simplejson:
+            /home/me/env/lib/python3.9/site-packages/simplejson
+            /home/me/env/lib/python3.9/site-packages/simplejson-2.2.1-py3.9.egg-info
+         Proceed (Y/n)? y
+            Successfully uninstalled simplejson
diff --git a/docs/html/cli/pip_wheel.rst b/docs/html/cli/pip_wheel.rst
new file mode 100644
index 00000000000..bf371f28506
--- /dev/null
+++ b/docs/html/cli/pip_wheel.rst
@@ -0,0 +1,72 @@
+
+.. _`pip wheel`:
+
+=========
+pip wheel
+=========
+
+
+
+Usage
+=====
+
+.. tab:: Unix/macOS
+
+   .. pip-command-usage:: wheel "python -m pip"
+
+.. tab:: Windows
+
+   .. pip-command-usage:: wheel "py -m pip"
+
+
+Description
+===========
+
+.. pip-command-description:: wheel
+
+
+Build System Interface
+----------------------
+
+This is now covered in :doc:`../reference/build-system/index`.
+
+Options
+=======
+
+.. pip-command-options:: wheel
+
+.. pip-index-options:: wheel
+
+
+Examples
+========
+
+#. Build wheels for a requirement (and all its dependencies), and then install
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage
+         python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage
+         py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage
+
+#. Build a wheel for a package from source
+
+   .. tab:: Unix/macOS
+
+      .. code-block:: shell
+
+         python -m pip wheel --no-binary SomePackage SomePackage
+
+   .. tab:: Windows
+
+      .. code-block:: shell
+
+         py -m pip wheel --no-binary SomePackage SomePackage
diff --git a/docs/html/conf.py b/docs/html/conf.py
index f81cb6b7dac..cc967e0ba3c 100644
--- a/docs/html/conf.py
+++ b/docs/html/conf.py
@@ -1,325 +1,133 @@
-# pip documentation build configuration file, created by
-# sphinx-quickstart on Tue Apr 22 22:08:49 2008
-#
-# This file is execfile()d with the current directory set to its containing dir
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
+"""Sphinx configuration file for pip's documentation."""
 
 import glob
 import os
 import pathlib
 import re
 import sys
+from typing import List, Tuple
 
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
+# Add the docs/ directory to sys.path, because pip_sphinxext.py is there.
 docs_dir = os.path.dirname(os.path.dirname(__file__))
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, docs_dir)
-# sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
 
-# -- General configuration ----------------------------------------------------
+# -- General configuration ------------------------------------------------------------
 
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-# extensions = ['sphinx.ext.autodoc']
 extensions = [
-    # native:
-    'sphinx.ext.extlinks',
-    'sphinx.ext.intersphinx',
-    # third-party:
-    'sphinx_inline_tabs',
-    'sphinxcontrib.towncrier',
-    # in-tree:
-    'docs_feedback_sphinxext',
-    'pip_sphinxext',
+    # first-party extensions
+    "sphinx.ext.autodoc",
+    "sphinx.ext.todo",
+    "sphinx.ext.extlinks",
+    "sphinx.ext.intersphinx",
+    # our extensions
+    "pip_sphinxext",
+    # third-party extensions
+    "myst_parser",
+    "sphinx_copybutton",
+    "sphinx_inline_tabs",
+    "sphinxcontrib.towncrier",
 ]
 
-# intersphinx
-intersphinx_cache_limit = 0
-intersphinx_mapping = {
-    'pypug': ('https://packaging.python.org/', None),
-    'pypa': ('https://www.pypa.io/en/latest/', None),
-}
-
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = []
-
-# The suffix of source filenames.
-source_suffix = '.rst'
-
-# The encoding of source files.
-# source_encoding = 'utf-8'
-
-# The master toctree document.
-master_doc = 'index'
-
 # General information about the project.
-project = 'pip'
-copyright = '2008-2020, PyPA'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-
-version = release = 'dev'
-
-# Readthedocs seems to install pip as an egg (via setup.py install) which
-# is somehow resulting in "import pip" picking up an older copy of pip.
-# Rather than trying to force RTD to install pip properly, we'll simply
-# read the version direct from the __init__.py file. (Yes, this is
-# fragile, but it works...)
-
-pip_init = os.path.join(docs_dir, '..', 'src', 'pip', '__init__.py')
-with open(pip_init) as f:
+project = "pip"
+copyright = "The pip developers"
+
+# Find the version and release information.
+# We have a single source of truth for our version number: pip's __init__.py file.
+# This next bit of code reads from it.
+file_with_version = os.path.join(docs_dir, "..", "src", "pip", "__init__.py")
+with open(file_with_version) as f:
     for line in f:
         m = re.match(r'__version__ = "(.*)"', line)
         if m:
             __version__ = m.group(1)
             # The short X.Y version.
-            version = '.'.join(__version__.split('.')[:2])
+            version = ".".join(__version__.split(".")[:2])
             # The full version, including alpha/beta/rc tags.
             release = __version__
             break
+    else:  # AKA no-break
+        version = release = "dev"
 
-# We have this here because readthedocs plays tricks sometimes and there seems
-# to be a heisenbug, related to the version of pip discovered. This is here to
-# help debug that if someone decides to do that in the future.
 print("pip version:", version)
 print("pip release:", release)
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-# language = None
+# -- Options for myst-parser ----------------------------------------------------------
 
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-# today = ''
-# Else, today_fmt is used as the format for a strftime call.
-today_fmt = '%B %d, %Y'
+myst_enable_extensions = ["deflist"]
+myst_heading_anchors = 3
 
-# List of documents that shouldn't be included in the build.
-# unused_docs = []
+# -- Options for smartquotes ----------------------------------------------------------
 
-# List of directories, relative to source directory, that shouldn't be searched
-# for source files.
-exclude_patterns = ['build/']
-
-# The reST default role (used for this markup: `text`) to use for all documents
-# default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-# add_function_parentheses = True
+# Disable the conversion of dashes so that long options like "--find-links" won't
+# render as "-find-links" if included in the text.The default of "qDe" converts normal
+# quote characters ('"' and "'"), en and em dashes ("--" and "---"), and ellipses "..."
+smartquotes_action = "qe"
 
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-# add_module_names = True
+# -- Options for intersphinx ----------------------------------------------------------
 
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-# show_authors = False
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "pypug": ("https://packaging.python.org", None),
+}
 
-# A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
+# -- Options for extlinks -------------------------------------------------------------
 
 extlinks = {
-    'issue': ('https://github.com/pypa/pip/issues/%s', '#'),
-    'pull': ('https://github.com/pypa/pip/pull/%s', 'PR #'),
-    'pypi': ('https://pypi.org/project/%s/', ''),
+    "issue": ("https://github.com/pypa/pip/issues/%s", "#"),
+    "pull": ("https://github.com/pypa/pip/pull/%s", "PR #"),
+    "pypi": ("https://pypi.org/project/%s/", ""),
 }
 
-# Turn off sphinx build warnings because of sphinx tabs during man pages build
-sphinx_tabs_nowarn = True
-
-# -- Options for HTML output --------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  Major themes that come with
-# Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = "furo"
+# -- Options for towncrier_draft extension --------------------------------------------
 
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-html_theme_options = {}
+towncrier_draft_autoversion_mode = "draft"  # or: 'sphinx-release', 'sphinx-version'
+towncrier_draft_include_empty = True
+towncrier_draft_working_directory = pathlib.Path(docs_dir).parent
+# Not yet supported: towncrier_draft_config_path = 'pyproject.toml'  # relative to cwd
 
-# Add any paths that contain custom themes here, relative to this directory.
+# -- Options for HTML -----------------------------------------------------------------
 
-# The name for this set of Sphinx documents.  If None, it defaults to
-# " v documentation".
+html_theme = "furo"
 html_title = f"{project} documentation v{release}"
 
-# A shorter title for the navigation bar.  Default is the same as html_title.
-# html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-# html_logo = '_static/piplogo.png'
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-# html_favicon = 'favicon.png'
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = []
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%b %d, %Y'
-
-# If true, the Docutils Smart Quotes transform (originally based on
-# SmartyPants) will be used to convert characters like quotes and dashes
-# to typographically correct entities.  The default is True.
-smartquotes = True
-
-# This string, for use with Docutils 0.14 or later, customizes the
-# SmartQuotes transform. The default of "qDe" converts normal quote
-# characters ('"' and "'"), en and em dashes ("--" and "---"), and
-# ellipses "...".
-#    For now, we disable the conversion of dashes so that long options
-# like "--find-links" won't render as "-find-links" if included in the
-# text in places where monospaced type can't be used. For example, backticks
-# can't be used inside roles like :ref:`--no-index <--no-index>` because
-# of nesting.
-smartquotes_action = "qe"
-
-# Custom sidebar templates, maps document names to template names.
-html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-# html_additional_pages = {}
-
-# If false, no module index is generated.
+# Disable the generation of the various indexes
 html_use_modindex = False
-
-# If false, no index is generated.
 html_use_index = False
 
-# If true, the index is split into individual pages for each letter.
-# html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-html_show_sourcelink = False
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a  tag referring to it.  The value of this option must be the
-# base URL from which the finished HTML is served.
-# html_use_opensearch = ''
-
-# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = ''
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'pipdocs'
-
-
-# -- Options for LaTeX output -------------------------------------------------
-
-# The paper size ('letter' or 'a4').
-# latex_paper_size = 'letter'
-
-# The font size ('10pt', '11pt' or '12pt').
-# latex_font_size = '10pt'
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual])
-latex_documents = [
-    (
-        'index',
-        'pip.tex',
-        'pip Documentation',
-        'pip developers',
-        'manual',
-    ),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-# latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-# latex_use_parts = False
-
-# Additional stuff for the LaTeX preamble.
-# latex_preamble = ''
+# -- Options for Manual Pages ---------------------------------------------------------
 
-# Documents to append as an appendix to all manuals.
-# latex_appendices = []
-
-# If false, no module index is generated.
-# latex_use_modindex = True
-
-# -- Options for Manual Pages -------------------------------------------------
 
 # List of manual pages generated
-man_pages = [
-    (
-        'index',
-        'pip',
-        'package manager for Python packages',
-        'pip developers',
-        1
-    )
-]
-
-
-def to_document_name(path, base_dir):
-    """Convert a provided path to a Sphinx "document name".
-    """
-    relative_path = os.path.relpath(path, base_dir)
-    root, _ = os.path.splitext(relative_path)
-    return root.replace(os.sep, '/')
-
-
-# Here, we crawl the entire man/commands/ directory and list every file with
-# appropriate name and details
-man_dir = os.path.join(docs_dir, 'man')
-raw_subcommands = glob.glob(os.path.join(man_dir, 'commands/*.rst'))
-if not raw_subcommands:
-    raise FileNotFoundError(
-        'The individual subcommand manpages could not be found!'
-    )
-for fname in raw_subcommands:
-    fname_base = to_document_name(fname, man_dir)
-    outname = 'pip-' + fname_base.split('/')[1]
-    description = 'description of {} command'.format(
-        outname.replace('-', ' ')
-    )
-
-    man_pages.append((fname_base, outname, description, 'pip developers', 1))
-
-# -- Options for docs_feedback_sphinxext --------------------------------------
-
-# NOTE: Must be one of 'attention', 'caution', 'danger', 'error', 'hint',
-# NOTE: 'important', 'note', 'tip', 'warning' or 'admonition'.
-docs_feedback_admonition_type = 'important'
-docs_feedback_big_doc_lines = 50  # bigger docs will have a banner on top
-docs_feedback_email = 'Docs UX Team '
-docs_feedback_excluded_documents = {  # these won't have any banners
-    'news', 'reference/index',
-}
-docs_feedback_questions_list = (
-    'What problem were you trying to solve when you came to this page?',
-    'What content was useful?',
-    'What content was not useful?',
-)
-
-# -- Options for towncrier_draft extension -----------------------------------
-
-towncrier_draft_autoversion_mode = 'draft'  # or: 'sphinx-release', 'sphinx-version'
-towncrier_draft_include_empty = True
-towncrier_draft_working_directory = pathlib.Path(docs_dir).parent
-# Not yet supported: towncrier_draft_config_path = 'pyproject.toml'  # relative to cwd
+def determine_man_pages() -> List[Tuple[str, str, str, str, int]]:
+    """Determine which man pages need to be generated."""
+
+    def to_document_name(path: str, base_dir: str) -> str:
+        """Convert a provided path to a Sphinx "document name"."""
+        relative_path = os.path.relpath(path, base_dir)
+        root, _ = os.path.splitext(relative_path)
+        return root.replace(os.sep, "/")
+
+    # Crawl the entire man/commands/ directory and list every file with appropriate
+    # name and details.
+    man_dir = os.path.join(docs_dir, "man")
+    raw_subcommands = glob.glob(os.path.join(man_dir, "commands/*.rst"))
+    if not raw_subcommands:
+        raise FileNotFoundError(
+            "The individual subcommand manpages could not be found!"
+        )
+
+    retval = [
+        ("index", "pip", "package manager for Python packages", "pip developers", 1),
+    ]
+    for fname in raw_subcommands:
+        fname_base = to_document_name(fname, man_dir)
+        outname = "pip-" + fname_base.split("/")[1]
+        description = "description of {} command".format(outname.replace("-", " "))
+
+        retval.append((fname_base, outname, description, "pip developers", 1))
+
+    return retval
+
+
+man_pages = determine_man_pages()
diff --git a/docs/html/cookbook.rst b/docs/html/cookbook.rst
deleted file mode 100644
index efd76af150b..00000000000
--- a/docs/html/cookbook.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-:orphan:
-
-========
-Cookbook
-========
-
-This content is now covered in the :doc:`User Guide `
diff --git a/docs/html/copyright.rst b/docs/html/copyright.rst
index fd0212f53ec..0e2ede5ee05 100644
--- a/docs/html/copyright.rst
+++ b/docs/html/copyright.rst
@@ -6,4 +6,4 @@ Copyright
 
 pip and this documentation is:
 
-Copyright © 2008-2020 The pip developers (see `AUTHORS.txt `_ file). All rights reserved.
+Copyright © 2008-2020 The pip developers (see `AUTHORS.txt `_ file). All rights reserved.
diff --git a/docs/html/development/architecture/anatomy.rst b/docs/html/development/architecture/anatomy.rst
index 46bba448944..2f1ab0fe8bb 100644
--- a/docs/html/development/architecture/anatomy.rst
+++ b/docs/html/development/architecture/anatomy.rst
@@ -30,7 +30,6 @@ The ``README``, license, ``pyproject.toml``, ``setup.py``, and so on are in the
 * ``.gitignore``
 * ``.mailmap``
 * ``.readthedocs.yml``
-* ``.travis.yml``
 * ``docs/`` *[documentation, built with Sphinx]*
 
   * ``html/`` *[sources to HTML documentation avail. online]*
@@ -47,14 +46,12 @@ The ``README``, license, ``pyproject.toml``, ``setup.py``, and so on are in the
 
   * ``__init__.py``
   * ``conftest.py``
-  * ``data/`` *[test data for running tests -- pesudo package index in it!  Lots of small packages that are invalid or are valid. Test fixtures.  Used by functional tests]*
+  * ``data/`` *[test data for running tests -- pseudo package index in it!  Lots of small packages that are invalid or are valid. Test fixtures.  Used by functional tests]*
   * ``functional/`` *[functional tests of pip’s CLI -- end-to-end, invoke pip in subprocess & check results of execution against desired result. This also is what makes test suite slow]*
   * ``lib/`` *[helpers for tests]*
   * ``unit/`` *[unit tests -- fast and small and nice!]*
-  * ``yaml/`` *[resolver tests! They’re written in YAML. This folder just contains .yaml files -- actual code for reading/running them is in lib/yaml.py . This is fine!]*
 
-* ``tools`` *[misc development workflow tools, like requirements files & Travis CI files & helpers for tox]*
-* ``.azure-pipelines``
+* ``tools`` *[misc development workflow tools, like requirements files & CI files & helpers for tox]*
 * ``.github``
 * ``.tox``
 
@@ -105,5 +102,5 @@ Within ``src/``:
 
 .. _`tracking issue`: https://github.com/pypa/pip/issues/6831
 .. _GitHub repository: https://github.com/pypa/pip/
-.. _tox.ini: https://github.com/pypa/pip/blob/master/tox.ini
+.. _tox.ini: https://github.com/pypa/pip/blob/main/tox.ini
 .. _improving the pip dependency resolver: https://github.com/pypa/pip/issues/988
diff --git a/docs/html/development/architecture/package-finding.rst b/docs/html/development/architecture/package-finding.rst
index 18545275de2..0b64d420d93 100644
--- a/docs/html/development/architecture/package-finding.rst
+++ b/docs/html/development/architecture/package-finding.rst
@@ -101,7 +101,7 @@ One of ``PackageFinder``'s main top-level methods is
 1. Calls its ``find_all_candidates()`` method, which gathers all
    possible package links by reading and parsing the index URL's and
    locations provided by the user (the :ref:`LinkCollector
-   ` class's ``collect_links()`` method), constructs a
+   ` class's ``collect_sources()`` method), constructs a
    :ref:`LinkEvaluator ` object to filter out some of
    those links, and then returns a list of ``InstallationCandidates`` (aka
    candidates for install). This corresponds to steps 1-3 of the
@@ -131,7 +131,7 @@ responsible for collecting the raw list of "links" to package files
 The ``LinkCollector`` class takes into account the user's :ref:`--find-links
 `, :ref:`--extra-index-url `,
 and related options when deciding which locations to collect links from. The
-class's main method is the ``collect_links()`` method. The :ref:`PackageFinder
+class's main method is the ``collect_sources()`` method. The :ref:`PackageFinder
 ` class invokes this method as the first step of its
 ``find_all_candidates()`` method.
 
diff --git a/docs/html/development/architecture/upgrade-options.rst b/docs/html/development/architecture/upgrade-options.rst
index 6196413ef93..76c7d1fc00c 100644
--- a/docs/html/development/architecture/upgrade-options.rst
+++ b/docs/html/development/architecture/upgrade-options.rst
@@ -30,7 +30,8 @@ candidate.
 ``--upgrade-strategy``
 
 This option affects which packages are allowed to be installed. It is only
-relevant if ``--upgrade`` is specified. The base behaviour is to allow
+relevant if ``--upgrade`` is specified (except for the ``to-satisfy-only``
+option mentioned below). The base behaviour is to allow
 packages specified on pip's command line to be upgraded. This option controls
 what *other* packages can be upgraded:
 
@@ -43,9 +44,15 @@ what *other* packages can be upgraded:
   pip command or a requirement file (i.e, they are direct requirements), or
   an upgraded parent needs a later version of the dependency than is
   currently installed.
-* ``to-satisfy-only`` (**undocumented**) - packages are not upgraded (not
-  even direct requirements) unless the currently installed version fails to
-  satisfy a requirement (either explicitly specified or a dependency).
+* ``to-satisfy-only`` (**undocumented, please avoid**) - packages are not
+  upgraded (not even direct requirements) unless the currently installed
+  version fails to satisfy a requirement (either explicitly specified or a
+  dependency).
+
+  * This is actually the "default" upgrade strategy when ``--upgrade`` is
+    *not set*, i.e. ``pip install AlreadyInstalled`` and
+    ``pip install --upgrade --upgrade-strategy=to-satisfy-only AlreadyInstalled``
+    yield the same behavior.
 
 ``--force-reinstall``
 
diff --git a/docs/html/development/ci.rst b/docs/html/development/ci.rst
index 5c33231b1b2..ac65f816594 100644
--- a/docs/html/development/ci.rst
+++ b/docs/html/development/ci.rst
@@ -1,7 +1,8 @@
 .. note::
 
-    This section of the documentation is currently being written. pip
-    developers welcome your help to complete this documentation. If
+    This section of the documentation is currently out of date.
+
+    pip developers welcome your help to update this documentation. If
     you're interested in helping out, please let us know in the
     `tracking issue`_, or just submit a pull request and mention it in
     that tracking issue.
@@ -17,16 +18,17 @@ Supported interpreters
 
 pip support a variety of Python interpreters:
 
-- CPython 3.6
 - CPython 3.7
 - CPython 3.8
+- CPython 3.9
+- CPython 3.10
 - Latest PyPy3
 
 on different operating systems:
 
 - Linux
 - Windows
-- MacOS
+- macOS
 
 and on different architectures:
 
@@ -60,15 +62,9 @@ interpreters.
 Services
 ========
 
-pip test suite and checks are distributed on three different platforms that
-provides free executors for open source packages:
-
-- `GitHub Actions`_ (Used for code quality and development tasks)
-- `Azure DevOps CI`_ (Used for tests)
-- `Travis CI`_ (Used for PyPy tests)
+pip test suite and checks are distributed on `GitHub Actions`_ which provides
+free executors for open source packages.
 
-.. _`Travis CI`: https://travis-ci.org/
-.. _`Azure DevOps CI`: https://azure.microsoft.com/en-us/services/devops/
 .. _`GitHub Actions`: https://github.com/features/actions
 
 
@@ -81,9 +77,9 @@ Developer tasks
 ======== =============== ================ ================== =============
    OS          docs            lint           vendoring        packaging
 ======== =============== ================ ================== =============
-Linux     Travis, Github  Travis, Github    Travis, Github       Azure
-Windows       Github           Github           Github           Azure
-MacOS         Github           Github           Github           Azure
+Linux         GitHub           GitHub           GitHub           GitHub
+Windows       GitHub           GitHub           GitHub           GitHub
+macOS         GitHub           GitHub           GitHub           GitHub
 ======== =============== ================ ================== =============
 
 Actual testing
@@ -92,52 +88,61 @@ Actual testing
 +------------------------------+---------------+-----------------+
 |       **interpreter**        |   **unit**    | **integration** |
 +-----------+----------+-------+---------------+-----------------+
+|           |   x86    | CP3.7 |               |                 |
 |           |          +-------+---------------+-----------------+
-|           |          | CP3.6 |   Azure       |                 |
+|           |          | CP3.8 |               |                 |
 |           |          +-------+---------------+-----------------+
-|           |   x86    | CP3.7 |   Azure       |                 |
+|           |          | CP3.9 |               |                 |
 |           |          +-------+---------------+-----------------+
-|           |          | CP3.8 |   Azure       |                 |
+|           |          | CP3.10|               |                 |
 |           |          +-------+---------------+-----------------+
 |           |          | PyPy3 |               |                 |
 |  Windows  +----------+-------+---------------+-----------------+
-|           |          | CP3.6 |   Azure       |                 |
+|           |   x64    | CP3.7 |   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |   x64    | CP3.7 |   Azure       |                 |
+|           |          | CP3.8 |               |                 |
+|           |          +-------+---------------+-----------------+
+|           |          | CP3.9 |               |                 |
 |           |          +-------+---------------+-----------------+
-|           |          | CP3.8 |   Azure       |   Azure         |
+|           |          | CP3.10|   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
 |           |          | PyPy3 |               |                 |
 +-----------+----------+-------+---------------+-----------------+
-|           |          | CP3.6 |               |                 |
-|           |          +-------+---------------+-----------------+
 |           |   x86    | CP3.7 |               |                 |
 |           |          +-------+---------------+-----------------+
 |           |          | CP3.8 |               |                 |
 |           |          +-------+---------------+-----------------+
+|           |          | CP3.9 |               |                 |
+|           |          +-------+---------------+-----------------+
 |           |          | PyPy3 |               |                 |
 |   Linux   +----------+-------+---------------+-----------------+
-|           |          | CP3.6 |   Azure       |   Azure         |
+|           |   x64    | CP3.7 |   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |   x64    | CP3.7 |   Azure       |   Azure         |
+|           |          | CP3.8 |   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |          | CP3.8 |   Azure       |   Azure         |
+|           |          | CP3.9 |   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |          | PyPy3 |   Travis      |   Travis        |
-+-----------+----------+-------+---------------+-----------------+
-|           |          | CP3.6 |               |                 |
+|           |          | CP3.10|   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |   x86    | CP3.7 |               |                 |
+|           |          | PyPy3 |               |                 |
++-----------+----------+-------+---------------+-----------------+
+|           |  arm64   | CP3.7 |               |                 |
 |           |          +-------+---------------+-----------------+
 |           |          | CP3.8 |               |                 |
 |           |          +-------+---------------+-----------------+
+|           |          | CP3.9 |               |                 |
+|           |          +-------+---------------+-----------------+
+|           |          | CP3.10|               |                 |
+|           |          +-------+---------------+-----------------+
 |           |          | PyPy3 |               |                 |
-|   MacOS   +----------+-------+---------------+-----------------+
-|           |          | CP3.6 |   Azure       |   Azure         |
+|   macOS   +----------+-------+---------------+-----------------+
+|           |   x64    | CP3.7 |   GitHub      |   GitHub        |
+|           |          +-------+---------------+-----------------+
+|           |          | CP3.8 |   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |   x64    | CP3.7 |   Azure       |   Azure         |
+|           |          | CP3.9 |   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
-|           |          | CP3.8 |   Azure       |   Azure         |
+|           |          | CP3.10|   GitHub      |   GitHub        |
 |           |          +-------+---------------+-----------------+
 |           |          | PyPy3 |               |                 |
 +-----------+----------+-------+---------------+-----------------+
diff --git a/docs/html/development/configuration.rst b/docs/html/development/configuration.rst
deleted file mode 100644
index 8615065aa6b..00000000000
--- a/docs/html/development/configuration.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-:orphan:
-
-=============
-Configuration
-=============
-
-This content is now covered in the :ref:`Configuration` section of the :doc:`User Guide `.
diff --git a/docs/html/development/contributing.rst b/docs/html/development/contributing.rst
index 63eb4c33ee7..e443bfa57cd 100644
--- a/docs/html/development/contributing.rst
+++ b/docs/html/development/contributing.rst
@@ -11,7 +11,7 @@ We have an in-progress guide to the
 Submitting Pull Requests
 ========================
 
-Submit pull requests against the ``master`` branch, providing a good
+Submit pull requests against the ``main`` branch, providing a good
 description of what you're doing and why. You must have legal permission to
 distribute any code you contribute to pip and it must be available under the
 MIT License.
@@ -39,9 +39,8 @@ separately, as a "formatting cleanup" PR, if needed.
 Automated Testing
 =================
 
-All pull requests and merges to 'master' branch are tested using `Travis CI`_,
-`Azure Pipelines`_ and `GitHub Actions`_ based on our `.travis.yml`_,
-`.azure-pipelines`_ and `.github/workflows`_ files. More details about pip's
+All pull requests and merges to 'main' branch are tested using `GitHub
+Actions`_ based on our `.github/workflows`_ files. More details about pip's
 Continuous Integration can be found in the `CI Documentation`_
 
 
@@ -131,8 +130,8 @@ updating deprecation policy, etc.
 Updating your branch
 ====================
 
-As you work, you might need to update your local master branch up-to-date with
-the ``master`` branch in the main pip repository, which moves forward as the
+As you work, you might need to update your local main branch up-to-date with
+the ``main`` branch in the main pip repository, which moves forward as the
 maintainers merge pull requests. Most people working on the project use the
 following workflow.
 
@@ -160,24 +159,24 @@ First, fetch the latest changes from the main pip repository, ``upstream``:
 
     git fetch upstream
 
-Then, check out your local ``master`` branch, and rebase the changes on top of
+Then, check out your local ``main`` branch, and rebase the changes on top of
 it:
 
 .. code-block:: console
 
-    git checkout master
-    git rebase upstream/master
+    git checkout main
+    git rebase upstream/main
 
 At this point, you might have to `resolve merge conflicts`_. Once this is done,
-push the updates you have just made to your local ``master`` branch to your
+push the updates you have just made to your local ``main`` branch to your
 ``origin`` repository on GitHub:
 
 .. code-block:: console
 
-    git checkout master
-    git push origin master
+    git checkout main
+    git push origin main
 
-Now your local ``master`` branch and the ``master`` branch in your ``origin``
+Now your local ``main`` branch and the ``main`` branch in your ``origin``
 repo have been updated with the most recent changes from the main pip
 repository.
 
@@ -187,10 +186,10 @@ To keep your branches updated, the process is similar:
 
     git checkout awesome-feature
     git fetch upstream
-    git rebase upstream/master
+    git rebase upstream/main
 
 Now your branch has been updated with the latest changes from the
-``master`` branch on the upstream pip repository.
+``main`` branch on the upstream pip repository.
 
 It's good practice to back up your branches by pushing them to your
 ``origin`` on GitHub as you are working on them. To push a branch,
@@ -230,7 +229,7 @@ If you get an error message like this:
 
 Try force-pushing your branch with ``push -f``.
 
-The ``master`` branch in the main pip repository gets updated frequently, so
+The ``main`` branch in the main pip repository gets updated frequently, so
 you might have to update your branch at least once while you are working on it.
 
 Thank you for your contribution!
@@ -264,12 +263,8 @@ will initiate a vote among the existing maintainers.
 
 .. _`Studies have shown`: https://www.kessler.de/prd/smartbear/BestPracticesForPeerCodeReview.pdf
 .. _`resolve merge conflicts`: https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line
-.. _`Travis CI`: https://travis-ci.org/
-.. _`Azure Pipelines`: https://azure.microsoft.com/en-in/services/devops/pipelines/
 .. _`GitHub Actions`: https://github.com/features/actions
-.. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml
-.. _`.azure-pipelines`: https://github.com/pypa/pip/blob/master/.azure-pipelines
-.. _`.github/workflows`: https://github.com/pypa/pip/blob/master/.github/workflows
+.. _`.github/workflows`: https://github.com/pypa/pip/blob/main/.github/workflows
 .. _`CI Documentation`: https://pip.pypa.io/en/latest/development/ci/
 .. _`towncrier`: https://pypi.org/project/towncrier/
 .. _`Testing the next-gen pip dependency resolver`: https://pradyunsg.me/blog/2020/03/27/pip-resolver-testing/
diff --git a/docs/html/development/getting-started.rst b/docs/html/development/getting-started.rst
index 1d625613b73..730f5ece08f 100644
--- a/docs/html/development/getting-started.rst
+++ b/docs/html/development/getting-started.rst
@@ -27,8 +27,8 @@ Development Environment
 pip is a command line application written in Python. For developing pip,
 you should `install Python`_ on your computer.
 
-For developing pip, you need to install :pypi:`tox`. Often, you can run
-``python -m pip install tox`` to install and use it.
+For developing pip, you need to install :pypi:`nox`. Often, you can run
+``python -m pip install nox`` to install and use it.
 
 
 Running pip From Source Tree
@@ -42,8 +42,8 @@ You can then invoke your local source tree pip normally.
 
     .. code-block:: shell
 
-        virtualenv venv # You can also use "python -m venv venv" from python3.3+
-        source venv/bin/activate
+        python -m venv .venv
+        source .venv/bin/activate
         python -m pip install -e .
         python -m pip --version
 
@@ -51,17 +51,17 @@ You can then invoke your local source tree pip normally.
 
     .. code-block:: shell
 
-        virtualenv venv # You can also use "py -m venv venv" from python3.3+
-        venv\Scripts\activate
+        py -m venv .venv
+        .venv\Scripts\activate
         py -m pip install -e .
         py -m pip --version
 
 Running Tests
 =============
 
-pip's tests are written using the :pypi:`pytest` test framework, :pypi:`mock`
-and :pypi:`pretend`. :pypi:`tox` is used to automate the setup and execution of
-pip's tests.
+pip's tests are written using the :pypi:`pytest` test framework and
+:mod:`unittest.mock`. :pypi:`nox` is used to automate the setup and execution
+of pip's tests.
 
 It is preferable to run the tests in parallel for better experience during development,
 since the tests can take a long time to finish when run sequentially.
@@ -70,38 +70,39 @@ To run tests:
 
 .. code-block:: console
 
-    $ tox -e py36 -- -n auto
+    $ nox -s test-3.10 -- -n auto
 
 To run tests without parallelization, run:
 
 .. code-block:: console
 
-    $ tox -e py36
+    $ nox -s test-3.10
 
-The example above runs tests against Python 3.6. You can also use other
-versions like ``py39`` and ``pypy3``.
+The example above runs tests against Python 3.10. You can also use other
+versions like ``3.9`` and ``pypy3``.
 
-``tox`` has been configured to forward any additional arguments it is given to
+``nox`` has been configured to forward any additional arguments it is given to
 ``pytest``. This enables the use of pytest's `rich CLI`_. As an example, you
 can select tests using the various ways that pytest provides:
 
 .. code-block:: console
 
     $ # Using file name
-    $ tox -e py36 -- tests/functional/test_install.py
+    $ nox -s test-3.10 -- tests/functional/test_install.py
     $ # Using markers
-    $ tox -e py36 -- -m unit
+    $ nox -s test-3.10 -- -m unit
     $ # Using keywords
-    $ tox -e py36 -- -k "install and not wheel"
+    $ nox -s test-3.10 -- -k "install and not wheel"
 
-Running pip's test suite requires supported version control tools (subversion,
-bazaar, git, and mercurial) to be installed. If you are missing one of the VCS
-tools, you can tell pip to skip those tests:
+Running pip's entire test suite requires supported version control tools
+(subversion, bazaar, git, and mercurial) to be installed. If you are missing
+any of these VCS, those tests should be skipped automatically. You can also
+explicitly tell pytest to skip those tests:
 
 .. code-block:: console
 
-    $ tox -e py36 -- -k "not svn"
-    $ tox -e py36 -- -k "not (svn or git)"
+    $ nox -s test-3.10 -- -k "not svn"
+    $ nox -s test-3.10 -- -k "not (svn or git)"
 
 
 Running Linters
@@ -115,7 +116,7 @@ To use linters locally, run:
 
 .. code-block:: console
 
-    $ tox -e lint
+    $ nox -s lint
 
 .. note::
 
@@ -125,6 +126,25 @@ To use linters locally, run:
     readability problems.
 
 
+Running pip under a debugger
+============================
+
+In order to debug pip's behavior, you can run it under a debugger like so:
+
+.. code-block:: console
+
+    $ python -m pdb -m pip --debug ...
+
+
+Replace the ``...`` with arguments you'd like to run pip with. Give PDB the
+``c`` ("continue") command afterwards, to run the process.
+
+The ``--debug`` flag disables pip's exception handler, which would normally
+catch all unhandled exceptions. With this flag, pip will let these exceptions
+propagate outside of its main subroutine, letting them get caught by the
+debugger. This way you'll be able to debug an exception post-mortem via PDB.
+
+
 Building Documentation
 ======================
 
@@ -135,7 +155,7 @@ To build it locally, run:
 
 .. code-block:: console
 
-    $ tox -e docs
+    $ nox -s docs
 
 The built documentation can be found in the ``docs/build`` folder.
 
diff --git a/docs/html/development/index.rst b/docs/html/development/index.rst
index 47907584919..31df114ae1c 100644
--- a/docs/html/development/index.rst
+++ b/docs/html/development/index.rst
@@ -7,7 +7,7 @@ of all forms. The sections below will help you get started with development,
 testing, and documentation.
 
 You can also join ``#pypa`` (general packaging discussion and user support) and
-``#pypa-dev`` (discussion about development of packaging tools) `on Freenode`_,
+``#pypa-dev`` (discussion about development of packaging tools) `on Libera.chat`_,
 or the `distutils-sig mailing list`_, to ask questions or get involved.
 
 .. toctree::
@@ -26,5 +26,5 @@ or the `distutils-sig mailing list`_, to ask questions or get involved.
     pip's development documentation has been rearranged and some older
     references might be broken.
 
-.. _`on Freenode`: https://webchat.freenode.net/?channels=%23pypa-dev,pypa
+.. _`on Libera.chat`: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev
 .. _`distutils-sig mailing list`: https://mail.python.org/mailman3/lists/distutils-sig.python.org/
diff --git a/docs/html/development/issue-triage.rst b/docs/html/development/issue-triage.rst
index 9b5e5cc1c3e..7a67a0a89aa 100644
--- a/docs/html/development/issue-triage.rst
+++ b/docs/html/development/issue-triage.rst
@@ -137,7 +137,7 @@ The lifecycle of an issue (bug or support) generally looks like:
 #. waiting for triage (marked with label ``triage``)
 #. confirming issue - some discussion with the user, gathering
    details, trying to reproduce the issue (may be marked with a specific
-   category, ``S: awaiting-respose``, ``S: discussion-needed``, or
+   category, ``S: awaiting-response``, ``S: discussion-needed``, or
    ``S: need-repro``)
 #. confirmed - the issue is pretty consistently reproducible in a
    straightforward way, or a mechanism that could be causing the issue has been
@@ -229,7 +229,7 @@ Examples:
   (`link `__)
 - get-pip on system with no ``/usr/lib64``
   (`link `__)
-- reproducing with ``pip`` from master branch
+- reproducing with ``pip`` from current development branch
   (`link `__)
 
 
@@ -285,7 +285,7 @@ An issue may be considered resolved and closed when:
     - already tracked by another issue
 
   - A project-specific issue has been identified and the issue no
-    longer occurs as of the latest commit on the master branch.
+    longer occurs as of the latest commit on the main branch.
 
 - An enhancement or feature request no longer has a proponent and the maintainers
   don't think it's worth keeping open.
diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst
index a133e57f20c..188d3e87bec 100644
--- a/docs/html/development/release-process.rst
+++ b/docs/html/development/release-process.rst
@@ -7,7 +7,7 @@ Release process
 Release Cadence
 ===============
 
-The pip project has a release cadence of releasing whatever is on ``master``
+The pip project has a release cadence of releasing whatever is on ``main``
 every 3 months. This gives users a predictable pattern for when releases
 are going to happen and prevents locking up improvements for fixes for long
 periods of time, while still preventing massively fracturing the user base
@@ -22,8 +22,8 @@ The release manager may, at their discretion, choose whether or not there
 will be a pre-release period for a release, and if there is may extend that
 period into the next month if needed.
 
-Because releases are made direct from the ``master`` branch, it is essential
-that ``master`` is always in a releasable state. It is acceptable to merge
+Because releases are made direct from the ``main`` branch, it is essential
+that ``main`` is always in a releasable state. It is acceptable to merge
 PRs that partially implement a new feature, but only if the partially
 implemented version is usable in that state (for example, with reduced
 functionality or disabled by default). In the case where a merged PR is found
@@ -116,15 +116,17 @@ Release Process
 Creating a new release
 ----------------------
 
-#. Checkout the current pip ``master`` branch.
 #. Ensure you have the latest ``nox`` installed.
+#. Create a new ``release/YY.N`` branch off ``main`` and switch to it.
 #. Prepare for release using ``nox -s prepare-release -- YY.N``.
    This will update the relevant files and tag the correct commit.
+#. Submit the ``release/YY.N`` branch as a pull request and ensure CI passes.
+   Merge the changes back into ``main`` and pull them back locally.
 #. Build the release artifacts using ``nox -s build-release -- YY.N``.
    This will checkout the tag, generate the distribution files to be
-   uploaded and checkout the master branch again.
+   uploaded and checkout the main branch again.
 #. Upload the release to PyPI using ``nox -s upload-release -- YY.N``.
-#. Push all of the changes including the tag.
+#. Push the tag created by ``prepare-release``.
 #. Regenerate the ``get-pip.py`` script in the `get-pip repository`_ (as
    documented there) and commit the results.
 #. Submit a Pull Request to `CPython`_ adding the new version of pip (and upgrading
@@ -155,20 +157,20 @@ Creating a bug-fix release
 
 Sometimes we need to release a bugfix release of the form ``YY.N.Z+1``. In
 order to create one of these the changes should already be merged into the
-``master`` branch.
+``main`` branch.
 
 #. Create a new ``release/YY.N.Z+1`` branch off of the ``YY.N`` tag using the
    command ``git checkout -b release/YY.N.Z+1 YY.N``.
-#. Cherry pick the fixed commits off of the ``master`` branch, fixing any
+#. Cherry pick the fixed commits off of the ``main`` branch, fixing any
    conflicts.
 #. Run ``nox -s prepare-release -- YY.N.Z+1``.
-#. Merge master into your release branch and drop the news files that have been
+#. Merge main into your release branch and drop the news files that have been
    included in your release (otherwise they would also appear in the ``YY.N+1``
    changelog)
 #. Push the ``release/YY.N.Z+1`` branch to github and submit a PR for it against
-   the ``master`` branch and wait for the tests to run.
-#. Once tests run, merge the ``release/YY.N.Z+1`` branch into master, and follow
-   the above release process starting with step 4.
+   the ``main`` branch and wait for the tests to run.
+#. Once tests run, merge the ``release/YY.N.Z+1`` branch into ``main``, and
+   follow the above release process starting with step 5.
 
 .. _`get-pip repository`: https://github.com/pypa/get-pip
 .. _`psf-salt repository`: https://github.com/python/psf-salt
diff --git a/docs/html/getting-started.md b/docs/html/getting-started.md
new file mode 100644
index 00000000000..0967b0eb99f
--- /dev/null
+++ b/docs/html/getting-started.md
@@ -0,0 +1,102 @@
+# Getting Started
+
+To get started with using pip, you should [install Python] on your system.
+
+[install Python]: https://realpython.com/installing-python/
+
+## Ensure you have a working pip
+
+As a first step, you should check that you have a working Python with pip
+installed. This can be done by running the following commands and making
+sure that the output looks similar.
+
+```{pip-cli}
+$ python --version
+Python 3.N.N
+$ pip --version
+pip X.Y.Z from ... (python 3.N.N)
+```
+
+If that worked, congratulations! You have a working pip in your environment.
+
+If you got output that does not look like the sample above, please read
+the {doc}`installation` page. It provides guidance on how to install pip
+within a Python environment that doesn't have it.
+
+## Common tasks
+
+### Install a package
+
+```{pip-cli}
+$ pip install sampleproject
+[...]
+Successfully installed sampleproject
+```
+
+By default, pip will fetch packages from [Python Package Index][PyPI], a
+repository of software for the Python programming language where anyone can
+upload packages.
+
+[PyPI]: https://pypi.org/
+
+### Install a package from GitHub
+
+```{pip-cli}
+$ pip install git+https://github.com/pypa/sampleproject.git@main
+[...]
+Successfully installed sampleproject
+```
+
+See {doc}`topics/vcs-support` for more information about this syntax.
+
+### Install a package from a distribution file
+
+pip can install directly from distribution files as well. They come in 2 forms:
+
+- {term}`source distribution ` (usually shortened to "sdist")
+- {term}`wheel distribution ` (usually shortened to "wheel")
+
+```{pip-cli}
+$ pip install sampleproject-1.0.tar.gz
+[...]
+Successfully installed sampleproject
+$ pip install sampleproject-1.0-py3-none-any.whl
+[...]
+Successfully installed sampleproject
+```
+
+### Install multiple packages using a requirements file
+
+Many Python projects use {file}`requirements.txt` files, to specify the
+list of packages that need to be installed for the project to run. To install
+the packages listed in that file, you can run:
+
+```{pip-cli}
+$ pip install -r requirements.txt
+[...]
+Successfully installed sampleproject
+```
+
+### Upgrade a package
+
+```{pip-cli}
+$ pip install --upgrade sampleproject
+   [...]
+Successfully installed sampleproject
+```
+
+### Uninstall a package
+
+```{pip-cli}
+$ pip uninstall sampleproject
+Uninstalling sampleproject:
+   [...]
+Proceed (Y/n)? y
+Successfully uninstalled sampleproject
+```
+
+## Next Steps
+
+It is recommended to learn about what virtual environments are and how to use
+them. This is covered in the ["Installing Packages"](pypug:tutorials/installing-packages)
+tutorial on packaging.python.org.
diff --git a/docs/html/index.md b/docs/html/index.md
new file mode 100644
index 00000000000..34a01744996
--- /dev/null
+++ b/docs/html/index.md
@@ -0,0 +1,50 @@
+---
+hide-toc: true
+---
+
+# pip
+
+pip is the [package installer for Python][recommended]. You can use it to
+install packages from the [Python Package Index][pypi] and other indexes.
+
+```{toctree}
+:hidden:
+
+getting-started
+installation
+user_guide
+topics/index
+reference/index
+cli/index
+```
+
+```{toctree}
+:caption: Project
+:hidden:
+
+development/index
+ux_research_design
+news
+Code of Conduct 
+GitHub 
+```
+
+If you want to learn about how to use pip, check out the following resources:
+
+- [Getting Started](getting-started)
+- [Python Packaging User Guide](https://packaging.python.org)
+
+If you find bugs, need help, or want to talk to the developers, use our mailing
+lists or chat rooms:
+
+- [GitHub Issues][issue-tracker]
+- [Discourse channel][packaging-discourse]
+- [User IRC][irc-pypa]
+- [Development IRC][irc-pypa-dev]
+
+[recommended]: https://packaging.python.org/guides/tool-recommendations/
+[pypi]: https://pypi.org/
+[issue-tracker]: https://github.com/pypa/pip/issues/
+[packaging-discourse]: https://discuss.python.org/c/packaging/14
+[irc-pypa]: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa
+[irc-pypa-dev]: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev
diff --git a/docs/html/index.rst b/docs/html/index.rst
deleted file mode 100644
index 1ac460bd9d9..00000000000
--- a/docs/html/index.rst
+++ /dev/null
@@ -1,63 +0,0 @@
-==================================
-pip - The Python Package Installer
-==================================
-
-pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.
-
-Please take a look at our documentation for how to install and use pip:
-
-.. toctree::
-   :maxdepth: 1
-
-   quickstart
-   installing
-   user_guide
-   reference/index
-   development/index
-   ux_research_design
-   news
-
-.. warning::
-
-   In pip 20.3, we've `made a big improvement to the heart of pip`_;
-   :ref:`Resolver changes 2020`. We want your input, so `sign up for
-   our user experience research studies`_ to help us do it right.
-
-.. warning::
-
-   pip 21.0, in January 2021, will remove Python 2 support, per pip's
-   :ref:`Python 2 Support` policy. Please migrate to Python 3.
-
-If you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:
-
-* `Issue tracking`_
-* `Discourse channel`_
-* `User IRC`_
-
-If you want to get involved, head over to GitHub to get the source code, and feel free to jump on the developer mailing lists and chat rooms:
-
-* `GitHub page`_
-* `Development mailing list`_
-* `Development IRC`_
-
-
-Code of Conduct
-===============
-
-Everyone interacting in the pip project's codebases, issue trackers, chat
-rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.
-
-.. _package installer: https://packaging.python.org/guides/tool-recommendations/
-.. _Python Package Index: https://pypi.org
-.. _made a big improvement to the heart of pip: https://pyfound.blogspot.com/2020/11/pip-20-3-new-resolver.html
-.. _sign up for our user experience research studies: https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html
-.. _Installation: https://pip.pypa.io/en/stable/installing.html
-.. _Documentation: https://pip.pypa.io/en/stable/
-.. _Changelog: https://pip.pypa.io/en/stable/news.html
-.. _GitHub page: https://github.com/pypa/pip
-.. _Issue tracking: https://github.com/pypa/pip/issues
-.. _Discourse channel: https://discuss.python.org/c/packaging
-.. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/
-.. _User IRC: https://webchat.freenode.net/?channels=%23pypa
-.. _Development IRC: https://webchat.freenode.net/?channels=%23pypa-dev
-.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md
diff --git a/docs/html/installation.md b/docs/html/installation.md
new file mode 100644
index 00000000000..d9b9a2d9fef
--- /dev/null
+++ b/docs/html/installation.md
@@ -0,0 +1,89 @@
+# Installation
+
+Usually, pip is automatically installed if you are:
+
+- working in a
+  {ref}`virtual environment `
+- using Python downloaded from [python.org](https://www.python.org)
+- using Python that has not been modified by a redistributor to remove
+  {mod}`ensurepip`
+
+## Supported Methods
+
+If your Python environment does not have pip installed, there are 2 mechanisms
+to install pip supported directly by pip's maintainers:
+
+- [`ensurepip`](#ensurepip)
+- [`get-pip.py`](#get-pippy)
+
+### `ensurepip`
+
+Python comes with an {mod}`ensurepip` module[^python], which can install pip in
+a Python environment.
+
+```{pip-cli}
+$ python -m ensurepip --upgrade
+```
+
+More details about how {mod}`ensurepip` works and how it can be used, is
+available in the standard library documentation.
+
+### `get-pip.py`
+
+This is a Python script that uses some bootstrapping logic to install
+pip.
+
+- Download the script, from .
+- Open a terminal/command prompt, `cd` to the folder containing the
+  `get-pip.py` file and run:
+
+  ```{pip-cli}
+  $ python get-pip.py
+  ```
+
+More details about this script can be found in [pypa/get-pip]'s README.
+
+[pypa/get-pip]: https://github.com/pypa/get-pip
+
+## Alternative Methods
+
+Depending on how you installed Python, there might be other mechanisms
+available to you for installing pip such as
+{ref}`using Linux package managers `.
+
+These mechanisms are provided by redistributors of pip, who may have modified
+pip to change its behaviour. This has been a frequent source of user confusion,
+since it causes a mismatch between documented behaviour in this documentation
+and how pip works after those modifications.
+
+If you face issues when using Python and pip installed using these mechanisms,
+it is recommended to request for support from the relevant provider (eg: Linux
+distro community, cloud provider support channels, etc).
+
+## Upgrading `pip`
+
+Upgrading your `pip` by running:
+
+```{pip-cli}
+$ pip install --upgrade pip
+```
+
+(compatibility-requirements)=
+
+## Compatibility
+
+The current version of pip works on:
+
+- Windows, Linux and MacOS.
+- CPython 3.7, 3.8, 3.9, 3.10 and latest PyPy3.
+
+pip is tested to work on the latest patch version of the Python interpreter,
+for each of the minor versions listed above. Previous patch versions are
+supported on a best effort approach.
+
+Other operating systems and Python versions are not supported by pip's
+maintainers.
+
+Users who are on unsupported platforms should be aware that if they hit issues, they may have to resolve them for themselves. If they received pip from a source which provides support for their platform, they should request pip support from that source.
+
+[^python]: The `ensurepip` module was added to the Python standard library in Python 3.4.
diff --git a/docs/html/installing.rst b/docs/html/installing.rst
index 9e2c7051ef3..e8d86f3441c 100644
--- a/docs/html/installing.rst
+++ b/docs/html/installing.rst
@@ -1,230 +1,11 @@
-.. _`Installation`:
+:orphan:
 
-============
-Installation
-============
+.. meta::
 
-Do I need to install pip?
-=========================
+  :http-equiv=refresh: 3; url=../installation/
 
-pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4
-downloaded from `python.org `_ or if you are working
-in a :ref:`Virtual Environment `
-created by :ref:`pypug:virtualenv` or :ref:`venv `. Just make sure
-to :ref:`upgrade pip `.
+This page has moved
+===================
 
-Use the following command to check whether pip is installed:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip --version
-      pip X.Y.Z from .../site-packages/pip (python X.Y)
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip --version
-      pip X.Y.Z from ...\site-packages\pip (python X.Y)
-
-Using Linux Package Managers
-============================
-
-.. warning::
-
-   If you installed Python from a package manager on Linux, you should always
-   install pip for that Python installation using the same source.
-
-See `pypug:Installing pip/setuptools/wheel with Linux Package Managers `_
-in the Python Packaging User Guide.
-
-Here are ways to contact a few Linux package maintainers if you run into
-problems:
-
-* `Deadsnakes PPA `_
-* `Debian Python Team `_ (for general
-  issues related to ``apt``)
-* `Red Hat Bugzilla `_
-
-pip developers do not have control over how Linux distributions handle pip
-installations, and are unable to provide solutions to related issues in
-general.
-
-Using ensurepip
-===============
-
-Python >=3.4 can self-bootstrap pip with the built-in
-:ref:`ensurepip ` module. Refer to the standard library
-documentation for more details. Make sure to :ref:`upgrade pip `
-after ``ensurepip`` installs pip.
-
-See the `Using Linux Package Managers`_ section if your Python reports
-``No module named ensurepip`` on Debian and derived systems (e.g. Ubuntu).
-
-
-.. _`get-pip`:
-
-Installing with get-pip.py
-==========================
-
-.. warning::
-
-   Be cautious if you are using a Python install that is managed by your operating
-   system or another package manager. ``get-pip.py`` does not coordinate with
-   those tools, and may leave your system in an inconsistent state.
-
-To manually install pip, securely [1]_ download ``get-pip.py`` by following
-this link: `get-pip.py
-`_. Alternatively, use ``curl``::
-
- curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-
-Then run the following command in the folder where you
-have downloaded ``get-pip.py``:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python get-pip.py
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py get-pip.py
-
-``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel`
-if they are not already. :ref:`pypug:setuptools` is required to install
-:term:`source distributions `.  Both are
-required in order to build a :ref:`Wheel cache` (which improves installation
-speed), although neither are required to install pre-built :term:`wheels
-`.
-
-.. note::
-
-   The get-pip.py script is supported on the same python version as pip.
-   For the now unsupported Python 2.6, alternate script is available
-   `here `__.
-
-
-get-pip.py options
-------------------
-
-.. option:: --no-setuptools
-
-    If set, do not attempt to install :ref:`pypug:setuptools`
-
-.. option:: --no-wheel
-
-    If set, do not attempt to install :ref:`pypug:wheel`
-
-
-``get-pip.py`` allows :ref:`pip install options ` and the :ref:`general options `. Below are
-some examples:
-
-Install from local copies of pip and setuptools:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python get-pip.py --no-index --find-links=/local/copies
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py get-pip.py --no-index --find-links=/local/copies
-
-Install to the user site [3]_:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python get-pip.py --user
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py get-pip.py --user
-
-Install behind a proxy:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python get-pip.py --proxy="http://[user:passwd@]proxy.server:port"
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py get-pip.py --proxy="http://[user:passwd@]proxy.server:port"
-
-``get-pip.py`` can also be used to install a specified combination of ``pip``,
-``setuptools``, and ``wheel`` using the same requirements syntax as pip:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0
-
-.. _`Upgrading pip`:
-
-Upgrading pip
-=============
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install -U pip
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install -U pip
-
-
-.. _compatibility-requirements:
-
-Python and OS Compatibility
-===========================
-
-pip works with CPython versions 3.6, 3.7, 3.8 and also PyPy.
-
-This means pip works on the latest patch version of each of these minor
-versions. Previous patch versions are supported on a best effort approach.
-
-pip works on Unix/Linux, macOS, and Windows.
-
-
-----
-
-.. [1] "Secure" in this context means using a modern browser or a
-       tool like ``curl`` that verifies SSL certificates when downloading from
-       https URLs.
-
-.. [2] Beginning with pip v1.5.1, ``get-pip.py`` stopped requiring setuptools to
-       be installed first.
-
-.. [3] The pip developers are considering making ``--user`` the default for all
-       installs, including ``get-pip.py`` installs of pip, but at this time,
-       ``--user`` installs for pip itself, should not be considered to be fully
-       tested or endorsed. For discussion, see `Issue 1668
-       `_.
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`installation`
diff --git a/docs/html/logic.rst b/docs/html/logic.rst
deleted file mode 100644
index 189169a8c54..00000000000
--- a/docs/html/logic.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-:orphan:
-
-================
-Internal Details
-================
-
-This content is now covered in the :doc:`Architecture section `.
diff --git a/docs/html/news.rst b/docs/html/news.rst
index 8b54a02e637..af1d10479a5 100644
--- a/docs/html/news.rst
+++ b/docs/html/news.rst
@@ -7,6 +7,6 @@ Changelog
     Major and minor releases of pip also include changes listed within
     prior beta releases.
 
-.. towncrier-draft-entries:: |release|, unreleased as on
+.. towncrier-draft-entries:: Not yet released
 
-.. include:: ../../NEWS.rst
+.. pip-news-include:: ../../NEWS.rst
diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst
index 96602a7b316..4385f4a7394 100644
--- a/docs/html/quickstart.rst
+++ b/docs/html/quickstart.rst
@@ -1,136 +1,11 @@
-==========
-Quickstart
-==========
+:orphan:
 
-First, :doc:`install pip `.
+.. meta::
 
-Install a package from `PyPI`_:
+  :http-equiv=refresh: 3; url=../getting-started/
 
-.. tab:: Unix/macOS
+This page has moved
+===================
 
-   .. code-block:: console
-
-      $ python -m pip install SomePackage
-      [...]
-      Successfully installed SomePackage
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip install SomePackage
-      [...]
-      Successfully installed SomePackage
-
-
-Install a package that's already been downloaded from `PyPI`_ or
-obtained from elsewhere. This is useful if the target machine does not have a
-network connection:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl
-      [...]
-      Successfully installed SomePackage
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl
-      [...]
-      Successfully installed SomePackage
-
-Show what files were installed:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip show --files SomePackage
-      Name: SomePackage
-      Version: 1.0
-      Location: /my/env/lib/pythonx.x/site-packages
-      Files:
-      ../somepackage/__init__.py
-      [...]
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip show --files SomePackage
-      Name: SomePackage
-      Version: 1.0
-      Location: /my/env/lib/pythonx.x/site-packages
-      Files:
-      ../somepackage/__init__.py
-      [...]
-
-List what packages are outdated:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip list --outdated
-      SomePackage (Current: 1.0 Latest: 2.0)
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip list --outdated
-      SomePackage (Current: 1.0 Latest: 2.0)
-
-Upgrade a package:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip install --upgrade SomePackage
-      [...]
-      Found existing installation: SomePackage 1.0
-      Uninstalling SomePackage:
-      Successfully uninstalled SomePackage
-      Running setup.py install for SomePackage
-      Successfully installed SomePackage
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip install --upgrade SomePackage
-      [...]
-      Found existing installation: SomePackage 1.0
-      Uninstalling SomePackage:
-      Successfully uninstalled SomePackage
-      Running setup.py install for SomePackage
-      Successfully installed SomePackage
-
-Uninstall a package:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip uninstall SomePackage
-      Uninstalling SomePackage:
-      /my/env/lib/pythonx.x/site-packages/somepackage
-      Proceed (y/n)? y
-      Successfully uninstalled SomePackage
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip uninstall SomePackage
-      Uninstalling SomePackage:
-         /my/env/lib/pythonx.x/site-packages/somepackage
-      Proceed (y/n)? y
-      Successfully uninstalled SomePackage
-
-.. _PyPI: https://pypi.org/
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`getting-started`
diff --git a/docs/html/reference/build-system/index.md b/docs/html/reference/build-system/index.md
new file mode 100644
index 00000000000..ed43fec3778
--- /dev/null
+++ b/docs/html/reference/build-system/index.md
@@ -0,0 +1,127 @@
+(build-interface)=
+
+# Build System Interface
+
+When dealing with installable source distributions of a package, pip does not
+directly handle the build process for the package. This responsibility is
+delegated to "build backends" -- also known as "build systems". This means
+that pip needs an interface, to interact with these build backends.
+
+There are two main interfaces that pip uses for these interactions:
+
+```{toctree}
+:hidden:
+
+pyproject-toml
+setup-py
+```
+
+
+[`pyproject.toml` based](pyproject-toml)
+: Standards-backed interface, that has explicit declaration and management of
+  build dependencies.
+
+[`setup.py` based](setup-py)
+: Legacy interface, that we're working to migrate users away from. Has no good
+  mechanisms to declare build dependencies.
+
+
+Details on the individual interfaces can be found on their dedicated pages,
+linked above. This document covers the nuances around which build system
+interface pip will use for a project, as well as details that apply to all
+the build system interfaces that pip may use.
+
+## Determining which build system interface is used
+
+Currently, pip uses the `pyproject.toml` based build system interface, if a
+`pyproject.toml` file exists. If not, the legacy build system interface is used.
+The intention is to switch to using the `pyproject.toml` build system interface
+unconditionally and to drop support for the legacy build system interface at
+some point in the future.
+
+When performing a build, pip will mention which build system interface it is
+using. Typically, this will take the form of a message like:
+
+```none
+Building wheel for pip (pyproject.toml)... done
+```
+
+```none
+Building wheel for pip (setup.py)... done
+```
+
+The content in the brackets, refers to which build system interface is being
+used.
+
+```{versionchanged} 21.3
+The output uses "pyproject.toml" instead of "PEP 517" to refer to be
+`pyproject.toml` based build system interface.
+```
+
+## Controlling which build system interface is used
+
+The [`--use-pep517`](install_--use-pep517) flag (and corresponding environment
+variable: `PIP_USE_PEP517`) can be used to force all packages to build using
+the `pyproject.toml` based build system interface. There is no way to force
+the use of the legacy build system interface.
+
+(controlling-setup_requires)=
+
+## Controlling `setup_requires`
+
+```{hint}
+This is only relevant for projects that use setuptools as the build backend,
+and use the `setup_requires` keyword argument in their setup.py file.
+```
+
+The `setup_requires` argument in `setup.py` is used to specify build-time
+dependencies for a package. This has been superseded by the
+`build-system.requires` key in `pyproject.toml` files (per {pep}`518`).
+However, there are situations where you might encounter a package that uses
+`setup_requires` (eg: the package has not been updated to use the newer
+approach yet!).
+
+If you control the package, consider adding a `pyproject.toml` file to utilise
+the modern build system interface. That avoids invoking the problematic
+behaviour by deferring to pip for the installations.
+
+For the end users, the best solution for dealing with packages with
+`setup_requires` is to install the packages listed in `setup_requires`
+beforehand, using a prior `pip install` command. This is because there is no
+way to control how these dependencies are located by `easy_install`, or how
+setuptools will invoke `pip` using pip's command line options -- which makes it
+tricky to get things working appropriately.
+
+If you wish to ensure that `easy_install` invocations do not reach out to PyPI,
+you will need to configure its behaviour using a
+[`distutils` configuration file][distutils-config]. Here are some examples:
+
+- To have the dependency located at an alternate index with `easy_install`
+
+  ```ini
+  [easy_install]
+  index_url = https://my.index-mirror.com
+  ```
+
+- To have the dependency located from a local directory and not crawl PyPI, add this:
+
+  ```ini
+  [easy_install]
+  allow_hosts = ''
+  find_links = file:///path/to/local/archives/
+  ```
+
+```{admonition} Historical context
+`setuptools < 52.0` will use `easy_install` to try to fulfill `setup_requires`
+dependencies, which can result in weird failures -- `easy_install` does not
+understand many of the modern Python packaging standards, and will usually
+attempt to install incompatible package versions or to build packages
+incorrectly. It also generates improper script wrappers, which don't do the
+right thing in many situations.
+
+Newer versions of `setuptools` will use `pip` for these installations, but have
+limited ability to pass through any command line arguments. This can also result
+in weird failures and subtly-incorrect behaviour.
+```
+
+[distutils-config]: https://docs.python.org/3/install/index.html#distutils-configuration-files
diff --git a/docs/html/reference/build-system/pyproject-toml.md b/docs/html/reference/build-system/pyproject-toml.md
new file mode 100644
index 00000000000..ee93df034a7
--- /dev/null
+++ b/docs/html/reference/build-system/pyproject-toml.md
@@ -0,0 +1,146 @@
+# `pyproject.toml`
+
+```{versionadded} 10.0
+
+```
+
+Modern Python packages can contain a `pyproject.toml` file, first introduced in
+{pep}`518` and later expanded in {pep}`517`, {pep}`621` and {pep}`660`.
+This file contains build system requirements and information, which are used by
+pip to build the package.
+
+## Build process
+
+The overall process for building a package is:
+
+- Create an isolated build environment.
+- Populate the build environment with build dependencies.
+- Generate the package's metadata, if necessary and possible.
+- Generate a wheel for the package.
+
+The wheel can then be used to perform an installation, if necessary.
+
+### Build Isolation
+
+For building packages using this interface, pip uses an _isolated environment_.
+That is, pip will install build-time Python dependencies in a temporary
+directory which will be added to `sys.path` for the build commands. This ensures
+that build requirements are handled independently of the user's runtime
+environment.
+
+For example, a project that needs an older version of setuptools to build can
+still be installed, even if the user has an newer version installed (and
+without silently replacing that version).
+
+### Build-time dependencies
+
+Introduced in {pep}`518`, the `build-system.requires` key in the
+`pyproject.toml` file is a list of requirement specifiers for build-time
+dependencies of a package.
+
+```toml
+[build-system]
+requires = ["setuptools ~= 58.0", "cython ~= 0.29.0"]
+```
+
+It is also possible for a build backend to provide dynamically calculated
+build dependencies, using {pep}`517`'s `get_requires_for_build_wheel` hook. This
+hook will be called by pip, and dependencies it describes will also be installed
+in the build environment. For example, newer versions of setuptools expose the
+contents of `setup_requires` to pip via this hook.
+
+### Metadata Generation
+
+```{versionadded} 19.0
+
+```
+
+Once the build environment has been created and populated with build-time
+dependencies, `pip` will usually need metadata about a package (name, version,
+dependencies, and more).
+
+If {pep}`517`'s `prepare_metadata_for_build_wheel` hook is provided by the
+build backend, that will be used to generate the packages' metadata. Otherwise,
+a wheel will be generated (as described below) and the metadata contained
+within such a wheel will be used.
+
+### Wheel Generation
+
+```{versionadded} 19.0
+
+```
+
+For generating a wheel, pip uses the {pep}`517` `build_wheel` hook that has
+to be provided by the build backend. The build backend will generate a wheel,
+which may involve compiling extension code written in C/C++ (or other
+languages).
+
+Wheels generated using this mechanism can be [cached](wheel-caching) for reuse,
+to speed up future installations.
+
+### Editable Installation
+
+```{versionadded} 21.3
+
+```
+
+For performing editable installs, pip will use {pep}`660`
+`build_wheel_for_editable` hook that has to be provided by the build backend.
+The wheels generated using this mechanism are not cached.
+
+```{admonition} Compatibility fallback
+If this hook is missing on the build backend _and_ there's a `setup.py` file
+in the project, pip will fallback to the legacy setup.py-based editable
+installation.
+
+This is considered a stopgap solution until setuptools adds support for
+{pep}`660`, at which point this functionality will be removed; following pip's
+regular {ref}`deprecation policy `.
+```
+
+## Build output
+
+It is the responsibility of the build backend to ensure that the output is
+in the correct encoding, as described in {pep}`517`. This likely involves
+dealing with [the same challenges as pip has for legacy builds](build-output).
+
+## Fallback Behaviour
+
+If a project does not have a `pyproject.toml` file containing a `build-system`
+section, it will be assumed to have the following backend settings:
+
+```toml
+[build-system]
+requires = ["setuptools>=40.8.0", "wheel"]
+build-backend = "setuptools.build_meta:__legacy__"
+```
+
+If a project has a `build-system` section but no `build-backend`, then:
+
+- It is expected to include `setuptools` and `wheel` as build requirements. An
+  error is reported if the available version of `setuptools` is not recent
+  enough.
+
+- The `setuptools.build_meta:__legacy__` build backend will be used.
+
+## Disabling build isolation
+
+This can be disabled using the `--no-build-isolation` flag -- users supplying
+this flag are responsible for ensuring the build environment is managed
+appropriately, including ensuring that all required build-time dependencies are
+installed, since pip does not manage build-time dependencies when this flag is
+passed.
+
+## Historical notes
+
+As this feature was incrementally rolled out, there have been various notable
+changes and improvements in it.
+
+- setuptools 40.8.0 is the first version of setuptools that offers a
+  {pep}`517` backend that closely mimics directly executing `setup.py`.
+- Prior to pip 18.0, pip only supports installing build requirements from
+  wheels, and does not support the use of environment markers and extras (only
+  version specifiers are respected).
+- Prior to pip 18.1, build dependencies using `.pth` files are not properly
+  supported; as a result namespace packages do not work under Python 3.2 and
+  earlier.
diff --git a/docs/html/reference/build-system/setup-py.md b/docs/html/reference/build-system/setup-py.md
new file mode 100644
index 00000000000..53917b8a4c8
--- /dev/null
+++ b/docs/html/reference/build-system/setup-py.md
@@ -0,0 +1,133 @@
+# `setup.py` (legacy)
+
+Prior to the introduction of pyproject.toml-based builds (in {pep}`517` and
+{pep}`518`), pip had only supported installing packages using `setup.py` files
+that were built using {pypi}`setuptools`.
+
+The interface documented here is retained currently solely for legacy purposes,
+until the migration to `pyproject.toml`-based builds can be completed.
+
+```{caution}
+The arguments and syntax of the various invocations of `setup.py` made by
+pip, are considered an implementation detail that is strongly coupled with
+{pypi}`setuptools`. This build system interface is not meant to be used by any
+other build backend, which should be based on the {doc}`pyproject-toml` build
+system interface instead.
+
+Further, projects should _not_ expect to rely on there being any form of
+backward compatibility guarantees around the `setup.py` interface.
+```
+
+## Build process
+
+The overall process for building a package is:
+
+- Generate the package's metadata.
+- Generate a wheel for the package.
+  - If this fails and we're trying to install the package, attempt a direct
+    installation.
+
+The wheel can then be used to perform an installation, if necessary.
+
+### Metadata Generation
+
+As a first step, `pip` needs to get metadata about a package (name, version,
+dependencies, and more). It collects this by calling `setup.py egg_info`.
+
+The `egg_info` command generates the metadata for the package, which pip can
+then consume and proceed to gather all the dependencies of the package. Once
+the dependency resolution process is complete, pip will proceed to the next
+stage of the build process for these packages.
+
+### Wheel Generation
+
+When provided with a {term}`pypug:source distribution (or "sdist")` for a
+package, pip will attempt to build a {term}`pypug:wheel`. Since wheel
+distributions can be [cached](wheel-caching), this can greatly speed up future
+installations for the package.
+
+This is done by calling `setup.py bdist_wheel` which requires the {pypi}`wheel`
+package to be installed.
+
+If this wheel generation is successful (this can include compiling C/C++ code,
+depending on the package), the generated wheel is added to pip's wheel cache
+and will be used for this installation. The built wheel is cached locally
+by pip to avoid repeated identical builds.
+
+If this wheel generation fails, pip runs `setup.py clean` to clean up any build
+artifacts that may have been generated. After that, pip will attempt a direct
+installation.
+
+### Direct Installation
+
+When all else fails, pip will invoke `setup.py install` to install a package
+using setuptools' mechanisms to perform the installation. This is currently the
+last-resort fallback for projects that cannot be built into wheels, and may not
+be supported in the future.
+
+### Editable Installation
+
+For installing packages in "editable" mode
+({ref}`pip install --editable `), pip will invoke
+`setup.py develop`, which will use setuptools' mechanisms to perform an
+editable/development installation.
+
+## Setuptools Injection
+
+To support projects that directly use `distutils`, pip injects `setuptools` into
+`sys.modules` before invoking `setup.py`. This injection should be transparent
+to `distutils`-based projects.
+
+## Customising the build
+
+The `--global-option` and `--build-option` arguments to the `pip install`
+and `pip wheel` inject additional arguments into the `setup.py` command
+(`--build-option` is only available in `pip wheel`).
+
+```{attention}
+The use of `--global-option` and `--build-option` is highly setuptools
+specific, and is considered more an accident of the current implementation than
+a supported interface. It is documented here for completeness. These flags will
+not be supported, once this build system interface is dropped.
+```
+
+These arguments are included in the command as follows:
+
+```
+python setup.py  BUILD COMMAND 
+```
+
+The options are passed unmodified, and presently offer direct access to the
+distutils command line. For example:
+
+```{pip-cli}
+$ pip wheel --global-option bdist_ext --global-option -DFOO wheel
+```
+
+will result in pip invoking:
+
+```
+setup.py bdist_ext -DFOO bdist_wheel -d TARGET
+```
+
+This passes a preprocessor symbol to the extension build.
+
+(build-output)=
+
+## Build Output
+
+Any output produced by the build system will be read by pip (for display to the
+user if requested). In order to correctly read the build system output, pip
+requires that the output is written in a well-defined encoding, specifically
+the encoding the user has configured for text output (which can be obtained in
+Python using `locale.getpreferredencoding`). If the configured encoding is
+ASCII, pip assumes UTF-8 (to account for the behaviour of some Unix systems).
+
+Build systems should ensure that any tools they invoke (compilers, etc) produce
+output in the correct encoding. In practice - and in particular on Windows,
+where tools are inconsistent in their use of the "OEM" and "ANSI" codepages -
+this may not always be possible. pip will therefore attempt to recover cleanly
+if presented with incorrectly encoded build tool output, by translating
+unexpected byte sequences to Python-style hexadecimal escape sequences
+(`"\x80\xff"`, etc). However, it is still possible for output to be displayed
+using an incorrect encoding (mojibake).
diff --git a/docs/html/reference/index.md b/docs/html/reference/index.md
new file mode 100644
index 00000000000..855dc79b37a
--- /dev/null
+++ b/docs/html/reference/index.md
@@ -0,0 +1,12 @@
+# Reference
+
+Reference provides information about various file formats, interfaces and
+interoperability standards that pip utilises/implements.
+
+```{toctree}
+:titlesonly:
+
+build-system/index
+requirement-specifiers
+requirements-file-format
+```
diff --git a/docs/html/reference/index.rst b/docs/html/reference/index.rst
deleted file mode 100644
index d21b7a9801a..00000000000
--- a/docs/html/reference/index.rst
+++ /dev/null
@@ -1,21 +0,0 @@
-===============
-Reference Guide
-===============
-
-.. toctree::
-   :maxdepth: 2
-
-   pip
-   pip_install
-   pip_download
-   pip_uninstall
-   pip_freeze
-   pip_list
-   pip_show
-   pip_search
-   pip_cache
-   pip_check
-   pip_config
-   pip_wheel
-   pip_hash
-   pip_debug
diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst
index 1f52630f69f..53b1c9e0d41 100644
--- a/docs/html/reference/pip.rst
+++ b/docs/html/reference/pip.rst
@@ -1,255 +1,11 @@
-===
-pip
-===
+:orphan:
 
+.. meta::
 
-Usage
-*****
+  :http-equiv=refresh: 3; url=../../cli/pip/
 
-.. tab:: Unix/macOS
+This page has moved
+===================
 
-    .. code-block:: shell
-
-        python -m pip  [options]
-
-.. tab:: Windows
-
-    .. code-block:: shell
-
-        py -m pip  [options]
-
-Description
-***********
-
-
-.. _`Logging`:
-
-
-Logging
-=======
-
-Console logging
-~~~~~~~~~~~~~~~
-
-pip offers :ref:`-v, --verbose <--verbose>` and :ref:`-q, --quiet <--quiet>`
-to control the console log level. By default, some messages (error and warnings)
-are colored in the terminal. If you want to suppress the colored output use
-:ref:`--no-color <--no-color>`.
-
-
-.. _`FileLogging`:
-
-File logging
-~~~~~~~~~~~~
-
-pip offers the :ref:`--log <--log>` option for specifying a file where a maximum
-verbosity log will be kept.  This option is empty by default. This log appends
-to previous logging.
-
-Like all pip options, ``--log`` can also be set as an environment variable, or
-placed into the pip config file.  See the :ref:`Configuration` section.
-
-.. _`exists-action`:
-
---exists-action option
-======================
-
-This option specifies default behavior when path already exists.
-Possible cases: downloading files or checking out repositories for installation,
-creating archives. If ``--exists-action`` is not defined, pip will prompt
-when decision is needed.
-
-*(s)witch*
-    Only relevant to VCS checkout. Attempt to switch the checkout
-    to the appropriate URL and/or revision.
-*(i)gnore*
-    Abort current operation (e.g. don't copy file, don't create archive,
-    don't modify a checkout).
-*(w)ipe*
-    Delete the file or VCS checkout before trying to create, download, or checkout a new one.
-*(b)ackup*
-    Rename the file or checkout to ``{name}{'.bak' * n}``, where n is some number
-    of ``.bak`` extensions, such that the file didn't exist at some point.
-    So the most recent backup will be the one with the largest number after ``.bak``.
-*(a)abort*
-    Abort pip and return non-zero exit status.
-
-.. _`build-interface`:
-
-
-Build System Interface
-======================
-
-pip builds packages by invoking the build system. By default, builds will use
-``setuptools``, but if a project specifies a different build system using a
-``pyproject.toml`` file, as per :pep:`517`, pip will use that instead.  As well
-as package building, the build system is also invoked to install packages
-direct from source.  This is handled by invoking the build system to build a
-wheel, and then installing from that wheel.  The built wheel is cached locally
-by pip to avoid repeated identical builds.
-
-The current interface to the build system is via the ``setup.py`` command line
-script - all build actions are defined in terms of the specific ``setup.py``
-command line that will be run to invoke the required action.
-
-Setuptools Injection
-~~~~~~~~~~~~~~~~~~~~
-
-When :pep:`517` is not used, the supported build system is ``setuptools``.
-However, not all packages use ``setuptools`` in their build scripts. To support
-projects that use "pure ``distutils``", pip injects ``setuptools`` into
-``sys.modules`` before invoking ``setup.py``. The injection should be
-transparent to ``distutils``-based projects, but 3rd party build tools wishing
-to provide a ``setup.py`` emulating the commands pip requires may need to be
-aware that it takes place.
-
-Projects using :pep:`517` *must* explicitly use setuptools - pip does not do
-the above injection process in this case.
-
-Build System Output
-~~~~~~~~~~~~~~~~~~~
-
-Any output produced by the build system will be read by pip (for display to the
-user if requested). In order to correctly read the build system output, pip
-requires that the output is written in a well-defined encoding, specifically
-the encoding the user has configured for text output (which can be obtained in
-Python using ``locale.getpreferredencoding``). If the configured encoding is
-ASCII, pip assumes UTF-8 (to account for the behaviour of some Unix systems).
-
-Build systems should ensure that any tools they invoke (compilers, etc) produce
-output in the correct encoding. In practice - and in particular on Windows,
-where tools are inconsistent in their use of the "OEM" and "ANSI" codepages -
-this may not always be possible. pip will therefore attempt to recover cleanly
-if presented with incorrectly encoded build tool output, by translating
-unexpected byte sequences to Python-style hexadecimal escape sequences
-(``"\x80\xff"``, etc). However, it is still possible for output to be displayed
-using an incorrect encoding (mojibake).
-
-Under :pep:`517`, handling of build tool output is the backend's responsibility,
-and pip simply displays the output produced by the backend. (Backends, however,
-will likely still have to address the issues described above).
-
-PEP 517 and 518 Support
-~~~~~~~~~~~~~~~~~~~~~~~
-
-As of version 10.0, pip supports projects declaring dependencies that are
-required at install time using a ``pyproject.toml`` file, in the form described
-in :pep:`518`. When building a project, pip will install the required
-dependencies locally, and make them available to the build process.
-Furthermore, from version 19.0 onwards, pip supports projects specifying the
-build backend they use in ``pyproject.toml``, in the form described in
-:pep:`517`.
-
-When making build requirements available, pip does so in an *isolated
-environment*. That is, pip does not install those requirements into the user's
-``site-packages``, but rather installs them in a temporary directory which it
-adds to the user's ``sys.path`` for the duration of the build. This ensures
-that build requirements are handled independently of the user's runtime
-environment. For example, a project that needs a recent version of setuptools
-to build can still be installed, even if the user has an older version
-installed (and without silently replacing that version).
-
-In certain cases, projects (or redistributors) may have workflows that
-explicitly manage the build environment. For such workflows, build isolation
-can be problematic. If this is the case, pip provides a
-``--no-build-isolation`` flag to disable build isolation. Users supplying this
-flag are responsible for ensuring the build environment is managed
-appropriately (including ensuring that all required build dependencies are
-installed).
-
-By default, pip will continue to use the legacy (direct ``setup.py`` execution
-based) build processing for projects that do not have a ``pyproject.toml`` file.
-Projects with a ``pyproject.toml`` file will use a :pep:`517` backend. Projects
-with a ``pyproject.toml`` file, but which don't have a ``build-system`` section,
-will be assumed to have the following backend settings::
-
-    [build-system]
-    requires = ["setuptools>=40.8.0", "wheel"]
-    build-backend = "setuptools.build_meta:__legacy__"
-
-.. note::
-
-    ``setuptools`` 40.8.0 is the first version of setuptools that offers a
-    :pep:`517` backend that closely mimics directly executing ``setup.py``.
-
-If a project has ``[build-system]``, but no ``build-backend``, pip will also use
-``setuptools.build_meta:__legacy__``, but will expect the project requirements
-to include ``setuptools`` and ``wheel`` (and will report an error if the
-installed version of ``setuptools`` is not recent enough).
-
-If a user wants to explicitly request :pep:`517` handling even though a project
-doesn't have a ``pyproject.toml`` file, this can be done using the
-``--use-pep517`` command line option. Similarly, to request legacy processing
-even though ``pyproject.toml`` is present, the ``--no-use-pep517`` option is
-available (although obviously it is an error to choose ``--no-use-pep517`` if
-the project has no ``setup.py``, or explicitly requests a build backend). As
-with other command line flags, pip recognises the ``PIP_USE_PEP517``
-environment veriable and a ``use-pep517`` config file option (set to true or
-false) to set this option globally. Note that overriding pip's choice of
-whether to use :pep:`517` processing in this way does *not* affect whether pip
-will use an isolated build environment (which is controlled via
-``--no-build-isolation`` as noted above).
-
-Except in the case noted above (projects with no :pep:`518` ``[build-system]``
-section in ``pyproject.toml``), pip will never implicitly install a build
-system. Projects **must** ensure that the correct build system is listed in
-their ``requires`` list (this applies even if pip assumes that the
-``setuptools`` backend is being used, as noted above).
-
-.. _pep-518-limitations:
-
-**Historical Limitations**:
-
-* ``pip<18.0``: only supports installing build requirements from wheels, and
-  does not support the use of environment markers and extras (only version
-  specifiers are respected).
-
-* ``pip<18.1``: build dependencies using .pth files are not properly supported;
-  as a result namespace packages do not work under Python 3.2 and earlier.
-
-Future Developments
-~~~~~~~~~~~~~~~~~~~
-
-:pep:`426` notes that the intention is to add hooks to project metadata in
-version 2.1 of the metadata spec, to explicitly define how to build a project
-from its source. Once this version of the metadata spec is final, pip will
-migrate to using that interface. At that point, the ``setup.py`` interface
-documented here will be retained solely for legacy purposes, until projects
-have migrated.
-
-Specifically, applications should *not* expect to rely on there being any form
-of backward compatibility guarantees around the ``setup.py`` interface.
-
-
-Build Options
-~~~~~~~~~~~~~
-
-The ``--global-option`` and ``--build-option`` arguments to the ``pip install``
-and ``pip wheel`` inject additional arguments into the ``setup.py`` command
-(``--build-option`` is only available in ``pip wheel``).  These arguments are
-included in the command as follows:
-
-.. tab:: Unix/macOS
-
-    .. code-block:: console
-
-        python setup.py  BUILD COMMAND 
-
-.. tab:: Windows
-
-    .. code-block:: shell
-
-        py setup.py  BUILD COMMAND 
-
-The options are passed unmodified, and presently offer direct access to the
-distutils command line. Use of ``--global-option`` and ``--build-option``
-should be considered as build system dependent, and may not be supported in the
-current form if support for alternative build systems is added to pip.
-
-
-.. _`General Options`:
-
-General Options
-***************
-
-.. pip-general-options::
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip`
diff --git a/docs/html/reference/pip_cache.rst b/docs/html/reference/pip_cache.rst
index 0a23c510d6f..a9cbd69dae5 100644
--- a/docs/html/reference/pip_cache.rst
+++ b/docs/html/reference/pip_cache.rst
@@ -1,27 +1,11 @@
+:orphan:
 
-.. _`pip cache`:
+.. meta::
 
-pip cache
----------
+  :http-equiv=refresh: 3; url=../../cli/pip_cache/
 
+This page has moved
+===================
 
-Usage
-*****
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: cache "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: cache "py -m pip"
-
-Description
-***********
-
-.. pip-command-description:: cache
-
-Options
-*******
-
-.. pip-command-options:: cache
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_cache`
diff --git a/docs/html/reference/pip_check.rst b/docs/html/reference/pip_check.rst
index 268cf9a143c..5bb7fc84fcb 100644
--- a/docs/html/reference/pip_check.rst
+++ b/docs/html/reference/pip_check.rst
@@ -1,87 +1,11 @@
-.. _`pip check`:
+:orphan:
 
-=========
-pip check
-=========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_check/
 
-Usage
-=====
+This page has moved
+===================
 
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: check "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: check "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: check
-
-
-Examples
-========
-
-#. If all dependencies are compatible:
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip check
-         No broken requirements found.
-         $ echo $?
-         0
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip check
-         No broken requirements found.
-         C:\> echo %errorlevel%
-         0
-
-#. If a package is missing:
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip check
-         pyramid 1.5.2 requires WebOb, which is not installed.
-         $ echo $?
-         1
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip check
-         pyramid 1.5.2 requires WebOb, which is not installed.
-         C:\> echo %errorlevel%
-         1
-
-#. If a package has the wrong version:
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip check
-         pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8.
-         $ echo $?
-         1
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip check
-         pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8.
-         C:\> echo %errorlevel%
-         1
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_check`
diff --git a/docs/html/reference/pip_config.rst b/docs/html/reference/pip_config.rst
index 8b2f846304f..31a048a513a 100644
--- a/docs/html/reference/pip_config.rst
+++ b/docs/html/reference/pip_config.rst
@@ -1,30 +1,11 @@
+:orphan:
 
-.. _`pip config`:
+.. meta::
 
-==========
-pip config
-==========
+  :http-equiv=refresh: 3; url=../../cli/pip_config/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: config "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: config "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: config
-
-
-Options
-=======
-
-.. pip-command-options:: config
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_config`
diff --git a/docs/html/reference/pip_debug.rst b/docs/html/reference/pip_debug.rst
index 4023533c905..b0de682751f 100644
--- a/docs/html/reference/pip_debug.rst
+++ b/docs/html/reference/pip_debug.rst
@@ -1,35 +1,11 @@
-.. _`pip debug`:
+:orphan:
 
-=========
-pip debug
-=========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_debug/
 
-Usage
-=====
+This page has moved
+===================
 
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: debug "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: debug "py -m pip"
-
-
-.. warning::
-
-    This command is only meant for debugging.
-    Its options and outputs are provisional and may change without notice.
-
-
-Description
-===========
-
-.. pip-command-description:: debug
-
-
-Options
-=======
-
-.. pip-command-options:: debug
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_debug`
diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst
index 4f15314d765..d54a7bec554 100644
--- a/docs/html/reference/pip_download.rst
+++ b/docs/html/reference/pip_download.rst
@@ -1,226 +1,11 @@
+:orphan:
 
-.. _`pip download`:
+.. meta::
 
-============
-pip download
-============
+  :http-equiv=refresh: 3; url=../../cli/pip_download/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: download "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: download "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: download
-
-Overview
---------
-
-``pip download`` does the same resolution and downloading as ``pip install``,
-but instead of installing the dependencies, it collects the downloaded
-distributions into the directory provided (defaulting to the current
-directory). This directory can later be passed as the value to ``pip install
---find-links`` to facilitate offline or locked down package installation.
-
-``pip download`` with the ``--platform``, ``--python-version``,
-``--implementation``, and ``--abi`` options provides the ability to fetch
-dependencies for an interpreter and system other than the ones that pip is
-running on. ``--only-binary=:all:`` or ``--no-deps`` is required when using any
-of these options. It is important to note that these options all default to the
-current system/interpreter, and not to the most restrictive constraints (e.g.
-platform any, abi none, etc). To avoid fetching dependencies that happen to
-match the constraint of the current interpreter (but not your target one), it
-is recommended to specify all of these options if you are specifying one of
-them. Generic dependencies (e.g. universal wheels, or dependencies with no
-platform, abi, or implementation constraints) will still match an over-
-constrained download requirement.
-
-
-
-Options
-=======
-
-.. pip-command-options:: download
-
-.. pip-index-options:: download
-
-
-Examples
-========
-
-#. Download a package and all of its dependencies
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip download SomePackage
-         python -m pip download -d . SomePackage  # equivalent to above
-         python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip download SomePackage
-         py -m pip download -d . SomePackage  # equivalent to above
-         py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage
-
-
-#. Download a package and all of its dependencies with OSX specific interpreter constraints.
-   This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible,
-   this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``,
-   etc.
-   It will also match deps with platform ``any``. Also force the interpreter version to ``27``
-   (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``).
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip download \
-            --only-binary=:all: \
-            --platform macosx-10_10_x86_64 \
-            --python-version 27 \
-            --implementation cp \
-            SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip download ^
-            --only-binary=:all: ^
-            --platform macosx-10_10_x86_64 ^
-            --python-version 27 ^
-            --implementation cp ^
-            SomePackage
-
-#. Download a package and its dependencies with linux specific constraints.
-   Force the interpreter to be any minor version of py3k, and only accept
-   ``cp34m`` or ``none`` as the abi.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip download \
-            --only-binary=:all: \
-            --platform linux_x86_64 \
-            --python-version 3 \
-            --implementation cp \
-            --abi cp34m \
-            SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip download ^
-            --only-binary=:all: ^
-            --platform linux_x86_64 ^
-            --python-version 3 ^
-            --implementation cp ^
-            --abi cp34m ^
-            SomePackage
-
-#. Force platform, implementation, and abi agnostic deps.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip download \
-            --only-binary=:all: \
-            --platform any \
-            --python-version 3 \
-            --implementation py \
-            --abi none \
-            SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip download ^
-            --only-binary=:all: ^
-            --platform any ^
-            --python-version 3 ^
-            --implementation py ^
-            --abi none ^
-            SomePackage
-
-#. Even when overconstrained, this will still correctly fetch the pip universal wheel.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip download \
-            --only-binary=:all: \
-            --platform linux_x86_64 \
-            --python-version 33 \
-            --implementation cp \
-            --abi cp34m \
-            pip>=8
-
-      .. code-block:: console
-
-         $ ls pip-8.1.1-py2.py3-none-any.whl
-         pip-8.1.1-py2.py3-none-any.whl
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip download ^
-            --only-binary=:all: ^
-            --platform linux_x86_64 ^
-            --python-version 33 ^
-            --implementation cp ^
-            --abi cp34m ^
-            pip>=8
-
-      .. code-block:: console
-
-         C:\> dir pip-8.1.1-py2.py3-none-any.whl
-         pip-8.1.1-py2.py3-none-any.whl
-
-#. Download a package supporting one of several ABIs and platforms.
-    This is useful when fetching wheels for a well-defined interpreter, whose
-    supported ABIs and platforms are known and fixed, different than the one pip is
-    running under.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip download \
-            --only-binary=:all: \
-            --platform manylinux1_x86_64 --platform linux_x86_64 --platform any \
-            --python-version 36 \
-            --implementation cp \
-            --abi cp36m --abi cp36 --abi abi3 --abi none \
-            SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:> py -m pip download ^
-            --only-binary=:all: ^
-            --platform manylinux1_x86_64 --platform linux_x86_64 --platform any ^
-            --python-version 36 ^
-            --implementation cp ^
-            --abi cp36m --abi cp36 --abi abi3 --abi none ^
-            SomePackage
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_download`
diff --git a/docs/html/reference/pip_freeze.rst b/docs/html/reference/pip_freeze.rst
index 352f7d32168..1cf31d5d708 100644
--- a/docs/html/reference/pip_freeze.rst
+++ b/docs/html/reference/pip_freeze.rst
@@ -1,74 +1,11 @@
+:orphan:
 
-.. _`pip freeze`:
+.. meta::
 
-==========
-pip freeze
-==========
+  :http-equiv=refresh: 3; url=../../cli/pip_freeze/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: freeze "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: freeze "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: freeze
-
-
-Options
-=======
-
-.. pip-command-options:: freeze
-
-
-Examples
-========
-
-#. Generate output suitable for a requirements file.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip freeze
-         docutils==0.11
-         Jinja2==2.7.2
-         MarkupSafe==0.19
-         Pygments==1.6
-         Sphinx==1.2.2
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip freeze
-         docutils==0.11
-         Jinja2==2.7.2
-         MarkupSafe==0.19
-         Pygments==1.6
-         Sphinx==1.2.2
-
-#. Generate a requirements file and then install from it in another environment.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         env1/bin/python -m pip freeze > requirements.txt
-         env2/bin/python -m pip install -r requirements.txt
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         env1\bin\python -m pip freeze > requirements.txt
-         env2\bin\python -m pip install -r requirements.txt
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_freeze`
diff --git a/docs/html/reference/pip_hash.rst b/docs/html/reference/pip_hash.rst
index 7df0d5a4f13..6112bec5fa3 100644
--- a/docs/html/reference/pip_hash.rst
+++ b/docs/html/reference/pip_hash.rst
@@ -1,72 +1,11 @@
-.. _`pip hash`:
+:orphan:
 
-========
-pip hash
-========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_hash/
 
-Usage
-=====
+This page has moved
+===================
 
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: hash "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: hash "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: hash
-
-Overview
---------
-
-``pip hash`` is a convenient way to get a hash digest for use with
-:ref:`hash-checking mode`, especially for packages with multiple archives. The
-error message from ``pip install --require-hashes ...`` will give you one
-hash, but, if there are multiple archives (like source and binary ones), you
-will need to manually download and compute a hash for the others. Otherwise, a
-spurious hash mismatch could occur when :ref:`pip install` is passed a
-different set of options, like :ref:`--no-binary `.
-
-
-Options
-=======
-
-.. pip-command-options:: hash
-
-
-Example
-=======
-
-Compute the hash of a downloaded archive:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip download SomePackage
-      Collecting SomePackage
-         Downloading SomePackage-2.2.tar.gz
-         Saved ./pip_downloads/SomePackage-2.2.tar.gz
-      Successfully downloaded SomePackage
-      $ python -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz
-      ./pip_downloads/SomePackage-2.2.tar.gz:
-      --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip download SomePackage
-      Collecting SomePackage
-         Downloading SomePackage-2.2.tar.gz
-         Saved ./pip_downloads/SomePackage-2.2.tar.gz
-      Successfully downloaded SomePackage
-      C:\> py -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz
-      ./pip_downloads/SomePackage-2.2.tar.gz:
-      --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_hash`
diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst
index 1b53513266d..580900cfb93 100644
--- a/docs/html/reference/pip_install.rst
+++ b/docs/html/reference/pip_install.rst
@@ -1,1199 +1,11 @@
-.. _`pip install`:
+:orphan:
 
-===========
-pip install
-===========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_install/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: install "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: install "py -m pip"
-
-
-
-Description
-===========
-
-.. pip-command-description:: install
-
-Overview
---------
-
-pip install has several stages:
-
-1. Identify the base requirements. The user supplied arguments are processed
-   here.
-2. Resolve dependencies. What will be installed is determined here.
-3. Build wheels. All the dependencies that can be are built into wheels.
-4. Install the packages (and uninstall anything being upgraded/replaced).
-
-Note that ``pip install`` prefers to leave the installed version as-is
-unless ``--upgrade`` is specified.
-
-Argument Handling
------------------
-
-When looking at the items to be installed, pip checks what type of item
-each is, in the following order:
-
-1. Project or archive URL.
-2. Local directory (which must contain a ``setup.py``, or pip will report
-   an error).
-3. Local file (a sdist or wheel format archive, following the naming
-   conventions for those formats).
-4. A requirement, as specified in :pep:`440`.
-
-Each item identified is added to the set of requirements to be satisfied by
-the install.
-
-Working Out the Name and Version
---------------------------------
-
-For each candidate item, pip needs to know the project name and version. For
-wheels (identified by the ``.whl`` file extension) this can be obtained from
-the filename, as per the Wheel spec. For local directories, or explicitly
-specified sdist files, the ``setup.py egg_info`` command is used to determine
-the project metadata. For sdists located via an index, the filename is parsed
-for the name and project version (this is in theory slightly less reliable
-than using the ``egg_info`` command, but avoids downloading and processing
-unnecessary numbers of files).
-
-Any URL may use the ``#egg=name`` syntax (see :ref:`VCS Support`) to
-explicitly state the project name.
-
-Satisfying Requirements
------------------------
-
-Once pip has the set of requirements to satisfy, it chooses which version of
-each requirement to install using the simple rule that the latest version that
-satisfies the given constraints will be installed (but see :ref:`here 
`
-for an exception regarding pre-release versions). Where more than one source of
-the chosen version is available, it is assumed that any source is acceptable
-(as otherwise the versions would differ).
-
-Installation Order
-------------------
-
-.. note::
-
-   This section is only about installation order of runtime dependencies, and
-   does not apply to build dependencies (those are specified using PEP 518).
-
-As of v6.1.0, pip installs dependencies before their dependents, i.e. in
-"topological order."  This is the only commitment pip currently makes related
-to order.  While it may be coincidentally true that pip will install things in
-the order of the install arguments or in the order of the items in a
-requirements file, this is not a promise.
-
-In the event of a dependency cycle (aka "circular dependency"), the current
-implementation (which might possibly change later) has it such that the first
-encountered member of the cycle is installed last.
-
-For instance, if quux depends on foo which depends on bar which depends on baz,
-which depends on foo:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip install quux
-      ...
-      Installing collected packages baz, bar, foo, quux
-
-      $ python -m pip install bar
-      ...
-      Installing collected packages foo, baz, bar
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip install quux
-      ...
-      Installing collected packages baz, bar, foo, quux
-
-      C:\> py -m pip install bar
-      ...
-      Installing collected packages foo, baz, bar
-
-
-Prior to v6.1.0, pip made no commitments about install order.
-
-The decision to install topologically is based on the principle that
-installations should proceed in a way that leaves the environment usable at each
-step. This has two main practical benefits:
-
-1. Concurrent use of the environment during the install is more likely to work.
-2. A failed install is less likely to leave a broken environment.  Although pip
-   would like to support failure rollbacks eventually, in the mean time, this is
-   an improvement.
-
-Although the new install order is not intended to replace (and does not replace)
-the use of ``setup_requires`` to declare build dependencies, it may help certain
-projects install from sdist (that might previously fail) that fit the following
-profile:
-
-1. They have build dependencies that are also declared as install dependencies
-   using ``install_requires``.
-2. ``python setup.py egg_info`` works without their build dependencies being
-   installed.
-3. For whatever reason, they don't or won't declare their build dependencies using
-   ``setup_requires``.
-
-
-.. _`Requirements File Format`:
-
-Requirements File Format
-------------------------
-
-Each line of the requirements file indicates something to be installed,
-and like arguments to :ref:`pip install`, the following forms are supported::
-
-    [[--option]...]
-     [; markers] [[--option]...]
-    
-    [-e] 
-    [-e] 
-
-For details on requirement specifiers, see :ref:`Requirement Specifiers`.
-
-See the :ref:`pip install Examples` for examples of all these forms.
-
-A line that begins with ``#`` is treated as a comment and ignored. Whitespace
-followed by a ``#`` causes the ``#`` and the remainder of the line to be
-treated as a comment.
-
-A line ending in an unescaped ``\`` is treated as a line continuation
-and the newline following it is effectively ignored.
-
-Comments are stripped *after* line continuations are processed.
-
-To interpret the requirements file in UTF-8 format add a comment
-``# -*- coding: utf-8 -*-`` to the first or second line of the file.
-
-The following options are supported:
-
-.. pip-requirements-file-options-ref-list::
-
-Please note that the above options are global options, and should be specified on their individual lines.
-The options which can be applied to individual requirements are
-:ref:`--install-option `, :ref:`--global-option ` and ``--hash``.
-
-For example, to specify :ref:`--pre `, :ref:`--no-index ` and two
-:ref:`--find-links ` locations:
-
-::
-
---pre
---no-index
---find-links /my/local/archives
---find-links http://some.archives.com/archives
-
-
-If you wish, you can refer to other requirements files, like this::
-
-    -r more_requirements.txt
-
-You can also refer to :ref:`constraints files `, like this::
-
-    -c some_constraints.txt
-
-.. _`Using Environment Variables`:
-
-Using Environment Variables
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Since version 10, pip supports the use of environment variables inside the
-requirements file. You can now store sensitive data (tokens, keys, etc.) in
-environment variables and only specify the variable name for your requirements,
-letting pip lookup the value at runtime. This approach aligns with the commonly
-used `12-factor configuration pattern `_.
-
-You have to use the POSIX format for variable names including brackets around
-the uppercase name as shown in this example: ``${API_TOKEN}``. pip will attempt
-to find the corresponding environment variable defined on the host system at
-runtime.
-
-.. note::
-
-   There is no support for other variable expansion syntaxes such as
-   ``$VARIABLE`` and ``%VARIABLE%``.
-
-
-.. _`Example Requirements File`:
-
-Example Requirements File
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Use ``pip install -r example-requirements.txt`` to install::
-
-    #
-    ####### example-requirements.txt #######
-    #
-    ###### Requirements without Version Specifiers ######
-    nose
-    nose-cov
-    beautifulsoup4
-    #
-    ###### Requirements with Version Specifiers ######
-    #   See https://www.python.org/dev/peps/pep-0440/#version-specifiers
-    docopt == 0.6.1             # Version Matching. Must be version 0.6.1
-    keyring >= 4.1.1            # Minimum version 4.1.1
-    coverage != 3.5             # Version Exclusion. Anything except version 3.5
-    Mopidy-Dirble ~= 1.1        # Compatible release. Same as >= 1.1, == 1.*
-    #
-    ###### Refer to other requirements files ######
-    -r other-requirements.txt
-    #
-    #
-    ###### A particular file ######
-    ./downloads/numpy-1.9.2-cp34-none-win32.whl
-    http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl
-    #
-    ###### Additional Requirements without Version Specifiers ######
-    #   Same as 1st section, just here to show that you can put things in any order.
-    rejected
-    green
-    #
-
-.. _`Requirement Specifiers`:
-
-Requirement Specifiers
-----------------------
-
-pip supports installing from a package index using a :term:`requirement
-specifier `. Generally speaking, a requirement
-specifier is composed of a project name followed by optional :term:`version
-specifiers `.  :pep:`508` contains a full specification
-of the format of a requirement. Since version 18.1 pip supports the
-``url_req``-form specification.
-
-Some examples:
-
- ::
-
-  SomeProject
-  SomeProject == 1.3
-  SomeProject >=1.2,<2.0
-  SomeProject[foo, bar]
-  SomeProject~=1.4.2
-
-Since version 6.0, pip also supports specifiers containing `environment markers
-`__ like so:
-
- ::
-
-  SomeProject ==5.4 ; python_version < '3.8'
-  SomeProject; sys_platform == 'win32'
-
-Since version 19.1, pip also supports `direct references
-`__ like so:
-
- ::
-
-  SomeProject @ file:///somewhere/...
-
-Environment markers are supported in the command line and in requirements files.
-
-.. note::
-
-   Use quotes around specifiers in the shell when using ``>``, ``<``, or when
-   using environment markers. Don't use quotes in requirement files. [1]_
-
-
-.. _`Per-requirement Overrides`:
-
-Per-requirement Overrides
--------------------------
-
-Since version 7.0 pip supports controlling the command line options given to
-``setup.py`` via requirements files. This disables the use of wheels (cached or
-otherwise) for that package, as ``setup.py`` does not exist for wheels.
-
-The ``--global-option`` and ``--install-option`` options are used to pass
-options to ``setup.py``. For example:
-
- ::
-
-    FooProject >= 1.2 --global-option="--no-user-cfg" \
-                      --install-option="--prefix='/usr/local'" \
-                      --install-option="--no-compile"
-
-The above translates roughly into running FooProject's ``setup.py``
-script as:
-
- ::
-
-   python setup.py --no-user-cfg install --prefix='/usr/local' --no-compile
-
-Note that the only way of giving more than one option to ``setup.py``
-is through multiple ``--global-option`` and ``--install-option``
-options, as shown in the example above. The value of each option is
-passed as a single argument to the ``setup.py`` script. Therefore, a
-line such as the following is invalid and would result in an
-installation error.
-
-::
-
-   # Invalid. Please use '--install-option' twice as shown above.
-   FooProject >= 1.2 --install-option="--prefix=/usr/local --no-compile"
-
-
-.. _`Pre Release Versions`:
-
-Pre-release Versions
---------------------
-
-Starting with v1.4, pip will only install stable versions as specified by
-`pre-releases`_ by default. If a version cannot be parsed as a compliant :pep:`440`
-version then it is assumed to be a pre-release.
-
-If a Requirement specifier includes a pre-release or development version
-(e.g. ``>=0.0.dev0``) then pip will allow pre-release and development versions
-for that requirement. This does not include the != flag.
-
-The ``pip install`` command also supports a :ref:`--pre ` flag
-that enables installation of pre-releases and development releases.
-
-
-.. _pre-releases: https://www.python.org/dev/peps/pep-0440/#handling-of-pre-releases
-
-
-.. _`VCS Support`:
-
-VCS Support
------------
-
-pip supports installing from Git, Mercurial, Subversion and Bazaar, and detects
-the type of VCS using URL prefixes: ``git+``, ``hg+``, ``svn+``, and ``bzr+``.
-
-pip requires a working VCS command on your path: ``git``, ``hg``, ``svn``, or
-``bzr``.
-
-VCS projects can be installed in :ref:`editable mode ` (using
-the :ref:`--editable ` option) or not.
-
-* For editable installs, the clone location by default is ``/src/SomeProject`` in virtual environments, and
-  ``/src/SomeProject``
-  for global installs.  The :ref:`--src ` option can be used to
-  modify this location.
-* For non-editable installs, the project is built locally in a temp dir and then
-  installed normally. Note that if a satisfactory version of the package is
-  already installed, the VCS source will not overwrite it without an
-  ``--upgrade`` flag. VCS requirements pin the package version (specified
-  in the ``setup.py`` file) of the target commit, not necessarily the commit
-  itself.
-* The :ref:`pip freeze` subcommand will record the VCS requirement specifier
-  (referencing a specific commit) if and only if the install is done using the
-  editable option.
-
-The "project name" component of the URL suffix ``egg=``
-is used by pip in its dependency logic to identify the project prior
-to pip downloading and analyzing the metadata. For projects
-where ``setup.py`` is not in the root of project, the "subdirectory" component
-is used. The value of the "subdirectory" component should be a path starting
-from the root of the project to where ``setup.py`` is located.
-
-If your repository layout is::
-
-   pkg_dir
-   ├── setup.py  # setup.py for package "pkg"
-   └── some_module.py
-   other_dir
-   └── some_file
-   some_other_file
-
-Then, to install from this repository, the syntax would be:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir"
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir"
-
-
-Git
-^^^
-
-pip currently supports cloning over ``git``, ``git+http``, ``git+https``,
-``git+ssh``, ``git+git`` and ``git+file``.
-
-.. warning::
-
-    Note that the use of ``git``, ``git+git``, and ``git+http`` is discouraged.
-    The former two use `the Git Protocol`_, which lacks authentication, and HTTP is
-    insecure due to lack of TLS based encryption.
-
-Here are the supported forms::
-
-    [-e] git+http://git.example.com/MyProject#egg=MyProject
-    [-e] git+https://git.example.com/MyProject#egg=MyProject
-    [-e] git+ssh://git.example.com/MyProject#egg=MyProject
-    [-e] git+file:///home/user/projects/MyProject#egg=MyProject
-
-Passing a branch name, a commit hash, a tag name or a git ref is possible like so::
-
-    [-e] git+https://git.example.com/MyProject.git@master#egg=MyProject
-    [-e] git+https://git.example.com/MyProject.git@v1.0#egg=MyProject
-    [-e] git+https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709#egg=MyProject
-    [-e] git+https://git.example.com/MyProject.git@refs/pull/123/head#egg=MyProject
-
-When passing a commit hash, specifying a full hash is preferable to a partial
-hash because a full hash allows pip to operate more efficiently (e.g. by
-making fewer network calls).
-
-.. _`the Git Protocol`: https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols
-
-Mercurial
-^^^^^^^^^
-
-The supported schemes are: ``hg+file``, ``hg+http``, ``hg+https``,
-``hg+static-http``, and ``hg+ssh``.
-
-Here are the supported forms::
-
-    [-e] hg+http://hg.myproject.org/MyProject#egg=MyProject
-    [-e] hg+https://hg.myproject.org/MyProject#egg=MyProject
-    [-e] hg+ssh://hg.myproject.org/MyProject#egg=MyProject
-    [-e] hg+file:///home/user/projects/MyProject#egg=MyProject
-
-You can also specify a revision number, a revision hash, a tag name or a local
-branch name like so::
-
-    [-e] hg+http://hg.example.com/MyProject@da39a3ee5e6b#egg=MyProject
-    [-e] hg+http://hg.example.com/MyProject@2019#egg=MyProject
-    [-e] hg+http://hg.example.com/MyProject@v1.0#egg=MyProject
-    [-e] hg+http://hg.example.com/MyProject@special_feature#egg=MyProject
-
-Subversion
-^^^^^^^^^^
-
-pip supports the URL schemes ``svn``, ``svn+svn``, ``svn+http``, ``svn+https``, ``svn+ssh``.
-
-Here are some of the supported forms::
-
-    [-e] svn+https://svn.example.com/MyProject#egg=MyProject
-    [-e] svn+ssh://svn.example.com/MyProject#egg=MyProject
-    [-e] svn+ssh://user@svn.example.com/MyProject#egg=MyProject
-
-You can also give specific revisions to an SVN URL, like so::
-
-    [-e] svn+svn://svn.example.com/svn/MyProject#egg=MyProject
-    [-e] svn+http://svn.example.com/svn/MyProject/trunk@2019#egg=MyProject
-
-which will check out revision 2019.  ``@{20080101}`` would also check
-out the revision from 2008-01-01. You can only check out specific
-revisions using ``-e svn+...``.
-
-Bazaar
-^^^^^^
-
-pip supports Bazaar using the ``bzr+http``, ``bzr+https``, ``bzr+ssh``,
-``bzr+sftp``, ``bzr+ftp`` and ``bzr+lp`` schemes.
-
-Here are the supported forms::
-
-    [-e] bzr+http://bzr.example.com/MyProject/trunk#egg=MyProject
-    [-e] bzr+sftp://user@example.com/MyProject/trunk#egg=MyProject
-    [-e] bzr+ssh://user@example.com/MyProject/trunk#egg=MyProject
-    [-e] bzr+ftp://user@example.com/MyProject/trunk#egg=MyProject
-    [-e] bzr+lp:MyProject#egg=MyProject
-
-Tags or revisions can be installed like so::
-
-    [-e] bzr+https://bzr.example.com/MyProject/trunk@2019#egg=MyProject
-    [-e] bzr+http://bzr.example.com/MyProject/trunk@v1.0#egg=MyProject
-
-Using Environment Variables
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Since version 10, pip also makes it possible to use environment variables which
-makes it possible to reference private repositories without having to store
-access tokens in the requirements file. For example, a private git repository
-allowing Basic Auth for authentication can be refenced like this::
-
-    [-e] git+http://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
-    [-e] git+https://${AUTH_USER}:${AUTH_PASSWORD}@git.example.com/MyProject#egg=MyProject
-
-.. note::
-
-   Only ``${VARIABLE}`` is supported, other formats like ``$VARIABLE`` or
-   ``%VARIABLE%`` won't work.
-
-Finding Packages
-----------------
-
-pip searches for packages on `PyPI`_ using the
-`HTTP simple interface `_,
-which is documented `here `_
-and `there `_.
-
-pip offers a number of package index options for modifying how packages are
-found.
-
-pip looks for packages in a number of places: on PyPI (if not disabled via
-``--no-index``), in the local filesystem, and in any additional repositories
-specified via ``--find-links`` or ``--index-url``. There is no ordering in
-the locations that are searched. Rather they are all checked, and the "best"
-match for the requirements (in terms of version number - see :pep:`440` for
-details) is selected.
-
-See the :ref:`pip install Examples`.
-
-
-.. _`SSL Certificate Verification`:
-
-SSL Certificate Verification
-----------------------------
-
-Starting with v1.3, pip provides SSL certificate verification over https, to
-prevent man-in-the-middle attacks against PyPI downloads.
-
-
-.. _`Caching`:
-
-Caching
--------
-
-Starting with v6.0, pip provides an on-by-default cache which functions
-similarly to that of a web browser. While the cache is on by default and is
-designed do the right thing by default you can disable the cache and always
-access PyPI by utilizing the ``--no-cache-dir`` option.
-
-When making any HTTP request pip will first check its local cache to determine
-if it has a suitable response stored for that request which has not expired. If
-it does then it simply returns that response and doesn't make the request.
-
-If it has a response stored, but it has expired, then it will attempt to make a
-conditional request to refresh the cache which will either return an empty
-response telling pip to simply use the cached item (and refresh the expiration
-timer) or it will return a whole new response which pip can then store in the
-cache.
-
-While this cache attempts to minimize network activity, it does not prevent
-network access altogether. If you want a local install solution that
-circumvents accessing PyPI, see :ref:`Installing from local packages`.
-
-The default location for the cache directory depends on the operating system:
-
-Unix
-  :file:`~/.cache/pip` and it respects the ``XDG_CACHE_HOME`` directory.
-macOS
-  :file:`~/Library/Caches/pip`.
-Windows
-  :file:`\\pip\\Cache`
-
-Run ``pip cache dir`` to show the cache directory and see :ref:`pip cache` to
-inspect and manage pip’s cache.
-
-
-.. _`Wheel cache`:
-
-Wheel Cache
-^^^^^^^^^^^
-
-pip will read from the subdirectory ``wheels`` within the pip cache directory
-and use any packages found there. This is disabled via the same
-``--no-cache-dir`` option that disables the HTTP cache. The internal structure
-of that is not part of the pip API. As of 7.0, pip makes a subdirectory for
-each sdist that wheels are built from and places the resulting wheels inside.
-
-As of version 20.0, pip also caches wheels when building from an immutable Git
-reference (i.e. a commit hash).
-
-pip attempts to choose the best wheels from those built in preference to
-building a new wheel. Note that this means when a package has both optional
-C extensions and builds ``py`` tagged wheels when the C extension can't be built
-that pip will not attempt to build a better wheel for Pythons that would have
-supported it, once any generic wheel is built. To correct this, make sure that
-the wheels are built with Python specific tags - e.g. pp on PyPy.
-
-When no wheels are found for an sdist, pip will attempt to build a wheel
-automatically and insert it into the wheel cache.
-
-
-.. _`hash-checking mode`:
-
-Hash-Checking Mode
-------------------
-
-Since version 8.0, pip can check downloaded package archives against local
-hashes to protect against remote tampering. To verify a package against one or
-more hashes, add them to the end of the line::
-
-    FooProject == 1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 \
-                      --hash=sha256:486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65260e9cb8a7
-
-(The ability to use multiple hashes is important when a package has both
-binary and source distributions or when it offers binary distributions for a
-variety of platforms.)
-
-The recommended hash algorithm at the moment is sha256, but stronger ones are
-allowed, including all those supported by ``hashlib``. However, weaker ones
-such as md5, sha1, and sha224 are excluded to avoid giving a false sense of
-security.
-
-Hash verification is an all-or-nothing proposition. Specifying a ``--hash``
-against any requirement not only checks that hash but also activates a global
-*hash-checking mode*, which imposes several other security restrictions:
-
-* Hashes are required for all requirements. This is because a partially-hashed
-  requirements file is of little use and thus likely an error: a malicious
-  actor could slip bad code into the installation via one of the unhashed
-  requirements. Note that hashes embedded in URL-style requirements via the
-  ``#md5=...`` syntax suffice to satisfy this rule (regardless of hash
-  strength, for legacy reasons), though you should use a stronger
-  hash like sha256 whenever possible.
-* Hashes are required for all dependencies. An error results if there is a
-  dependency that is not spelled out and hashed in the requirements file.
-* Requirements that take the form of project names (rather than URLs or local
-  filesystem paths) must be pinned to a specific version using ``==``. This
-  prevents a surprising hash mismatch upon the release of a new version
-  that matches the requirement specifier.
-* ``--egg`` is disallowed, because it delegates installation of dependencies
-  to setuptools, giving up pip's ability to enforce any of the above.
-
-.. _`--require-hashes`:
-
-Hash-checking mode can be forced on with the ``--require-hashes`` command-line
-option:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: console
-
-      $ python -m pip install --require-hashes -r requirements.txt
-      ...
-      Hashes are required in --require-hashes mode (implicitly on when a hash is
-      specified for any package). These requirements were missing hashes,
-      leaving them open to tampering. These are the hashes the downloaded
-      archives actually had. You can add lines like these to your requirements
-      files to prevent tampering.
-         pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa
-         more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0
-
-.. tab:: Windows
-
-   .. code-block:: console
-
-      C:\> py -m pip install --require-hashes -r requirements.txt
-      ...
-      Hashes are required in --require-hashes mode (implicitly on when a hash is
-      specified for any package). These requirements were missing hashes,
-      leaving them open to tampering. These are the hashes the downloaded
-      archives actually had. You can add lines like these to your requirements
-      files to prevent tampering.
-         pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa
-         more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0
-
-
-This can be useful in deploy scripts, to ensure that the author of the
-requirements file provided hashes. It is also a convenient way to bootstrap
-your list of hashes, since it shows the hashes of the packages it fetched. It
-fetches only the preferred archive for each package, so you may still need to
-add hashes for alternatives archives using :ref:`pip hash`: for instance if
-there is both a binary and a source distribution.
-
-The :ref:`wheel cache ` is disabled in hash-checking mode to
-prevent spurious hash mismatch errors. These would otherwise occur while
-installing sdists that had already been automatically built into cached wheels:
-those wheels would be selected for installation, but their hashes would not
-match the sdist ones from the requirements file. A further complication is that
-locally built wheels are nondeterministic: contemporary modification times make
-their way into the archive, making hashes unpredictable across machines and
-cache flushes. Compilation of C code adds further nondeterminism, as many
-compilers include random-seeded values in their output. However, wheels fetched
-from index servers are the same every time. They land in pip's HTTP cache, not
-its wheel cache, and are used normally in hash-checking mode. The only downside
-of having the wheel cache disabled is thus extra build time for sdists, and
-this can be solved by making sure pre-built wheels are available from the index
-server.
-
-Hash-checking mode also works with :ref:`pip download` and :ref:`pip wheel`. A
-:ref:`comparison of hash-checking mode with other repeatability strategies
-` is available in the User Guide.
-
-.. warning::
-
-   Beware of the ``setup_requires`` keyword arg in :file:`setup.py`. The
-   (rare) packages that use it will cause those dependencies to be downloaded
-   by setuptools directly, skipping pip's hash-checking. If you need to use
-   such a package, see :ref:`Controlling
-   setup_requires`.
-
-.. warning::
-
-   Be careful not to nullify all your security work when you install your
-   actual project by using setuptools directly: for example, by calling
-   ``python setup.py install``, ``python setup.py develop``, or
-   ``easy_install``. Setuptools will happily go out and download, unchecked,
-   anything you missed in your requirements file—and it’s easy to miss things
-   as your project evolves. To be safe, install your project using pip and
-   :ref:`--no-deps `.
-
-   Instead of ``python setup.py develop``, use...
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --no-deps -e .
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --no-deps -e .
-
-
-   Instead of ``python setup.py install``, use...
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --no-deps .
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --no-deps .
-
-Hashes from PyPI
-^^^^^^^^^^^^^^^^
-
-PyPI provides an MD5 hash in the fragment portion of each package download URL,
-like ``#md5=123...``, which pip checks as a protection against download
-corruption. Other hash algorithms that have guaranteed support from ``hashlib``
-are also supported here: sha1, sha224, sha384, sha256, and sha512. Since this
-hash originates remotely, it is not a useful guard against tampering and thus
-does not satisfy the ``--require-hashes`` demand that every package have a
-local hash.
-
-
-Local project installs
-----------------------
-
-pip supports installing local project in both regular mode and editable mode.
-You can install local projects by specifying the project path to pip:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install path/to/SomeProject
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install path/to/SomeProject
-
-During regular installation, pip will copy the entire project directory to a
-temporary location and install from there. The exception is that pip will
-exclude .tox and .nox directories present in the top level of the project from
-being copied.
-
-
-.. _`editable-installs`:
-
-"Editable" Installs
-^^^^^^^^^^^^^^^^^^^
-
-"Editable" installs are fundamentally `"setuptools develop mode"
-`_
-installs.
-
-You can install local projects or VCS projects in "editable" mode:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install -e path/to/SomeProject
-      python -m pip install -e git+http://repo/my_project.git#egg=SomeProject
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install -e path/to/SomeProject
-      py -m pip install -e git+http://repo/my_project.git#egg=SomeProject
-
-
-(See the :ref:`VCS Support` section above for more information on VCS-related syntax.)
-
-For local projects, the "SomeProject.egg-info" directory is created relative to
-the project path.  This is one advantage over just using ``setup.py develop``,
-which creates the "egg-info" directly relative the current working directory.
-
-
-.. _`controlling-setup-requires`:
-
-Controlling setup_requires
---------------------------
-
-Setuptools offers the ``setup_requires`` `setup() keyword
-`_
-for specifying dependencies that need to be present in order for the
-``setup.py`` script to run.  Internally, Setuptools uses ``easy_install``
-to fulfill these dependencies.
-
-pip has no way to control how these dependencies are located.  None of the
-package index options have an effect.
-
-The solution is to configure a "system" or "personal" `Distutils configuration
-file
-`_ to
-manage the fulfillment.
-
-For example, to have the dependency located at an alternate index, add this:
-
-::
-
-  [easy_install]
-  index_url = https://my.index-mirror.com
-
-To have the dependency located from a local directory and not crawl PyPI, add this:
-
-::
-
-  [easy_install]
-  allow_hosts = ''
-  find_links = file:///path/to/local/archives/
-
-
-Build System Interface
-----------------------
-
-In order for pip to install a package from source, ``setup.py`` must implement
-the following commands::
-
-    setup.py egg_info [--egg-base XXX]
-    setup.py install --record XXX [--single-version-externally-managed] [--root XXX] [--compile|--no-compile] [--install-headers XXX]
-
-The ``egg_info`` command should create egg metadata for the package, as
-described in the setuptools documentation at
-https://setuptools.readthedocs.io/en/latest/setuptools.html#egg-info-create-egg-metadata-and-set-build-tags
-
-The ``install`` command should implement the complete process of installing the
-package to the target directory XXX.
-
-To install a package in "editable" mode (``pip install -e``), ``setup.py`` must
-implement the following command::
-
-    setup.py develop --no-deps
-
-This should implement the complete process of installing the package in
-"editable" mode.
-
-All packages will be attempted to built into wheels::
-
-    setup.py bdist_wheel -d XXX
-
-One further ``setup.py`` command is invoked by ``pip install``::
-
-    setup.py clean
-
-This command is invoked to clean up temporary commands from the build. (TODO:
-Investigate in more detail when this command is required).
-
-No other build system commands are invoked by the ``pip install`` command.
-
-Installing a package from a wheel does not invoke the build system at all.
-
-.. _PyPI: https://pypi.org/
-.. _setuptools extras: https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies
-
-
-
-.. _`pip install Options`:
-
-
-Options
-=======
-
-.. pip-command-options:: install
-
-.. pip-index-options:: install
-
-
-.. _`pip install Examples`:
-
-
-Examples
-========
-
-#. Install ``SomePackage`` and its dependencies from `PyPI`_ using :ref:`Requirement Specifiers`
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install SomePackage            # latest version
-         python -m pip install SomePackage==1.0.4     # specific version
-         python -m pip install 'SomePackage>=1.0.4'   # minimum version
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install SomePackage            # latest version
-         py -m pip install SomePackage==1.0.4     # specific version
-         py -m pip install 'SomePackage>=1.0.4'   # minimum version
-
-
-#. Install a list of requirements specified in a file.  See the :ref:`Requirements files `.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install -r requirements.txt
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install -r requirements.txt
-
-
-#. Upgrade an already installed ``SomePackage`` to the latest from PyPI.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --upgrade SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --upgrade SomePackage
-
-
-#. Install a local project in "editable" mode. See the section on :ref:`Editable Installs `.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install -e .                # project in current directory
-         python -m pip install -e path/to/project  # project in another directory
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install -e .                 # project in current directory
-         py -m pip install -e path/to/project   # project in another directory
-
-
-#. Install a project from VCS
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1
-
-
-#. Install a project from VCS in "editable" mode. See the sections on :ref:`VCS Support ` and :ref:`Editable Installs `.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage          # from git
-         python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage            # from mercurial
-         python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage         # from svn
-         python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage  # from 'feature' branch
-         python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage          # from git
-         py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage            # from mercurial
-         py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage         # from svn
-         py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage  # from 'feature' branch
-         py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory
-
-#. Install a package with `setuptools extras`_.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install SomePackage[PDF]
-         python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path"
-         python -m pip install .[PDF]  # project in current directory
-         python -m pip install SomePackage[PDF]==3.0
-         python -m pip install SomePackage[PDF,EPUB]  # multiple extras
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install SomePackage[PDF]
-         py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path"
-         py -m pip install .[PDF]  # project in current directory
-         py -m pip install SomePackage[PDF]==3.0
-         py -m pip install SomePackage[PDF,EPUB]  # multiple extras
-
-#. Install a particular source archive file.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install ./downloads/SomePackage-1.0.4.tar.gz
-         python -m pip install http://my.package.repo/SomePackage-1.0.4.zip
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install ./downloads/SomePackage-1.0.4.tar.gz
-         py -m pip install http://my.package.repo/SomePackage-1.0.4.zip
-
-#. Install a particular source archive file following :pep:`440` direct references.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl
-         python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl"
-         python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl
-         py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl"
-         py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz
-
-#. Install from alternative package repositories.
-
-   Install from a different index, and not `PyPI`_
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --index-url http://my.package.repo/simple/ SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --index-url http://my.package.repo/simple/ SomePackage
-
-   Search an additional index during install, in addition to `PyPI`_
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --extra-index-url http://my.package.repo/simple SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --extra-index-url http://my.package.repo/simple SomePackage
-
-   Install from a local flat directory containing archives (and don't scan indexes):
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --no-index --find-links=file:///local/dir/ SomePackage
-         python -m pip install --no-index --find-links=/local/dir/ SomePackage
-         python -m pip install --no-index --find-links=relative/dir/ SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --no-index --find-links=file:///local/dir/ SomePackage
-         py -m pip install --no-index --find-links=/local/dir/ SomePackage
-         py -m pip install --no-index --find-links=relative/dir/ SomePackage
-
-
-#. Find pre-release and development versions, in addition to stable versions.  By default, pip only finds stable versions.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install --pre SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install --pre SomePackage
-
-
-#. Install packages from source.
-
-   Do not use any binary packages
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install SomePackage1 SomePackage2 --no-binary :all:
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install SomePackage1 SomePackage2 --no-binary :all:
-
-   Specify ``SomePackage1`` to be installed from source:
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1
-
-----
-
-.. [1] This is true with the exception that pip v7.0 and v7.0.1 required quotes
-       around specifiers containing environment markers in requirement files.
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_install`
diff --git a/docs/html/reference/pip_list.rst b/docs/html/reference/pip_list.rst
index 5119a804c0d..3768baf60d6 100644
--- a/docs/html/reference/pip_list.rst
+++ b/docs/html/reference/pip_list.rst
@@ -1,201 +1,11 @@
-.. _`pip list`:
+:orphan:
 
-========
-pip list
-========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_list/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: list "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: list "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: list
-
-
-Options
-=======
-
-.. pip-command-options:: list
-
-.. pip-index-options:: list
-
-
-Examples
-========
-
-#. List installed packages.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list
-         docutils (0.10)
-         Jinja2 (2.7.2)
-         MarkupSafe (0.18)
-         Pygments (1.6)
-         Sphinx (1.2.1)
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list
-         docutils (0.10)
-         Jinja2 (2.7.2)
-         MarkupSafe (0.18)
-         Pygments (1.6)
-         Sphinx (1.2.1)
-
-#. List outdated packages (excluding editables), and the latest version available.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list --outdated
-         docutils (Current: 0.10 Latest: 0.11)
-         Sphinx (Current: 1.2.1 Latest: 1.2.2)
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list --outdated
-         docutils (Current: 0.10 Latest: 0.11)
-         Sphinx (Current: 1.2.1 Latest: 1.2.2)
-
-#. List installed packages with column formatting.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list --format columns
-         Package Version
-         ------- -------
-         docopt  0.6.2
-         idlex   1.13
-         jedi    0.9.0
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list --format columns
-         Package Version
-         ------- -------
-         docopt  0.6.2
-         idlex   1.13
-         jedi    0.9.0
-
-#. List outdated packages with column formatting.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list -o --format columns
-         Package    Version Latest Type
-         ---------- ------- ------ -----
-         retry      0.8.1   0.9.1  wheel
-         setuptools 20.6.7  21.0.0 wheel
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list -o --format columns
-         Package    Version Latest Type
-         ---------- ------- ------ -----
-         retry      0.8.1   0.9.1  wheel
-         setuptools 20.6.7  21.0.0 wheel
-
-#. List packages that are not dependencies of other packages. Can be combined with
-   other options.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list --outdated --not-required
-         docutils (Current: 0.10 Latest: 0.11)
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list --outdated --not-required
-         docutils (Current: 0.10 Latest: 0.11)
-
-#. Use legacy formatting
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list --format=legacy
-         colorama (0.3.7)
-         docopt (0.6.2)
-         idlex (1.13)
-         jedi (0.9.0)
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list --format=legacy
-         colorama (0.3.7)
-         docopt (0.6.2)
-         idlex (1.13)
-         jedi (0.9.0)
-
-#. Use json formatting
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list --format=json
-         [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ...
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list --format=json
-         [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ...
-
-#. Use freeze formatting
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip list --format=freeze
-         colorama==0.3.7
-         docopt==0.6.2
-         idlex==1.13
-         jedi==0.9.0
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip list --format=freeze
-         colorama==0.3.7
-         docopt==0.6.2
-         idlex==1.13
-         jedi==0.9.0
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_list`
diff --git a/docs/html/reference/pip_search.rst b/docs/html/reference/pip_search.rst
index 9905a1bafac..0a7532ee79d 100644
--- a/docs/html/reference/pip_search.rst
+++ b/docs/html/reference/pip_search.rst
@@ -1,52 +1,11 @@
-.. _`pip search`:
+:orphan:
 
-==========
-pip search
-==========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_search/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: search "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: search "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: search
-
-
-Options
-=======
-
-.. pip-command-options:: search
-
-
-Examples
-========
-
-#. Search for "peppercorn"
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip search peppercorn
-         pepperedform    - Helpers for using peppercorn with formprocess.
-         peppercorn      - A library for converting a token stream into [...]
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip search peppercorn
-         pepperedform    - Helpers for using peppercorn with formprocess.
-         peppercorn      - A library for converting a token stream into [...]
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_search`
diff --git a/docs/html/reference/pip_show.rst b/docs/html/reference/pip_show.rst
index b603f786fd9..b2ce3c7d8c3 100644
--- a/docs/html/reference/pip_show.rst
+++ b/docs/html/reference/pip_show.rst
@@ -1,154 +1,11 @@
-.. _`pip show`:
+:orphan:
 
-========
-pip show
-========
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_show/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: show "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: show "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: show
-
-
-Options
-=======
-
-.. pip-command-options:: show
-
-
-Examples
-========
-
-#. Show information about a package:
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip show sphinx
-         Name: Sphinx
-         Version: 1.4.5
-         Summary: Python documentation generator
-         Home-page: http://sphinx-doc.org/
-         Author: Georg Brandl
-         Author-email: georg@python.org
-         License: BSD
-         Location: /my/env/lib/python2.7/site-packages
-         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip show sphinx
-         Name: Sphinx
-         Version: 1.4.5
-         Summary: Python documentation generator
-         Home-page: http://sphinx-doc.org/
-         Author: Georg Brandl
-         Author-email: georg@python.org
-         License: BSD
-         Location: /my/env/lib/python2.7/site-packages
-         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
-
-#. Show all information about a package
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip show --verbose sphinx
-         Name: Sphinx
-         Version: 1.4.5
-         Summary: Python documentation generator
-         Home-page: http://sphinx-doc.org/
-         Author: Georg Brandl
-         Author-email: georg@python.org
-         License: BSD
-         Location: /my/env/lib/python2.7/site-packages
-         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
-         Metadata-Version: 2.0
-         Installer:
-         Classifiers:
-            Development Status :: 5 - Production/Stable
-            Environment :: Console
-            Environment :: Web Environment
-            Intended Audience :: Developers
-            Intended Audience :: Education
-            License :: OSI Approved :: BSD License
-            Operating System :: OS Independent
-            Programming Language :: Python
-            Programming Language :: Python :: 2
-            Programming Language :: Python :: 3
-            Framework :: Sphinx
-            Framework :: Sphinx :: Extension
-            Framework :: Sphinx :: Theme
-            Topic :: Documentation
-            Topic :: Documentation :: Sphinx
-            Topic :: Text Processing
-            Topic :: Utilities
-         Entry-points:
-            [console_scripts]
-            sphinx-apidoc = sphinx.apidoc:main
-            sphinx-autogen = sphinx.ext.autosummary.generate:main
-            sphinx-build = sphinx:main
-            sphinx-quickstart = sphinx.quickstart:main
-            [distutils.commands]
-            build_sphinx = sphinx.setup_command:BuildDoc
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip show --verbose sphinx
-         Name: Sphinx
-         Version: 1.4.5
-         Summary: Python documentation generator
-         Home-page: http://sphinx-doc.org/
-         Author: Georg Brandl
-         Author-email: georg@python.org
-         License: BSD
-         Location: /my/env/lib/python2.7/site-packages
-         Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six
-         Metadata-Version: 2.0
-         Installer:
-         Classifiers:
-            Development Status :: 5 - Production/Stable
-            Environment :: Console
-            Environment :: Web Environment
-            Intended Audience :: Developers
-            Intended Audience :: Education
-            License :: OSI Approved :: BSD License
-            Operating System :: OS Independent
-            Programming Language :: Python
-            Programming Language :: Python :: 2
-            Programming Language :: Python :: 3
-            Framework :: Sphinx
-            Framework :: Sphinx :: Extension
-            Framework :: Sphinx :: Theme
-            Topic :: Documentation
-            Topic :: Documentation :: Sphinx
-            Topic :: Text Processing
-            Topic :: Utilities
-         Entry-points:
-            [console_scripts]
-            sphinx-apidoc = sphinx.apidoc:main
-            sphinx-autogen = sphinx.ext.autosummary.generate:main
-            sphinx-build = sphinx:main
-            sphinx-quickstart = sphinx.quickstart:main
-            [distutils.commands]
-            build_sphinx = sphinx.setup_command:BuildDoc
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_show`
diff --git a/docs/html/reference/pip_uninstall.rst b/docs/html/reference/pip_uninstall.rst
index e6eeb5ebf6a..db84476c859 100644
--- a/docs/html/reference/pip_uninstall.rst
+++ b/docs/html/reference/pip_uninstall.rst
@@ -1,58 +1,11 @@
-.. _`pip uninstall`:
+:orphan:
 
-=============
-pip uninstall
-=============
+.. meta::
 
+  :http-equiv=refresh: 3; url=../../cli/pip_uninstall/
 
+This page has moved
+===================
 
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: uninstall "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: uninstall "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: uninstall
-
-
-Options
-=======
-
-.. pip-command-options:: uninstall
-
-
-Examples
-========
-
-#. Uninstall a package.
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: console
-
-         $ python -m pip uninstall simplejson
-         Uninstalling simplejson:
-            /home/me/env/lib/python3.9/site-packages/simplejson
-            /home/me/env/lib/python3.9/site-packages/simplejson-2.2.1-py3.9.egg-info
-         Proceed (y/n)? y
-            Successfully uninstalled simplejson
-
-   .. tab:: Windows
-
-      .. code-block:: console
-
-         C:\> py -m pip uninstall simplejson
-         Uninstalling simplejson:
-            /home/me/env/lib/python3.9/site-packages/simplejson
-            /home/me/env/lib/python3.9/site-packages/simplejson-2.2.1-py3.9.egg-info
-         Proceed (y/n)? y
-            Successfully uninstalled simplejson
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_uninstall`
diff --git a/docs/html/reference/pip_wheel.rst b/docs/html/reference/pip_wheel.rst
index c2a9543fc99..06861f60763 100644
--- a/docs/html/reference/pip_wheel.rst
+++ b/docs/html/reference/pip_wheel.rst
@@ -1,125 +1,11 @@
+:orphan:
 
-.. _`pip wheel`:
+.. meta::
 
-=========
-pip wheel
-=========
+  :http-equiv=refresh: 3; url=../../cli/pip_wheel/
 
+This page has moved
+===================
 
-
-Usage
-=====
-
-.. tab:: Unix/macOS
-
-   .. pip-command-usage:: wheel "python -m pip"
-
-.. tab:: Windows
-
-   .. pip-command-usage:: wheel "py -m pip"
-
-
-Description
-===========
-
-.. pip-command-description:: wheel
-
-
-Build System Interface
-----------------------
-
-In order for pip to build a wheel, ``setup.py`` must implement the
-``bdist_wheel`` command with the following syntax:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python setup.py bdist_wheel -d TARGET
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py setup.py bdist_wheel -d TARGET
-
-
-This command must create a wheel compatible with the invoking Python
-interpreter, and save that wheel in the directory TARGET.
-
-No other build system commands are invoked by the ``pip wheel`` command.
-
-Customising the build
-^^^^^^^^^^^^^^^^^^^^^
-
-It is possible using ``--global-option`` to include additional build commands
-with their arguments in the ``setup.py`` command. This is currently the only
-way to influence the building of C extensions from the command line. For
-example:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel
-
-
-will result in a build command of
-
-::
-
-    setup.py bdist_ext -DFOO bdist_wheel -d TARGET
-
-which passes a preprocessor symbol to the extension build.
-
-Such usage is considered highly build-system specific and more an accident of
-the current implementation than a supported interface.
-
-
-
-Options
-=======
-
-.. pip-command-options:: wheel
-
-.. pip-index-options:: wheel
-
-
-Examples
-========
-
-#. Build wheels for a requirement (and all its dependencies), and then install
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage
-         python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage
-         py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage
-
-#. Build a wheel for a package from source
-
-   .. tab:: Unix/macOS
-
-      .. code-block:: shell
-
-         python -m pip wheel --no-binary SomePackage SomePackage
-
-   .. tab:: Windows
-
-      .. code-block:: shell
-
-         py -m pip wheel --no-binary SomePackage SomePackage
+You should be redirected automatically in 3 seconds. If that didn't
+work, here's a link: :doc:`../cli/pip_wheel`
diff --git a/docs/html/reference/requirement-specifiers.md b/docs/html/reference/requirement-specifiers.md
new file mode 100644
index 00000000000..d1449e5ef3a
--- /dev/null
+++ b/docs/html/reference/requirement-specifiers.md
@@ -0,0 +1,61 @@
+(Requirement Specifiers)=
+
+# Requirement Specifiers
+
+pip supports installing from a package index using a {term}`requirement specifier `. Generally speaking, a requirement specifier is composed of a project name followed by optional {term}`version specifiers `.
+
+{pep}`508` contains a full specification of the format of a requirement.
+
+```{versionadded} 6.0
+Support for environment markers.
+```
+
+```{versionadded} 19.1
+Support for the direct URL reference form.
+```
+
+## Overview
+
+A requirement specifier comes in two forms:
+
+- name-based, which is composed of:
+
+  - a package name (eg: `requests`)
+  - optionally, a set of "extras" that serve to install optional dependencies (eg: `security`)
+  - optionally, constraints to apply on the version of the package
+  - optionally, environment markers
+
+- URL-based, which is composed of:
+
+  - a package name (eg: `requests`)
+  - optionally, a set of "extras" that serve to install optional dependencies (eg: `security`)
+  - a URL for the package
+  - optionally, environment markers
+
+## Examples
+
+A few example name-based requirement specifiers:
+
+```
+SomeProject
+SomeProject == 1.3
+SomeProject >= 1.2, < 2.0
+SomeProject[foo, bar]
+SomeProject ~= 1.4.2
+SomeProject == 5.4 ; python_version < '3.8'
+SomeProject ; sys_platform == 'win32'
+requests [security] >= 2.8.1, == 2.8.* ; python_version < "2.7"
+```
+
+```{note}
+Use quotes around specifiers in the shell when using `>`, `<`, or when using environment markers.
+
+Do _not_ use quotes in requirement files. There is only one exception: pip v7.0 and v7.0.1 (from May 2015) required quotes around specifiers containing environment markers in requirement files.
+```
+
+A few example URL-based requirement specifiers:
+
+```none
+pip @ https://github.com/pypa/pip/archive/22.0.2.zip
+requests [security] @ https://github.com/psf/requests/archive/refs/heads/main.zip ; python_version >= "3.11"
+```
diff --git a/docs/html/reference/requirements-file-format.md b/docs/html/reference/requirements-file-format.md
new file mode 100644
index 00000000000..cf1d434eb6a
--- /dev/null
+++ b/docs/html/reference/requirements-file-format.md
@@ -0,0 +1,184 @@
+(requirements-file-format)=
+
+# Requirements File Format
+
+Requirements files serve as a list of items to be installed by pip, when
+using {ref}`pip install`. Files that use this format are often called
+"pip requirements.txt files", since `requirements.txt` is usually what
+these files are named (although, that is not a requirement).
+
+```{note}
+The requirements file format is closely tied to a number of internal details of
+pip (e.g., pip's command line options). The basic format is relatively stable
+and portable but the full syntax, as described here, is only intended for
+consumption by pip, and other tools should take that into account before using
+it for their own purposes.
+```
+
+## Example
+
+```
+# This is a comment, to show how #-prefixed lines are ignored.
+# It is possible to specify requirements as plain names.
+pytest
+pytest-cov
+beautifulsoup4
+
+# The syntax supported here is the same as that of requirement specifiers.
+docopt == 0.6.1
+requests [security] >= 2.8.1, == 2.8.* ; python_version < "2.7"
+urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
+
+# It is possible to refer to other requirement files or constraints files.
+-r other-requirements.txt
+-c constraints.txt
+
+# It is possible to refer to specific local distribution paths.
+./downloads/numpy-1.9.2-cp34-none-win32.whl
+
+# It is possible to refer to URLs.
+http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl
+```
+
+## Structure
+
+Each line of the requirements file indicates something to be installed,
+or arguments to {ref}`pip install`. The following forms are supported:
+
+- `[[--option]...]`
+- ``
+- ``
+- `[-e] `
+- `[-e] `
+
+For details on requirement specifiers, see {ref}`Requirement Specifiers`. For
+examples of all these forms, see {ref}`pip install Examples`.
+
+### Encoding
+
+Requirements files are `utf-8` encoding by default and also support
+{pep}`263` style comments to change the encoding (i.e.
+`# -*- coding:  -*-`).
+
+### Line continuations
+
+A line ending in an unescaped `\` is treated as a line continuation
+and the newline following it is effectively ignored.
+
+### Comments
+
+A line that begins with `#` is treated as a comment and ignored. Whitespace
+followed by a `#` causes the `#` and the remainder of the line to be
+treated as a comment.
+
+Comments are stripped _after_ line continuations are processed.
+
+## Supported options
+
+Requirements files only supports certain pip install options, which are listed
+below.
+
+### Global options
+
+The following options have an effect on the _entire_ `pip install` run, and
+must be specified on their individual lines.
+
+```{eval-rst}
+.. pip-requirements-file-options-ref-list::
+```
+
+````{admonition} Example
+To specify {ref}`--pre `, {ref}`--no-index `
+and two {ref}`--find-links ` locations:
+
+```
+--pre
+--no-index
+--find-links /my/local/archives
+--find-links http://some.archives.com/archives
+```
+````
+
+(per-requirement-options)=
+
+### Per-requirement options
+
+```{versionadded} 7.0
+
+```
+
+The options which can be applied to individual requirements are:
+
+- {ref}`--install-option `
+- {ref}`--global-option `
+- `--hash` (for {ref}`Hash-checking mode`)
+
+## Referring to other requirements files
+
+If you wish, you can refer to other requirements files, like this:
+
+```
+-r more_requirements.txt
+```
+
+You can also refer to {ref}`constraints files `, like this:
+
+```
+-c some_constraints.txt
+```
+
+## Using environment variables
+
+```{versionadded} 10.0
+
+```
+
+pip supports the use of environment variables inside the
+requirements file.
+
+You have to use the POSIX format for variable names including brackets around
+the uppercase name as shown in this example: `${API_TOKEN}`. pip will attempt
+to find the corresponding environment variable defined on the host system at
+runtime.
+
+```{note}
+There is no support for other variable expansion syntaxes such as `$VARIABLE`
+and `%VARIABLE%`.
+```
+
+You can now store sensitive data (tokens, keys, etc.) in environment variables
+and only specify the variable name for your requirements, letting pip lookup
+the value at runtime. This approach aligns with the commonly used
+[12-factor configuration pattern](https://12factor.net/config).
+
+
+## Influencing the build system
+
+```{danger}
+This disables the use of wheels (cached or otherwise). This could mean that builds will be slower, less deterministic, less reliable and may not behave correctly upon installation.
+
+This mechanism is only preserved for backwards compatibility and should be considered deprecated. A future release of pip may drop these options.
+```
+
+The `--global-option` and `--install-option` options are used to pass options to `setup.py`.
+
+```{attention}
+These options are highly coupled with how pip invokes setuptools using the {doc}`../reference/build-system/setup-py` build system interface. It is not compatible with newer {doc}`../reference/build-system/pyproject-toml` build system interface.
+
+This is will not work with other build-backends or newer setup.cfg-only projects.
+```
+
+If you have a declaration like:
+
+    FooProject >= 1.2 --global-option="--no-user-cfg" \
+                      --install-option="--prefix='/usr/local'" \
+                      --install-option="--no-compile"
+
+The above translates roughly into running FooProject's `setup.py` script as:
+
+    python setup.py --no-user-cfg install --prefix='/usr/local' --no-compile
+
+Note that the only way of giving more than one option to `setup.py` is through multiple `--global-option` and `--install-option` options, as shown in the example above. The value of each option is passed as a single argument to the `setup.py` script. Therefore, a line such as the following is invalid and would result in an installation error.
+
+    # Invalid. Please use '--install-option' twice as shown above.
+    FooProject >= 1.2 --install-option="--prefix=/usr/local --no-compile"
diff --git a/docs/html/topics/authentication.md b/docs/html/topics/authentication.md
new file mode 100644
index 00000000000..981aab5abd7
--- /dev/null
+++ b/docs/html/topics/authentication.md
@@ -0,0 +1,83 @@
+# Authentication
+
+## Basic HTTP authentication
+
+pip supports basic HTTP-based authentication credentials. This is done by
+providing the username (and optionally password) in the URL:
+
+```
+https://username:password@pypi.company.com/simple
+```
+
+For indexes that only require single-part authentication tokens, provide the
+token as the "username" and do not provide a password:
+
+```
+https://0123456789abcdef@pypi.company.com/simple
+```
+
+### Percent-encoding special characters
+
+```{versionadded} 10.0
+```
+
+Certain special characters are not valid in the credential part of a URL.
+If the user or password part of your login credentials contain any of these
+[special characters][reserved-chars], then they must be percent-encoded. As an
+example, for a user with username `user` and password `he//o` accessing a
+repository at `pypi.company.com/simple`, the URL with credentials would look
+like:
+
+```
+https://user:he%2F%2Fo@pypi.company.com/simple
+```
+
+[reserved-chars]: https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters
+
+## netrc support
+
+pip supports loading credentials from a user's `.netrc` file. If no credentials
+are part of the URL, pip will attempt to get authentication credentials for the
+URL's hostname from the user's `.netrc` file. This behaviour comes from the
+underlying use of {pypi}`requests`, which in turn delegates it to the
+[Python standard library's `netrc` module][netrc-std-lib].
+
+```{note}
+As mentioned in the [standard library documentation for netrc][netrc-std-lib],
+only ASCII characters are allowed in `.netrc` files. Whitespace and
+non-printable characters are not allowed in passwords.
+```
+
+Below is an example `.netrc`, for the host `example.com`, with a user named
+`daniel`, using the password `qwerty`:
+
+```
+machine example.com
+login daniel
+password qwerty
+```
+
+More information about the `.netrc` file format can be found in the GNU [`ftp`
+man pages][netrc-docs].
+
+[netrc-docs]: https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
+[netrc-std-lib]: https://docs.python.org/3/library/netrc.html
+
+## Keyring Support
+
+pip supports loading credentials stored in your keyring using the
+{pypi}`keyring` library.
+
+```bash
+$ pip install keyring  # install keyring from PyPI
+$ echo "your-password" | keyring set pypi.company.com your-username
+$ pip install your-package --index-url https://pypi.company.com/
+```
+
+Note that `keyring` (the Python package) needs to be installed separately from
+pip. This can create a bootstrapping issue if you need the credentials stored in
+the keyring to download and install keyring.
+
+It is, thus, expected that users that wish to use pip's keyring support have
+some mechanism for downloading and installing {pypi}`keyring` in their Python
+environment.
diff --git a/docs/html/topics/caching.md b/docs/html/topics/caching.md
new file mode 100644
index 00000000000..929ac3541df
--- /dev/null
+++ b/docs/html/topics/caching.md
@@ -0,0 +1,145 @@
+# Caching
+
+```{versionadded} 6.0
+
+```
+
+pip provides an on-by-default caching, designed to reduce the amount of time
+spent on duplicate downloads and builds.
+
+## What is cached
+
+### HTTP responses
+
+This cache functions like a web browser cache.
+
+When making any HTTP request, pip will first check its local cache to determine
+if it has a suitable response stored for that request which has not expired. If
+it does then it returns that response and doesn't re-download the content.
+
+If it has a response stored but it has expired, then it will attempt to make a
+conditional request to refresh the cache which will either return an empty
+response telling pip to simply use the cached item (and refresh the expiration
+timer) or it will return a whole new response which pip can then store in the
+cache.
+
+While this cache attempts to minimize network activity, it does not prevent
+network access altogether. If you want a local install solution that
+circumvents accessing PyPI, see {ref}`Installing from local packages`.
+
+(wheel-caching)=
+
+### Locally built wheels
+
+pip attempts to use wheels from its local wheel cache whenever possible.
+
+This means that if there is a cached wheel for the same version of a specific
+package name, pip will use that wheel instead of rebuilding the project.
+
+When no wheels are found for a source distribution, pip will attempt to build a
+wheel using the package's build system. If the build is successful, this wheel
+is added to the cache and used in subsequent installs for the same package
+version.
+
+Wheels built from source distributions provided to pip as a direct path (such
+as `pip install .`) are not cached across runs, though they may be reused within
+the same `pip` execution.
+
+```{versionchanged} 20.0
+pip now caches wheels when building from an immutable Git reference
+(i.e. a commit hash).
+```
+
+## Where is the cache stored
+
+```{caution}
+The exact filesystem structure of pip's cache's contents is considered to be
+an implementation detail and may change between any two versions of pip.
+```
+
+### `pip cache dir`
+
+```{versionadded} 20.1
+
+```
+
+You can use `pip cache dir` to get the cache directory that pip is currently configured to use.
+
+### Default paths
+
+````{tab} Unix
+```
+~/.cache/pip
+```
+
+pip will also respect `XDG_CACHE_HOME`.
+````
+
+````{tab} MacOS
+```
+~/Library/Caches/pip
+```
+````
+
+````{tab} Windows
+```
+%LocalAppData%\pip\Cache
+```
+````
+
+## Avoiding caching
+
+pip tries to use its cache whenever possible, and it is designed do the right
+thing by default.
+
+In some cases, pip's caching behaviour can be undesirable. As an example, if you
+have package with optional C extensions, that generates a pure Python wheel
+when the C extension can’t be built, pip will use that cached wheel even when
+you later invoke it from an environment that could have built those optional C
+extensions. This is because pip is seeing a cached wheel for that matches the
+package being built, and pip assumes that the result of building a package from
+a package index is deterministic.
+
+The recommended approach for dealing with these situations is to directly
+install from a source distribution instead of letting pip auto-discover the
+package when it is trying to install. Installing directly from a source
+distribution will make pip build a wheel, regardless of whether there is a
+matching cached wheel. This usually means doing something like:
+
+```{pip-cli}
+$ pip download sampleproject==1.0.0 --no-binary :all:
+$ pip install sampleproject-1.0.0.tar.gz
+```
+
+It is also a good idea to remove the offending cached wheel using the
+{ref}`pip cache` command.
+
+## Cache management
+
+The {ref}`pip cache` command can be used to manage pip's cache.
+
+### General overview
+
+`pip cache info` provides an overview of the contents of pip's cache, such as the total size and location of various parts of it.
+
+### Removing a single package
+
+`pip cache remove setuptools` removes all wheel files related to setuptools from pip's cache.
+
+### Removing the cache
+
+`pip cache purge` will clear all wheel files from pip's cache.
+
+### Listing cached files
+
+`pip cache list` will list all wheel files from pip's cache.
+
+`pip cache list setuptools` will list all setuptools-related wheel files from pip's cache.
+
+## Disabling caching
+
+pip's caching behaviour is disabled by passing the `--no-cache-dir` option.
+
+It is, however, recommended to **NOT** disable pip's caching. Doing so can
+significantly slow down pip (due to repeated operations and package builds)
+and result in significantly more network usage.
diff --git a/docs/html/topics/configuration.md b/docs/html/topics/configuration.md
new file mode 100644
index 00000000000..2d63ac1413f
--- /dev/null
+++ b/docs/html/topics/configuration.md
@@ -0,0 +1,225 @@
+# Configuration
+
+pip allows a user to change its behaviour via 3 mechanisms:
+
+- command line options
+- environment variables
+- configuration files
+
+This page explains how the configuration files and environment variables work,
+and how they are related to pip's various command line options.
+
+## Configuration Files
+
+Configuration files can change the default values for command line option.
+They are written using a standard INI style configuration files.
+
+pip has 3 "levels" of configuration files:
+
+- `global`: system-wide configuration file, shared across users.
+- `user`: per-user configuration file.
+- `site`: per-environment configuration file; i.e. per-virtualenv.
+
+### Location
+
+pip's configuration files are located in fairly standard locations. This
+location is different on different operating systems, and has some additional
+complexity for backwards compatibility reasons.
+
+```{tab} Unix
+
+Global
+: In a "pip" subdirectory of any of the paths set in the environment variable
+  `XDG_CONFIG_DIRS` (if it exists), for example {file}`/etc/xdg/pip/pip.conf`.
+
+  This will be followed by loading {file}`/etc/pip.conf`.
+
+User
+: {file}`$HOME/.config/pip/pip.conf`, which respects the `XDG_CONFIG_HOME` environment variable.
+
+  The legacy "per-user" configuration file is also loaded, if it exists: {file}`$HOME/.pip/pip.conf`.
+
+Site
+: {file}`$VIRTUAL_ENV/pip.conf`
+```
+
+```{tab} MacOS
+
+Global
+: {file}`/Library/Application Support/pip/pip.conf`
+
+User
+: {file}`$HOME/Library/Application Support/pip/pip.conf`
+  if directory `$HOME/Library/Application Support/pip` exists
+  else {file}`$HOME/.config/pip/pip.conf`
+
+  The legacy "per-user" configuration file is also loaded, if it exists: {file}`$HOME/.pip/pip.conf`.
+
+Site
+: {file}`$VIRTUAL_ENV/pip.conf`
+```
+
+```{tab} Windows
+
+Global
+: * On Windows 7 and later: {file}`C:\\ProgramData\\pip\\pip.ini`
+    (hidden but writeable)
+  * On Windows Vista: Global configuration is not supported.
+  * On Windows XP:
+    {file}`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini`
+
+User
+: {file}`%APPDATA%\\pip\\pip.ini`
+
+  The legacy "per-user" configuration file is also loaded, if it exists: {file}`%HOME%\\pip\\pip.ini`
+
+Site
+: {file}`%VIRTUAL_ENV%\\pip.ini`
+```
+
+### `PIP_CONFIG_FILE`
+
+Additionally, the environment variable `PIP_CONFIG_FILE` can be used to specify
+a configuration file that's loaded first, and whose values are overridden by
+the values set in the aforementioned files. Setting this to {any}`os.devnull`
+disables the loading of _all_ configuration files.
+
+### Loading order
+
+When multiple configuration files are found, pip combines them in the following
+order:
+
+- `PIP_CONFIG_FILE`, if given.
+- Global
+- User
+- Site
+
+Each file read overrides any values read from previous files, so if the
+global timeout is specified in both the global file and the per-user file
+then the latter value will be used.
+
+### Naming
+
+The names of the settings are derived from the long command line option.
+
+As an example, if you want to use a different package index (`--index-url`) and
+set the HTTP timeout (`--default-timeout`) to 60 seconds, your config file would
+look like this:
+
+```ini
+[global]
+timeout = 60
+index-url = https://download.zope.org/ppix
+```
+
+### Per-command section
+
+Each subcommand can be configured optionally in its own section. This overrides
+the global setting with the same name.
+
+As an example, if you want to decrease the `timeout` to `10` seconds when
+running the {ref}`pip freeze`, and use `60` seconds for all other commands:
+
+```ini
+[global]
+timeout = 60
+
+[freeze]
+timeout = 10
+```
+
+### Boolean options
+
+Boolean options like `--ignore-installed` or `--no-dependencies` can be set
+like this:
+
+```ini
+[install]
+ignore-installed = true
+no-dependencies = yes
+```
+
+To enable the boolean options `--no-compile`, `--no-warn-script-location` and
+`--no-cache-dir`, falsy values have to be used:
+
+```ini
+[global]
+no-cache-dir = false
+
+[install]
+no-compile = no
+no-warn-script-location = false
+```
+
+### Repeatable options
+
+For options which can be repeated like `--verbose` and `--quiet`, a
+non-negative integer can be used to represent the level to be specified:
+
+```ini
+[global]
+quiet = 0
+verbose = 2
+```
+
+It is possible to append values to a section within a configuration file. This
+is applicable to appending options like `--find-links` or `--trusted-host`,
+which can be written on multiple lines:
+
+```ini
+[global]
+find-links =
+    http://download.example.com
+
+[install]
+find-links =
+    http://mirror1.example.com
+    http://mirror2.example.com
+
+trusted-host =
+    mirror1.example.com
+    mirror2.example.com
+```
+
+This enables users to add additional values in the order of entry for such
+command line arguments.
+
+## Environment Variables
+
+pip's command line options can be set with environment variables using the
+format `PIP_` . Dashes (`-`) have to be replaced with
+underscores (`_`).
+
+- `PIP_DEFAULT_TIMEOUT=60` is the same as `--default-timeout=60`
+- ```
+  PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com"
+  ```
+
+  is the same as
+
+  ```
+  --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com
+  ```
+
+Repeatable options that do not take a value (such as `--verbose`) can be
+specified using the number of repetitions:
+
+- `PIP_VERBOSE=3` is the same as `pip install -vvv`
+
+```{note}
+Environment variables set to an empty string (like with `export X=` on Unix) will **not** be treated as false.
+Use `no`, `false` or `0` instead.
+```
+
+## Precedence / Override order
+
+Command line options have override environment variables, which override the
+values in a configuration file. Within the configuration file, values in
+command-specific sections over values in the global section.
+
+Examples:
+
+- `--host=foo` overrides `PIP_HOST=foo`
+- `PIP_HOST=foo` overrides a config file with `[global] host = foo`
+- A command specific section in the config file `[] host = bar`
+  overrides the option with same name in the `[global]` config file section.
diff --git a/docs/html/topics/dependency-resolution.md b/docs/html/topics/dependency-resolution.md
new file mode 100644
index 00000000000..7dd9848b021
--- /dev/null
+++ b/docs/html/topics/dependency-resolution.md
@@ -0,0 +1,317 @@
+# Dependency Resolution
+
+pip is capable of determining and installing the dependencies of packages. The
+process of determining which version of a dependency to install is known as
+dependency resolution. This behaviour can be disabled by passing
+{any}`--no-deps` to {any}`pip install`.
+
+## How it works
+
+When a user does a `pip install` (e.g. `pip install tea`), pip needs to work
+out the package's dependencies (e.g. `spoon`, `hot-water`, `tea-leaves` etc.)
+and what the versions of each of those dependencies it should install.
+
+At the start of a `pip install` run, pip does not have all the dependency
+information of the requested packages. It needs to work out the dependencies
+of the requested packages, the dependencies of those dependencies, and so on.
+Over the course of the dependency resolution process, pip will need to download
+distribution files of the packages which are used to get the dependencies of a
+package.
+
+## Backtracking
+
+```{versionchanged} 20.3
+pip's dependency resolver is now capable of backtracking.
+```
+
+During dependency resolution, pip needs to make assumptions about the package
+versions it needs to install and, later, check these assumptions were not
+incorrect. When pip finds that an assumption it made earlier is incorrect, it
+has to backtrack, which means also discarding some of the work that has already
+been done, and going back to choose another path.
+
+This can look like pip downloading multiple versions of the same package,
+since pip explicitly presents each download to the user. The backtracking of
+choices made during is not unexpected behaviour or a bug. It is part of how
+dependency resolution for Python packages works.
+
+````{admonition} Example
+The user requests `pip install tea`. The package `tea` declares a dependency on
+`hot-water`, `spoon`, `cup`, amongst others.
+
+pip starts by picking the most recent version of `tea` and get the list of
+dependencies of that version of `tea`. It will then repeat the process for
+those packages, picking the most recent version of `spoon` and then `cup`. Now,
+pip notices that the version of `cup` it has chosen is not compatible with the
+version of `spoon` it has chosen. Thus, pip will "go back" (backtrack) and try
+to use another version of `cup`. If it is successful, it will continue onto the
+next package (like `sugar`). Otherwise, it will continue to backtrack on `cup`
+until it finds a version of `cup` that is compatible with all the other
+packages.
+
+This can look like:
+
+```console
+$ pip install tea
+Collecting tea
+  Downloading tea-1.9.8-py2.py3-none-any.whl (346 kB)
+     |████████████████████████████████| 346 kB 10.4 MB/s
+Collecting spoon==2.27.0
+  Downloading spoon-2.27.0-py2.py3-none-any.whl (312 kB)
+     |████████████████████████████████| 312 kB 19.2 MB/s
+Collecting cup>=1.6.0
+  Downloading cup-3.22.0-py2.py3-none-any.whl (397 kB)
+     |████████████████████████████████| 397 kB 28.2 MB/s
+INFO: pip is looking at multiple versions of this package to determine
+which version is compatible with other requirements.
+This could take a while.
+  Downloading cup-3.21.0-py2.py3-none-any.whl (395 kB)
+     |████████████████████████████████| 395 kB 27.0 MB/s
+  Downloading cup-3.20.0-py2.py3-none-any.whl (394 kB)
+     |████████████████████████████████| 394 kB 24.4 MB/s
+  Downloading cup-3.19.1-py2.py3-none-any.whl (394 kB)
+     |████████████████████████████████| 394 kB 21.3 MB/s
+  Downloading cup-3.19.0-py2.py3-none-any.whl (394 kB)
+     |████████████████████████████████| 394 kB 26.2 MB/s
+  Downloading cup-3.18.0-py2.py3-none-any.whl (393 kB)
+     |████████████████████████████████| 393 kB 22.1 MB/s
+  Downloading cup-3.17.0-py2.py3-none-any.whl (382 kB)
+     |████████████████████████████████| 382 kB 23.8 MB/s
+  Downloading cup-3.16.0-py2.py3-none-any.whl (376 kB)
+     |████████████████████████████████| 376 kB 27.5 MB/s
+  Downloading cup-3.15.1-py2.py3-none-any.whl (385 kB)
+     |████████████████████████████████| 385 kB 30.4 MB/s
+INFO: pip is looking at multiple versions of this package to determine
+which version is compatible with other requirements.
+This could take a while.
+  Downloading cup-3.15.0-py2.py3-none-any.whl (378 kB)
+     |████████████████████████████████| 378 kB 21.4 MB/s
+  Downloading cup-3.14.0-py2.py3-none-any.whl (372 kB)
+     |████████████████████████████████| 372 kB 21.1 MB/s
+```
+
+These multiple `Downloading cup-{version}` lines show that pip is backtracking
+choices it is making during dependency resolution.
+````
+
+If pip starts backtracking during dependency resolution, it does not know how
+many choices it will reconsider, and how much computation would be needed.
+
+For the user, this means it can take a long time to complete when pip starts
+backtracking. In the case where a package has a lot of versions, arriving at a
+good candidate can take a lot of time. The amount of time depends on the
+package size, the number of versions pip must try, and various other factors.
+
+Backtracking reduces the risk that installing a new package will accidentally
+break an existing installed package, and so reduces the risk that your
+environment gets messed up. To do this, pip has to do more work, to find out
+which version of a package is a good candidate to install.
+
+## Possible ways to reduce backtracking
+
+There is no one-size-fits-all answer to situations where pip is backtracking
+excessively during dependency resolution. There are ways to reduce the
+degree to which pip might backtrack though. Nearly all of these approaches
+require some amount of trial and error.
+
+### Allow pip to complete its backtracking
+
+In most cases, pip will complete the backtracking process successfully.
+This could take a very long time to complete, so this may not be your
+preferred option.
+
+However, it is a possible that pip will not be able to find a set of
+compatible versions. For this, pip will try every possible combination that
+it needs to and determine that there is no compatible set.
+
+If you'd prefer not to wait, you can interrupt pip (Ctrl+c) and try the
+strategies listed below.
+
+### Reduce the number of versions pip is trying to use
+
+It is usually a good idea to add constraints the package(s) that pip is backtracking on (e.g. in the above example - `cup`).
+
+You could try something like:
+
+```{pip-cli}
+$ pip install tea "cup >= 3.13"
+```
+
+This will reduce the number of versions of `cup` it tries, and
+possibly reduce the time pip takes to install.
+
+There is a possibility that the addition constraint is incorrect. When this
+happens, the reduced search space makes it easier for pip to more quickly
+determine what caused the conflict and present that to the user. It could also
+result in pip backtracking on a different package due to some other conflict.
+
+### Use constraint files or lockfiles
+
+This option is a progression of the previous section. It requires users to know
+how to inspect:
+
+- the packages they're trying to install
+- the package release frequency and compatibility policies
+- their release notes and changelogs from past versions
+
+During deployment, you can create a lockfile stating the exact package and
+version number for for each dependency of that package. You can create this
+with [pip-tools](https://github.com/jazzband/pip-tools/).
+
+This means the "work" is done once during development process, and thus
+will avoid performing dependency resolution during deployment.
+
+## Dealing with dependency conflicts
+
+This section provides practical suggestions to pip users who encounter
+a `ResolutionImpossible` error, where pip cannot install their specified
+packages due to conflicting dependencies.
+
+### Understanding your error message
+
+When you get a `ResolutionImpossible` error, you might see something
+like this:
+
+```{pip-cli}
+$ pip install "pytest < 4.6" pytest-cov==2.12.1
+[regular pip output]
+ERROR: Cannot install pytest-cov==2.12.1 and pytest<4.6 because these package versions have conflicting dependencies.
+
+The conflict is caused by:
+    The user requested pytest<4.6
+    pytest-cov 2.12.1 depends on pytest>=4.6
+```
+
+In this example, pip cannot install the packages requested because they are
+asking for conflicting versions of pytest.
+
+- `pytest-cov` version `2.12.1`, requires `pytest` with a version or equal to
+  `4.6`.
+- `package_tea` version `4.3.0` depends on version `2.3.1` of
+  `package_water`
+
+Sometimes these messages are straightforward to read, because they use
+commonly understood comparison operators to specify the required version
+(e.g. `<` or `>`).
+
+However, Python packaging also supports some more complex ways for
+specifying package versions (e.g. `~=` or `*`):
+
+| Operator | Description                                                    | Example                                             |
+| -------- | -------------------------------------------------------------- | --------------------------------------------------- |
+| `>`      | Any version greater than the specified version.                | `>3.1`: any version greater than `3.1`.             |
+| `<`      | Any version less than the specified version.                   | `<3.1`: any version less than `3.1`.                |
+| `<=`     | Any version less than or equal to the specified version.       | `<=3.1`: any version less than or equal to `3.1`.   |
+| `>=`     | Any version greater than or equal to the specified version.    | `>=3.1`: version `3.1` and greater.                 |
+| `==`     | Exactly the specified version.                                 | `==3.1`: only `3.1`.                                |
+| `!=`     | Any version not equal to the specified version.                | `!=3.1`: any version other than `3.1`.              |
+| `~=`     | Any compatible{sup}`1` version.                                | `~=3.1`: any version compatible{sup}`1` with `3.1`. |
+| `*`      | Can be used at the end of a version number to represent _all_. | `==3.1.*`: any version that starts with `3.1`.      |
+
+{sup}`1` Compatible versions are higher versions that only differ in the final segment.
+`~=3.1.2` is equivalent to `>=3.1.2, ==3.1.*`. `~=3.1` is equivalent to `>=3.1, ==3.*`.
+
+The detailed specification of supported comparison operators can be
+found in {pep}`440`.
+
+### Possible solutions
+
+The solution to your error will depend on your individual use case. Here
+are some things to try:
+
+#### Audit your top level requirements
+
+As a first step, it is useful to audit your project and remove any
+unnecessary or out of date requirements (e.g. from your `setup.py` or
+`requirements.txt` files). Removing these can significantly reduce the
+complexity of your dependency tree, thereby reducing opportunities for
+conflicts to occur.
+
+#### Loosen your top level requirements
+
+Sometimes the packages that you have asked pip to install are
+incompatible because you have been too strict when you specified the
+package version.
+
+In our first example both `package_coffee` and `package_tea` have been
+_pinned_ to use specific versions
+(`package_coffee==0.44.1b0 package_tea==4.3.0`).
+
+To find a version of both `package_coffee` and `package_tea` that depend on
+the same version of `package_water`, you might consider:
+
+- Loosening the range of packages that you are prepared to install
+  (e.g. `pip install "package_coffee>0.44.*" "package_tea>4.0.0"`)
+- Asking pip to install _any_ version of `package_coffee` and `package_tea`
+  by removing the version specifiers altogether (e.g.
+  `pip install package_coffee package_tea`)
+
+In the second case, pip will automatically find a version of both
+`package_coffee` and `package_tea` that depend on the same version of
+`package_water`, installing:
+
+- `package_coffee 0.46.0b0`, which depends on `package_water 2.6.1`
+- `package_tea 4.3.0` which _also_ depends on `package_water 2.6.1`
+
+If you want to prioritize one package over another, you can add version
+specifiers to _only_ the more important package:
+
+```{pip-cli}
+$ pip install package_coffee==0.44.1b0 package_tea
+```
+
+This will result in:
+
+- `package_coffee 0.44.1b0`, which depends on `package_water 2.6.1`
+- `package_tea 4.1.3` which also depends on `package_water 2.6.1`
+
+Now that you have resolved the issue, you can repin the compatible
+package versions as required.
+
+#### Loosen the requirements of your dependencies
+
+Assuming that you cannot resolve the conflict by loosening the version
+of the package you require (as above), you can try to fix the issue on
+your _dependency_ by:
+
+- Requesting that the package maintainers loosen _their_ dependencies
+- Forking the package and loosening the dependencies yourself
+
+```{warning}
+If you choose to fork the package yourself, you are _opting out_ of
+any support provided by the package maintainers. Proceed at your own risk!
+```
+
+#### All requirements are appropriate, but a solution does not exist
+
+Sometimes it's simply impossible to find a combination of package
+versions that do not conflict. Welcome to [dependency hell].
+
+In this situation, you could consider:
+
+- Using an alternative package, if that is acceptable for your project.
+  See [Awesome Python] for similar packages.
+- Refactoring your project to reduce the number of dependencies (for
+  example, by breaking up a monolithic code base into smaller pieces).
+
+### Getting help
+
+If none of the suggestions above work for you, we recommend that you ask
+for help on:
+
+- [Python user Discourse](https://discuss.python.org/c/users/7)
+- [Python user forums](https://www.python.org/community/forums/)
+- [Python developers Slack channel](https://pythondev.slack.com/)
+- [Python IRC](https://www.python.org/community/irc/)
+- [Stack Overflow](https://stackoverflow.com/questions/tagged/python)
+
+See ["How do I ask a good question?"] for tips on asking for help.
+
+Unfortunately, **the pip team cannot provide support for individual
+dependency conflict errors**. Please _only_ open a ticket on
+[pip's issue tracker](https://github.com/pypa/pip/issues) if you believe
+that your problem has exposed a bug in pip.
+
+["how do i ask a good question?"]: https://stackoverflow.com/help/how-to-ask
+[awesome python]: https://python.libhunt.com/
+[dependency hell]: https://en.wikipedia.org/wiki/Dependency_hell
diff --git a/docs/html/topics/index.md b/docs/html/topics/index.md
new file mode 100644
index 00000000000..011205a111d
--- /dev/null
+++ b/docs/html/topics/index.md
@@ -0,0 +1,21 @@
+# Topic Guides
+
+These pages provide detailed information on individual topics.
+
+```{note}
+This section of the documentation is currently being fleshed out. See
+{issue}`9475` for more details.
+```
+
+```{toctree}
+:maxdepth: 1
+
+authentication
+caching
+configuration
+dependency-resolution
+local-project-installs
+repeatable-installs
+secure-installs
+vcs-support
+```
diff --git a/docs/html/topics/local-project-installs.md b/docs/html/topics/local-project-installs.md
new file mode 100644
index 00000000000..331afadb8c3
--- /dev/null
+++ b/docs/html/topics/local-project-installs.md
@@ -0,0 +1,65 @@
+# Local project installs
+
+It is extremely common to have a project, available in a folder/directory on your computer [^1] that you wish to install.
+
+With pip, depending on your usecase, there are two ways to do this:
+
+- A regular install
+- An editable install
+
+## Regular installs
+
+You can install local projects by specifying the project path to pip:
+
+```{pip-cli}
+$ pip install path/to/SomeProject
+```
+
+This will install the project into the Python that pip is associated with, in a manner similar to how it would actually be installed.
+
+This is what should be used in CI system and for deployments, since it most closely mirrors how a package would get installed if you build a distribution and installed from it (because that's _exactly_ what it does).
+
+(editable-installs)=
+
+## Editable installs
+
+You can install local projects in "editable" mode:
+
+```{pip-cli}
+$ pip install -e path/to/SomeProject
+```
+
+Editable installs allow you to install your project without copying any files. Instead, the files in the development directory are added to Python's import path. This approach is well suited for development and is also known as a "development installation".
+
+With an editable install, you only need to perform a re-installation if you change the project metadata (eg: version, what scripts need to be generated etc). You will still need to run build commands when you need to perform a compilation for non-Python code in the project (eg: C extensions).
+
+```{caution}
+It is possible to see behaviour differences between regular installs vs editable installs. In case you distribute the project as a "distribution package", users will see the behaviour of regular installs -- thus, it is important to ensure that regular installs work correctly.
+```
+
+```{note}
+This is functionally the same as [setuptools' develop mode], and that's precisely the mechanism used for setuptools-based projects.
+
+There are two advantages over using `setup.py develop` directly:
+
+- This works with non-setuptools build-backends as well.
+- The ".egg-info" directory is created relative to the project path, when using pip. This is generally a better location than setuptools, which dumps it in the current working directory.
+```
+
+[setuptools' develop mode]: https://setuptools.readthedocs.io/en/latest/userguide/development_mode.html
+
+## Build artifacts
+
+```{versionchanged} 21.3
+The project being installed is no longer copied to a temporary directory before invoking the build system.
+```
+
+This behaviour change has several consequences:
+
+- Local project builds will now be significantly faster, for certain kinds of projects and on systems with slow I/O (eg: via network attached storage or overly aggressive antivirus software).
+- Certain build backends (eg: `setuptools`) will litter the project directory with secondary build artifacts (eg: `.egg-info` directories).
+- Certain build backends (eg: `setuptools`) may not be able to perform with parallel builds anymore, since they previously relied on the fact that pip invoked them in a separate directory for each build.
+
+A `--use-deprecated=out-of-tree-build` option is available, until pip 22.1, as a mechanism to aid users with transitioning to the newer model of in-tree-builds.
+
+[^1]: Specifically, the current machine's filesystem.
diff --git a/docs/html/topics/repeatable-installs.md b/docs/html/topics/repeatable-installs.md
new file mode 100644
index 00000000000..c6e8f9689e4
--- /dev/null
+++ b/docs/html/topics/repeatable-installs.md
@@ -0,0 +1,98 @@
+# Repeatable Installs
+
+pip can be used to achieve various levels of repeatable environments. This page
+walks through increasingly stricter definitions of what "repeatable" means.
+
+## Pinning the package versions
+
+Pinning package versions of your dependencies in the requirements file
+protects you from bugs or incompatibilities in newly released versions:
+
+```
+SomePackage == 1.2.3
+DependencyOfSomePackage == 4.5.6
+```
+
+```{note}
+Pinning refers to using the `==` operator to require the package to be a
+specific version.
+```
+
+A requirements file, containing pinned package versions can be generated using
+{ref}`pip freeze`. This would not only the top-level packages, but also all of
+their transitive dependencies. Performing the installation using
+{ref}`--no-deps ` would provide an extra dose of insurance
+against installing anything not explicitly listed.
+
+This strategy is easy to implement and works across OSes and architectures.
+However, it trusts the locations you're fetching the packages from (like PyPI)
+and the certificate authority chain. It also relies on those locations not
+allowing packages to change without a version increase. (PyPI does protect
+against this.)
+
+## Hash-checking
+
+Beyond pinning version numbers, you can add hashes against which to verify
+downloaded packages:
+
+```none
+FooProject == 1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
+```
+
+This protects against a compromise of PyPI or the HTTPS certificate chain. It
+also guards against a package changing without its version number changing (on
+indexes that allow this). This approach is a good fit for automated server
+deployments.
+
+Hash-checking mode is a labour-saving alternative to running a private index
+server containing approved packages: it removes the need to upload packages,
+maintain ACLs, and keep an audit trail (which a VCS gives you on the
+requirements file for free). It can also substitute for a vendored library,
+providing easier upgrades and less VCS noise. It does not, of course,
+provide the availability benefits of a private index or a vendored library.
+
+[pip-tools] is a package that builds upon pip, and provides a good workflow for
+managing and generating requirements files.
+
+[pip-tools]: https://github.com/jazzband/pip-tools#readme
+
+## Using a wheelhouse (AKA Installation Bundles)
+
+{ref}`pip wheel` can be used to generate and package all of a project's
+dependencies, with all the compilation performed, into a single directory that
+can be converted into a single archive. This archive then allows installation
+when index servers are unavailable and avoids time-consuming recompilation.
+
+````{admonition} Example
+Creating the bundle, on a modern Unix system:
+
+```
+$ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX)
+$ python -m pip wheel -r requirements.txt --wheel-dir=$tempdir
+$ cwd=`pwd`
+$ (cd "$tempdir"; tar -cjvf "$cwd/bundled.tar.bz2" *)
+```
+
+Installing from the bundle, on a modern Unix system:
+
+```
+$ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX)
+$ (cd $tempdir; tar -xvf /path/to/bundled.tar.bz2)
+$ python -m pip install --force-reinstall --no-index --no-deps $tempdir/*
+```
+````
+
+Note that such a wheelhouse contains compiled packages, which are typically
+OS and architecture-specific, so these archives are not necessarily portable
+across machines.
+
+Hash-checking mode can also be used along with this method (since this uses a
+requirements file as well), to ensure that future archives are built with
+identical packages.
+
+```{warning}
+Beware of the `setup_requires` keyword arg in {file}`setup.py`. The (rare)
+packages that use it will cause those dependencies to be downloaded by
+setuptools directly, skipping pip's protections. If you need to use such a
+package, see {ref}`Controlling setup_requires `.
+```
diff --git a/docs/html/topics/secure-installs.md b/docs/html/topics/secure-installs.md
new file mode 100644
index 00000000000..f012842b2ac
--- /dev/null
+++ b/docs/html/topics/secure-installs.md
@@ -0,0 +1,100 @@
+# Secure installs
+
+By default, pip does not perform any checks to protect against remote tampering and involves running arbitrary code from distributions. It is, however, possible to use pip in a manner that changes these behaviours, to provide a more secure installation mechanism.
+
+This can be achieved by doing the following:
+
+- Enable {ref}`Hash-checking mode`, by passing {any}`--require-hashes`
+- Disallow source distributions, by passing {any}`--only-binary :all: <--only-binary>`
+
+(Hash-checking mode)=
+
+## Hash-checking Mode
+
+```{versionadded} 8.0
+
+```
+
+This mode uses local hashes, embedded in a requirements.txt file, to protect against remote tampering and network issues. These hashes are specified using a `--hash` [per requirement option](per-requirement-options).
+
+Note that hash-checking is an all-or-nothing proposition. Specifying `--hash` against _any_ requirement will activate this mode globally.
+
+To add hashes for a package, add them to line as follows:
+
+```
+FooProject == 1.2 \
+  --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 \
+  --hash=sha256:486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65260e9cb8a7
+```
+
+### Additional restrictions
+
+- Hashes are required for _all_ requirements.
+
+  This is because a partially-hashed requirements file is of little use and thus likely an error: a malicious actor could slip bad code into the installation via one of the unhashed requirements.
+
+  Note that hashes embedded in URL-style requirements via the `#md5=...` syntax suffice to satisfy this rule (regardless of hash strength, for legacy reasons), though you should use a stronger hash like sha256 whenever possible.
+
+- Hashes are required for _all_ dependencies.
+
+  If there is a dependency that is not spelled out and hashed in the requirements file, it will result in an error.
+
+- Requirements must be pinned (either to a URL, filesystem path or using `==`).
+
+  This prevents a surprising hash mismatch upon the release of a new version that matches the requirement specifier.
+
+### Forcing Hash-checking mode
+
+It is possible to force the hash checking mode to be enabled, by passing `--require-hashes` command-line option.
+
+This can be useful in deploy scripts, to ensure that the author of the requirements file provided hashes. It is also a convenient way to bootstrap your list of hashes, since it shows the hashes of the packages it fetched. It fetches only the preferred archive for each package, so you may still need to add hashes for alternatives archives using {ref}`pip hash`: for instance if there is both a binary and a source distribution.
+
+### Hash algorithms
+
+The recommended hash algorithm at the moment is sha256, but stronger ones are allowed, including all those supported by `hashlib`. However, weaker ones such as md5, sha1, and sha224 are excluded to avoid giving a false sense of security.
+
+### Multiple hashes per package
+
+It is possible to use multiple hashes for each package. This is important when a package offers binary distributions for a variety of platforms or when it is important to allow both binary and source distributions.
+
+### Interaction with caching
+
+The {ref}`locally-built wheel cache ` is disabled in hash-checking mode to prevent spurious hash mismatch errors.
+
+These would otherwise occur while installing sdists that had already been automatically built into cached wheels: those wheels would be selected for installation, but their hashes would not match the sdist ones from the requirements file.
+
+A further complication is that locally built wheels are nondeterministic: contemporary modification times make their way into the archive, making hashes unpredictable across machines and cache flushes. Compilation of C code adds further nondeterminism, as many compilers include random-seeded values in their output.
+
+However, wheels fetched from index servers are required to be the same every time. They land in pip's HTTP cache, not its wheel cache, and are used normally in hash-checking mode. The only downside of having the wheel cache disabled is thus extra build time for sdists, and this can be solved by making sure pre-built wheels are available from the index server.
+
+### Using hashes from PyPI (or other index servers)
+
+PyPI (and certain other index servers) provides a hash for the distribution, in the fragment portion of each download URL, like `#sha256=123...`, which pip checks as a protection against download corruption.
+
+Other hash algorithms that have guaranteed support from `hashlib` are also supported here: sha1, sha224, sha384, sha256, and sha512. Since this hash originates remotely, it is not a useful guard against tampering and thus does not satisfy the `--require-hashes` demand that every package have a local hash.
+
+## Repeatable installs
+
+Hash-checking mode also works with {ref}`pip download` and {ref}`pip wheel`. See {doc}`../topics/repeatable-installs` for a comparison of hash-checking mode with other repeatability strategies.
+
+```{warning}
+Beware of the `setup_requires` keyword arg in {file}`setup.py`. The (rare) packages that use it will cause those dependencies to be downloaded by setuptools directly, skipping pip's hash-checking. If you need to use such a package, see {ref}`controlling setup_requires `.
+```
+
+## Do not use setuptools directly
+
+Be careful not to nullify all your security work by installing your actual project by using setuptools' deprecated interfaces directly: for example, by calling `python setup.py install`, `python setup.py develop`, or `easy_install`.
+
+These will happily go out and download, unchecked, anything you missed in your requirements file and it’s easy to miss things as your project evolves. To be safe, install your project using pip and {any}`--no-deps`.
+
+Instead of `python setup.py install`, use:
+
+```{pip-cli}
+$ pip install --no-deps .
+```
+
+Instead of `python setup.py develop`, use:
+
+```{pip-cli}
+$ pip install --no-deps -e .
+```
diff --git a/docs/html/topics/vcs-support.md b/docs/html/topics/vcs-support.md
new file mode 100644
index 00000000000..d42c5c92e40
--- /dev/null
+++ b/docs/html/topics/vcs-support.md
@@ -0,0 +1,162 @@
+# VCS Support
+
+pip supports installing from various version control systems (VCS).
+This support requires a working executable to be available (for the version
+control system being used). It is used through URL prefixes:
+
+- Git -- `git+`
+- Mercurial -- `hg+`
+- Subversion -- `svn+`
+- Bazaar -- `bzr+`
+
+## Supported VCS
+
+### Git
+
+The supported schemes are `git+file`, `git+https`, `git+ssh`, `git+http`,
+`git+git` and `git`. Here are some of the supported forms:
+
+```none
+MyProject @ git+ssh://git.example.com/MyProject
+MyProject @ git+file:///home/user/projects/MyProject
+MyProject @ git+https://git.example.com/MyProject
+```
+
+```{warning}
+The use of `git`, `git+git`, and `git+http` schemes is discouraged.
+The former two use [the Git Protocol], which lacks authentication, and HTTP is
+insecure due to lack of TLS based encryption.
+```
+
+[the Git Protocol]: https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols
+
+It is also possible to specify a "git ref" such as branch name, a commit hash or
+a tag name:
+
+```none
+MyProject @ git+https://git.example.com/MyProject.git@master
+MyProject @ git+https://git.example.com/MyProject.git@v1.0
+MyProject @ git+https://git.example.com/MyProject.git@da39a3ee5e6b4b0d3255bfef95601890afd80709
+MyProject @ git+https://git.example.com/MyProject.git@refs/pull/123/head
+```
+
+When passing a commit hash, specifying a full hash is preferable to a partial
+hash because a full hash allows pip to operate more efficiently (e.g. by
+making fewer network calls).
+
+### Mercurial
+
+The supported schemes are `hg+file`, `hg+http`, `hg+https`, `hg+ssh`
+and `hg+static-http`. Here are some of the supported forms:
+
+```
+MyProject @ hg+http://hg.myproject.org/MyProject
+MyProject @ hg+https://hg.myproject.org/MyProject
+MyProject @ hg+ssh://hg.myproject.org/MyProject
+MyProject @ hg+file:///home/user/projects/MyProject
+```
+
+It is also possible to specify a revision number, a revision hash, a tag name
+or a local branch name:
+
+```none
+MyProject @ hg+http://hg.example.com/MyProject@da39a3ee5e6b
+MyProject @ hg+http://hg.example.com/MyProject@2019
+MyProject @ hg+http://hg.example.com/MyProject@v1.0
+MyProject @ hg+http://hg.example.com/MyProject@special_feature
+```
+
+### Subversion
+
+The supported schemes are `svn`, `svn+svn`, `svn+http`, `svn+https` and
+`svn+ssh`. Here are some of the supported forms:
+
+```none
+MyProject @svn+https://svn.example.com/MyProject
+MyProject @svn+ssh://svn.example.com/MyProject
+MyProject @svn+ssh://user@svn.example.com/MyProject
+```
+
+You can also give specific revisions to an SVN URL, like so:
+
+```none
+MyProject @ -e svn+http://svn.example.com/svn/MyProject/trunk@2019
+MyProject @ -e svn+http://svn.example.com/svn/MyProject/trunk@{20080101}
+```
+
+Note that you need to use [Editable VCS installs](#editable-vcs-installs) for
+using specific revisions from Subversion.
+
+### Bazaar
+
+The supported schemes are `bzr+http`, `bzr+https`, `bzr+ssh`, `bzr+sftp`,
+`bzr+ftp` and `bzr+lp`. Here are the supported forms:
+
+```none
+MyProject @ bzr+http://bzr.example.com/MyProject/trunk
+MyProject @ bzr+sftp://user@example.com/MyProject/trunk
+MyProject @ bzr+ssh://user@example.com/MyProject/trunk
+MyProject @ bzr+ftp://user@example.com/MyProject/trunk
+MyProject @ bzr+lp:MyProject
+```
+
+Tags or revisions can be installed like so:
+
+```none
+MyProject @ bzr+https://bzr.example.com/MyProject/trunk@2019
+MyProject @ bzr+http://bzr.example.com/MyProject/trunk@v1.0
+```
+
+(editable-vcs-installs)=
+
+## Editable VCS installs
+
+VCS projects can be installed in {ref}`editable mode ` (using
+the {ref}`--editable ` option) or not.
+
+- The default clone location (for editable installs) is:
+
+  - `/src/SomeProject` in virtual environments
+  - `/src/SomeProject` for global Python installs
+
+  The {ref}`--src ` option can be used to modify this location.
+
+- For non-editable installs, the project is built locally in a temp dir and then
+  installed normally.
+
+Note that if a satisfactory version of the package is already installed, the
+VCS source will not overwrite it without an `--upgrade` flag. Further, pip
+looks at the package version, at the target revision to determine what action to
+take on the VCS requirement (not the commit itself).
+
+The {ref}`pip freeze` subcommand will record the VCS requirement specifier
+(referencing a specific commit) only if the install is done with the editable
+option.
+
+## URL fragments
+
+pip looks at 2 fragments for VCS URLs:
+
+- `egg`: For specifying the "project name" for use in pip's dependency
+  resolution logic. eg: `egg=project_name`
+- `subdirectory`: For specifying the path to the Python package, when it is not
+  in the root of the VCS directory. eg: `pkg_dir`
+
+````{admonition} Example
+If your repository layout is:
+
+```
+pkg_dir
+├── setup.py  # setup.py for package "pkg"
+└── some_module.py
+other_dir
+└── some_file
+some_other_file
+```
+
+Then, to install from this repository, the syntax would be:
+
+```{pip-cli}
+$ pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir"
+```
+````
diff --git a/docs/html/usage.rst b/docs/html/usage.rst
deleted file mode 100644
index ab1e9737f1c..00000000000
--- a/docs/html/usage.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-:orphan:
-
-=====
-Usage
-=====
-
-The "Usage" section is now covered in the :doc:`Reference Guide `
diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst
index 415c9b1e71c..aa5d41c8cfe 100644
--- a/docs/html/user_guide.rst
+++ b/docs/html/user_guide.rst
@@ -53,7 +53,7 @@ Specifiers`
 
       py -m pip install SomePackage            # latest version
       py -m pip install SomePackage==1.0.4     # specific version
-      py -m pip install 'SomePackage>=1.0.4'     # minimum version
+      py -m pip install 'SomePackage>=1.0.4'   # minimum version
 
 For more information and examples, see the :ref:`pip install` reference.
 
@@ -63,72 +63,17 @@ For more information and examples, see the :ref:`pip install` reference.
 Basic Authentication Credentials
 ================================
 
-pip supports basic authentication credentials. Basically, in the URL there is
-a username and password separated by ``:``.
-
-``https://[username[:password]@]pypi.company.com/simple``
-
-Certain special characters are not valid in the authentication part of URLs.
-If the user or password part of your login credentials contain any of the
-special characters
-`here `_
-then they must be percent-encoded. For example, for a
-user with username "user" and password "he//o" accessing a repository at
-pypi.company.com, the index URL with credentials would look like:
-
-``https://user:he%2F%2Fo@pypi.company.com``
-
-Support for percent-encoded authentication in index URLs was added in pip 10.0.0
-(in `#3236 `_). Users that must use authentication
-for their Python repository on systems with older pip versions should make the latest
-get-pip.py available in their environment to bootstrap pip to a recent-enough version.
-
-For indexes that only require single-part authentication tokens, provide the token
-as the "username" and do not provide a password, for example -
-
-``https://0123456789abcdef@pypi.company.com``
-
+This is now covered in :doc:`topics/authentication`.
 
 netrc Support
 -------------
 
-If no credentials are part of the URL, pip will attempt to get authentication credentials
-for the URL’s hostname from the user’s .netrc file. This behaviour comes from the underlying
-use of `requests`_ which in turn delegates it to the `Python standard library`_.
-
-The .netrc file contains login and initialization information used by the auto-login process.
-It resides in the user's home directory. The .netrc file format is simple. You specify lines
-with a machine name and follow that with lines for the login and password that are
-associated with that machine. Machine name is the hostname in your URL.
-
-An example .netrc for the host example.com with a user named 'daniel', using the password
-'qwerty' would look like:
-
-.. code-block:: shell
-
-   machine example.com
-   login daniel
-   password qwerty
-
-As mentioned in the `standard library docs `_,
-only ASCII characters are allowed. Whitespace and non-printable characters are not allowed in passwords.
-
+This is now covered in :doc:`topics/authentication`.
 
 Keyring Support
 ---------------
 
-pip also supports credentials stored in your keyring using the `keyring`_
-library. Note that ``keyring`` will need to be installed separately, as pip
-does not come with it included.
-
-.. code-block:: shell
-
-   pip install keyring
-   echo your-password | keyring set pypi.company.com your-username
-   pip install your-package --extra-index-url https://pypi.company.com/
-
-.. _keyring: https://pypi.org/project/keyring/
-
+This is now covered in :doc:`topics/authentication`.
 
 Using a Proxy Server
 ====================
@@ -139,7 +84,7 @@ in many corporate environments requires an outbound HTTP proxy server.
 pip can be configured to connect through a proxy server in various ways:
 
 * using the ``--proxy`` command-line option to specify a proxy in the form
-  ``[user:passwd@]proxy.server:port``
+  ``scheme://[user:passwd@]proxy.server:port``
 * using ``proxy`` in a :ref:`config-file`
 * by setting the standard environment-variables ``http_proxy``, ``https_proxy``
   and ``no_proxy``.
@@ -168,7 +113,7 @@ installed using :ref:`pip install` like so:
 
       py -m pip install -r requirements.txt
 
-Details on the format of the files are here: :ref:`Requirements File Format`.
+Details on the format of the files are here: :ref:`requirements-file-format`.
 
 Logically, a Requirements file is just a list of :ref:`pip install` arguments
 placed in a file. Note that you should not rely on the items in the file being
@@ -177,7 +122,7 @@ installed by pip in any particular order.
 In practice, there are 4 common uses of Requirements files:
 
 1. Requirements files are used to hold the result from :ref:`pip freeze` for the
-   purpose of achieving :ref:`repeatable installations `.  In
+   purpose of achieving :doc:`topics/repeatable-installs`.  In
    this case, your requirement file contains a pinned version of everything that
    was installed when ``pip freeze`` was run.
 
@@ -235,12 +180,12 @@ In practice, there are 4 common uses of Requirements files:
 
 It's important to be clear that pip determines package dependencies using
 `install_requires metadata
-`_,
+`_,
 not by discovering ``requirements.txt`` files embedded in projects.
 
 See also:
 
-* :ref:`Requirements File Format`
+* :ref:`requirements-file-format`
 * :ref:`pip freeze`
 * `"setup.py vs requirements.txt" (an article by Donald Stufft)
   `_
@@ -254,9 +199,11 @@ Constraints Files
 
 Constraints files are requirements files that only control which version of a
 requirement is installed, not whether it is installed or not. Their syntax and
-contents is nearly identical to :ref:`Requirements Files`. There is one key
-difference: Including a package in a constraints file does not trigger
-installation of the package.
+contents is a subset of :ref:`Requirements Files`, with several kinds of syntax
+not allowed: constraints must have a name, they cannot be editable, and they
+cannot specify extras. In terms of semantics, there is one key difference:
+Including a package in a constraints file does not trigger installation of the
+package.
 
 Use a constraints file like so:
 
@@ -324,6 +271,26 @@ To install directly from a wheel archive:
 
       py -m pip install SomePackage-1.0-py2.py3-none-any.whl
 
+To include optional dependencies provided in the ``provides_extras``
+metadata in the wheel, you must add quotes around the install target
+name:
+
+.. tab:: Unix/macOS
+
+   .. code-block:: shell
+
+      python -m pip install './somepackage-1.0-py2.py3-none-any.whl[my-extras]'
+
+.. tab:: Windows
+
+   .. code-block:: shell
+
+      py -m pip install './somepackage-1.0-py2.py3-none-any.whl[my-extras]'
+
+.. note::
+
+    In the future, the ``path[extras]`` syntax may become deprecated. It is
+    recommended to use PEP 508 syntax wherever possible.
 
 For the cases where wheels are not available, pip offers :ref:`pip wheel` as a
 convenience, to build wheels for all your requirements and dependencies.
@@ -490,242 +457,26 @@ For more information and examples, see the :ref:`pip search` reference.
 Configuration
 =============
 
+This is now covered in :doc:`topics/configuration`.
+
 .. _config-file:
 
 Config file
 -----------
 
-pip allows you to set all command line option defaults in a standard ini
-style config file.
-
-The names and locations of the configuration files vary slightly across
-platforms. You may have per-user, per-virtualenv or global (shared amongst
-all users) configuration:
-
-**Per-user**:
-
-* On Unix the default configuration file is: :file:`$HOME/.config/pip/pip.conf`
-  which respects the ``XDG_CONFIG_HOME`` environment variable.
-* On macOS the configuration file is
-  :file:`$HOME/Library/Application Support/pip/pip.conf`
-  if directory ``$HOME/Library/Application Support/pip`` exists
-  else :file:`$HOME/.config/pip/pip.conf`.
-* On Windows the configuration file is :file:`%APPDATA%\\pip\\pip.ini`.
-
-There is also a legacy per-user configuration file which is also respected.
-To find its location:
-
-* On Unix and macOS the configuration file is: :file:`$HOME/.pip/pip.conf`
-* On Windows the configuration file is: :file:`%HOME%\\pip\\pip.ini`
-
-You can set a custom path location for this config file using the environment
-variable ``PIP_CONFIG_FILE``.
-
-**Inside a virtualenv**:
-
-* On Unix and macOS the file is :file:`$VIRTUAL_ENV/pip.conf`
-* On Windows the file is: :file:`%VIRTUAL_ENV%\\pip.ini`
-
-**Global**:
-
-* On Unix the file may be located in :file:`/etc/pip.conf`. Alternatively
-  it may be in a "pip" subdirectory of any of the paths set in the
-  environment variable ``XDG_CONFIG_DIRS`` (if it exists), for example
-  :file:`/etc/xdg/pip/pip.conf`.
-* On macOS the file is: :file:`/Library/Application Support/pip/pip.conf`
-* On Windows XP the file is:
-  :file:`C:\\Documents and Settings\\All Users\\Application Data\\pip\\pip.ini`
-* On Windows 7 and later the file is hidden, but writeable at
-  :file:`C:\\ProgramData\\pip\\pip.ini`
-* Global configuration is not supported on Windows Vista.
-
-The global configuration file is shared by all Python installations.
-
-If multiple configuration files are found by pip then they are combined in
-the following order:
-
-1. The global file is read
-2. The per-user file is read
-3. The virtualenv-specific file is read
-
-Each file read overrides any values read from previous files, so if the
-global timeout is specified in both the global file and the per-user file
-then the latter value will be used.
-
-The names of the settings are derived from the long command line option, e.g.
-if you want to use a different package index (``--index-url``) and set the
-HTTP timeout (``--default-timeout``) to 60 seconds your config file would
-look like this:
-
-.. code-block:: ini
-
-    [global]
-    timeout = 60
-    index-url = https://download.zope.org/ppix
-
-Each subcommand can be configured optionally in its own section so that every
-global setting with the same name will be overridden; e.g. decreasing the
-``timeout`` to ``10`` seconds when running the ``freeze``
-(:ref:`pip freeze`) command and using
-``60`` seconds for all other commands is possible with:
-
-.. code-block:: ini
-
-    [global]
-    timeout = 60
-
-    [freeze]
-    timeout = 10
-
-
-Boolean options like ``--ignore-installed`` or ``--no-dependencies`` can be
-set like this:
-
-.. code-block:: ini
-
-    [install]
-    ignore-installed = true
-    no-dependencies = yes
-
-To enable the boolean options ``--no-compile``, ``--no-warn-script-location``
-and ``--no-cache-dir``, falsy values have to be used:
-
-.. code-block:: ini
-
-    [global]
-    no-cache-dir = false
-
-    [install]
-    no-compile = no
-    no-warn-script-location = false
-
-For options which can be repeated like ``--verbose`` and ``--quiet``,
-a non-negative integer can be used to represent the level to be specified:
-
-.. code-block:: ini
-
-    [global]
-    quiet = 0
-    verbose = 2
-
-It is possible to append values to a section within a configuration file such as the pip.ini file.
-This is applicable to appending options like ``--find-links`` or ``--trusted-host``,
-which can be written on multiple lines:
-
-.. code-block:: ini
-
-    [global]
-    find-links =
-        http://download.example.com
-
-    [install]
-    find-links =
-        http://mirror1.example.com
-        http://mirror2.example.com
-
-    trusted-host =
-        mirror1.example.com
-        mirror2.example.com
-
-This enables users to add additional values in the order of entry for such command line arguments.
-
+This is now covered in :doc:`topics/configuration`.
 
 Environment Variables
 ---------------------
 
-pip's command line options can be set with environment variables using the
-format ``PIP_`` . Dashes (``-``) have to be replaced with
-underscores (``_``).
-
-For example, to set the default timeout:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      export PIP_DEFAULT_TIMEOUT=60
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      set PIP_DEFAULT_TIMEOUT=60
-
-This is the same as passing the option to pip directly:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip --default-timeout=60 [...]
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip --default-timeout=60 [...]
-
-For command line options which can be repeated, use a space to separate
-multiple values. For example:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com"
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com"
-
-is the same as calling:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com
-
-Options that do not take a value, but can be repeated (such as ``--verbose``)
-can be specified using the number of repetitions, so::
-
-    export PIP_VERBOSE=3
-
-is the same as calling::
-
-    pip install -vvv
-
-.. note::
-
-   Environment variables set to be empty string will not be treated as false.
-   Please use ``no``, ``false`` or ``0`` instead.
-
+This is now covered in :doc:`topics/configuration`.
 
 .. _config-precedence:
 
 Config Precedence
 -----------------
 
-Command line options have precedence over environment variables, which have
-precedence over the config file.
-
-Within the config file, command specific sections have precedence over the
-global section.
-
-Examples:
-
-- ``--host=foo`` overrides ``PIP_HOST=foo``
-- ``PIP_HOST=foo`` overrides a config file with ``[global] host = foo``
-- A command specific section in the config file ``[] host = bar``
-  overrides the option with same name in the ``[global]`` config file section
+This is now covered in :doc:`topics/configuration`.
 
 
 Command Completion
@@ -825,6 +576,21 @@ strategies supported:
 The default strategy is ``only-if-needed``. This was changed in pip 10.0 due to
 the breaking nature of ``eager`` when upgrading conflicting dependencies.
 
+It is important to note that ``--upgrade`` affects *direct requirements* (e.g.
+those specified on the command-line or via a requirements file) while
+``--upgrade-strategy`` affects *indirect requirements* (dependencies of direct
+requirements).
+
+As an example, say ``SomePackage`` has a dependency, ``SomeDependency``, and
+both of them are already installed but are not the latest available versions:
+
+- ``pip install SomePackage``: will not upgrade the existing ``SomePackage`` or
+  ``SomeDependency``.
+- ``pip install --upgrade SomePackage``: will upgrade ``SomePackage``, but not
+  ``SomeDependency`` (unless a minimum requirement is not met).
+- ``pip install --upgrade SomePackage --upgrade-strategy=eager``: upgrades both
+  ``SomePackage`` and ``SomeDependency``.
+
 As an historic note, an earlier "fix" for getting the ``only-if-needed``
 behaviour was:
 
@@ -1016,522 +782,14 @@ is the latest version:
 Ensuring Repeatability
 ======================
 
-pip can achieve various levels of repeatability:
-
-Pinned Version Numbers
-----------------------
-
-Pinning the versions of your dependencies in the requirements file
-protects you from bugs or incompatibilities in newly released versions::
-
-    SomePackage == 1.2.3
-    DependencyOfSomePackage == 4.5.6
-
-Using :ref:`pip freeze` to generate the requirements file will ensure that not
-only the top-level dependencies are included but their sub-dependencies as
-well, and so on. Perform the installation using :ref:`--no-deps
-` for an extra dose of insurance against installing
-anything not explicitly listed.
-
-This strategy is easy to implement and works across OSes and architectures.
-However, it trusts PyPI and the certificate authority chain. It
-also relies on indices and find-links locations not allowing
-packages to change without a version increase. (PyPI does protect
-against this.)
-
-Hash-checking Mode
-------------------
-
-Beyond pinning version numbers, you can add hashes against which to verify
-downloaded packages::
-
-    FooProject == 1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
-
-This protects against a compromise of PyPI or the HTTPS
-certificate chain. It also guards against a package changing
-without its version number changing (on indexes that allow this).
-This approach is a good fit for automated server deployments.
-
-Hash-checking mode is a labor-saving alternative to running a private index
-server containing approved packages: it removes the need to upload packages,
-maintain ACLs, and keep an audit trail (which a VCS gives you on the
-requirements file for free). It can also substitute for a vendor library,
-providing easier upgrades and less VCS noise. It does not, of course,
-provide the availability benefits of a private index or a vendor library.
-
-For more, see
-:ref:`pip install\'s discussion of hash-checking mode `.
-
-.. _`Installation Bundle`:
-
-Installation Bundles
---------------------
-
-Using :ref:`pip wheel`, you can bundle up all of a project's dependencies, with
-any compilation done, into a single archive. This allows installation when
-index servers are unavailable and avoids time-consuming recompilation. Create
-an archive like this::
-
-    $ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX)
-    $ python -m pip wheel -r requirements.txt --wheel-dir=$tempdir
-    $ cwd=`pwd`
-    $ (cd "$tempdir"; tar -cjvf "$cwd/bundled.tar.bz2" *)
-
-You can then install from the archive like this::
-
-    $ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX)
-    $ (cd $tempdir; tar -xvf /path/to/bundled.tar.bz2)
-    $ python -m pip install --force-reinstall --ignore-installed --upgrade --no-index --no-deps $tempdir/*
-
-Note that compiled packages are typically OS- and architecture-specific, so
-these archives are not necessarily portable across machines.
-
-Hash-checking mode can be used along with this method to ensure that future
-archives are built with identical packages.
-
-.. warning::
-
-    Finally, beware of the ``setup_requires`` keyword arg in :file:`setup.py`.
-    The (rare) packages that use it will cause those dependencies to be
-    downloaded by setuptools directly, skipping pip's protections. If you need
-    to use such a package, see :ref:`Controlling
-    setup_requires`.
+This is now covered in :doc:`../topics/repeatable-installs`.
 
 .. _`Fixing conflicting dependencies`:
 
 Fixing conflicting dependencies
 ===============================
 
-The purpose of this section of documentation is to provide practical suggestions to
-pip users who encounter an error where pip cannot install their
-specified packages due to conflicting dependencies (a
-``ResolutionImpossible`` error).
-
-This documentation is specific to the new resolver, which is the
-default behavior in pip 20.3 and later. If you are using pip 20.2, you
-can invoke the new resolver by using the flag
-``--use-feature=2020-resolver``.
-
-Understanding your error message
---------------------------------
-
-When you get a ``ResolutionImpossible`` error, you might see something
-like this:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install package_coffee==0.44.1 package_tea==4.3.0
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install package_coffee==0.44.1 package_tea==4.3.0
-
-::
-
-   Due to conflicting dependencies pip cannot install
-   package_coffee and package_tea:
-   - package_coffee depends on package_water<3.0.0,>=2.4.2
-   - package_tea depends on package_water==2.3.1
-
-In this example, pip cannot install the packages you have requested,
-because they each depend on different versions of the same package
-(``package_water``):
-
-- ``package_coffee`` version ``0.44.1`` depends on a version of
-  ``package_water`` that is less than ``3.0.0`` but greater than or equal to
-  ``2.4.2``
-- ``package_tea`` version ``4.3.0`` depends on version ``2.3.1`` of
-  ``package_water``
-
-Sometimes these messages are straightforward to read, because they use
-commonly understood comparison operators to specify the required version
-(e.g. ``<`` or ``>``).
-
-However, Python packaging also supports some more complex ways for
-specifying package versions (e.g. ``~=`` or ``*``):
-
-+----------+---------------------------------+--------------------------------+
-| Operator | Description                     | Example                        |
-+==========+=================================+================================+
-|  ``>``   | Any version greater than        | ``>3.1``: any version          |
-|          | the specified version.          | greater than ``3.1``.          |
-+----------+---------------------------------+--------------------------------+
-|  ``<``   | Any version less than           | ``<3.1``: any version          |
-|          | the specified version.          | less than ``3.1``.             |
-+----------+---------------------------------+--------------------------------+
-|  ``<=``  | Any version less than or        | ``<=3.1``: any version         |
-|          | equal to the specified version. | less than or equal to ``3.1``. |
-+----------+---------------------------------+--------------------------------+
-|  ``>=``  | Any version greater than or     | ``>=3.1``:                     |
-|          | equal to the specified version. | version ``3.1`` and greater.   |
-+----------+---------------------------------+--------------------------------+
-|  ``==``  | Exactly the specified version.  | ``==3.1``: only ``3.1``.       |
-+----------+---------------------------------+--------------------------------+
-|  ``!=``  | Any version not equal           | ``!=3.1``: any version         |
-|          | to the specified version.       | other than ``3.1``.            |
-+----------+---------------------------------+--------------------------------+
-|  ``~=``  | Any compatible release.         | ``~=3.1``: version ``3.1``     |
-|          | Compatible releases are         | or later, but not              |
-|          | releases that are within the    | version ``4.0`` or later.      |
-|          | same major or minor version,    | ``~=3.1.2``: version ``3.1.2`` |
-|          | assuming the package author     | or later, but not              |
-|          | is using semantic versioning.   | version ``3.2.0`` or later.    |
-+----------+---------------------------------+--------------------------------+
-|  ``*``   | Can be used at the end of       | ``==3.1.*``: any version       |
-|          | a version number to represent   | that starts with ``3.1``.      |
-|          | *all*,                          | Equivalent to ``~=3.1.0``.     |
-+----------+---------------------------------+--------------------------------+
-
-The detailed specification of supported comparison operators can be
-found in :pep:`440`.
-
-Possible solutions
-------------------
-
-The solution to your error will depend on your individual use case. Here
-are some things to try:
-
-1. Audit your top level requirements
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-As a first step it is useful to audit your project and remove any
-unnecessary or out of date requirements (e.g. from your ``setup.py`` or
-``requirements.txt`` files). Removing these can significantly reduce the
-complexity of your dependency tree, thereby reducing opportunities for
-conflicts to occur.
-
-2. Loosen your top level requirements
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Sometimes the packages that you have asked pip to install are
-incompatible because you have been too strict when you specified the
-package version.
-
-In our first example both ``package_coffee`` and ``package_tea`` have been
-*pinned* to use specific versions
-(``package_coffee==0.44.1b0 package_tea==4.3.0``).
-
-To find a version of both ``package_coffee`` and ``package_tea`` that depend on
-the same version of ``package_water``, you might consider:
-
--  Loosening the range of packages that you are prepared to install
-   (e.g. ``pip install "package_coffee>0.44.*" "package_tea>4.0.0"``)
--  Asking pip to install *any* version of ``package_coffee`` and ``package_tea``
-   by removing the version specifiers altogether (e.g.
-   ``python -m pip install package_coffee package_tea``)
-
-In the second case, pip will automatically find a version of both
-``package_coffee`` and ``package_tea`` that depend on the same version of
-``package_water``, installing:
-
--  ``package_coffee 0.46.0b0``, which depends on ``package_water 2.6.1``
--  ``package_tea 4.3.0`` which *also* depends on ``package_water 2.6.1``
-
-If you want to prioritize one package over another, you can add version
-specifiers to *only* the more important package:
-
-.. tab:: Unix/macOS
-
-   .. code-block:: shell
-
-      python -m pip install package_coffee==0.44.1b0 package_tea
-
-.. tab:: Windows
-
-   .. code-block:: shell
-
-      py -m pip install package_coffee==0.44.1b0 package_tea
-
-This will result in:
-
-- ``package_coffee 0.44.1b0``, which depends on ``package_water 2.6.1``
-- ``package_tea 4.1.3`` which also depends on ``package_water 2.6.1``
-
-Now that you have resolved the issue, you can repin the compatible
-package versions as required.
-
-3. Loosen the requirements of your dependencies
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Assuming that you cannot resolve the conflict by loosening the version
-of the package you require (as above), you can try to fix the issue on
-your *dependency* by:
-
--  Requesting that the package maintainers loosen *their* dependencies
--  Forking the package and loosening the dependencies yourself
-
-.. warning::
-
-   If you choose to fork the package yourself, you are *opting out* of
-   any support provided by the package maintainers. Proceed at your own risk!
-
-4. All requirements are loose, but a solution does not exist
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Sometimes it's simply impossible to find a combination of package
-versions that do not conflict. Welcome to `dependency hell`_.
-
-In this situation, you could consider:
-
--  Using an alternative package, if that is acceptable for your project.
-   See `Awesome Python`_ for similar packages.
--  Refactoring your project to reduce the number of dependencies (for
-   example, by breaking up a monolithic code base into smaller pieces)
-
-.. _`Getting help`:
-
-Getting help
-------------
-
-If none of the suggestions above work for you, we recommend that you ask
-for help on:
-
--  `Python user Discourse`_
--  `Python user forums`_
--  `Python developers Slack channel`_
--  `Python IRC`_
--  `Stack Overflow`_
-
-See `"How do I ask a good question?"`_ for tips on asking for help.
-
-Unfortunately, **the pip team cannot provide support for individual
-dependency conflict errors**. Please *only* open a ticket on the `pip
-issue tracker`_ if you believe that your problem has exposed a bug in pip.
-
-.. _dependency hell: https://en.wikipedia.org/wiki/Dependency_hell
-.. _Awesome Python: https://python.libhunt.com/
-.. _Python user Discourse: https://discuss.python.org/c/users/7
-.. _Python user forums: https://www.python.org/community/forums/
-.. _Python developers Slack channel: https://pythondev.slack.com/
-.. _Python IRC: https://www.python.org/community/irc/
-.. _Stack Overflow: https://stackoverflow.com/questions/tagged/python
-.. _"How do I ask a good question?": https://stackoverflow.com/help/how-to-ask
-.. _pip issue tracker: https://github.com/pypa/pip/issues
-
-.. _`Dependency resolution backtracking`:
-
-Dependency resolution backtracking
-==================================
-
-Or more commonly known as *"Why does pip download multiple versions of
-the same package over and over again during an install?"*.
-
-The purpose of this section is to provide explanation of why
-backtracking happens, and practical suggestions to pip users who
-encounter it during a ``pip install``.
-
-What is backtracking?
----------------------
-
-Backtracking is not a bug, or an unexpected behaviour. It is part of the
-way pip's dependency resolution process works.
-
-During a pip install (e.g. ``pip install tea``), pip needs to work out
-the package's dependencies (e.g. ``spoon``, ``hot-water``, ``cup`` etc.), the
-versions of each of these packages it needs to install. For each package
-pip needs to decide which version is a good candidate to install.
-
-A "good candidate" means a version of each package that is compatible with all
-the other package versions being installed at the same time.
-
-In the case where a package has a lot of versions, arriving at a good
-candidate can take a lot of time. (The amount of time depends on the
-package size, the number of versions pip must try, and other concerns.)
-
-How does backtracking work?
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-When doing a pip install, pip starts by making assumptions about the
-packages it needs to install. During the install process it needs to check these
-assumptions as it goes along.
-
-When pip finds that an assumption is incorrect, it has to try another approach
-(backtrack), which means discarding some of the work that has already been done,
-and going back to choose another path.
-
-For example; The user requests ``pip install tea``. ```tea`` has dependencies of
-``cup``, ``hot-water``, ``spoon`` amongst others.
-
-pip starts by installing a version of ``cup``. If it finds out it isn’t
-compatible (with the other package versions) it needs to “go back”
-(backtrack) and download an older version.
-
-It then tries to install that version. If it is successful, it will continue
-onto the next package. If not it will continue to backtrack until it finds a
-compatible version.
-
-This backtrack behaviour can end in 2 ways - either 1) it will
-successfully find a set of packages it can install (good news!), or 2) it will
-eventually display a `resolution impossible `__ error
-message (not so good).
-
-If pip starts backtracking during dependency resolution, it does not
-know how long it will backtrack, and how much computation would be
-needed. For the user this means it can take a long time to complete.
-
-Why does backtracking occur?
-----------------------------
-
-With the release of the new resolver (:ref:`Resolver changes 2020`), pip is now
-more strict in the package versions it installs when a user runs a
-``pip install`` command.
-
-Pip needs to backtrack because initially, it doesn't have all the information it
-needs to work out the correct set of packages. This is because package indexes
-don't provide full package dependency information before you have downloaded
-the package.
-
-This new resolver behaviour means that pip works harder to find out which
-version of a package is a good candidate to install. It reduces the risk that
-installing a new package will accidentally break an existing installed package,
-and so reduces the risk that your environment gets messed up.
-
-What does this behaviour look like?
------------------------------------
-
-Right now backtracking behaviour looks like this:
-
-::
-
-   $ pip install tea==1.9.8
-   Collecting tea==1.9.8
-     Downloading tea-1.9.8-py2.py3-none-any.whl (346 kB)
-        |████████████████████████████████| 346 kB 10.4 MB/s
-   Collecting spoon==2.27.0
-     Downloading spoon-2.27.0-py2.py3-none-any.whl (312 kB)
-        |████████████████████████████████| 312 kB 19.2 MB/s
-   Collecting hot-water>=0.1.9
-   Downloading hot-water-0.1.13-py3-none-any.whl (9.3 kB)
-   Collecting cup>=1.6.0
-     Downloading cup-3.22.0-py2.py3-none-any.whl (397 kB)
-        |████████████████████████████████| 397 kB 28.2 MB/s
-   INFO: pip is looking at multiple versions of this package to determine
-   which version is compatible with other requirements.
-   This could take a while.
-     Downloading cup-3.21.0-py2.py3-none-any.whl (395 kB)
-        |████████████████████████████████| 395 kB 27.0 MB/s
-     Downloading cup-3.20.0-py2.py3-none-any.whl (394 kB)
-        |████████████████████████████████| 394 kB 24.4 MB/s
-     Downloading cup-3.19.1-py2.py3-none-any.whl (394 kB)
-        |████████████████████████████████| 394 kB 21.3 MB/s
-     Downloading cup-3.19.0-py2.py3-none-any.whl (394 kB)
-        |████████████████████████████████| 394 kB 26.2 MB/s
-     Downloading cup-3.18.0-py2.py3-none-any.whl (393 kB)
-        |████████████████████████████████| 393 kB 22.1 MB/s
-     Downloading cup-3.17.0-py2.py3-none-any.whl (382 kB)
-        |████████████████████████████████| 382 kB 23.8 MB/s
-     Downloading cup-3.16.0-py2.py3-none-any.whl (376 kB)
-        |████████████████████████████████| 376 kB 27.5 MB/s
-     Downloading cup-3.15.1-py2.py3-none-any.whl (385 kB)
-        |████████████████████████████████| 385 kB 30.4 MB/s
-   INFO: pip is looking at multiple versions of this package to determine
-   which version is compatible with other requirements.
-   This could take a while.
-     Downloading cup-3.15.0-py2.py3-none-any.whl (378 kB)
-        |████████████████████████████████| 378 kB 21.4 MB/s
-     Downloading cup-3.14.0-py2.py3-none-any.whl (372 kB)
-        |████████████████████████████████| 372 kB 21.1 MB/s
-     Downloading cup-3.13.1-py2.py3-none-any.whl (381 kB)
-        |████████████████████████████████| 381 kB 21.8 MB/s
-   This is taking longer than usual. You might need to provide the
-   dependency resolver with stricter constraints to reduce runtime.
-   If you want to abort this run, you can press Ctrl + C to do so.
-     Downloading cup-3.13.0-py2.py3-none-any.whl (374 kB)
-
-In the above sample output, pip had to download multiple versions of
-package ``cup`` - cup-3.22.0 to cup-3.13.0 - to find a version that will be
-compatible with the other packages - ``spoon``, ``hot-water``, etc.
-
-These multiple ``Downloading cup-version`` lines show pip backtracking.
-
-Possible ways to reduce backtracking occurring
-----------------------------------------------
-
-It's important to mention backtracking behaviour is expected during a
-``pip install`` process. What pip is trying to do is complicated - it is
-working through potentially millions of package versions to identify the
-compatible versions.
-
-There is no guaranteed solution to backtracking but you can reduce it -
-here are a number of ways.
-
-.. _1-allow-pip-to-complete-its-backtracking:
-
-1. Allow pip to complete its backtracking
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-In most cases, pip will complete the backtracking process successfully.
-It is possible this could take a very long time to complete - this may
-not be your preferred option.
-
-However, there is a possibility pip will not be able to find a set of
-compatible versions.
-
-If you'd prefer not to wait, you can interrupt pip (ctrl and c) and use
-:ref:`Constraints Files`: to reduce the number of package versions it tries.
-
-.. _2-reduce-the-versions-of-the-backtracking-package:
-
-2. Reduce the number of versions pip will try to backtrack through
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-If pip is backtracking more than you'd like, the next option is to
-constrain the number of package versions it tries.
-
-A first good candidate for this constraining is the package(s) it is
-backtracking on (e.g. in the above example - ``cup``).
-
-You could try:
-
-``pip install tea "cup > 3.13"``
-
-This will reduce the number of versions of ``cup`` it tries, and
-possibly reduce the time pip takes to install.
-
-There is a possibility that if you're wrong (in this case an older
-version would have worked) then you missed the chance to use it. This
-can be trial and error.
-
-.. _3-use-constraint-files-or-lockfiles:
-
-3. Use constraint files or lockfiles
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-This option is a progression of 2 above. It requires users to know how
-to inspect:
-
--  the packages they're trying to install
--  the package release frequency and compatibility policies
--  their release notes and changelogs from past versions
-
-During deployment, you can create a lockfile stating the exact package and
-version number for for each dependency of that package. You can create this
-with `pip-tools `__.
-
-This means the "work" is done once during development process, and so
-will save users this work during deployment.
-
-The pip team is not available to provide support in helping you create a
-suitable constraints file.
-
-.. _4-be-more-strict-on-package-dependencies-during-development:
-
-4. Be more strict on package dependencies during development
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-For package maintainers during software development, give pip some help by
-creating constraint files for the dependency tree. This will reduce the
-number of versions it will try.
-
-Getting help
-------------
-
-If none of the suggestions above work for you, we recommend that you ask
-for help. :ref:`Getting help`.
+This is now covered in :doc:`../topics/dependency-resolution`.
 
 .. _`Using pip from your program`:
 
@@ -1699,7 +957,7 @@ errors. Specifically:
 Per our :ref:`Python 2 Support` policy, pip 20.3 users who are using
 Python 2 will use the legacy resolver by default. Python 2 users
 should upgrade to Python 3 as soon as possible, since in pip 21.0 in
-January 2021, pip will drop support for Python 2 altogether.
+January 2021, pip dropped support for Python 2 altogether.
 
 
 How to upgrade and migrate
@@ -1750,9 +1008,9 @@ How to upgrade and migrate
 
 4. **Troubleshoot and try these workarounds if necessary.**
 
-   -  If pip is taking longer to install packages, read
-      :ref:`Dependency resolution backtracking` for ways to reduce the
-      time pip spends backtracking due to dependency conflicts.
+   -  If pip is taking longer to install packages, read :doc:`Dependency
+      resolution backtracking ` for ways to
+      reduce the time pip spends backtracking due to dependency conflicts.
    -  If you don't want pip to actually resolve dependencies, use the
       ``--no-deps`` option. This is useful when you have a set of package
       versions that work together in reality, even though their metadata says
@@ -1782,7 +1040,7 @@ Setups to test with special attention
 
 *    Continuous integration/continuous deployment setups
 
-*    Installing from any kind of version control systems (i.e., Git, Subversion, Mercurial, or CVS), per :ref:`VCS Support`
+*    Installing from any kind of version control systems (i.e., Git, Subversion, Mercurial, or CVS), per :doc:`topics/vcs-support`
 
 *    Installing from source code held in local directories
 
@@ -1857,9 +1115,11 @@ We plan for the resolver changeover to proceed as follows, using
      environments, pip defaults to the old resolver, and the new one is
      available using the flag ``--use-feature=2020-resolver``.
 
-*    pip 21.0: pip uses new resolver, and the old resolver is no longer
-     available. Python 2 support is removed per our :ref:`Python 2
-     Support` policy.
+*    pip 21.0: pip uses new resolver by default, and the old resolver is
+     no longer supported. It will be removed after a currently undecided
+     amount of time, as the removal is dependent on pip's volunteer
+     maintainers' availability. Python 2 support is removed per our
+     :ref:`Python 2 Support` policy.
 
 Since this work will not change user-visible behavior described in the
 pip documentation, this change is not covered by the :ref:`Deprecation
@@ -1885,6 +1145,4 @@ announcements on the `low-traffic packaging announcements list`_ and
 .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/
 .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform
 .. _the official Python blog: https://blog.python.org/
-.. _requests: https://requests.readthedocs.io/en/master/user/authentication/#netrc-authentication
-.. _Python standard library: https://docs.python.org/3/library/netrc.html
 .. _Python Windows launcher: https://docs.python.org/3/using/windows.html#launcher
diff --git a/docs/pip_sphinxext.py b/docs/pip_sphinxext.py
index df4390d8103..f398b7d0973 100644
--- a/docs/pip_sphinxext.py
+++ b/docs/pip_sphinxext.py
@@ -1,32 +1,90 @@
 """pip sphinx extensions"""
 
 import optparse
+import pathlib
+import re
 import sys
 from textwrap import dedent
+from typing import Dict, Iterable, Iterator, List, Optional, Union
 
-from docutils import nodes
+from docutils import nodes, statemachine
 from docutils.parsers import rst
-from docutils.statemachine import ViewList
+from docutils.statemachine import StringList, ViewList
+from sphinx.application import Sphinx
 
 from pip._internal.cli import cmdoptions
 from pip._internal.commands import commands_dict, create_command
 from pip._internal.req.req_file import SUPPORTED_OPTIONS
 
 
+class PipNewsInclude(rst.Directive):
+    required_arguments = 1
+
+    def _is_version_section_title_underline(
+        self, prev: Optional[str], curr: str
+    ) -> bool:
+        """Find a ==== line that marks the version section title."""
+        if prev is None:
+            return False
+        if re.match(r"^=+$", curr) is None:
+            return False
+        if len(curr) < len(prev):
+            return False
+        return True
+
+    def _iter_lines_with_refs(self, lines: Iterable[str]) -> Iterator[str]:
+        """Transform the input lines to add a ref before each section title.
+
+        This is done by looking one line ahead and locate a title's underline,
+        and add a ref before the title text.
+
+        Dots in the version is converted into dash, and a ``v`` is prefixed.
+        This makes Sphinx use them as HTML ``id`` verbatim without generating
+        auto numbering (which would make the the anchors unstable).
+        """
+        prev = None
+        for line in lines:
+            # Transform the previous line to include an explicit ref.
+            if self._is_version_section_title_underline(prev, line):
+                assert prev is not None
+                vref = prev.split(None, 1)[0].replace(".", "-")
+                yield f".. _`v{vref}`:"
+                yield ""  # Empty line between ref and the title.
+            if prev is not None:
+                yield prev
+            prev = line
+        if prev is not None:
+            yield prev
+
+    def run(self) -> List[nodes.Node]:
+        source = self.state_machine.input_lines.source(
+            self.lineno - self.state_machine.input_offset - 1,
+        )
+        path = (
+            pathlib.Path(source).resolve().parent.joinpath(self.arguments[0]).resolve()
+        )
+        include_lines = statemachine.string2lines(
+            path.read_text(encoding="utf-8"),
+            self.state.document.settings.tab_width,
+            convert_whitespace=True,
+        )
+        include_lines = list(self._iter_lines_with_refs(include_lines))
+        self.state_machine.insert_input(include_lines, str(path))
+        return []
+
+
 class PipCommandUsage(rst.Directive):
     required_arguments = 1
     optional_arguments = 3
 
-    def run(self):
+    def run(self) -> List[nodes.Node]:
         cmd = create_command(self.arguments[0])
-        cmd_prefix = 'python -m pip'
+        cmd_prefix = "python -m pip"
         if len(self.arguments) > 1:
             cmd_prefix = " ".join(self.arguments[1:])
             cmd_prefix = cmd_prefix.strip('"')
             cmd_prefix = cmd_prefix.strip("'")
-        usage = dedent(
-            cmd.usage.replace('%prog', f'{cmd_prefix} {cmd.name}')
-        ).strip()
+        usage = dedent(cmd.usage.replace("%prog", f"{cmd_prefix} {cmd.name}")).strip()
         node = nodes.literal_block(usage, usage)
         return [node]
 
@@ -34,26 +92,28 @@ def run(self):
 class PipCommandDescription(rst.Directive):
     required_arguments = 1
 
-    def run(self):
+    def run(self) -> List[nodes.Node]:
         node = nodes.paragraph()
         node.document = self.state.document
         desc = ViewList()
         cmd = create_command(self.arguments[0])
+        assert cmd.__doc__ is not None
         description = dedent(cmd.__doc__)
-        for line in description.split('\n'):
+        for line in description.split("\n"):
             desc.append(line, "")
         self.state.nested_parse(desc, 0, node)
         return [node]
 
 
 class PipOptions(rst.Directive):
-
-    def _format_option(self, option, cmd_name=None):
+    def _format_option(
+        self, option: optparse.Option, cmd_name: Optional[str] = None
+    ) -> List[str]:
         bookmark_line = (
-            ".. _`{cmd_name}_{option._long_opts[0]}`:"
-            if cmd_name else
-            ".. _`{option._long_opts[0]}`:"
-        ).format(**locals())
+            f".. _`{cmd_name}_{option._long_opts[0]}`:"
+            if cmd_name
+            else f".. _`{option._long_opts[0]}`:"
+        )
         line = ".. option:: "
         if option._short_opts:
             line += option._short_opts[0]
@@ -62,22 +122,26 @@ def _format_option(self, option, cmd_name=None):
         elif option._long_opts:
             line += option._long_opts[0]
         if option.takes_value():
-            metavar = option.metavar or option.dest.lower()
+            metavar = option.metavar or option.dest
+            assert metavar is not None
             line += f" <{metavar.lower()}>"
         # fix defaults
-        opt_help = option.help.replace('%default', str(option.default))
+        assert option.help is not None
+        opt_help = option.help.replace("%default", str(option.default))
         # fix paths with sys.prefix
         opt_help = opt_help.replace(sys.prefix, "")
         return [bookmark_line, "", line, "", "    " + opt_help, ""]
 
-    def _format_options(self, options, cmd_name=None):
+    def _format_options(
+        self, options: Iterable[optparse.Option], cmd_name: Optional[str] = None
+    ) -> None:
         for option in options:
             if option.help == optparse.SUPPRESS_HELP:
                 continue
             for line in self._format_option(option, cmd_name):
                 self.view_list.append(line, "")
 
-    def run(self):
+    def run(self) -> List[nodes.Node]:
         node = nodes.paragraph()
         node.document = self.state.document
         self.view_list = ViewList()
@@ -87,19 +151,17 @@ def run(self):
 
 
 class PipGeneralOptions(PipOptions):
-    def process_options(self):
-        self._format_options(
-            [o() for o in cmdoptions.general_group['options']]
-        )
+    def process_options(self) -> None:
+        self._format_options([o() for o in cmdoptions.general_group["options"]])
 
 
 class PipIndexOptions(PipOptions):
     required_arguments = 1
 
-    def process_options(self):
+    def process_options(self) -> None:
         cmd_name = self.arguments[0]
         self._format_options(
-            [o() for o in cmdoptions.index_group['options']],
+            [o() for o in cmdoptions.index_group["options"]],
             cmd_name=cmd_name,
         )
 
@@ -107,7 +169,7 @@ def process_options(self):
 class PipCommandOptions(PipOptions):
     required_arguments = 1
 
-    def process_options(self):
+    def process_options(self) -> None:
         cmd = create_command(self.arguments[0])
         self._format_options(
             cmd.parser.option_groups[0].option_list,
@@ -116,49 +178,132 @@ def process_options(self):
 
 
 class PipReqFileOptionsReference(PipOptions):
-
-    def determine_opt_prefix(self, opt_name):
+    def determine_opt_prefix(self, opt_name: str) -> str:
         for command in commands_dict:
             cmd = create_command(command)
             if cmd.cmd_opts.has_option(opt_name):
                 return command
 
-        raise KeyError(f'Could not identify prefix of opt {opt_name}')
+        raise KeyError(f"Could not identify prefix of opt {opt_name}")
 
-    def process_options(self):
+    def process_options(self) -> None:
         for option in SUPPORTED_OPTIONS:
-            if getattr(option, 'deprecated', False):
+            if getattr(option, "deprecated", False):
                 continue
 
             opt = option()
             opt_name = opt._long_opts[0]
             if opt._short_opts:
-                short_opt_name = '{}, '.format(opt._short_opts[0])
+                short_opt_name = "{}, ".format(opt._short_opts[0])
             else:
-                short_opt_name = ''
+                short_opt_name = ""
 
-            if option in cmdoptions.general_group['options']:
-                prefix = ''
+            if option in cmdoptions.general_group["options"]:
+                prefix = ""
             else:
-                prefix = '{}_'.format(self.determine_opt_prefix(opt_name))
+                prefix = "{}_".format(self.determine_opt_prefix(opt_name))
 
             self.view_list.append(
-                '*  :ref:`{short}{long}<{prefix}{opt_name}>`'.format(
+                "*  :ref:`{short}{long}<{prefix}{opt_name}>`".format(
                     short=short_opt_name,
                     long=opt_name,
                     prefix=prefix,
-                    opt_name=opt_name
+                    opt_name=opt_name,
                 ),
-                "\n"
+                "\n",
             )
 
 
-def setup(app):
-    app.add_directive('pip-command-usage', PipCommandUsage)
-    app.add_directive('pip-command-description', PipCommandDescription)
-    app.add_directive('pip-command-options', PipCommandOptions)
-    app.add_directive('pip-general-options', PipGeneralOptions)
-    app.add_directive('pip-index-options', PipIndexOptions)
+class PipCLIDirective(rst.Directive):
+    """
+    - Only works when used in a MyST document.
+    - Requires sphinx-inline-tabs' tab directive.
+    """
+
+    has_content = True
+    optional_arguments = 1
+
+    def run(self) -> List[nodes.Node]:
+        node = nodes.paragraph()
+        node.document = self.state.document
+
+        os_variants = {
+            "Linux": {
+                "highlighter": "console",
+                "executable": "python",
+                "prompt": "$",
+            },
+            "MacOS": {
+                "highlighter": "console",
+                "executable": "python",
+                "prompt": "$",
+            },
+            "Windows": {
+                "highlighter": "doscon",
+                "executable": "py",
+                "prompt": "C:>",
+            },
+        }
+
+        if self.arguments:
+            assert self.arguments == ["in-a-venv"]
+            in_virtual_environment = True
+        else:
+            in_virtual_environment = False
+
+        lines = []
+        # Create a tab for each OS
+        for os, variant in os_variants.items():
+
+            # Unpack the values
+            prompt = variant["prompt"]
+            highlighter = variant["highlighter"]
+            if in_virtual_environment:
+                executable = "python"
+                pip_spelling = "pip"
+            else:
+                executable = variant["executable"]
+                pip_spelling = f"{executable} -m pip"
+
+            # Substitute the various "prompts" into the correct variants
+            substitution_pipeline = [
+                (
+                    r"(^|(?<=\n))\$ python",
+                    f"{prompt} {executable}",
+                ),
+                (
+                    r"(^|(?<=\n))\$ pip",
+                    f"{prompt} {pip_spelling}",
+                ),
+            ]
+            content = self.block_text
+            for pattern, substitution in substitution_pipeline:
+                content = re.sub(pattern, substitution, content)
+
+            # Write the tab
+            lines.append(f"````{{tab}} {os}")
+            lines.append(f"```{highlighter}")
+            lines.append(f"{content}")
+            lines.append("```")
+            lines.append("````")
+
+        string_list = StringList(lines)
+        self.state.nested_parse(string_list, 0, node)
+        return [node]
+
+
+def setup(app: Sphinx) -> Dict[str, Union[bool, str]]:
+    app.add_directive("pip-command-usage", PipCommandUsage)
+    app.add_directive("pip-command-description", PipCommandDescription)
+    app.add_directive("pip-command-options", PipCommandOptions)
+    app.add_directive("pip-general-options", PipGeneralOptions)
+    app.add_directive("pip-index-options", PipIndexOptions)
     app.add_directive(
-        'pip-requirements-file-options-ref-list', PipReqFileOptionsReference
+        "pip-requirements-file-options-ref-list", PipReqFileOptionsReference
     )
+    app.add_directive("pip-news-include", PipNewsInclude)
+    app.add_directive("pip-cli", PipCLIDirective)
+    return {
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
diff --git a/tools/requirements/docs.txt b/docs/requirements.txt
similarity index 59%
rename from tools/requirements/docs.txt
rename to docs/requirements.txt
index a5aae67c106..fa3a7390c15 100644
--- a/tools/requirements/docs.txt
+++ b/docs/requirements.txt
@@ -1,7 +1,10 @@
-sphinx == 3.2.1
+sphinx ~= 4.2, != 4.4.0
+towncrier
 furo
+myst_parser
+sphinx-copybutton
 sphinx-inline-tabs
-sphinxcontrib-towncrier
+sphinxcontrib-towncrier >= 0.2.0a0
 
 # `docs.pipext` uses pip's internals to generate documentation. So, we install
 # the current directory to make it work.
diff --git a/news/10560.bugfix.rst b/news/10560.bugfix.rst
new file mode 100644
index 00000000000..988e8f3b244
--- /dev/null
+++ b/news/10560.bugfix.rst
@@ -0,0 +1 @@
+Fix conditional checks to prevent ``pip.exe`` from trying to modify itself, on Windows.
diff --git a/news/10630.trivial.rst b/news/10630.trivial.rst
new file mode 100644
index 00000000000..85e471d51d8
--- /dev/null
+++ b/news/10630.trivial.rst
@@ -0,0 +1 @@
+Replace the git protocol with https in tests.
diff --git a/news/10696.bugfix.rst b/news/10696.bugfix.rst
new file mode 100644
index 00000000000..f3b913532a6
--- /dev/null
+++ b/news/10696.bugfix.rst
@@ -0,0 +1 @@
+Fix uninstall editable from Windows junction link.
diff --git a/news/10812.feature.rst b/news/10812.feature.rst
new file mode 100644
index 00000000000..a8fbc37d9aa
--- /dev/null
+++ b/news/10812.feature.rst
@@ -0,0 +1,2 @@
+Improve error message when ``pip config edit`` is provided an editor that
+doesn't exist.
diff --git a/news/10899.doc.rst b/news/10899.doc.rst
new file mode 100644
index 00000000000..7e4f8872e28
--- /dev/null
+++ b/news/10899.doc.rst
@@ -0,0 +1 @@
+Add more dedicated topic and reference pages to the documentation.
diff --git a/news/10936.doc.rst b/news/10936.doc.rst
new file mode 100644
index 00000000000..390b67c24c9
--- /dev/null
+++ b/news/10936.doc.rst
@@ -0,0 +1 @@
+Capitalise Y as the default for "Proceed (y/n)?" when uninstalling.
diff --git a/news/10950.bugfix.rst b/news/10950.bugfix.rst
new file mode 100644
index 00000000000..e74e4ab9c18
--- /dev/null
+++ b/news/10950.bugfix.rst
@@ -0,0 +1 @@
+Disable brotli import in vendored urllib3 so brotli could be uninstalled/upgraded by pip.
diff --git a/news/10951.doc.rst b/news/10951.doc.rst
new file mode 100644
index 00000000000..f132abd111f
--- /dev/null
+++ b/news/10951.doc.rst
@@ -0,0 +1 @@
+Add ``scheme://`` requirement to ``--proxy`` option's description
diff --git a/news/10972.doc.rst b/news/10972.doc.rst
new file mode 100644
index 00000000000..c98aecd5485
--- /dev/null
+++ b/news/10972.doc.rst
@@ -0,0 +1,2 @@
+The wheel command now references the build interface section instead of stating the legacy
+setuptools behavior as the default.
diff --git a/news/1884.feature.rst b/news/1884.feature.rst
deleted file mode 100644
index 4b0b4180c35..00000000000
--- a/news/1884.feature.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add ``--ignore-requires-python`` support to pip download.
diff --git a/news/30e2240e-e268-4519-bee7-6f79bc4cf489.trivial.rst b/news/30e2240e-e268-4519-bee7-6f79bc4cf489.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/3ba38d39-9189-4bc3-bc87-bf598f1c1064.trivial.rst b/news/3ba38d39-9189-4bc3-bc87-bf598f1c1064.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/43602ba6-8a59-425c-9a97-9c8e87e28ddb.trivial.rst b/news/43602ba6-8a59-425c-9a97-9c8e87e28ddb.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/49254991-9583-470e-a263-b196acf4072b.trivial.rst b/news/49254991-9583-470e-a263-b196acf4072b.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/4a85b5f1-5f9a-4f6b-8960-3334570ae591.trivial.rst b/news/4a85b5f1-5f9a-4f6b-8960-3334570ae591.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/4ec6e91c-91d6-475c-959a-83485cafa4b2.trivial.rst b/news/4ec6e91c-91d6-475c-959a-83485cafa4b2.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/54754cb1-2151-45c3-baa0-b87e50d7e56d.trivial.rst b/news/54754cb1-2151-45c3-baa0-b87e50d7e56d.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/6148.removal.rst b/news/6148.removal.rst
deleted file mode 100644
index cf6d85e70ba..00000000000
--- a/news/6148.removal.rst
+++ /dev/null
@@ -1 +0,0 @@
-Drop support for Python 2.
diff --git a/news/738a71b0-98f9-4e1f-a541-af95fb990af9.trivial.rst b/news/738a71b0-98f9-4e1f-a541-af95fb990af9.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/7502.removal.rst b/news/7502.removal.rst
deleted file mode 100644
index 9f03366ed8e..00000000000
--- a/news/7502.removal.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Remove support for legacy wheel cache entries that were created with pip
-versions older than 20.0.
diff --git a/news/7ced09a1-9af6-4190-8249-05a6328e379e.trivial.rst b/news/7ced09a1-9af6-4190-8249-05a6328e379e.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/7edb0afc-938e-457b-ae6e-0905e7443b2f.trivial.rst b/news/7edb0afc-938e-457b-ae6e-0905e7443b2f.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/80B9D718-3054-41C7-B920-78348DFD01A6.trivial.rst b/news/80B9D718-3054-41C7-B920-78348DFD01A6.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/857785f2-1d4e-4067-9b4b-acc6ae741aef.trivial.rst b/news/857785f2-1d4e-4067-9b4b-acc6ae741aef.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/86c319cb-0539-41a0-871b-4ffe72765f6f.trivial.rst b/news/86c319cb-0539-41a0-871b-4ffe72765f6f.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/8802.removal.rst b/news/8802.removal.rst
deleted file mode 100644
index 79d8e508166..00000000000
--- a/news/8802.removal.rst
+++ /dev/null
@@ -1 +0,0 @@
-Modernise the codebase after Python 2.
diff --git a/news/8876.bugfix.rst b/news/8876.bugfix.rst
deleted file mode 100644
index 98250dc9745..00000000000
--- a/news/8876.bugfix.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fixed hanging VCS subprocess calls when the VCS outputs a large amount of data
-on stderr. Restored logging of VCS errors that was inadvertently removed in pip
-20.2.
diff --git a/news/8a225e3d-998e-4924-92e6-2ab2173159f8.trivial.rst b/news/8a225e3d-998e-4924-92e6-2ab2173159f8.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/9172.doc.rst b/news/9172.doc.rst
deleted file mode 100644
index fc0063766b2..00000000000
--- a/news/9172.doc.rst
+++ /dev/null
@@ -1 +0,0 @@
-Render the unreleased pip version change notes on the news page in docs.
diff --git a/news/9180.bugfix.rst b/news/9180.bugfix.rst
deleted file mode 100644
index e597c1ad90a..00000000000
--- a/news/9180.bugfix.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix error when an existing incompatibility is unable to be applied to a backtracked state.
diff --git a/news/9189.removal.rst b/news/9189.removal.rst
deleted file mode 100644
index 79928cbb15a..00000000000
--- a/news/9189.removal.rst
+++ /dev/null
@@ -1 +0,0 @@
-Drop support for Python 3.5.
diff --git a/news/9203.bugfix.rst b/news/9203.bugfix.rst
deleted file mode 100644
index 38320218fbb..00000000000
--- a/news/9203.bugfix.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-New resolver: Discard a faulty distribution, instead of quitting outright.
-This implementation is taken from 20.2.2, with a fix that always makes the
-resolver iterate through candidates from indexes lazily, to avoid downloading
-candidates we do not need.
diff --git a/news/9206.feature.rst b/news/9206.feature.rst
deleted file mode 100644
index 90cd2cf99fb..00000000000
--- a/news/9206.feature.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-``pip wheel`` now verifies the built wheel contains valid metadata, and can be
-installed by a subsequent ``pip install``. This can be disabled with
-``--no-verify``.
diff --git a/news/9243.bugfix.rst b/news/9243.bugfix.rst
new file mode 100644
index 00000000000..2e588c90534
--- /dev/null
+++ b/news/9243.bugfix.rst
@@ -0,0 +1 @@
+Filter available distributions using hash declarations from constraints files.
diff --git a/news/9246.bugfix.rst b/news/9246.bugfix.rst
deleted file mode 100644
index 93f7f18f9f5..00000000000
--- a/news/9246.bugfix.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-New resolver: Discard a source distribution if it fails to generate metadata,
-instead of quitting outright. This implementation is taken from 20.2.2, with a
-fix that always makes the resolver iterate through candidates from indexes
-lazily, to avoid downloading candidates we do not need.
diff --git a/news/9315.feature.rst b/news/9315.feature.rst
deleted file mode 100644
index 64d1f25338b..00000000000
--- a/news/9315.feature.rst
+++ /dev/null
@@ -1 +0,0 @@
-Improve presentation of XMLRPC errors in pip search.
diff --git a/news/9343.doc.rst b/news/9343.doc.rst
deleted file mode 100644
index 1e4f91aec4c..00000000000
--- a/news/9343.doc.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix broken email link in docs feedback banners.
diff --git a/news/9615.feature.rst b/news/9615.feature.rst
new file mode 100644
index 00000000000..075a6cd4295
--- /dev/null
+++ b/news/9615.feature.rst
@@ -0,0 +1 @@
+Explains why specified version cannot be retrieved when *Requires-Python* is not satisfied.
diff --git a/news/9691.bugfix.rst b/news/9691.bugfix.rst
new file mode 100644
index 00000000000..6a07d49c83a
--- /dev/null
+++ b/news/9691.bugfix.rst
@@ -0,0 +1 @@
+Fix pip install issues using a proxy due to an inconsistency in how Requests is currently handling variable precedence in session.
diff --git a/news/9794.feature.rst b/news/9794.feature.rst
new file mode 100644
index 00000000000..a68fad76a31
--- /dev/null
+++ b/news/9794.feature.rst
@@ -0,0 +1 @@
+Validate build dependencies when using ``--no-build-isolation``.
diff --git a/news/9DE59242-2198-4760-B5A7-B5A6BB98ECA2.trivial.rst b/news/9DE59242-2198-4760-B5A7-B5A6BB98ECA2.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/a13640e3-bfd3-46ae-b4b6-bcb9784303b4.trivial.rst b/news/a13640e3-bfd3-46ae-b4b6-bcb9784303b4.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/a9950589-8b92-4ec1-a3a1-a6657cf6fd5b.trivial.rst b/news/a9950589-8b92-4ec1-a3a1-a6657cf6fd5b.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/ae7bdce7-d6f3-4f30-9192-6a8e69027d6a.trivial.rst b/news/ae7bdce7-d6f3-4f30-9192-6a8e69027d6a.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/b034ad46-e6b0-48b1-8b26-1145d611d082.trivial.rst b/news/b034ad46-e6b0-48b1-8b26-1145d611d082.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/b5e475d5-38b6-48cc-8136-0c32d3ace838.trivial.rst b/news/b5e475d5-38b6-48cc-8136-0c32d3ace838.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/bb86bc866fdc4257a445e0df09dd7e64.trivial.rst b/news/bb86bc866fdc4257a445e0df09dd7e64.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/d96bbdcd-059a-4ea4-b02e-343f8b51aad5.trivial.rst b/news/d96bbdcd-059a-4ea4-b02e-343f8b51aad5.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/dc9e5ecc-9fc9-4762-914e-34014e8d09bf.trivial.rst b/news/dc9e5ecc-9fc9-4762-914e-34014e8d09bf.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/ea24fc60-675c-4104-9825-39d1ee0a20b7.trivial.rst b/news/ea24fc60-675c-4104-9825-39d1ee0a20b7.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/f0af302f-aef7-4323-8332-819f0be13d79.trivial.rst b/news/f0af302f-aef7-4323-8332-819f0be13d79.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/f0b01f0a-f673-4c1b-9959-3196b6c000e9.trivial.rst b/news/f0b01f0a-f673-4c1b-9959-3196b6c000e9.trivial.rst
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/news/resolvelib.vendor.rst b/news/resolvelib.vendor.rst
deleted file mode 100644
index 680da3be1e7..00000000000
--- a/news/resolvelib.vendor.rst
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade resolvelib to 0.5.4.
diff --git a/noxfile.py b/noxfile.py
index 2c0a48ecbea..5d5f426c564 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -1,20 +1,20 @@
 """Automation using nox.
 """
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import glob
 import os
 import shutil
 import sys
 from pathlib import Path
+from typing import Iterator, List, Tuple
 
 import nox
 
+# fmt: off
 sys.path.append(".")
-from tools.automation import release  # isort:skip  # noqa
+from tools import release  # isort:skip  # noqa
 sys.path.pop()
+# fmt: on
 
 nox.options.reuse_existing_virtualenvs = True
 nox.options.sessions = ["lint"]
@@ -24,16 +24,16 @@
     "protected-pip": "tools/tox_pip.py",
 }
 REQUIREMENTS = {
-    "docs": "tools/requirements/docs.txt",
-    "tests": "tools/requirements/tests.txt",
-    "common-wheels": "tools/requirements/tests-common_wheels.txt",
+    "docs": "docs/requirements.txt",
+    "tests": "tests/requirements.txt",
+    "common-wheels": "tests/requirements-common_wheels.txt",
 }
 
 AUTHORS_FILE = "AUTHORS.txt"
 VERSION_FILE = "src/pip/__init__.py"
 
 
-def run_with_protected_pip(session, *arguments):
+def run_with_protected_pip(session: nox.Session, *arguments: str) -> None:
     """Do a session.run("pip", *arguments), using a "protected" pip.
 
     This invokes a wrapper script, that forwards calls to original virtualenv
@@ -43,11 +43,10 @@ def run_with_protected_pip(session, *arguments):
     env = {"VIRTUAL_ENV": session.virtualenv.location}
 
     command = ("python", LOCATIONS["protected-pip"]) + arguments
-    kwargs = {"env": env, "silent": True}
-    session.run(*command, **kwargs)
+    session.run(*command, env=env, silent=True)
 
 
-def should_update_common_wheels():
+def should_update_common_wheels() -> bool:
     # If the cache hasn't been created, create it.
     if not os.path.exists(LOCATIONS["common-wheels"]):
         return True
@@ -66,36 +65,35 @@ def should_update_common_wheels():
 
 # -----------------------------------------------------------------------------
 # Development Commands
-#   These are currently prototypes to evaluate whether we want to switch over
-#   completely to nox for all our automation. Contributors should prefer using
-#   `tox -e ...` until this note is removed.
 # -----------------------------------------------------------------------------
-@nox.session(python=["3.6", "3.7", "3.8", "3.9", "pypy3"])
-def test(session):
+@nox.session(python=["3.7", "3.8", "3.9", "3.10", "pypy3"])
+def test(session: nox.Session) -> None:
     # Get the common wheels.
     if should_update_common_wheels():
+        # fmt: off
         run_with_protected_pip(
             session,
             "wheel",
             "-w", LOCATIONS["common-wheels"],
             "-r", REQUIREMENTS["common-wheels"],
         )
+        # fmt: on
     else:
-        msg = (
-            "Re-using existing common-wheels at {}."
-            .format(LOCATIONS["common-wheels"])
-        )
+        msg = f"Re-using existing common-wheels at {LOCATIONS['common-wheels']}."
         session.log(msg)
 
     # Build source distribution
     sdist_dir = os.path.join(session.virtualenv.location, "sdist")
     if os.path.exists(sdist_dir):
         shutil.rmtree(sdist_dir, ignore_errors=True)
+
+    # fmt: off
     session.run(
-        "python", "setup.py", "sdist",
-        "--formats=zip", "--dist-dir", sdist_dir,
+        "python", "setup.py", "sdist", "--formats=zip", "--dist-dir", sdist_dir,
         silent=True,
     )
+    # fmt: on
+
     generated_files = os.listdir(sdist_dir)
     assert len(generated_files) == 1
     generated_sdist = os.path.join(sdist_dir, generated_files[0])
@@ -112,19 +110,27 @@ def test(session):
     # Run the tests
     #   LC_CTYPE is set to get UTF-8 output inside of the subprocesses that our
     #   tests use.
-    session.run("pytest", *arguments, env={"LC_CTYPE": "en_US.UTF-8"})
+    session.run(
+        "pytest",
+        *arguments,
+        env={
+            "LC_CTYPE": "en_US.UTF-8",
+            "SETUPTOOLS_USE_DISTUTILS": "stdlib",
+        },
+    )
 
 
 @nox.session
-def docs(session):
+def docs(session: nox.Session) -> None:
     session.install("-e", ".")
     session.install("-r", REQUIREMENTS["docs"])
 
-    def get_sphinx_build_command(kind):
+    def get_sphinx_build_command(kind: str) -> List[str]:
         # Having the conf.py in the docs/html is weird but needed because we
         # can not use a different configuration directory vs source directory
         # on RTD currently. So, we'll pass "-c docs/html" here.
         # See https://github.com/rtfd/readthedocs.org/issues/1543.
+        # fmt: off
         return [
             "sphinx-build",
             "-W",
@@ -134,13 +140,29 @@ def get_sphinx_build_command(kind):
             "docs/" + kind,
             "docs/build/" + kind,
         ]
+        # fmt: on
 
     session.run(*get_sphinx_build_command("html"))
     session.run(*get_sphinx_build_command("man"))
 
 
+@nox.session(name="docs-live")
+def docs_live(session: nox.Session) -> None:
+    session.install("-e", ".")
+    session.install("-r", REQUIREMENTS["docs"], "sphinx-autobuild")
+
+    session.run(
+        "sphinx-autobuild",
+        "-d=docs/build/doctrees/livehtml",
+        "-b=dirhtml",
+        "docs/html",
+        "docs/build/livehtml",
+        *session.posargs,
+    )
+
+
 @nox.session
-def lint(session):
+def lint(session: nox.Session) -> None:
     session.install("pre-commit")
 
     if session.posargs:
@@ -149,25 +171,25 @@ def lint(session):
         args = ["--all-files", "--show-diff-on-failure"]
 
     session.run("pre-commit", "run", *args)
-    session.run(
-        "pre-commit", "run", "-c", ".pre-commit-config-slow.yaml", *args
-    )
 
 
 @nox.session
-def vendoring(session):
-    session.install("vendoring>=0.3.0")
+def vendoring(session: nox.Session) -> None:
+    session.install("vendoring~=1.2.0")
 
     if "--upgrade" not in session.posargs:
-        session.run("vendoring", "sync", ".", "-v")
+        session.run("vendoring", "sync", "-v")
         return
 
-    def pinned_requirements(path):
-        for line in path.read_text().splitlines():
-            one, two = line.split("==", 1)
+    def pinned_requirements(path: Path) -> Iterator[Tuple[str, str]]:
+        for line in path.read_text().splitlines(keepends=False):
+            one, sep, two = line.partition("==")
+            if not sep:
+                continue
             name = one.strip()
-            version = two.split("#")[0].strip()
-            yield name, version
+            version = two.split("#", 1)[0].strip()
+            if name and version:
+                yield name, version
 
     vendor_txt = Path("src/pip/_vendor/vendor.txt")
     for name, old_version in pinned_requirements(vendor_txt):
@@ -205,11 +227,26 @@ def pinned_requirements(path):
         release.commit_file(session, ".", message=message)
 
 
+@nox.session
+def coverage(session: nox.Session) -> None:
+    if not os.path.exists("./.coverage-output"):
+        os.mkdir("./.coverage-output")
+    session.run(
+        "pytest",
+        "--cov=pip",
+        "--cov-config=./setup.cfg",
+        env={
+            "COVERAGE_OUTPUT_DIR": "./.coverage-output",
+            "COVERAGE_PROCESS_START": "./setup.cfg",
+        },
+    )
+
+
 # -----------------------------------------------------------------------------
 # Release Commands
 # -----------------------------------------------------------------------------
 @nox.session(name="prepare-release")
-def prepare_release(session):
+def prepare_release(session: nox.Session) -> None:
     version = release.get_version_from_arguments(session)
     if not version:
         session.error("Usage: nox -s prepare-release -- ")
@@ -221,9 +258,7 @@ def prepare_release(session):
     session.log(f"# Updating {AUTHORS_FILE}")
     release.generate_authors(AUTHORS_FILE)
     if release.modified_files_in_git():
-        release.commit_file(
-            session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}",
-        )
+        release.commit_file(session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}")
     else:
         session.log(f"# No changes to {AUTHORS_FILE}")
 
@@ -244,7 +279,7 @@ def prepare_release(session):
 
 
 @nox.session(name="build-release")
-def build_release(session):
+def build_release(session: nox.Session) -> None:
     version = release.get_version_from_arguments(session)
     if not version:
         session.error("Usage: nox -s build-release -- YY.N[.P]")
@@ -269,13 +304,13 @@ def build_release(session):
 
         tmp_dist_paths = (build_dir / p for p in tmp_dists)
         session.log(f"# Copying dists from {build_dir}")
-        os.makedirs('dist', exist_ok=True)
+        os.makedirs("dist", exist_ok=True)
         for dist, final in zip(tmp_dist_paths, tmp_dists):
             session.log(f"# Copying {dist} to {final}")
             shutil.copy(dist, final)
 
 
-def build_dists(session):
+def build_dists(session: nox.Session) -> List[str]:
     """Return dists with valid metadata."""
     session.log(
         "# Check if there's any Git-untracked files before building the wheel",
@@ -283,7 +318,7 @@ def build_dists(session):
 
     has_forbidden_git_untracked_files = any(
         # Don't report the environment this session is running in
-        not untracked_file.startswith('.nox/build-release/')
+        not untracked_file.startswith(".nox/build-release/")
         for untracked_file in release.get_git_untracked_files()
     )
     if has_forbidden_git_untracked_files:
@@ -303,7 +338,7 @@ def build_dists(session):
 
 
 @nox.session(name="upload-release")
-def upload_release(session):
+def upload_release(session: nox.Session) -> None:
     version = release.get_version_from_arguments(session)
     if not version:
         session.error("Usage: nox -s upload-release -- YY.N[.P]")
@@ -322,15 +357,13 @@ def upload_release(session):
             f"Remove dist/ and run 'nox -s build-release -- {version}'"
         )
     # Sanity check: Make sure the files are correctly named.
-    distfile_names = map(os.path.basename, distribution_files)
+    distfile_names = (os.path.basename(fn) for fn in distribution_files)
     expected_distribution_files = [
-        f"pip-{version}-py2.py3-none-any.whl",
+        f"pip-{version}-py3-none-any.whl",
         f"pip-{version}.tar.gz",
     ]
     if sorted(distfile_names) != sorted(expected_distribution_files):
-        session.error(
-            f"Distribution files do not seem to be for {version} release."
-        )
+        session.error(f"Distribution files do not seem to be for {version} release.")
 
     session.log("# Upload distributions")
     session.run("twine", "upload", *distribution_files)
diff --git a/pyproject.toml b/pyproject.toml
index 04f7258064e..a02457eeffd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,13 +3,19 @@ requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.towncrier]
+# For finding the __version__
 package = "pip"
 package_dir = "src"
+# For writing into the correct file
 filename = "NEWS.rst"
+# For finding the news fragments
 directory = "news/"
-title_format = "{version} ({project_date})"
+
+# For rendering properly for this project
 issue_format = "`#{issue} `_"
-template = "tools/automation/news/template.rst"
+template = "tools/news/template.rst"
+
+# Grouping of entries, within our changelog
 type = [
   { name = "Process",                   directory = "process", showcontent = true },
   { name = "Deprecations and Removals", directory = "removal", showcontent = true },
@@ -26,13 +32,14 @@ requirements = "src/pip/_vendor/vendor.txt"
 namespace = "pip._vendor"
 
 protected-files = ["__init__.py", "README.rst", "vendor.txt"]
-patches-dir = "tools/automation/vendoring/patches"
+patches-dir = "tools/vendoring/patches"
 
 [tool.vendoring.transformations]
 substitute = [
   # pkg_resource's vendored packages are directly vendored in pip.
   { match='pkg_resources\.extern', replace="pip._vendor" },
   { match='from \.extern', replace="from pip._vendor" },
+  { match='''\('pygments\.lexers\.''', replace="('pip._vendor.pygments.lexers." },
 ]
 drop = [
   # contains unnecessary scripts
@@ -44,18 +51,21 @@ drop = [
   "setuptools",
   "pkg_resources/_vendor/",
   "pkg_resources/extern/",
+  # trim vendored pygments styles and lexers
+  "pygments/styles/[!_]*.py",
+  '^pygments/lexers/(?!python|__init__|_mapping).*\.py$',
+  # trim rich's markdown support
+  "rich/markdown.py",
 ]
 
 [tool.vendoring.typing-stubs]
 six = ["six.__init__", "six.moves.__init__", "six.moves.configparser"]
-appdirs = []
-contextlib2 = []
+distro = []
 
 [tool.vendoring.license.directories]
 setuptools = "pkg_resources"
-msgpack-python = "msgpack"
 
 [tool.vendoring.license.fallback-urls]
-pytoml = "https://github.com/avakar/pytoml/raw/master/LICENSE"
-resolvelib = "https://github.com/sarugaku/resolvelib/raw/master/LICENSE"
+CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt"
+distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt"
 webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE"
diff --git a/setup.cfg b/setup.cfg
index 25850c4cefa..ae6aa38d8b2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -31,20 +31,32 @@ per-file-ignores =
     tests/*: B011
 
 [mypy]
-follow_imports = silent
 ignore_missing_imports = True
 disallow_untyped_defs = True
 disallow_any_generics = True
 warn_unused_ignores = True
 
-[mypy-pip/_vendor/*]
-follow_imports = skip
+[mypy-pip._vendor.*]
 ignore_errors = True
 
+# These vendored libraries use runtime magic to populate things and don't sit
+# well with static typing out of the box. Eventually we should provide correct
+# typing information for their public interface and remove these configs.
+[mypy-pip._vendor.colorama]
+follow_imports = skip
+[mypy-pip._vendor.pkg_resources]
+follow_imports = skip
+[mypy-pip._vendor.progress.*]
+follow_imports = skip
+[mypy-pip._vendor.requests.*]
+follow_imports = skip
+
 [tool:pytest]
-addopts = --ignore src/pip/_vendor --ignore tests/tests_cache -r aR
+addopts = --ignore src/pip/_vendor --ignore tests/tests_cache -r aR --color=yes
+xfail_strict = True
 markers =
     network: tests that need network
+    incompatible_with_sysconfig
     incompatible_with_test_venv
     incompatible_with_venv
     no_auto_tempdir_manager
@@ -54,7 +66,6 @@ markers =
     svn: VCS: Subversion
     mercurial: VCS: Mercurial
     git: VCS: git
-    yaml: yaml based tests
     search: tests for 'pip search'
 
 [coverage:run]
@@ -91,9 +102,7 @@ exclude_lines =
     # it.
     pragma: no cover
     # This excludes typing-specific code, which will be validated by mypy anyway.
-    if MYPY_CHECK_RUNNING
-    # Can be set to exclude e.g. `if PY2:` on Python 3
-    ${PIP_CI_COVERAGE_EXCLUDES}
+    if TYPE_CHECKING
 
 [metadata]
 license_file = LICENSE.txt
diff --git a/setup.py b/setup.py
index b3aaa361396..fbb6a48a69c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,38 +1,34 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import os
 import sys
 
 from setuptools import find_packages, setup
 
 
-def read(rel_path):
+def read(rel_path: str) -> str:
     here = os.path.abspath(os.path.dirname(__file__))
     # intentionally *not* adding an encoding option to open, See:
     #   https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
-    with open(os.path.join(here, rel_path), 'r') as fp:
+    with open(os.path.join(here, rel_path)) as fp:
         return fp.read()
 
 
-def get_version(rel_path):
+def get_version(rel_path: str) -> str:
     for line in read(rel_path).splitlines():
-        if line.startswith('__version__'):
+        if line.startswith("__version__"):
             # __version__ = "0.9"
             delim = '"' if '"' in line else "'"
             return line.split(delim)[1]
     raise RuntimeError("Unable to find version string.")
 
 
-long_description = read('README.rst')
+long_description = read("README.rst")
 
 setup(
     name="pip",
     version=get_version("src/pip/__init__.py"),
     description="The PyPA recommended tool for installing Python packages.",
     long_description=long_description,
-
-    license='MIT',
+    license="MIT",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
@@ -40,47 +36,49 @@ def get_version(rel_path):
         "Topic :: Software Development :: Build Tools",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3 :: Only"
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
     ],
-    url='https://pip.pypa.io/',
-    keywords='distutils easy_install egg setuptools wheel virtualenv',
+    url="https://pip.pypa.io/",
     project_urls={
         "Documentation": "https://pip.pypa.io",
         "Source": "https://github.com/pypa/pip",
         "Changelog": "https://pip.pypa.io/en/stable/news/",
     },
-
-    author='The pip developers',
-    author_email='distutils-sig@python.org',
-
+    author="The pip developers",
+    author_email="distutils-sig@python.org",
     package_dir={"": "src"},
     packages=find_packages(
         where="src",
         exclude=["contrib", "docs", "tests*", "tasks"],
     ),
     package_data={
+        "pip": ["py.typed"],
         "pip._vendor": ["vendor.txt"],
         "pip._vendor.certifi": ["*.pem"],
         "pip._vendor.requests": ["*.pem"],
         "pip._vendor.distlib._backport": ["sysconfig.cfg"],
-        "pip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"],
+        "pip._vendor.distlib": [
+            "t32.exe",
+            "t64.exe",
+            "t64-arm.exe",
+            "w32.exe",
+            "w64.exe",
+            "w64-arm.exe",
+        ],
     },
     entry_points={
         "console_scripts": [
             "pip=pip._internal.cli.main:main",
             "pip{}=pip._internal.cli.main:main".format(sys.version_info[0]),
-            "pip{}.{}=pip._internal.cli.main:main".format(
-                *sys.version_info[:2]
-            ),
+            "pip{}.{}=pip._internal.cli.main:main".format(*sys.version_info[:2]),
         ],
     },
-
     zip_safe=False,
-    python_requires='>=3.6',
+    python_requires=">=3.7",
 )
diff --git a/src/pip/__init__.py b/src/pip/__init__.py
index ae0fe9a9f24..519b304f090 100644
--- a/src/pip/__init__.py
+++ b/src/pip/__init__.py
@@ -1,14 +1,9 @@
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from typing import List, Optional
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
+__version__ = "22.1.dev0"
 
 
-__version__ = "21.0.dev0"
-
-
-def main(args=None):
-    # type: (Optional[List[str]]) -> int
+def main(args: Optional[List[str]] = None) -> int:
     """This is an internal API only meant for use by pip's own console scripts.
 
     For additional details, see https://github.com/pypa/pip/issues/7498.
diff --git a/src/pip/__main__.py b/src/pip/__main__.py
index 1005489f32f..fe34a7b7772 100644
--- a/src/pip/__main__.py
+++ b/src/pip/__main__.py
@@ -1,16 +1,17 @@
 import os
 import sys
+import warnings
 
 # Remove '' and current working directory from the first entry
 # of sys.path, if present to avoid using current directory
 # in pip commands check, freeze, install, list and show,
 # when invoked as python -m pip 
-if sys.path[0] in ('', os.getcwd()):
+if sys.path[0] in ("", os.getcwd()):
     sys.path.pop(0)
 
 # If we are running from a wheel, add the wheel to sys.path
 # This allows the usage python pip-*.whl/pip install pip-*.whl
-if __package__ == '':
+if __package__ == "":
     # __file__ is pip-*.whl/pip/__main__.py
     # first dirname call strips of '/__main__.py', second strips off '/pip'
     # Resulting path is the name of the wheel itself
@@ -18,7 +19,13 @@
     path = os.path.dirname(os.path.dirname(__file__))
     sys.path.insert(0, path)
 
-from pip._internal.cli.main import main as _main
+if __name__ == "__main__":
+    # Work around the error reported in #9540, pending a proper fix.
+    # Note: It is essential the warning filter is set *before* importing
+    #       pip, as the deprecation happens at import time, not runtime.
+    warnings.filterwarnings(
+        "ignore", category=DeprecationWarning, module=".*packaging\\.version"
+    )
+    from pip._internal.cli.main import main as _main
 
-if __name__ == '__main__':
     sys.exit(_main())
diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py
index a778e99488e..6afb5c627ce 100755
--- a/src/pip/_internal/__init__.py
+++ b/src/pip/_internal/__init__.py
@@ -1,12 +1,14 @@
+from typing import List, Optional
+
 import pip._internal.utils.inject_securetransport  # noqa
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils import _log
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
+# init_logging() must be called before any call to logging.getLogger()
+# which happens at import of most modules.
+_log.init_logging()
 
 
-def main(args=None):
-    # type: (Optional[List[str]]) -> int
+def main(args: (Optional[List[str]]) = None) -> int:
     """This is preserved for old console scripts that may still be referencing
     it.
 
diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py
index a587d9f7c8f..b15a8112dff 100644
--- a/src/pip/_internal/build_env.py
+++ b/src/pip/_internal/build_env.py
@@ -1,68 +1,85 @@
 """Build Environment used for isolation during sdist building
 """
 
+import contextlib
 import logging
 import os
+import pathlib
 import sys
 import textwrap
+import zipfile
 from collections import OrderedDict
-from distutils.sysconfig import get_python_lib
 from sysconfig import get_paths
+from types import TracebackType
+from typing import TYPE_CHECKING, Iterable, Iterator, List, Optional, Set, Tuple, Type
 
-from pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet
+from pip._vendor.certifi import where
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.version import Version
 
 from pip import __file__ as pip_location
 from pip._internal.cli.spinners import open_spinner
+from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
+from pip._internal.metadata import get_default_environment, get_environment
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from types import TracebackType
-    from typing import Iterable, List, Optional, Set, Tuple, Type
 
+if TYPE_CHECKING:
     from pip._internal.index.package_finder import PackageFinder
 
 logger = logging.getLogger(__name__)
 
 
 class _Prefix:
-
-    def __init__(self, path):
-        # type: (str) -> None
+    def __init__(self, path: str) -> None:
         self.path = path
         self.setup = False
         self.bin_dir = get_paths(
-            'nt' if os.name == 'nt' else 'posix_prefix',
-            vars={'base': path, 'platbase': path}
-        )['scripts']
-        # Note: prefer distutils' sysconfig to get the
-        # library paths so PyPy is correctly supported.
-        purelib = get_python_lib(plat_specific=False, prefix=path)
-        platlib = get_python_lib(plat_specific=True, prefix=path)
-        if purelib == platlib:
-            self.lib_dirs = [purelib]
-        else:
-            self.lib_dirs = [purelib, platlib]
+            "nt" if os.name == "nt" else "posix_prefix",
+            vars={"base": path, "platbase": path},
+        )["scripts"]
+        self.lib_dirs = get_prefixed_libs(path)
 
 
-class BuildEnvironment:
-    """Creates and manages an isolated environment to install build deps
+@contextlib.contextmanager
+def _create_standalone_pip() -> Iterator[str]:
+    """Create a "standalone pip" zip file.
+
+    The zip file's content is identical to the currently-running pip.
+    It will be used to install requirements into the build environment.
     """
+    source = pathlib.Path(pip_location).resolve().parent
 
-    def __init__(self):
-        # type: () -> None
-        temp_dir = TempDirectory(
-            kind=tempdir_kinds.BUILD_ENV, globally_managed=True
-        )
+    # Return the current instance if `source` is not a directory. We can't build
+    # a zip from this, and it likely means the instance is already standalone.
+    if not source.is_dir():
+        yield str(source)
+        return
+
+    with TempDirectory(kind="standalone-pip") as tmp_dir:
+        pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip")
+        kwargs = {}
+        if sys.version_info >= (3, 8):
+            kwargs["strict_timestamps"] = False
+        with zipfile.ZipFile(pip_zip, "w", **kwargs) as zf:
+            for child in source.rglob("*"):
+                zf.write(child, child.relative_to(source.parent).as_posix())
+        yield os.path.join(pip_zip, "pip")
+
+
+class BuildEnvironment:
+    """Creates and manages an isolated environment to install build deps"""
+
+    def __init__(self) -> None:
+        temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
 
-        self._prefixes = OrderedDict((
+        self._prefixes = OrderedDict(
             (name, _Prefix(os.path.join(temp_dir.path, name)))
-            for name in ('normal', 'overlay')
-        ))
+            for name in ("normal", "overlay")
+        )
 
-        self._bin_dirs = []  # type: List[str]
-        self._lib_dirs = []  # type: List[str]
+        self._bin_dirs: List[str] = []
+        self._lib_dirs: List[str] = []
         for prefix in reversed(list(self._prefixes.values())):
             self._bin_dirs.append(prefix.bin_dir)
             self._lib_dirs.extend(prefix.lib_dirs)
@@ -71,17 +88,17 @@ def __init__(self):
         # - ensure .pth files are honored
         # - prevent access to system site packages
         system_sites = {
-            os.path.normcase(site) for site in (
-                get_python_lib(plat_specific=False),
-                get_python_lib(plat_specific=True),
-            )
+            os.path.normcase(site) for site in (get_purelib(), get_platlib())
         }
-        self._site_dir = os.path.join(temp_dir.path, 'site')
+        self._site_dir = os.path.join(temp_dir.path, "site")
         if not os.path.exists(self._site_dir):
             os.mkdir(self._site_dir)
-        with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp:
-            fp.write(textwrap.dedent(
-                '''
+        with open(
+            os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
+        ) as fp:
+            fp.write(
+                textwrap.dedent(
+                    """
                 import os, site, sys
 
                 # First, drop system-sites related paths.
@@ -104,139 +121,180 @@ def __init__(self):
                 for path in {lib_dirs!r}:
                     assert not path in sys.path
                     site.addsitedir(path)
-                '''
-            ).format(system_sites=system_sites, lib_dirs=self._lib_dirs))
+                """
+                ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
+            )
 
-    def __enter__(self):
-        # type: () -> None
+    def __enter__(self) -> None:
         self._save_env = {
             name: os.environ.get(name, None)
-            for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH')
+            for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
         }
 
         path = self._bin_dirs[:]
-        old_path = self._save_env['PATH']
+        old_path = self._save_env["PATH"]
         if old_path:
             path.extend(old_path.split(os.pathsep))
 
         pythonpath = [self._site_dir]
 
-        os.environ.update({
-            'PATH': os.pathsep.join(path),
-            'PYTHONNOUSERSITE': '1',
-            'PYTHONPATH': os.pathsep.join(pythonpath),
-        })
+        os.environ.update(
+            {
+                "PATH": os.pathsep.join(path),
+                "PYTHONNOUSERSITE": "1",
+                "PYTHONPATH": os.pathsep.join(pythonpath),
+            }
+        )
 
     def __exit__(
         self,
-        exc_type,  # type: Optional[Type[BaseException]]
-        exc_val,  # type: Optional[BaseException]
-        exc_tb  # type: Optional[TracebackType]
-    ):
-        # type: (...) -> None
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
         for varname, old_value in self._save_env.items():
             if old_value is None:
                 os.environ.pop(varname, None)
             else:
                 os.environ[varname] = old_value
 
-    def check_requirements(self, reqs):
-        # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]
+    def check_requirements(
+        self, reqs: Iterable[str]
+    ) -> Tuple[Set[Tuple[str, str]], Set[str]]:
         """Return 2 sets:
-            - conflicting requirements: set of (installed, wanted) reqs tuples
-            - missing requirements: set of reqs
+        - conflicting requirements: set of (installed, wanted) reqs tuples
+        - missing requirements: set of reqs
         """
         missing = set()
         conflicting = set()
         if reqs:
-            ws = WorkingSet(self._lib_dirs)
-            for req in reqs:
-                try:
-                    if ws.find(Requirement.parse(req)) is None:
-                        missing.add(req)
-                except VersionConflict as e:
-                    conflicting.add((str(e.args[0].as_requirement()),
-                                     str(e.args[1])))
+            env = (
+                get_environment(self._lib_dirs)
+                if hasattr(self, "_lib_dirs")
+                else get_default_environment()
+            )
+            for req_str in reqs:
+                req = Requirement(req_str)
+                dist = env.get_distribution(req.name)
+                if not dist:
+                    missing.add(req_str)
+                    continue
+                if isinstance(dist.version, Version):
+                    installed_req_str = f"{req.name}=={dist.version}"
+                else:
+                    installed_req_str = f"{req.name}==={dist.version}"
+                if dist.version not in req.specifier:
+                    conflicting.add((installed_req_str, req_str))
+                # FIXME: Consider direct URL?
         return conflicting, missing
 
     def install_requirements(
         self,
-        finder,  # type: PackageFinder
-        requirements,  # type: Iterable[str]
-        prefix_as_string,  # type: str
-        message  # type: str
-    ):
-        # type: (...) -> None
+        finder: "PackageFinder",
+        requirements: Iterable[str],
+        prefix_as_string: str,
+        *,
+        kind: str,
+    ) -> None:
         prefix = self._prefixes[prefix_as_string]
         assert not prefix.setup
         prefix.setup = True
         if not requirements:
             return
-        args = [
-            sys.executable, os.path.dirname(pip_location), 'install',
-            '--ignore-installed', '--no-user', '--prefix', prefix.path,
-            '--no-warn-script-location',
-        ]  # type: List[str]
+        with contextlib.ExitStack() as ctx:
+            pip_runnable = ctx.enter_context(_create_standalone_pip())
+            self._install_requirements(
+                pip_runnable,
+                finder,
+                requirements,
+                prefix,
+                kind=kind,
+            )
+
+    @staticmethod
+    def _install_requirements(
+        pip_runnable: str,
+        finder: "PackageFinder",
+        requirements: Iterable[str],
+        prefix: _Prefix,
+        *,
+        kind: str,
+    ) -> None:
+        args: List[str] = [
+            sys.executable,
+            pip_runnable,
+            "install",
+            "--ignore-installed",
+            "--no-user",
+            "--prefix",
+            prefix.path,
+            "--no-warn-script-location",
+        ]
         if logger.getEffectiveLevel() <= logging.DEBUG:
-            args.append('-v')
-        for format_control in ('no_binary', 'only_binary'):
+            args.append("-v")
+        for format_control in ("no_binary", "only_binary"):
             formats = getattr(finder.format_control, format_control)
-            args.extend(('--' + format_control.replace('_', '-'),
-                         ','.join(sorted(formats or {':none:'}))))
+            args.extend(
+                (
+                    "--" + format_control.replace("_", "-"),
+                    ",".join(sorted(formats or {":none:"})),
+                )
+            )
 
         index_urls = finder.index_urls
         if index_urls:
-            args.extend(['-i', index_urls[0]])
+            args.extend(["-i", index_urls[0]])
             for extra_index in index_urls[1:]:
-                args.extend(['--extra-index-url', extra_index])
+                args.extend(["--extra-index-url", extra_index])
         else:
-            args.append('--no-index')
+            args.append("--no-index")
         for link in finder.find_links:
-            args.extend(['--find-links', link])
+            args.extend(["--find-links", link])
 
         for host in finder.trusted_hosts:
-            args.extend(['--trusted-host', host])
+            args.extend(["--trusted-host", host])
         if finder.allow_all_prereleases:
-            args.append('--pre')
+            args.append("--pre")
         if finder.prefer_binary:
-            args.append('--prefer-binary')
-        args.append('--')
+            args.append("--prefer-binary")
+        args.append("--")
         args.extend(requirements)
-        with open_spinner(message) as spinner:
-            call_subprocess(args, spinner=spinner)
+        extra_environ = {"_PIP_STANDALONE_CERT": where()}
+        with open_spinner(f"Installing {kind}") as spinner:
+            call_subprocess(
+                args,
+                command_desc=f"pip subprocess to install {kind}",
+                spinner=spinner,
+                extra_environ=extra_environ,
+            )
 
 
 class NoOpBuildEnvironment(BuildEnvironment):
-    """A no-op drop-in replacement for BuildEnvironment
-    """
+    """A no-op drop-in replacement for BuildEnvironment"""
 
-    def __init__(self):
-        # type: () -> None
+    def __init__(self) -> None:
         pass
 
-    def __enter__(self):
-        # type: () -> None
+    def __enter__(self) -> None:
         pass
 
     def __exit__(
         self,
-        exc_type,  # type: Optional[Type[BaseException]]
-        exc_val,  # type: Optional[BaseException]
-        exc_tb  # type: Optional[TracebackType]
-    ):
-        # type: (...) -> None
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
         pass
 
-    def cleanup(self):
-        # type: () -> None
+    def cleanup(self) -> None:
         pass
 
     def install_requirements(
         self,
-        finder,  # type: PackageFinder
-        requirements,  # type: Iterable[str]
-        prefix_as_string,  # type: str
-        message  # type: str
-    ):
-        # type: (...) -> None
+        finder: "PackageFinder",
+        requirements: Iterable[str],
+        prefix_as_string: str,
+        *,
+        kind: str,
+    ) -> None:
         raise NotImplementedError()
diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py
index 5e7db9c1f62..1d6df220118 100644
--- a/src/pip/_internal/cache.py
+++ b/src/pip/_internal/cache.py
@@ -5,29 +5,22 @@
 import json
 import logging
 import os
+from typing import Any, Dict, List, Optional, Set
 
-from pip._vendor.packaging.tags import interpreter_name, interpreter_version
+from pip._vendor.packaging.tags import Tag, interpreter_name, interpreter_version
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.exceptions import InvalidWheelFilename
+from pip._internal.models.format_control import FormatControl
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import path_to_url
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, List, Optional, Set
-
-    from pip._vendor.packaging.tags import Tag
-
-    from pip._internal.models.format_control import FormatControl
-
 logger = logging.getLogger(__name__)
 
 
-def _hash_dict(d):
-    # type: (Dict[str, str]) -> str
+def _hash_dict(d: Dict[str, str]) -> str:
     """Return a stable sha224 of a dictionary."""
     s = json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
     return hashlib.sha224(s.encode("ascii")).hexdigest()
@@ -37,15 +30,16 @@ class Cache:
     """An abstract class - provides cache directories for data from links
 
 
-        :param cache_dir: The root of the cache.
-        :param format_control: An object of FormatControl class to limit
-            binaries being read from the cache.
-        :param allowed_formats: which formats of files the cache should store.
-            ('binary' and 'source' are the only allowed values)
+    :param cache_dir: The root of the cache.
+    :param format_control: An object of FormatControl class to limit
+        binaries being read from the cache.
+    :param allowed_formats: which formats of files the cache should store.
+        ('binary' and 'source' are the only allowed values)
     """
 
-    def __init__(self, cache_dir, format_control, allowed_formats):
-        # type: (str, FormatControl, Set[str]) -> None
+    def __init__(
+        self, cache_dir: str, format_control: FormatControl, allowed_formats: Set[str]
+    ) -> None:
         super().__init__()
         assert not cache_dir or os.path.isabs(cache_dir)
         self.cache_dir = cache_dir or None
@@ -55,10 +49,8 @@ def __init__(self, cache_dir, format_control, allowed_formats):
         _valid_formats = {"source", "binary"}
         assert self.allowed_formats.union(_valid_formats) == _valid_formats
 
-    def _get_cache_path_parts(self, link):
-        # type: (Link) -> List[str]
-        """Get parts of part that must be os.path.joined with cache_dir
-        """
+    def _get_cache_path_parts(self, link: Link) -> List[str]:
+        """Get parts of part that must be os.path.joined with cache_dir"""
 
         # We want to generate an url to use as our cache key, we don't want to
         # just re-use the URL because it might have other items in the fragment
@@ -90,19 +82,12 @@ def _get_cache_path_parts(self, link):
 
         return parts
 
-    def _get_candidates(self, link, canonical_package_name):
-        # type: (Link, str) -> List[Any]
-        can_not_cache = (
-            not self.cache_dir or
-            not canonical_package_name or
-            not link
-        )
+    def _get_candidates(self, link: Link, canonical_package_name: str) -> List[Any]:
+        can_not_cache = not self.cache_dir or not canonical_package_name or not link
         if can_not_cache:
             return []
 
-        formats = self.format_control.get_allowed_formats(
-            canonical_package_name
-        )
+        formats = self.format_control.get_allowed_formats(canonical_package_name)
         if not self.allowed_formats.intersection(formats):
             return []
 
@@ -113,19 +98,16 @@ def _get_candidates(self, link, canonical_package_name):
                 candidates.append((candidate, path))
         return candidates
 
-    def get_path_for_link(self, link):
-        # type: (Link) -> str
-        """Return a directory to store cached items in for link.
-        """
+    def get_path_for_link(self, link: Link) -> str:
+        """Return a directory to store cached items in for link."""
         raise NotImplementedError()
 
     def get(
         self,
-        link,            # type: Link
-        package_name,    # type: Optional[str]
-        supported_tags,  # type: List[Tag]
-    ):
-        # type: (...) -> Link
+        link: Link,
+        package_name: Optional[str],
+        supported_tags: List[Tag],
+    ) -> Link:
         """Returns a link to a cached item if it exists, otherwise returns the
         passed link.
         """
@@ -133,15 +115,12 @@ def get(
 
 
 class SimpleWheelCache(Cache):
-    """A cache of wheels for future installs.
-    """
+    """A cache of wheels for future installs."""
 
-    def __init__(self, cache_dir, format_control):
-        # type: (str, FormatControl) -> None
+    def __init__(self, cache_dir: str, format_control: FormatControl) -> None:
         super().__init__(cache_dir, format_control, {"binary"})
 
-    def get_path_for_link(self, link):
-        # type: (Link) -> str
+    def get_path_for_link(self, link: Link) -> str:
         """Return a directory to store cached wheels for link
 
         Because there are M wheels for any one sdist, we provide a directory
@@ -163,20 +142,17 @@ def get_path_for_link(self, link):
 
     def get(
         self,
-        link,            # type: Link
-        package_name,    # type: Optional[str]
-        supported_tags,  # type: List[Tag]
-    ):
-        # type: (...) -> Link
+        link: Link,
+        package_name: Optional[str],
+        supported_tags: List[Tag],
+    ) -> Link:
         candidates = []
 
         if not package_name:
             return link
 
         canonical_package_name = canonicalize_name(package_name)
-        for wheel_name, wheel_dir in self._get_candidates(
-            link, canonical_package_name
-        ):
+        for wheel_name, wheel_dir in self._get_candidates(link, canonical_package_name):
             try:
                 wheel = Wheel(wheel_name)
             except InvalidWheelFilename:
@@ -185,7 +161,9 @@ def get(
                 logger.debug(
                     "Ignoring cached wheel %s for %s as it "
                     "does not match the expected distribution name %s.",
-                    wheel_name, link, package_name,
+                    wheel_name,
+                    link,
+                    package_name,
                 )
                 continue
             if not wheel.supported(supported_tags):
@@ -207,11 +185,9 @@ def get(
 
 
 class EphemWheelCache(SimpleWheelCache):
-    """A SimpleWheelCache that creates it's own temporary cache directory
-    """
+    """A SimpleWheelCache that creates it's own temporary cache directory"""
 
-    def __init__(self, format_control):
-        # type: (FormatControl) -> None
+    def __init__(self, format_control: FormatControl) -> None:
         self._temp_dir = TempDirectory(
             kind=tempdir_kinds.EPHEM_WHEEL_CACHE,
             globally_managed=True,
@@ -223,8 +199,8 @@ def __init__(self, format_control):
 class CacheEntry:
     def __init__(
         self,
-        link,  # type: Link
-        persistent,  # type: bool
+        link: Link,
+        persistent: bool,
     ):
         self.link = link
         self.persistent = persistent
@@ -237,27 +213,23 @@ class WheelCache(Cache):
     when a certain link is not found in the simple wheel cache first.
     """
 
-    def __init__(self, cache_dir, format_control):
-        # type: (str, FormatControl) -> None
-        super().__init__(cache_dir, format_control, {'binary'})
+    def __init__(self, cache_dir: str, format_control: FormatControl) -> None:
+        super().__init__(cache_dir, format_control, {"binary"})
         self._wheel_cache = SimpleWheelCache(cache_dir, format_control)
         self._ephem_cache = EphemWheelCache(format_control)
 
-    def get_path_for_link(self, link):
-        # type: (Link) -> str
+    def get_path_for_link(self, link: Link) -> str:
         return self._wheel_cache.get_path_for_link(link)
 
-    def get_ephem_path_for_link(self, link):
-        # type: (Link) -> str
+    def get_ephem_path_for_link(self, link: Link) -> str:
         return self._ephem_cache.get_path_for_link(link)
 
     def get(
         self,
-        link,            # type: Link
-        package_name,    # type: Optional[str]
-        supported_tags,  # type: List[Tag]
-    ):
-        # type: (...) -> Link
+        link: Link,
+        package_name: Optional[str],
+        supported_tags: List[Tag],
+    ) -> Link:
         cache_entry = self.get_cache_entry(link, package_name, supported_tags)
         if cache_entry is None:
             return link
@@ -265,11 +237,10 @@ def get(
 
     def get_cache_entry(
         self,
-        link,            # type: Link
-        package_name,    # type: Optional[str]
-        supported_tags,  # type: List[Tag]
-    ):
-        # type: (...) -> Optional[CacheEntry]
+        link: Link,
+        package_name: Optional[str],
+        supported_tags: List[Tag],
+    ) -> Optional[CacheEntry]:
         """Returns a CacheEntry with a link to a cached item if it exists or
         None. The cache entry indicates if the item was found in the persistent
         or ephemeral cache.
diff --git a/src/pip/_internal/cli/autocompletion.py b/src/pip/_internal/cli/autocompletion.py
index 329de602513..226fe84dc0d 100644
--- a/src/pip/_internal/cli/autocompletion.py
+++ b/src/pip/_internal/cli/autocompletion.py
@@ -5,36 +5,31 @@
 import os
 import sys
 from itertools import chain
+from typing import Any, Iterable, List, Optional
 
 from pip._internal.cli.main_parser import create_main_parser
 from pip._internal.commands import commands_dict, create_command
-from pip._internal.utils.misc import get_installed_distributions
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.metadata import get_default_environment
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Iterable, List, Optional
 
-
-def autocomplete():
-    # type: () -> None
-    """Entry Point for completion of main and subcommand options.
-    """
+def autocomplete() -> None:
+    """Entry Point for completion of main and subcommand options."""
     # Don't complete if user hasn't sourced bash_completion file.
-    if 'PIP_AUTO_COMPLETE' not in os.environ:
+    if "PIP_AUTO_COMPLETE" not in os.environ:
         return
-    cwords = os.environ['COMP_WORDS'].split()[1:]
-    cword = int(os.environ['COMP_CWORD'])
+    cwords = os.environ["COMP_WORDS"].split()[1:]
+    cword = int(os.environ["COMP_CWORD"])
     try:
         current = cwords[cword - 1]
     except IndexError:
-        current = ''
+        current = ""
 
     parser = create_main_parser()
     subcommands = list(commands_dict)
     options = []
 
     # subcommand
-    subcommand_name = None  # type: Optional[str]
+    subcommand_name: Optional[str] = None
     for word in cwords:
         if word in subcommands:
             subcommand_name = word
@@ -42,25 +37,36 @@ def autocomplete():
     # subcommand options
     if subcommand_name is not None:
         # special case: 'help' subcommand has no options
-        if subcommand_name == 'help':
+        if subcommand_name == "help":
             sys.exit(1)
         # special case: list locally installed dists for show and uninstall
-        should_list_installed = (
-            subcommand_name in ['show', 'uninstall'] and
-            not current.startswith('-')
-        )
+        should_list_installed = not current.startswith("-") and subcommand_name in [
+            "show",
+            "uninstall",
+        ]
         if should_list_installed:
-            installed = []
+            env = get_default_environment()
             lc = current.lower()
-            for dist in get_installed_distributions(local_only=True):
-                if dist.key.startswith(lc) and dist.key not in cwords[1:]:
-                    installed.append(dist.key)
+            installed = [
+                dist.canonical_name
+                for dist in env.iter_installed_distributions(local_only=True)
+                if dist.canonical_name.startswith(lc)
+                and dist.canonical_name not in cwords[1:]
+            ]
             # if there are no dists installed, fall back to option completion
             if installed:
                 for dist in installed:
                     print(dist)
                 sys.exit(1)
 
+        should_list_installables = (
+            not current.startswith("-") and subcommand_name == "install"
+        )
+        if should_list_installables:
+            for path in auto_complete_paths(current, "path"):
+                print(path)
+            sys.exit(1)
+
         subcommand = create_command(subcommand_name)
 
         for opt in subcommand.parser.option_list_all:
@@ -69,13 +75,15 @@ def autocomplete():
                     options.append((opt_str, opt.nargs))
 
         # filter out previously specified options from available options
-        prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]]
+        prev_opts = [x.split("=")[0] for x in cwords[1 : cword - 1]]
         options = [(x, v) for (x, v) in options if x not in prev_opts]
         # filter options by current input
         options = [(k, v) for k, v in options if k.startswith(current)]
         # get completion type given cwords and available subcommand options
         completion_type = get_path_completion_type(
-            cwords, cword, subcommand.parser.option_list_all,
+            cwords,
+            cword,
+            subcommand.parser.option_list_all,
         )
         # get completion files and directories if ``completion_type`` is
         # ````, ```` or ````
@@ -86,7 +94,7 @@ def autocomplete():
             opt_label = option[0]
             # append '=' to options which require args
             if option[1] and option[0][:2] == "--":
-                opt_label += '='
+                opt_label += "="
             print(opt_label)
     else:
         # show main parser options only when necessary
@@ -94,24 +102,23 @@ def autocomplete():
         opts = [i.option_list for i in parser.option_groups]
         opts.append(parser.option_list)
         flattened_opts = chain.from_iterable(opts)
-        if current.startswith('-'):
+        if current.startswith("-"):
             for opt in flattened_opts:
                 if opt.help != optparse.SUPPRESS_HELP:
                     subcommands += opt._long_opts + opt._short_opts
         else:
             # get completion type given cwords and all available options
-            completion_type = get_path_completion_type(cwords, cword,
-                                                       flattened_opts)
+            completion_type = get_path_completion_type(cwords, cword, flattened_opts)
             if completion_type:
-                subcommands = list(auto_complete_paths(current,
-                                                       completion_type))
+                subcommands = list(auto_complete_paths(current, completion_type))
 
-        print(' '.join([x for x in subcommands if x.startswith(current)]))
+        print(" ".join([x for x in subcommands if x.startswith(current)]))
     sys.exit(1)
 
 
-def get_path_completion_type(cwords, cword, opts):
-    # type: (List[str], int, Iterable[Any]) -> Optional[str]
+def get_path_completion_type(
+    cwords: List[str], cword: int, opts: Iterable[Any]
+) -> Optional[str]:
     """Get the type of path completion (``file``, ``dir``, ``path`` or None)
 
     :param cwords: same as the environmental variable ``COMP_WORDS``
@@ -119,28 +126,27 @@ def get_path_completion_type(cwords, cword, opts):
     :param opts: The available options to check
     :return: path completion type (``file``, ``dir``, ``path`` or None)
     """
-    if cword < 2 or not cwords[cword - 2].startswith('-'):
+    if cword < 2 or not cwords[cword - 2].startswith("-"):
         return None
     for opt in opts:
         if opt.help == optparse.SUPPRESS_HELP:
             continue
-        for o in str(opt).split('/'):
-            if cwords[cword - 2].split('=')[0] == o:
+        for o in str(opt).split("/"):
+            if cwords[cword - 2].split("=")[0] == o:
                 if not opt.metavar or any(
-                        x in ('path', 'file', 'dir')
-                        for x in opt.metavar.split('/')):
+                    x in ("path", "file", "dir") for x in opt.metavar.split("/")
+                ):
                     return opt.metavar
     return None
 
 
-def auto_complete_paths(current, completion_type):
-    # type: (str, str) -> Iterable[str]
+def auto_complete_paths(current: str, completion_type: str) -> Iterable[str]:
     """If ``completion_type`` is ``file`` or ``path``, list all regular files
     and directories starting with ``current``; otherwise only list directories
     starting with ``current``.
 
     :param current: The word to be completed
-    :param completion_type: path completion type(`file`, `path` or `dir`)i
+    :param completion_type: path completion type(``file``, ``path`` or ``dir``)
     :return: A generator of regular files and/or directories
     """
     directory, filename = os.path.split(current)
@@ -150,15 +156,16 @@ def auto_complete_paths(current, completion_type):
         return
     filename = os.path.normcase(filename)
     # list all files that start with ``filename``
-    file_list = (x for x in os.listdir(current_path)
-                 if os.path.normcase(x).startswith(filename))
+    file_list = (
+        x for x in os.listdir(current_path) if os.path.normcase(x).startswith(filename)
+    )
     for f in file_list:
         opt = os.path.join(current_path, f)
         comp_file = os.path.normcase(os.path.join(directory, f))
         # complete regular files when there is not ```` after option
         # complete directories when there is ````, ```` or
         # ````after option
-        if completion_type != 'dir' and os.path.isfile(opt):
+        if completion_type != "dir" and os.path.isfile(opt):
             yield comp_file
         elif os.path.isdir(opt):
-            yield os.path.join(comp_file, '')
+            yield os.path.join(comp_file, "")
diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py
index fac76bb12f7..78b96bb7070 100644
--- a/src/pip/_internal/cli/base_command.py
+++ b/src/pip/_internal/cli/base_command.py
@@ -1,11 +1,16 @@
 """Base Command class, and related routines"""
 
+import functools
 import logging
 import logging.config
 import optparse
 import os
 import sys
 import traceback
+from optparse import Values
+from typing import Any, Callable, List, Optional, Tuple
+
+from pip._vendor.rich import traceback as rich_traceback
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.command_context import CommandContextMixIn
@@ -19,57 +24,47 @@
 from pip._internal.exceptions import (
     BadCommand,
     CommandError,
+    DiagnosticPipError,
     InstallationError,
     NetworkConnectionError,
     PreviousBuildDirError,
     UninstallationError,
 )
-from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.filesystem import check_path_owner
 from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging
 from pip._internal.utils.misc import get_prog, normalize_path
+from pip._internal.utils.temp_dir import TempDirectoryTypeRegistry as TempDirRegistry
 from pip._internal.utils.temp_dir import global_tempdir_manager, tempdir_registry
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.virtualenv import running_under_virtualenv
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Any, List, Optional, Tuple
-
-    from pip._internal.utils.temp_dir import (
-        TempDirectoryTypeRegistry as TempDirRegistry,
-    )
-
-__all__ = ['Command']
+__all__ = ["Command"]
 
 logger = logging.getLogger(__name__)
 
 
 class Command(CommandContextMixIn):
-    usage = None  # type: str
-    ignore_require_venv = False  # type: bool
+    usage: str = ""
+    ignore_require_venv: bool = False
 
-    def __init__(self, name, summary, isolated=False):
-        # type: (str, str, bool) -> None
+    def __init__(self, name: str, summary: str, isolated: bool = False) -> None:
         super().__init__()
-        parser_kw = {
-            'usage': self.usage,
-            'prog': f'{get_prog()} {name}',
-            'formatter': UpdatingDefaultsHelpFormatter(),
-            'add_help_option': False,
-            'name': name,
-            'description': self.__doc__,
-            'isolated': isolated,
-        }
 
         self.name = name
         self.summary = summary
-        self.parser = ConfigOptionParser(**parser_kw)
+        self.parser = ConfigOptionParser(
+            usage=self.usage,
+            prog=f"{get_prog()} {name}",
+            formatter=UpdatingDefaultsHelpFormatter(),
+            add_help_option=False,
+            name=name,
+            description=self.__doc__,
+            isolated=isolated,
+        )
 
-        self.tempdir_registry = None  # type: Optional[TempDirRegistry]
+        self.tempdir_registry: Optional[TempDirRegistry] = None
 
         # Commands should add options to this option group
-        optgroup_name = f'{self.name.capitalize()} Options'
+        optgroup_name = f"{self.name.capitalize()} Options"
         self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)
 
         # Add the general options
@@ -81,39 +76,33 @@ def __init__(self, name, summary, isolated=False):
 
         self.add_options()
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         pass
 
-    def handle_pip_version_check(self, options):
-        # type: (Values) -> None
+    def handle_pip_version_check(self, options: Values) -> None:
         """
         This is a no-op so that commands by default do not do the pip version
         check.
         """
         # Make sure we do the pip version check if the index_group options
         # are present.
-        assert not hasattr(options, 'no_index')
+        assert not hasattr(options, "no_index")
 
-    def run(self, options, args):
-        # type: (Values, List[Any]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         raise NotImplementedError
 
-    def parse_args(self, args):
-        # type: (List[str]) -> Tuple[Any, Any]
+    def parse_args(self, args: List[str]) -> Tuple[Values, List[str]]:
         # factored out for testability
         return self.parser.parse_args(args)
 
-    def main(self, args):
-        # type: (List[str]) -> int
+    def main(self, args: List[str]) -> int:
         try:
             with self.main_context():
                 return self._main(args)
         finally:
             logging.shutdown()
 
-    def _main(self, args):
-        # type: (List[str]) -> int
+    def _main(self, args: List[str]) -> int:
         # We must initialize this before the tempdir manager, otherwise the
         # configuration would not be accessible by the time we clean up the
         # tempdir manager.
@@ -138,17 +127,15 @@ def _main(self, args):
         #       This also affects isolated builds and it should.
 
         if options.no_input:
-            os.environ['PIP_NO_INPUT'] = '1'
+            os.environ["PIP_NO_INPUT"] = "1"
 
         if options.exists_action:
-            os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)
+            os.environ["PIP_EXISTS_ACTION"] = " ".join(options.exists_action)
 
         if options.require_venv and not self.ignore_require_venv:
             # If a venv is required check if it can really be found
             if not running_under_virtualenv():
-                logger.critical(
-                    'Could not find an activated virtualenv (required).'
-                )
+                logger.critical("Could not find an activated virtualenv (required).")
                 sys.exit(VIRTUALENV_NOT_FOUND)
 
         if options.cache_dir:
@@ -158,69 +145,79 @@ def _main(self, args):
                     "The directory '%s' or its parent directory is not owned "
                     "or is not writable by the current user. The cache "
                     "has been disabled. Check the permissions and owner of "
-                    "that directory. If executing pip with sudo, you may want "
-                    "sudo's -H flag.",
+                    "that directory. If executing pip with sudo, you should "
+                    "use sudo's -H flag.",
                     options.cache_dir,
                 )
                 options.cache_dir = None
 
-        if getattr(options, "build_dir", None):
-            deprecated(
-                reason=(
-                    "The -b/--build/--build-dir/--build-directory "
-                    "option is deprecated and has no effect anymore."
-                ),
-                replacement=(
-                    "use the TMPDIR/TEMP/TMP environment variable, "
-                    "possibly combined with --no-clean"
-                ),
-                gone_in="21.1",
-                issue=8333,
-            )
-
-        if '2020-resolver' in options.features_enabled:
+        if "2020-resolver" in options.features_enabled:
             logger.warning(
                 "--use-feature=2020-resolver no longer has any effect, "
                 "since it is now the default dependency resolver in pip. "
                 "This will become an error in pip 21.0."
             )
 
+        def intercepts_unhandled_exc(
+            run_func: Callable[..., int]
+        ) -> Callable[..., int]:
+            @functools.wraps(run_func)
+            def exc_logging_wrapper(*args: Any) -> int:
+                try:
+                    status = run_func(*args)
+                    assert isinstance(status, int)
+                    return status
+                except DiagnosticPipError as exc:
+                    logger.error("[present-diagnostic] %s", exc)
+                    logger.debug("Exception information:", exc_info=True)
+
+                    return ERROR
+                except PreviousBuildDirError as exc:
+                    logger.critical(str(exc))
+                    logger.debug("Exception information:", exc_info=True)
+
+                    return PREVIOUS_BUILD_DIR_ERROR
+                except (
+                    InstallationError,
+                    UninstallationError,
+                    BadCommand,
+                    NetworkConnectionError,
+                ) as exc:
+                    logger.critical(str(exc))
+                    logger.debug("Exception information:", exc_info=True)
+
+                    return ERROR
+                except CommandError as exc:
+                    logger.critical("%s", exc)
+                    logger.debug("Exception information:", exc_info=True)
+
+                    return ERROR
+                except BrokenStdoutLoggingError:
+                    # Bypass our logger and write any remaining messages to
+                    # stderr because stdout no longer works.
+                    print("ERROR: Pipe to stdout was broken", file=sys.stderr)
+                    if level_number <= logging.DEBUG:
+                        traceback.print_exc(file=sys.stderr)
+
+                    return ERROR
+                except KeyboardInterrupt:
+                    logger.critical("Operation cancelled by user")
+                    logger.debug("Exception information:", exc_info=True)
+
+                    return ERROR
+                except BaseException:
+                    logger.critical("Exception:", exc_info=True)
+
+                    return UNKNOWN_ERROR
+
+            return exc_logging_wrapper
+
         try:
-            status = self.run(options, args)
-            assert isinstance(status, int)
-            return status
-        except PreviousBuildDirError as exc:
-            logger.critical(str(exc))
-            logger.debug('Exception information:', exc_info=True)
-
-            return PREVIOUS_BUILD_DIR_ERROR
-        except (InstallationError, UninstallationError, BadCommand,
-                NetworkConnectionError) as exc:
-            logger.critical(str(exc))
-            logger.debug('Exception information:', exc_info=True)
-
-            return ERROR
-        except CommandError as exc:
-            logger.critical('%s', exc)
-            logger.debug('Exception information:', exc_info=True)
-
-            return ERROR
-        except BrokenStdoutLoggingError:
-            # Bypass our logger and write any remaining messages to stderr
-            # because stdout no longer works.
-            print('ERROR: Pipe to stdout was broken', file=sys.stderr)
-            if level_number <= logging.DEBUG:
-                traceback.print_exc(file=sys.stderr)
-
-            return ERROR
-        except KeyboardInterrupt:
-            logger.critical('Operation cancelled by user')
-            logger.debug('Exception information:', exc_info=True)
-
-            return ERROR
-        except BaseException:
-            logger.critical('Exception:', exc_info=True)
-
-            return UNKNOWN_ERROR
+            if not options.debug_mode:
+                run = intercepts_unhandled_exc(self.run)
+            else:
+                run = self.run
+                rich_traceback.install(show_locals=True)
+            return run(options, args)
         finally:
             self.handle_pip_version_check(options)
diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py
index e16f42de610..64bd47d2b68 100644
--- a/src/pip/_internal/cli/cmdoptions.py
+++ b/src/pip/_internal/cli/cmdoptions.py
@@ -10,16 +10,17 @@
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
 
+import logging
 import os
 import textwrap
-import warnings
-from distutils.util import strtobool
 from functools import partial
-from optparse import SUPPRESS_HELP, Option, OptionGroup
+from optparse import SUPPRESS_HELP, Option, OptionGroup, OptionParser, Values
 from textwrap import dedent
+from typing import Any, Callable, Dict, Optional, Tuple
 
 from pip._vendor.packaging.utils import canonicalize_name
 
+from pip._internal.cli.parser import ConfigOptionParser
 from pip._internal.cli.progress_bars import BAR_TYPES
 from pip._internal.exceptions import CommandError
 from pip._internal.locations import USER_CACHE_DIR, get_src_prefix
@@ -27,17 +28,12 @@
 from pip._internal.models.index import PyPI
 from pip._internal.models.target_python import TargetPython
 from pip._internal.utils.hashes import STRONG_HASHES
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.misc import strtobool
 
-if MYPY_CHECK_RUNNING:
-    from optparse import OptionParser, Values
-    from typing import Any, Callable, Dict, Optional, Tuple
+logger = logging.getLogger(__name__)
 
-    from pip._internal.cli.parser import ConfigOptionParser
 
-
-def raise_option_error(parser, option, msg):
-    # type: (OptionParser, Option, str) -> None
+def raise_option_error(parser: OptionParser, option: Option, msg: str) -> None:
     """
     Raise an option parsing error using parser.error().
 
@@ -46,26 +42,26 @@ def raise_option_error(parser, option, msg):
       option: an Option instance.
       msg: the error text.
     """
-    msg = f'{option} error: {msg}'
-    msg = textwrap.fill(' '.join(msg.split()))
+    msg = f"{option} error: {msg}"
+    msg = textwrap.fill(" ".join(msg.split()))
     parser.error(msg)
 
 
-def make_option_group(group, parser):
-    # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup
+def make_option_group(group: Dict[str, Any], parser: ConfigOptionParser) -> OptionGroup:
     """
     Return an OptionGroup object
     group  -- assumed to be dict with 'name' and 'options' keys
     parser -- an optparse Parser
     """
-    option_group = OptionGroup(parser, group['name'])
-    for option in group['options']:
+    option_group = OptionGroup(parser, group["name"])
+    for option in group["options"]:
         option_group.add_option(option())
     return option_group
 
 
-def check_install_build_global(options, check_options=None):
-    # type: (Values, Optional[Values]) -> None
+def check_install_build_global(
+    options: Values, check_options: Optional[Values] = None
+) -> None:
     """Disable wheels if per-setup.py call options are set.
 
     :param options: The OptionParser options to update.
@@ -75,37 +71,37 @@ def check_install_build_global(options, check_options=None):
     if check_options is None:
         check_options = options
 
-    def getname(n):
-        # type: (str) -> Optional[Any]
+    def getname(n: str) -> Optional[Any]:
         return getattr(check_options, n, None)
+
     names = ["build_options", "global_options", "install_options"]
     if any(map(getname, names)):
         control = options.format_control
         control.disallow_binaries()
-        warnings.warn(
-            'Disabling all use of wheels due to the use of --build-option '
-            '/ --global-option / --install-option.', stacklevel=2,
+        logger.warning(
+            "Disabling all use of wheels due to the use of --build-option "
+            "/ --global-option / --install-option.",
         )
 
 
-def check_dist_restriction(options, check_target=False):
-    # type: (Values, bool) -> None
+def check_dist_restriction(options: Values, check_target: bool = False) -> None:
     """Function for determining if custom platform options are allowed.
 
     :param options: The OptionParser options.
     :param check_target: Whether or not to check if --target is being used.
     """
-    dist_restriction_set = any([
-        options.python_version,
-        options.platforms,
-        options.abis,
-        options.implementation,
-    ])
-
-    binary_only = FormatControl(set(), {':all:'})
+    dist_restriction_set = any(
+        [
+            options.python_version,
+            options.platforms,
+            options.abis,
+            options.implementation,
+        ]
+    )
+
+    binary_only = FormatControl(set(), {":all:"})
     sdist_dependencies_allowed = (
-        options.format_control != binary_only and
-        not options.ignore_dependencies
+        options.format_control != binary_only and not options.ignore_dependencies
     )
 
     # Installations or downloads using dist restrictions must not combine
@@ -128,13 +124,11 @@ def check_dist_restriction(options, check_target=False):
             )
 
 
-def _path_option_check(option, opt, value):
-    # type: (Option, str, str) -> str
+def _path_option_check(option: Option, opt: str, value: str) -> str:
     return os.path.expanduser(value)
 
 
-def _package_name_option_check(option, opt, value):
-    # type: (Option, str, str) -> str
+def _package_name_option_check(option: Option, opt: str, value: str) -> str:
     return canonicalize_name(value)
 
 
@@ -149,15 +143,28 @@ class PipOption(Option):
 # options #
 ###########
 
-help_ = partial(
+help_: Callable[..., Option] = partial(
     Option,
-    '-h', '--help',
-    dest='help',
-    action='help',
-    help='Show help.',
-)  # type: Callable[..., Option]
+    "-h",
+    "--help",
+    dest="help",
+    action="help",
+    help="Show help.",
+)
 
-isolated_mode = partial(
+debug_mode: Callable[..., Option] = partial(
+    Option,
+    "--debug",
+    dest="debug_mode",
+    action="store_true",
+    default=False,
+    help=(
+        "Let unhandled exceptions propagate outside the main subroutine, "
+        "instead of logging them to stderr."
+    ),
+)
+
+isolated_mode: Callable[..., Option] = partial(
     Option,
     "--isolated",
     dest="isolated_mode",
@@ -167,210 +174,224 @@ class PipOption(Option):
         "Run pip in an isolated mode, ignoring environment variables and user "
         "configuration."
     ),
-)  # type: Callable[..., Option]
+)
 
-require_virtualenv = partial(
+require_virtualenv: Callable[..., Option] = partial(
     Option,
-    # Run only if inside a virtualenv, bail if not.
-    '--require-virtualenv', '--require-venv',
-    dest='require_venv',
-    action='store_true',
+    "--require-virtualenv",
+    "--require-venv",
+    dest="require_venv",
+    action="store_true",
     default=False,
-    help=SUPPRESS_HELP
-)  # type: Callable[..., Option]
+    help=(
+        "Allow pip to only run in a virtual environment; "
+        "exit with an error otherwise."
+    ),
+)
 
-verbose = partial(
+verbose: Callable[..., Option] = partial(
     Option,
-    '-v', '--verbose',
-    dest='verbose',
-    action='count',
+    "-v",
+    "--verbose",
+    dest="verbose",
+    action="count",
     default=0,
-    help='Give more output. Option is additive, and can be used up to 3 times.'
-)  # type: Callable[..., Option]
+    help="Give more output. Option is additive, and can be used up to 3 times.",
+)
 
-no_color = partial(
+no_color: Callable[..., Option] = partial(
     Option,
-    '--no-color',
-    dest='no_color',
-    action='store_true',
+    "--no-color",
+    dest="no_color",
+    action="store_true",
     default=False,
     help="Suppress colored output.",
-)  # type: Callable[..., Option]
+)
 
-version = partial(
+version: Callable[..., Option] = partial(
     Option,
-    '-V', '--version',
-    dest='version',
-    action='store_true',
-    help='Show version and exit.',
-)  # type: Callable[..., Option]
+    "-V",
+    "--version",
+    dest="version",
+    action="store_true",
+    help="Show version and exit.",
+)
 
-quiet = partial(
+quiet: Callable[..., Option] = partial(
     Option,
-    '-q', '--quiet',
-    dest='quiet',
-    action='count',
+    "-q",
+    "--quiet",
+    dest="quiet",
+    action="count",
     default=0,
     help=(
-        'Give less output. Option is additive, and can be used up to 3'
-        ' times (corresponding to WARNING, ERROR, and CRITICAL logging'
-        ' levels).'
+        "Give less output. Option is additive, and can be used up to 3"
+        " times (corresponding to WARNING, ERROR, and CRITICAL logging"
+        " levels)."
     ),
-)  # type: Callable[..., Option]
+)
 
-progress_bar = partial(
+progress_bar: Callable[..., Option] = partial(
     Option,
-    '--progress-bar',
-    dest='progress_bar',
-    type='choice',
+    "--progress-bar",
+    dest="progress_bar",
+    type="choice",
     choices=list(BAR_TYPES.keys()),
-    default='on',
+    default="on",
     help=(
-        'Specify type of progress to be displayed [' +
-        '|'.join(BAR_TYPES.keys()) + '] (default: %default)'
+        "Specify type of progress to be displayed ["
+        + "|".join(BAR_TYPES.keys())
+        + "] (default: %default)"
     ),
-)  # type: Callable[..., Option]
+)
 
-log = partial(
+log: Callable[..., Option] = partial(
     PipOption,
-    "--log", "--log-file", "--local-log",
+    "--log",
+    "--log-file",
+    "--local-log",
     dest="log",
     metavar="path",
     type="path",
-    help="Path to a verbose appending log."
-)  # type: Callable[..., Option]
+    help="Path to a verbose appending log.",
+)
 
-no_input = partial(
+no_input: Callable[..., Option] = partial(
     Option,
     # Don't ask for input
-    '--no-input',
-    dest='no_input',
-    action='store_true',
+    "--no-input",
+    dest="no_input",
+    action="store_true",
     default=False,
-    help="Disable prompting for input."
-)  # type: Callable[..., Option]
+    help="Disable prompting for input.",
+)
 
-proxy = partial(
+proxy: Callable[..., Option] = partial(
     Option,
-    '--proxy',
-    dest='proxy',
-    type='str',
-    default='',
-    help="Specify a proxy in the form [user:passwd@]proxy.server:port."
-)  # type: Callable[..., Option]
+    "--proxy",
+    dest="proxy",
+    type="str",
+    default="",
+    help="Specify a proxy in the form scheme://[user:passwd@]proxy.server:port.",
+)
 
-retries = partial(
+retries: Callable[..., Option] = partial(
     Option,
-    '--retries',
-    dest='retries',
-    type='int',
+    "--retries",
+    dest="retries",
+    type="int",
     default=5,
     help="Maximum number of retries each connection should attempt "
-         "(default %default times).",
-)  # type: Callable[..., Option]
+    "(default %default times).",
+)
 
-timeout = partial(
+timeout: Callable[..., Option] = partial(
     Option,
-    '--timeout', '--default-timeout',
-    metavar='sec',
-    dest='timeout',
-    type='float',
+    "--timeout",
+    "--default-timeout",
+    metavar="sec",
+    dest="timeout",
+    type="float",
     default=15,
-    help='Set the socket timeout (default %default seconds).',
-)  # type: Callable[..., Option]
+    help="Set the socket timeout (default %default seconds).",
+)
 
 
-def exists_action():
-    # type: () -> Option
+def exists_action() -> Option:
     return Option(
         # Option when path already exist
-        '--exists-action',
-        dest='exists_action',
-        type='choice',
-        choices=['s', 'i', 'w', 'b', 'a'],
+        "--exists-action",
+        dest="exists_action",
+        type="choice",
+        choices=["s", "i", "w", "b", "a"],
         default=[],
-        action='append',
-        metavar='action',
+        action="append",
+        metavar="action",
         help="Default action when a path already exists: "
-             "(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.",
+        "(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.",
     )
 
 
-cert = partial(
+cert: Callable[..., Option] = partial(
     PipOption,
-    '--cert',
-    dest='cert',
-    type='path',
-    metavar='path',
-    help="Path to alternate CA bundle.",
-)  # type: Callable[..., Option]
-
-client_cert = partial(
+    "--cert",
+    dest="cert",
+    type="path",
+    metavar="path",
+    help=(
+        "Path to PEM-encoded CA certificate bundle. "
+        "If provided, overrides the default. "
+        "See 'SSL Certificate Verification' in pip documentation "
+        "for more information."
+    ),
+)
+
+client_cert: Callable[..., Option] = partial(
     PipOption,
-    '--client-cert',
-    dest='client_cert',
-    type='path',
+    "--client-cert",
+    dest="client_cert",
+    type="path",
     default=None,
-    metavar='path',
+    metavar="path",
     help="Path to SSL client certificate, a single file containing the "
-         "private key and the certificate in PEM format.",
-)  # type: Callable[..., Option]
+    "private key and the certificate in PEM format.",
+)
 
-index_url = partial(
+index_url: Callable[..., Option] = partial(
     Option,
-    '-i', '--index-url', '--pypi-url',
-    dest='index_url',
-    metavar='URL',
+    "-i",
+    "--index-url",
+    "--pypi-url",
+    dest="index_url",
+    metavar="URL",
     default=PyPI.simple_url,
     help="Base URL of the Python Package Index (default %default). "
-         "This should point to a repository compliant with PEP 503 "
-         "(the simple repository API) or a local directory laid out "
-         "in the same format.",
-)  # type: Callable[..., Option]
+    "This should point to a repository compliant with PEP 503 "
+    "(the simple repository API) or a local directory laid out "
+    "in the same format.",
+)
 
 
-def extra_index_url():
-    # type: () -> Option
+def extra_index_url() -> Option:
     return Option(
-        '--extra-index-url',
-        dest='extra_index_urls',
-        metavar='URL',
-        action='append',
+        "--extra-index-url",
+        dest="extra_index_urls",
+        metavar="URL",
+        action="append",
         default=[],
         help="Extra URLs of package indexes to use in addition to "
-             "--index-url. Should follow the same rules as "
-             "--index-url.",
+        "--index-url. Should follow the same rules as "
+        "--index-url.",
     )
 
 
-no_index = partial(
+no_index: Callable[..., Option] = partial(
     Option,
-    '--no-index',
-    dest='no_index',
-    action='store_true',
+    "--no-index",
+    dest="no_index",
+    action="store_true",
     default=False,
-    help='Ignore package index (only looking at --find-links URLs instead).',
-)  # type: Callable[..., Option]
+    help="Ignore package index (only looking at --find-links URLs instead).",
+)
 
 
-def find_links():
-    # type: () -> Option
+def find_links() -> Option:
     return Option(
-        '-f', '--find-links',
-        dest='find_links',
-        action='append',
+        "-f",
+        "--find-links",
+        dest="find_links",
+        action="append",
         default=[],
-        metavar='url',
+        metavar="url",
         help="If a URL or path to an html file, then parse for links to "
-             "archives such as sdist (.tar.gz) or wheel (.whl) files. "
-             "If a local path or file:// URL that's a directory,  "
-             "then look for archives in the directory listing. "
-             "Links to VCS project URLs are not supported.",
+        "archives such as sdist (.tar.gz) or wheel (.whl) files. "
+        "If a local path or file:// URL that's a directory, "
+        "then look for archives in the directory listing. "
+        "Links to VCS project URLs are not supported.",
     )
 
 
-def trusted_host():
-    # type: () -> Option
+def trusted_host() -> Option:
     return Option(
         "--trusted-host",
         dest="trusted_hosts",
@@ -378,140 +399,154 @@ def trusted_host():
         metavar="HOSTNAME",
         default=[],
         help="Mark this host or host:port pair as trusted, even though it "
-             "does not have valid or any HTTPS.",
+        "does not have valid or any HTTPS.",
     )
 
 
-def constraints():
-    # type: () -> Option
+def constraints() -> Option:
     return Option(
-        '-c', '--constraint',
-        dest='constraints',
-        action='append',
+        "-c",
+        "--constraint",
+        dest="constraints",
+        action="append",
         default=[],
-        metavar='file',
-        help='Constrain versions using the given constraints file. '
-        'This option can be used multiple times.'
+        metavar="file",
+        help="Constrain versions using the given constraints file. "
+        "This option can be used multiple times.",
     )
 
 
-def requirements():
-    # type: () -> Option
+def requirements() -> Option:
     return Option(
-        '-r', '--requirement',
-        dest='requirements',
-        action='append',
+        "-r",
+        "--requirement",
+        dest="requirements",
+        action="append",
         default=[],
-        metavar='file',
-        help='Install from the given requirements file. '
-        'This option can be used multiple times.'
+        metavar="file",
+        help="Install from the given requirements file. "
+        "This option can be used multiple times.",
     )
 
 
-def editable():
-    # type: () -> Option
+def editable() -> Option:
     return Option(
-        '-e', '--editable',
-        dest='editables',
-        action='append',
+        "-e",
+        "--editable",
+        dest="editables",
+        action="append",
         default=[],
-        metavar='path/url',
-        help=('Install a project in editable mode (i.e. setuptools '
-              '"develop mode") from a local project path or a VCS url.'),
+        metavar="path/url",
+        help=(
+            "Install a project in editable mode (i.e. setuptools "
+            '"develop mode") from a local project path or a VCS url.'
+        ),
     )
 
 
-def _handle_src(option, opt_str, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_src(option: Option, opt_str: str, value: str, parser: OptionParser) -> None:
     value = os.path.abspath(value)
     setattr(parser.values, option.dest, value)
 
 
-src = partial(
+src: Callable[..., Option] = partial(
     PipOption,
-    '--src', '--source', '--source-dir', '--source-directory',
-    dest='src_dir',
-    type='path',
-    metavar='dir',
+    "--src",
+    "--source",
+    "--source-dir",
+    "--source-directory",
+    dest="src_dir",
+    type="path",
+    metavar="dir",
     default=get_src_prefix(),
-    action='callback',
+    action="callback",
     callback=_handle_src,
-    help='Directory to check out editable projects into. '
+    help="Directory to check out editable projects into. "
     'The default in a virtualenv is "/src". '
-    'The default for global installs is "/src".'
-)  # type: Callable[..., Option]
+    'The default for global installs is "/src".',
+)
 
 
-def _get_format_control(values, option):
-    # type: (Values, Option) -> Any
+def _get_format_control(values: Values, option: Option) -> Any:
     """Get a format_control object."""
     return getattr(values, option.dest)
 
 
-def _handle_no_binary(option, opt_str, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_no_binary(
+    option: Option, opt_str: str, value: str, parser: OptionParser
+) -> None:
     existing = _get_format_control(parser.values, option)
     FormatControl.handle_mutual_excludes(
-        value, existing.no_binary, existing.only_binary,
+        value,
+        existing.no_binary,
+        existing.only_binary,
     )
 
 
-def _handle_only_binary(option, opt_str, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_only_binary(
+    option: Option, opt_str: str, value: str, parser: OptionParser
+) -> None:
     existing = _get_format_control(parser.values, option)
     FormatControl.handle_mutual_excludes(
-        value, existing.only_binary, existing.no_binary,
+        value,
+        existing.only_binary,
+        existing.no_binary,
     )
 
 
-def no_binary():
-    # type: () -> Option
+def no_binary() -> Option:
     format_control = FormatControl(set(), set())
     return Option(
-        "--no-binary", dest="format_control", action="callback",
-        callback=_handle_no_binary, type="str",
+        "--no-binary",
+        dest="format_control",
+        action="callback",
+        callback=_handle_no_binary,
+        type="str",
         default=format_control,
-        help='Do not use binary packages. Can be supplied multiple times, and '
-             'each time adds to the existing value. Accepts either ":all:" to '
-             'disable all binary packages, ":none:" to empty the set (notice '
-             'the colons), or one or more package names with commas between '
-             'them (no colons). Note that some packages are tricky to compile '
-             'and may fail to install when this option is used on them.',
+        help="Do not use binary packages. Can be supplied multiple times, and "
+        'each time adds to the existing value. Accepts either ":all:" to '
+        'disable all binary packages, ":none:" to empty the set (notice '
+        "the colons), or one or more package names with commas between "
+        "them (no colons). Note that some packages are tricky to compile "
+        "and may fail to install when this option is used on them.",
     )
 
 
-def only_binary():
-    # type: () -> Option
+def only_binary() -> Option:
     format_control = FormatControl(set(), set())
     return Option(
-        "--only-binary", dest="format_control", action="callback",
-        callback=_handle_only_binary, type="str",
+        "--only-binary",
+        dest="format_control",
+        action="callback",
+        callback=_handle_only_binary,
+        type="str",
         default=format_control,
-        help='Do not use source packages. Can be supplied multiple times, and '
-             'each time adds to the existing value. Accepts either ":all:" to '
-             'disable all source packages, ":none:" to empty the set, or one '
-             'or more package names with commas between them. Packages '
-             'without binary distributions will fail to install when this '
-             'option is used on them.',
+        help="Do not use source packages. Can be supplied multiple times, and "
+        'each time adds to the existing value. Accepts either ":all:" to '
+        'disable all source packages, ":none:" to empty the set, or one '
+        "or more package names with commas between them. Packages "
+        "without binary distributions will fail to install when this "
+        "option is used on them.",
     )
 
 
-platforms = partial(
+platforms: Callable[..., Option] = partial(
     Option,
-    '--platform',
-    dest='platforms',
-    metavar='platform',
-    action='append',
+    "--platform",
+    dest="platforms",
+    metavar="platform",
+    action="append",
     default=None,
-    help=("Only use wheels compatible with . Defaults to the "
-          "platform of the running system. Use this option multiple times to "
-          "specify multiple platforms supported by the target interpreter."),
-)  # type: Callable[..., Option]
+    help=(
+        "Only use wheels compatible with . Defaults to the "
+        "platform of the running system. Use this option multiple times to "
+        "specify multiple platforms supported by the target interpreter."
+    ),
+)
 
 
 # This was made a separate function for unit-testing purposes.
-def _convert_python_version(value):
-    # type: (str) -> Tuple[Tuple[int, ...], Optional[str]]
+def _convert_python_version(value: str) -> Tuple[Tuple[int, ...], Optional[str]]:
     """
     Convert a version string like "3", "37", or "3.7.3" into a tuple of ints.
 
@@ -522,9 +557,9 @@ def _convert_python_version(value):
         # The empty string is the same as not providing a value.
         return (None, None)
 
-    parts = value.split('.')
+    parts = value.split(".")
     if len(parts) > 3:
-        return ((), 'at most three version parts are allowed')
+        return ((), "at most three version parts are allowed")
 
     if len(parts) == 1:
         # Then we are in the case of "3" or "37".
@@ -535,86 +570,91 @@ def _convert_python_version(value):
     try:
         version_info = tuple(int(part) for part in parts)
     except ValueError:
-        return ((), 'each version part must be an integer')
+        return ((), "each version part must be an integer")
 
     return (version_info, None)
 
 
-def _handle_python_version(option, opt_str, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_python_version(
+    option: Option, opt_str: str, value: str, parser: OptionParser
+) -> None:
     """
     Handle a provided --python-version value.
     """
     version_info, error_msg = _convert_python_version(value)
     if error_msg is not None:
-        msg = (
-            'invalid --python-version value: {!r}: {}'.format(
-                value, error_msg,
-            )
+        msg = "invalid --python-version value: {!r}: {}".format(
+            value,
+            error_msg,
         )
         raise_option_error(parser, option=option, msg=msg)
 
     parser.values.python_version = version_info
 
 
-python_version = partial(
+python_version: Callable[..., Option] = partial(
     Option,
-    '--python-version',
-    dest='python_version',
-    metavar='python_version',
-    action='callback',
-    callback=_handle_python_version, type='str',
+    "--python-version",
+    dest="python_version",
+    metavar="python_version",
+    action="callback",
+    callback=_handle_python_version,
+    type="str",
     default=None,
-    help=dedent("""\
+    help=dedent(
+        """\
     The Python interpreter version to use for wheel and "Requires-Python"
     compatibility checks. Defaults to a version derived from the running
     interpreter. The version can be specified using up to three dot-separated
     integers (e.g. "3" for 3.0.0, "3.7" for 3.7.0, or "3.7.3"). A major-minor
     version can also be given as a string without dots (e.g. "37" for 3.7.0).
-    """),
-)  # type: Callable[..., Option]
+    """
+    ),
+)
 
 
-implementation = partial(
+implementation: Callable[..., Option] = partial(
     Option,
-    '--implementation',
-    dest='implementation',
-    metavar='implementation',
+    "--implementation",
+    dest="implementation",
+    metavar="implementation",
     default=None,
-    help=("Only use wheels compatible with Python "
-          "implementation , e.g. 'pp', 'jy', 'cp', "
-          " or 'ip'. If not specified, then the current "
-          "interpreter implementation is used.  Use 'py' to force "
-          "implementation-agnostic wheels."),
-)  # type: Callable[..., Option]
+    help=(
+        "Only use wheels compatible with Python "
+        "implementation , e.g. 'pp', 'jy', 'cp', "
+        " or 'ip'. If not specified, then the current "
+        "interpreter implementation is used.  Use 'py' to force "
+        "implementation-agnostic wheels."
+    ),
+)
 
 
-abis = partial(
+abis: Callable[..., Option] = partial(
     Option,
-    '--abi',
-    dest='abis',
-    metavar='abi',
-    action='append',
+    "--abi",
+    dest="abis",
+    metavar="abi",
+    action="append",
     default=None,
-    help=("Only use wheels compatible with Python abi , e.g. 'pypy_41'. "
-          "If not specified, then the current interpreter abi tag is used. "
-          "Use this option multiple times to specify multiple abis supported "
-          "by the target interpreter. Generally you will need to specify "
-          "--implementation, --platform, and --python-version when using this "
-          "option."),
-)  # type: Callable[..., Option]
+    help=(
+        "Only use wheels compatible with Python abi , e.g. 'pypy_41'. "
+        "If not specified, then the current interpreter abi tag is used. "
+        "Use this option multiple times to specify multiple abis supported "
+        "by the target interpreter. Generally you will need to specify "
+        "--implementation, --platform, and --python-version when using this "
+        "option."
+    ),
+)
 
 
-def add_target_python_options(cmd_opts):
-    # type: (OptionGroup) -> None
+def add_target_python_options(cmd_opts: OptionGroup) -> None:
     cmd_opts.add_option(platforms())
     cmd_opts.add_option(python_version())
     cmd_opts.add_option(implementation())
     cmd_opts.add_option(abis())
 
 
-def make_target_python(options):
-    # type: (Values) -> TargetPython
+def make_target_python(options: Values) -> TargetPython:
     target_python = TargetPython(
         platforms=options.platforms,
         py_version_info=options.python_version,
@@ -625,30 +665,30 @@ def make_target_python(options):
     return target_python
 
 
-def prefer_binary():
-    # type: () -> Option
+def prefer_binary() -> Option:
     return Option(
         "--prefer-binary",
         dest="prefer_binary",
         action="store_true",
         default=False,
-        help="Prefer older binary packages over newer source packages."
+        help="Prefer older binary packages over newer source packages.",
     )
 
 
-cache_dir = partial(
+cache_dir: Callable[..., Option] = partial(
     PipOption,
     "--cache-dir",
     dest="cache_dir",
     default=USER_CACHE_DIR,
     metavar="dir",
-    type='path',
-    help="Store the cache data in ."
-)  # type: Callable[..., Option]
+    type="path",
+    help="Store the cache data in .",
+)
 
 
-def _handle_no_cache_dir(option, opt, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_no_cache_dir(
+    option: Option, opt: str, value: str, parser: OptionParser
+) -> None:
     """
     Process a value provided for the --no-cache-dir option.
 
@@ -675,55 +715,48 @@ def _handle_no_cache_dir(option, opt, value, parser):
     parser.values.cache_dir = False
 
 
-no_cache = partial(
+no_cache: Callable[..., Option] = partial(
     Option,
     "--no-cache-dir",
     dest="cache_dir",
     action="callback",
     callback=_handle_no_cache_dir,
     help="Disable the cache.",
-)  # type: Callable[..., Option]
+)
 
-no_deps = partial(
+no_deps: Callable[..., Option] = partial(
     Option,
-    '--no-deps', '--no-dependencies',
-    dest='ignore_dependencies',
-    action='store_true',
+    "--no-deps",
+    "--no-dependencies",
+    dest="ignore_dependencies",
+    action="store_true",
     default=False,
     help="Don't install package dependencies.",
-)  # type: Callable[..., Option]
+)
 
-build_dir = partial(
-    PipOption,
-    '-b', '--build', '--build-dir', '--build-directory',
-    dest='build_dir',
-    type='path',
-    metavar='dir',
-    help=SUPPRESS_HELP,
-)  # type: Callable[..., Option]
-
-ignore_requires_python = partial(
+ignore_requires_python: Callable[..., Option] = partial(
     Option,
-    '--ignore-requires-python',
-    dest='ignore_requires_python',
-    action='store_true',
-    help='Ignore the Requires-Python information.'
-)  # type: Callable[..., Option]
+    "--ignore-requires-python",
+    dest="ignore_requires_python",
+    action="store_true",
+    help="Ignore the Requires-Python information.",
+)
 
-no_build_isolation = partial(
+no_build_isolation: Callable[..., Option] = partial(
     Option,
-    '--no-build-isolation',
-    dest='build_isolation',
-    action='store_false',
+    "--no-build-isolation",
+    dest="build_isolation",
+    action="store_false",
     default=True,
-    help='Disable isolation when building a modern source distribution. '
-         'Build dependencies specified by PEP 518 must be already installed '
-         'if this option is used.'
-)  # type: Callable[..., Option]
+    help="Disable isolation when building a modern source distribution. "
+    "Build dependencies specified by PEP 518 must be already installed "
+    "if this option is used.",
+)
 
 
-def _handle_no_use_pep517(option, opt, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_no_use_pep517(
+    option: Option, opt: str, value: str, parser: OptionParser
+) -> None:
     """
     Process a value provided for the --no-use-pep517 option.
 
@@ -746,194 +779,210 @@ def _handle_no_use_pep517(option, opt, value, parser):
     parser.values.use_pep517 = False
 
 
-use_pep517 = partial(
+use_pep517: Any = partial(
     Option,
-    '--use-pep517',
-    dest='use_pep517',
-    action='store_true',
+    "--use-pep517",
+    dest="use_pep517",
+    action="store_true",
     default=None,
-    help='Use PEP 517 for building source distributions '
-         '(use --no-use-pep517 to force legacy behaviour).'
-)  # type: Any
+    help="Use PEP 517 for building source distributions "
+    "(use --no-use-pep517 to force legacy behaviour).",
+)
 
-no_use_pep517 = partial(
+no_use_pep517: Any = partial(
     Option,
-    '--no-use-pep517',
-    dest='use_pep517',
-    action='callback',
+    "--no-use-pep517",
+    dest="use_pep517",
+    action="callback",
     callback=_handle_no_use_pep517,
     default=None,
-    help=SUPPRESS_HELP
-)  # type: Any
+    help=SUPPRESS_HELP,
+)
 
-install_options = partial(
+install_options: Callable[..., Option] = partial(
     Option,
-    '--install-option',
-    dest='install_options',
-    action='append',
-    metavar='options',
+    "--install-option",
+    dest="install_options",
+    action="append",
+    metavar="options",
     help="Extra arguments to be supplied to the setup.py install "
-         "command (use like --install-option=\"--install-scripts=/usr/local/"
-         "bin\"). Use multiple --install-option options to pass multiple "
-         "options to setup.py install. If you are using an option with a "
-         "directory path, be sure to use absolute path.",
-)  # type: Callable[..., Option]
-
-global_options = partial(
-    Option,
-    '--global-option',
-    dest='global_options',
-    action='append',
-    metavar='options',
+    'command (use like --install-option="--install-scripts=/usr/local/'
+    'bin"). Use multiple --install-option options to pass multiple '
+    "options to setup.py install. If you are using an option with a "
+    "directory path, be sure to use absolute path.",
+)
+
+build_options: Callable[..., Option] = partial(
+    Option,
+    "--build-option",
+    dest="build_options",
+    metavar="options",
+    action="append",
+    help="Extra arguments to be supplied to 'setup.py bdist_wheel'.",
+)
+
+global_options: Callable[..., Option] = partial(
+    Option,
+    "--global-option",
+    dest="global_options",
+    action="append",
+    metavar="options",
     help="Extra global options to be supplied to the setup.py "
-         "call before the install command.",
-)  # type: Callable[..., Option]
+    "call before the install or bdist_wheel command.",
+)
 
-no_clean = partial(
+no_clean: Callable[..., Option] = partial(
     Option,
-    '--no-clean',
-    action='store_true',
+    "--no-clean",
+    action="store_true",
     default=False,
-    help="Don't clean up build directories."
-)  # type: Callable[..., Option]
+    help="Don't clean up build directories.",
+)
 
-pre = partial(
+pre: Callable[..., Option] = partial(
     Option,
-    '--pre',
-    action='store_true',
+    "--pre",
+    action="store_true",
     default=False,
     help="Include pre-release and development versions. By default, "
-         "pip only finds stable versions.",
-)  # type: Callable[..., Option]
+    "pip only finds stable versions.",
+)
 
-disable_pip_version_check = partial(
+disable_pip_version_check: Callable[..., Option] = partial(
     Option,
     "--disable-pip-version-check",
     dest="disable_pip_version_check",
     action="store_true",
     default=False,
     help="Don't periodically check PyPI to determine whether a new version "
-         "of pip is available for download. Implied with --no-index.",
-)  # type: Callable[..., Option]
+    "of pip is available for download. Implied with --no-index.",
+)
 
 
-def _handle_merge_hash(option, opt_str, value, parser):
-    # type: (Option, str, str, OptionParser) -> None
+def _handle_merge_hash(
+    option: Option, opt_str: str, value: str, parser: OptionParser
+) -> None:
     """Given a value spelled "algo:digest", append the digest to a list
     pointed to in a dict by the algo name."""
     if not parser.values.hashes:
         parser.values.hashes = {}
     try:
-        algo, digest = value.split(':', 1)
+        algo, digest = value.split(":", 1)
     except ValueError:
-        parser.error('Arguments to {} must be a hash name '  # noqa
-                     'followed by a value, like --hash=sha256:'
-                     'abcde...'.format(opt_str))
+        parser.error(
+            "Arguments to {} must be a hash name "  # noqa
+            "followed by a value, like --hash=sha256:"
+            "abcde...".format(opt_str)
+        )
     if algo not in STRONG_HASHES:
-        parser.error('Allowed hash algorithms for {} are {}.'.format(  # noqa
-                     opt_str, ', '.join(STRONG_HASHES)))
+        parser.error(
+            "Allowed hash algorithms for {} are {}.".format(  # noqa
+                opt_str, ", ".join(STRONG_HASHES)
+            )
+        )
     parser.values.hashes.setdefault(algo, []).append(digest)
 
 
-hash = partial(
+hash: Callable[..., Option] = partial(
     Option,
-    '--hash',
+    "--hash",
     # Hash values eventually end up in InstallRequirement.hashes due to
     # __dict__ copying in process_line().
-    dest='hashes',
-    action='callback',
+    dest="hashes",
+    action="callback",
     callback=_handle_merge_hash,
-    type='string',
+    type="string",
     help="Verify that the package's archive matches this "
-         'hash before installing. Example: --hash=sha256:abcdef...',
-)  # type: Callable[..., Option]
+    "hash before installing. Example: --hash=sha256:abcdef...",
+)
 
 
-require_hashes = partial(
+require_hashes: Callable[..., Option] = partial(
     Option,
-    '--require-hashes',
-    dest='require_hashes',
-    action='store_true',
+    "--require-hashes",
+    dest="require_hashes",
+    action="store_true",
     default=False,
-    help='Require a hash to check each requirement against, for '
-         'repeatable installs. This option is implied when any package in a '
-         'requirements file has a --hash option.',
-)  # type: Callable[..., Option]
+    help="Require a hash to check each requirement against, for "
+    "repeatable installs. This option is implied when any package in a "
+    "requirements file has a --hash option.",
+)
 
 
-list_path = partial(
+list_path: Callable[..., Option] = partial(
     PipOption,
-    '--path',
-    dest='path',
-    type='path',
-    action='append',
-    help='Restrict to the specified installation path for listing '
-         'packages (can be used multiple times).'
-)  # type: Callable[..., Option]
+    "--path",
+    dest="path",
+    type="path",
+    action="append",
+    help="Restrict to the specified installation path for listing "
+    "packages (can be used multiple times).",
+)
 
 
-def check_list_path_option(options):
-    # type: (Values) -> None
+def check_list_path_option(options: Values) -> None:
     if options.path and (options.user or options.local):
-        raise CommandError(
-            "Cannot combine '--path' with '--user' or '--local'"
-        )
+        raise CommandError("Cannot combine '--path' with '--user' or '--local'")
 
 
-list_exclude = partial(
+list_exclude: Callable[..., Option] = partial(
     PipOption,
-    '--exclude',
-    dest='excludes',
-    action='append',
-    metavar='package',
-    type='package_name',
+    "--exclude",
+    dest="excludes",
+    action="append",
+    metavar="package",
+    type="package_name",
     help="Exclude specified package from the output",
-)  # type: Callable[..., Option]
+)
 
 
-no_python_version_warning = partial(
+no_python_version_warning: Callable[..., Option] = partial(
     Option,
-    '--no-python-version-warning',
-    dest='no_python_version_warning',
-    action='store_true',
+    "--no-python-version-warning",
+    dest="no_python_version_warning",
+    action="store_true",
     default=False,
-    help='Silence deprecation warnings for upcoming unsupported Pythons.',
-)  # type: Callable[..., Option]
+    help="Silence deprecation warnings for upcoming unsupported Pythons.",
+)
 
 
-use_new_feature = partial(
+use_new_feature: Callable[..., Option] = partial(
     Option,
-    '--use-feature',
-    dest='features_enabled',
-    metavar='feature',
-    action='append',
+    "--use-feature",
+    dest="features_enabled",
+    metavar="feature",
+    action="append",
     default=[],
-    choices=['2020-resolver', 'fast-deps'],
-    help='Enable new functionality, that may be backward incompatible.',
-)  # type: Callable[..., Option]
+    choices=["2020-resolver", "fast-deps", "in-tree-build"],
+    help="Enable new functionality, that may be backward incompatible.",
+)
 
-use_deprecated_feature = partial(
+use_deprecated_feature: Callable[..., Option] = partial(
     Option,
-    '--use-deprecated',
-    dest='deprecated_features_enabled',
-    metavar='feature',
-    action='append',
+    "--use-deprecated",
+    dest="deprecated_features_enabled",
+    metavar="feature",
+    action="append",
     default=[],
-    choices=['legacy-resolver'],
-    help=(
-        'Enable deprecated functionality, that will be removed in the future.'
-    ),
-)  # type: Callable[..., Option]
+    choices=[
+        "legacy-resolver",
+        "out-of-tree-build",
+        "backtrack-on-build-failures",
+        "html5lib",
+    ],
+    help=("Enable deprecated functionality, that will be removed in the future."),
+)
 
 
 ##########
 # groups #
 ##########
 
-general_group = {
-    'name': 'General Options',
-    'options': [
+general_group: Dict[str, Any] = {
+    "name": "General Options",
+    "options": [
         help_,
+        debug_mode,
         isolated_mode,
         require_virtualenv,
         verbose,
@@ -955,15 +1004,15 @@ def check_list_path_option(options):
         no_python_version_warning,
         use_new_feature,
         use_deprecated_feature,
-    ]
-}  # type: Dict[str, Any]
+    ],
+}
 
-index_group = {
-    'name': 'Package Index Options',
-    'options': [
+index_group: Dict[str, Any] = {
+    "name": "Package Index Options",
+    "options": [
         index_url,
         extra_index_url,
         no_index,
         find_links,
-    ]
-}  # type: Dict[str, Any]
+    ],
+}
diff --git a/src/pip/_internal/cli/command_context.py b/src/pip/_internal/cli/command_context.py
index ade14f2f677..ed68322376d 100644
--- a/src/pip/_internal/cli/command_context.py
+++ b/src/pip/_internal/cli/command_context.py
@@ -1,25 +1,17 @@
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
+from typing import ContextManager, Iterator, TypeVar
 
-from pip._vendor.contextlib2 import ExitStack
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import ContextManager, Iterator, TypeVar
-
-    _T = TypeVar('_T', covariant=True)
+_T = TypeVar("_T", covariant=True)
 
 
 class CommandContextMixIn:
-    def __init__(self):
-        # type: () -> None
+    def __init__(self) -> None:
         super().__init__()
         self._in_main_context = False
         self._main_context = ExitStack()
 
     @contextmanager
-    def main_context(self):
-        # type: () -> Iterator[None]
+    def main_context(self) -> Iterator[None]:
         assert not self._in_main_context
 
         self._in_main_context = True
@@ -29,8 +21,7 @@ def main_context(self):
         finally:
             self._in_main_context = False
 
-    def enter_context(self, context_provider):
-        # type: (ContextManager[_T]) -> _T
+    def enter_context(self, context_provider: ContextManager[_T]) -> _T:
         assert self._in_main_context
 
         return self._main_context.enter_context(context_provider)
diff --git a/src/pip/_internal/cli/main.py b/src/pip/_internal/cli/main.py
index ed59073072f..0e31221543a 100644
--- a/src/pip/_internal/cli/main.py
+++ b/src/pip/_internal/cli/main.py
@@ -4,16 +4,13 @@
 import logging
 import os
 import sys
+from typing import List, Optional
 
 from pip._internal.cli.autocompletion import autocomplete
 from pip._internal.cli.main_parser import parse_command
 from pip._internal.commands import create_command
 from pip._internal.exceptions import PipError
 from pip._internal.utils import deprecation
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
 
 logger = logging.getLogger(__name__)
 
@@ -44,8 +41,8 @@
 # call to main. As it is not safe to do any processing after calling
 # main, this should not be an issue in practice.
 
-def main(args=None):
-    # type: (Optional[List[str]]) -> int
+
+def main(args: Optional[List[str]] = None) -> int:
     if args is None:
         args = sys.argv[1:]
 
@@ -64,7 +61,7 @@ def main(args=None):
     # Needed for locale.getpreferredencoding(False) to work
     # in pip._internal.utils.encoding.auto_decode
     try:
-        locale.setlocale(locale.LC_ALL, '')
+        locale.setlocale(locale.LC_ALL, "")
     except locale.Error as e:
         # setlocale can apparently crash if locale are uninitialized
         logger.debug("Ignoring error %s when setting locale", e)
diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py
index fcee6a2c234..3666ab04ca6 100644
--- a/src/pip/_internal/cli/main_parser.py
+++ b/src/pip/_internal/cli/main_parser.py
@@ -3,35 +3,27 @@
 
 import os
 import sys
+from typing import List, Tuple
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
 from pip._internal.commands import commands_dict, get_similar_commands
 from pip._internal.exceptions import CommandError
 from pip._internal.utils.misc import get_pip_version, get_prog
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Tuple
-
 
 __all__ = ["create_main_parser", "parse_command"]
 
 
-def create_main_parser():
-    # type: () -> ConfigOptionParser
-    """Creates and returns the main parser for pip's CLI
-    """
-
-    parser_kw = {
-        'usage': '\n%prog  [options]',
-        'add_help_option': False,
-        'formatter': UpdatingDefaultsHelpFormatter(),
-        'name': 'global',
-        'prog': get_prog(),
-    }
+def create_main_parser() -> ConfigOptionParser:
+    """Creates and returns the main parser for pip's CLI"""
 
-    parser = ConfigOptionParser(**parser_kw)
+    parser = ConfigOptionParser(
+        usage="\n%prog  [options]",
+        add_help_option=False,
+        formatter=UpdatingDefaultsHelpFormatter(),
+        name="global",
+        prog=get_prog(),
+    )
     parser.disable_interspersed_args()
 
     parser.version = get_pip_version()
@@ -44,17 +36,16 @@ def create_main_parser():
     parser.main = True  # type: ignore
 
     # create command listing for description
-    description = [''] + [
-        '{name:27} {command_info.summary}'.format(**locals())
+    description = [""] + [
+        f"{name:27} {command_info.summary}"
         for name, command_info in commands_dict.items()
     ]
-    parser.description = '\n'.join(description)
+    parser.description = "\n".join(description)
 
     return parser
 
 
-def parse_command(args):
-    # type: (List[str]) -> Tuple[str, List[str]]
+def parse_command(args: List[str]) -> Tuple[str, List[str]]:
     parser = create_main_parser()
 
     # Note: parser calls disable_interspersed_args(), so the result of this
@@ -73,7 +64,7 @@ def parse_command(args):
         sys.exit()
 
     # pip || pip help -> print_help()
-    if not args_else or (args_else[0] == 'help' and len(args_else) == 1):
+    if not args_else or (args_else[0] == "help" and len(args_else) == 1):
         parser.print_help()
         sys.exit()
 
@@ -87,7 +78,7 @@ def parse_command(args):
         if guess:
             msg.append(f'maybe you meant "{guess}"')
 
-        raise CommandError(' - '.join(msg))
+        raise CommandError(" - ".join(msg))
 
     # all the args without the subcommand
     cmd_args = args[:]
diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py
index 3bc86d9a38a..a1c99a8cb30 100644
--- a/src/pip/_internal/cli/parser.py
+++ b/src/pip/_internal/cli/parser.py
@@ -1,20 +1,16 @@
 """Base option parser setup"""
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
 import optparse
 import shutil
 import sys
 import textwrap
-from distutils.util import strtobool
-
-from pip._vendor.contextlib2 import suppress
+from contextlib import suppress
+from typing import Any, Dict, Iterator, List, Tuple
 
 from pip._internal.cli.status_codes import UNKNOWN_ERROR
 from pip._internal.configuration import Configuration, ConfigurationError
-from pip._internal.utils.misc import redact_auth_from_url
+from pip._internal.utils.misc import redact_auth_from_url, strtobool
 
 logger = logging.getLogger(__name__)
 
@@ -22,17 +18,19 @@
 class PrettyHelpFormatter(optparse.IndentedHelpFormatter):
     """A prettier/less verbose help formatter for optparse."""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         # help position must be aligned with __init__.parseopts.description
-        kwargs['max_help_position'] = 30
-        kwargs['indent_increment'] = 1
-        kwargs['width'] = shutil.get_terminal_size()[0] - 2
+        kwargs["max_help_position"] = 30
+        kwargs["indent_increment"] = 1
+        kwargs["width"] = shutil.get_terminal_size()[0] - 2
         super().__init__(*args, **kwargs)
 
-    def format_option_strings(self, option):
+    def format_option_strings(self, option: optparse.Option) -> str:
         return self._format_option_strings(option)
 
-    def _format_option_strings(self, option, mvarfmt=' <{}>', optsep=', '):
+    def _format_option_strings(
+        self, option: optparse.Option, mvarfmt: str = " <{}>", optsep: str = ", "
+    ) -> str:
         """
         Return a comma-separated list of option strings and metavars.
 
@@ -50,52 +48,52 @@ def _format_option_strings(self, option, mvarfmt=' <{}>', optsep=', '):
             opts.insert(1, optsep)
 
         if option.takes_value():
+            assert option.dest is not None
             metavar = option.metavar or option.dest.lower()
             opts.append(mvarfmt.format(metavar.lower()))
 
-        return ''.join(opts)
+        return "".join(opts)
 
-    def format_heading(self, heading):
-        if heading == 'Options':
-            return ''
-        return heading + ':\n'
+    def format_heading(self, heading: str) -> str:
+        if heading == "Options":
+            return ""
+        return heading + ":\n"
 
-    def format_usage(self, usage):
+    def format_usage(self, usage: str) -> str:
         """
         Ensure there is only one newline between usage and the first heading
         if there is no description.
         """
-        msg = '\nUsage: {}\n'.format(
-            self.indent_lines(textwrap.dedent(usage), "  "))
+        msg = "\nUsage: {}\n".format(self.indent_lines(textwrap.dedent(usage), "  "))
         return msg
 
-    def format_description(self, description):
+    def format_description(self, description: str) -> str:
         # leave full control over description to us
         if description:
-            if hasattr(self.parser, 'main'):
-                label = 'Commands'
+            if hasattr(self.parser, "main"):
+                label = "Commands"
             else:
-                label = 'Description'
+                label = "Description"
             # some doc strings have initial newlines, some don't
-            description = description.lstrip('\n')
+            description = description.lstrip("\n")
             # some doc strings have final newlines and spaces, some don't
             description = description.rstrip()
             # dedent, then reindent
             description = self.indent_lines(textwrap.dedent(description), "  ")
-            description = f'{label}:\n{description}\n'
+            description = f"{label}:\n{description}\n"
             return description
         else:
-            return ''
+            return ""
 
-    def format_epilog(self, epilog):
+    def format_epilog(self, epilog: str) -> str:
         # leave full control over epilog to us
         if epilog:
             return epilog
         else:
-            return ''
+            return ""
 
-    def indent_lines(self, text, indent):
-        new_lines = [indent + line for line in text.split('\n')]
+    def indent_lines(self, text: str, indent: str) -> str:
+        new_lines = [indent + line for line in text.split("\n")]
         return "\n".join(new_lines)
 
 
@@ -108,14 +106,16 @@ class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter):
     Also redact auth from url type options
     """
 
-    def expand_default(self, option):
+    def expand_default(self, option: optparse.Option) -> str:
         default_values = None
         if self.parser is not None:
+            assert isinstance(self.parser, ConfigOptionParser)
             self.parser._update_defaults(self.parser.defaults)
+            assert option.dest is not None
             default_values = self.parser.defaults.get(option.dest)
         help_text = super().expand_default(option)
 
-        if default_values and option.metavar == 'URL':
+        if default_values and option.metavar == "URL":
             if isinstance(default_values, str):
                 default_values = [default_values]
 
@@ -124,15 +124,15 @@ def expand_default(self, option):
                 default_values = []
 
             for val in default_values:
-                help_text = help_text.replace(
-                    val, redact_auth_from_url(val))
+                help_text = help_text.replace(val, redact_auth_from_url(val))
 
         return help_text
 
 
 class CustomOptionParser(optparse.OptionParser):
-
-    def insert_option_group(self, idx, *args, **kwargs):
+    def insert_option_group(
+        self, idx: int, *args: Any, **kwargs: Any
+    ) -> optparse.OptionGroup:
         """Insert an OptionGroup at a given position."""
         group = self.add_option_group(*args, **kwargs)
 
@@ -142,7 +142,7 @@ def insert_option_group(self, idx, *args, **kwargs):
         return group
 
     @property
-    def option_list_all(self):
+    def option_list_all(self) -> List[optparse.Option]:
         """Get a list of all options, including those in option groups."""
         res = self.option_list[:]
         for i in self.option_groups:
@@ -155,34 +155,40 @@ class ConfigOptionParser(CustomOptionParser):
     """Custom option parser which updates its defaults by checking the
     configuration files and environmental variables"""
 
-    def __init__(self, *args, **kwargs):
-        self.name = kwargs.pop('name')
-
-        isolated = kwargs.pop("isolated", False)
+    def __init__(
+        self,
+        *args: Any,
+        name: str,
+        isolated: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        self.name = name
         self.config = Configuration(isolated)
 
         assert self.name
         super().__init__(*args, **kwargs)
 
-    def check_default(self, option, key, val):
+    def check_default(self, option: optparse.Option, key: str, val: Any) -> Any:
         try:
             return option.check_value(key, val)
         except optparse.OptionValueError as exc:
             print(f"An error occurred during configuration: {exc}")
             sys.exit(3)
 
-    def _get_ordered_configuration_items(self):
+    def _get_ordered_configuration_items(self) -> Iterator[Tuple[str, Any]]:
         # Configuration gives keys in an unordered manner. Order them.
         override_order = ["global", self.name, ":env:"]
 
         # Pool the options into different groups
-        section_items = {name: [] for name in override_order}
+        section_items: Dict[str, List[Tuple[str, Any]]] = {
+            name: [] for name in override_order
+        }
         for section_key, val in self.config.items():
             # ignore empty values
             if not val:
                 logger.debug(
                     "Ignoring configuration key '%s' as it's value is empty.",
-                    section_key
+                    section_key,
                 )
                 continue
 
@@ -195,7 +201,7 @@ def _get_ordered_configuration_items(self):
             for key, val in section_items[section]:
                 yield key, val
 
-    def _update_defaults(self, defaults):
+    def _update_defaults(self, defaults: Dict[str, Any]) -> Dict[str, Any]:
         """Updates the given defaults with values from the config files and
         the environ. Does a little special handling for certain types of
         options (lists)."""
@@ -206,7 +212,7 @@ def _update_defaults(self, defaults):
         # Then set the options with those values
         for key, val in self._get_ordered_configuration_items():
             # '--' because configuration supports only long names
-            option = self.get_option('--' + key)
+            option = self.get_option("--" + key)
 
             # Ignore options not present in this parser. E.g. non-globals put
             # in [global] by users that want them to apply to all applicable
@@ -214,31 +220,34 @@ def _update_defaults(self, defaults):
             if option is None:
                 continue
 
-            if option.action in ('store_true', 'store_false'):
+            assert option.dest is not None
+
+            if option.action in ("store_true", "store_false"):
                 try:
                     val = strtobool(val)
                 except ValueError:
                     self.error(
-                        '{} is not a valid value for {} option, '  # noqa
-                        'please specify a boolean value like yes/no, '
-                        'true/false or 1/0 instead.'.format(val, key)
+                        "{} is not a valid value for {} option, "  # noqa
+                        "please specify a boolean value like yes/no, "
+                        "true/false or 1/0 instead.".format(val, key)
                     )
-            elif option.action == 'count':
+            elif option.action == "count":
                 with suppress(ValueError):
                     val = strtobool(val)
                 with suppress(ValueError):
                     val = int(val)
                 if not isinstance(val, int) or val < 0:
                     self.error(
-                        '{} is not a valid value for {} option, '  # noqa
-                        'please instead specify either a non-negative integer '
-                        'or a boolean value like yes/no or false/true '
-                        'which is equivalent to 1/0.'.format(val, key)
+                        "{} is not a valid value for {} option, "  # noqa
+                        "please instead specify either a non-negative integer "
+                        "or a boolean value like yes/no or false/true "
+                        "which is equivalent to 1/0.".format(val, key)
                     )
-            elif option.action == 'append':
+            elif option.action == "append":
                 val = val.split()
                 val = [self.check_default(option, key, v) for v in val]
-            elif option.action == 'callback':
+            elif option.action == "callback":
+                assert option.callback is not None
                 late_eval.add(option.dest)
                 opt_str = option.get_opt_string()
                 val = option.convert_value(opt_str, val)
@@ -256,7 +265,7 @@ def _update_defaults(self, defaults):
         self.values = None
         return defaults
 
-    def get_default_values(self):
+    def get_default_values(self) -> optparse.Values:
         """Overriding to make updating the defaults after instantiation of
         the option parser possible, _update_defaults() does the dirty work."""
         if not self.process_default_values:
@@ -271,12 +280,13 @@ def get_default_values(self):
 
         defaults = self._update_defaults(self.defaults.copy())  # ours
         for option in self._get_all_options():
+            assert option.dest is not None
             default = defaults.get(option.dest)
             if isinstance(default, str):
                 opt_str = option.get_opt_string()
                 defaults[option.dest] = option.check_value(opt_str, default)
         return optparse.Values(defaults)
 
-    def error(self, msg):
+    def error(self, msg: str) -> None:
         self.print_usage(sys.stderr)
         self.exit(UNKNOWN_ERROR, f"{msg}\n")
diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py
index 59b01a6d0fd..ffa1964fc7b 100644
--- a/src/pip/_internal/cli/progress_bars.py
+++ b/src/pip/_internal/cli/progress_bars.py
@@ -1,17 +1,27 @@
+import functools
 import itertools
 import sys
 from signal import SIGINT, default_int_handler, signal
+from typing import Any, Callable, Iterator, Optional, Tuple
 
 from pip._vendor.progress.bar import Bar, FillingCirclesBar, IncrementalBar
 from pip._vendor.progress.spinner import Spinner
+from pip._vendor.rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    FileSizeColumn,
+    Progress,
+    ProgressColumn,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+    TransferSpeedColumn,
+)
 
 from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.logging import get_indentation
 from pip._internal.utils.misc import format_size
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, List
 
 try:
     from pip._vendor import colorama
@@ -20,9 +30,10 @@
 except Exception:
     colorama = None
 
+DownloadProgressRenderer = Callable[[Iterator[bytes]], Iterator[bytes]]
+
 
-def _select_progress_class(preferred, fallback):
-    # type: (Bar, Bar) -> Bar
+def _select_progress_class(preferred: Bar, fallback: Bar) -> Bar:
     encoding = getattr(preferred.file, "encoding", None)
 
     # If we don't know what encoding this file is in, then we'll just assume
@@ -49,7 +60,7 @@ def _select_progress_class(preferred, fallback):
         return preferred
 
 
-_BaseBar = _select_progress_class(IncrementalBar, Bar)  # type: Any
+_BaseBar: Any = _select_progress_class(IncrementalBar, Bar)
 
 
 class InterruptibleMixin:
@@ -70,8 +81,7 @@ class InterruptibleMixin:
        download has already completed, for example.
     """
 
-    def __init__(self, *args, **kwargs):
-        # type: (List[Any], Dict[Any, Any]) -> None
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         """
         Save the original SIGINT handler for later.
         """
@@ -88,8 +98,7 @@ def __init__(self, *args, **kwargs):
         if self.original_handler is None:
             self.original_handler = default_int_handler
 
-    def finish(self):
-        # type: () -> None
+    def finish(self) -> None:
         """
         Restore the original SIGINT handler after finishing.
 
@@ -111,9 +120,7 @@ def handle_sigint(self, signum, frame):  # type: ignore
 
 
 class SilentBar(Bar):
-
-    def update(self):
-        # type: () -> None
+    def update(self) -> None:
         pass
 
 
@@ -126,31 +133,24 @@ class BlueEmojiBar(IncrementalBar):
 
 
 class DownloadProgressMixin:
-
-    def __init__(self, *args, **kwargs):
-        # type: (List[Any], Dict[Any, Any]) -> None
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         # https://github.com/python/mypy/issues/5887
         super().__init__(*args, **kwargs)  # type: ignore
-        self.message = (" " * (
-            get_indentation() + 2
-        )) + self.message  # type: str
+        self.message: str = (" " * (get_indentation() + 2)) + self.message
 
     @property
-    def downloaded(self):
-        # type: () -> str
+    def downloaded(self) -> str:
         return format_size(self.index)  # type: ignore
 
     @property
-    def download_speed(self):
-        # type: () -> str
+    def download_speed(self) -> str:
         # Avoid zero division errors...
         if self.avg == 0.0:  # type: ignore
             return "..."
         return format_size(1 / self.avg) + "/s"  # type: ignore
 
     @property
-    def pretty_eta(self):
-        # type: () -> str
+    def pretty_eta(self) -> str:
         if self.eta:  # type: ignore
             return f"eta {self.eta_td}"  # type: ignore
         return ""
@@ -165,9 +165,7 @@ def iter(self, it):  # type: ignore
 
 
 class WindowsMixin:
-
-    def __init__(self, *args, **kwargs):
-        # type: (List[Any], Dict[Any, Any]) -> None
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         # The Windows terminal does not support the hide/show cursor ANSI codes
         # even with colorama. So we'll ensure that hide_cursor is False on
         # Windows.
@@ -195,16 +193,14 @@ def __init__(self, *args, **kwargs):
             self.file.flush = lambda: self.file.wrapped.flush()
 
 
-class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin,
-                              DownloadProgressMixin):
+class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, DownloadProgressMixin):
 
     file = sys.stdout
     message = "%(percent)d%%"
     suffix = "%(downloaded)s %(download_speed)s %(pretty_eta)s"
 
 
-class DefaultDownloadProgressBar(BaseDownloadProgressBar,
-                                 _BaseBar):
+class DefaultDownloadProgressBar(BaseDownloadProgressBar, _BaseBar):
     pass
 
 
@@ -212,45 +208,43 @@ class DownloadSilentBar(BaseDownloadProgressBar, SilentBar):
     pass
 
 
-class DownloadBar(BaseDownloadProgressBar,
-                  Bar):
+class DownloadBar(BaseDownloadProgressBar, Bar):
     pass
 
 
-class DownloadFillingCirclesBar(BaseDownloadProgressBar,
-                                FillingCirclesBar):
+class DownloadFillingCirclesBar(BaseDownloadProgressBar, FillingCirclesBar):
     pass
 
 
-class DownloadBlueEmojiProgressBar(BaseDownloadProgressBar,
-                                   BlueEmojiBar):
+class DownloadBlueEmojiProgressBar(BaseDownloadProgressBar, BlueEmojiBar):
     pass
 
 
-class DownloadProgressSpinner(WindowsMixin, InterruptibleMixin,
-                              DownloadProgressMixin, Spinner):
+class DownloadProgressSpinner(
+    WindowsMixin, InterruptibleMixin, DownloadProgressMixin, Spinner
+):
 
     file = sys.stdout
     suffix = "%(downloaded)s %(download_speed)s"
 
-    def next_phase(self):
-        # type: () -> str
+    def next_phase(self) -> str:
         if not hasattr(self, "_phaser"):
             self._phaser = itertools.cycle(self.phases)
         return next(self._phaser)
 
-    def update(self):
-        # type: () -> None
+    def update(self) -> None:
         message = self.message % self
         phase = self.next_phase()
         suffix = self.suffix % self
-        line = ''.join([
-            message,
-            " " if message else "",
-            phase,
-            " " if suffix else "",
-            suffix,
-        ])
+        line = "".join(
+            [
+                message,
+                " " if message else "",
+                phase,
+                " " if suffix else "",
+                suffix,
+            ]
+        )
 
         self.writeln(line)
 
@@ -260,12 +254,68 @@ def update(self):
     "on": (DefaultDownloadProgressBar, DownloadProgressSpinner),
     "ascii": (DownloadBar, DownloadProgressSpinner),
     "pretty": (DownloadFillingCirclesBar, DownloadProgressSpinner),
-    "emoji": (DownloadBlueEmojiProgressBar, DownloadProgressSpinner)
+    "emoji": (DownloadBlueEmojiProgressBar, DownloadProgressSpinner),
 }
 
 
-def DownloadProgressProvider(progress_bar, max=None):  # type: ignore
+def _legacy_progress_bar(
+    progress_bar: str, max: Optional[int]
+) -> DownloadProgressRenderer:
     if max is None or max == 0:
-        return BAR_TYPES[progress_bar][1]().iter
+        return BAR_TYPES[progress_bar][1]().iter  # type: ignore
     else:
         return BAR_TYPES[progress_bar][0](max=max).iter
+
+
+#
+# Modern replacement, for our legacy progress bars.
+#
+def _rich_progress_bar(
+    iterable: Iterator[bytes],
+    *,
+    bar_type: str,
+    size: int,
+) -> Iterator[bytes]:
+    assert bar_type == "on", "This should only be used in the default mode."
+
+    if not size:
+        total = float("inf")
+        columns: Tuple[ProgressColumn, ...] = (
+            TextColumn("[progress.description]{task.description}"),
+            SpinnerColumn("line", speed=1.5),
+            FileSizeColumn(),
+            TransferSpeedColumn(),
+            TimeElapsedColumn(),
+        )
+    else:
+        total = size
+        columns = (
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            DownloadColumn(),
+            TransferSpeedColumn(),
+            TextColumn("eta"),
+            TimeRemainingColumn(),
+        )
+
+    progress = Progress(*columns, refresh_per_second=30)
+    task_id = progress.add_task(" " * (get_indentation() + 2), total=total)
+    with progress:
+        for chunk in iterable:
+            yield chunk
+            progress.update(task_id, advance=len(chunk))
+
+
+def get_download_progress_renderer(
+    *, bar_type: str, size: Optional[int] = None
+) -> DownloadProgressRenderer:
+    """Get an object that can be used to render the download progress.
+
+    Returns a callable, that takes an iterable to "wrap".
+    """
+    if bar_type == "on":
+        return functools.partial(_rich_progress_bar, bar_type=bar_type, size=size)
+    elif bar_type == "off":
+        return iter  # no-op, when passed an iterator
+    else:
+        return _legacy_progress_bar(bar_type, size)
diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py
index 468b3cceab5..970deec2cb2 100644
--- a/src/pip/_internal/cli/req_command.py
+++ b/src/pip/_internal/cli/req_command.py
@@ -7,8 +7,12 @@
 
 import logging
 import os
+import sys
 from functools import partial
+from optparse import Values
+from typing import Any, List, Optional, Tuple
 
+from pip._internal.cache import WheelCache
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.command_context import CommandContextMixIn
@@ -16,7 +20,9 @@
 from pip._internal.index.collector import LinkCollector
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.selection_prefs import SelectionPreferences
+from pip._internal.models.target_python import TargetPython
 from pip._internal.network.session import PipSession
+from pip._internal.operations.build.build_tracker import BuildTracker
 from pip._internal.operations.prepare import RequirementPreparer
 from pip._internal.req.constructors import (
     install_req_from_editable,
@@ -25,21 +31,16 @@
     install_req_from_req_string,
 )
 from pip._internal.req.req_file import parse_requirements
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.resolution.base import BaseResolver
 from pip._internal.self_outdated_check import pip_self_version_check
-from pip._internal.utils.temp_dir import tempdir_kinds
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Any, List, Optional, Tuple
-
-    from pip._internal.cache import WheelCache
-    from pip._internal.models.target_python import TargetPython
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._internal.req.req_tracker import RequirementTracker
-    from pip._internal.resolution.base import BaseResolver
-    from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryTypeRegistry
-
+from pip._internal.utils.deprecation import deprecated
+from pip._internal.utils.temp_dir import (
+    TempDirectory,
+    TempDirectoryTypeRegistry,
+    tempdir_kinds,
+)
+from pip._internal.utils.virtualenv import running_under_virtualenv
 
 logger = logging.getLogger(__name__)
 
@@ -49,14 +50,13 @@ class SessionCommandMixin(CommandContextMixIn):
     """
     A class mixin for command classes needing _build_session().
     """
-    def __init__(self):
-        # type: () -> None
+
+    def __init__(self) -> None:
         super().__init__()
-        self._session = None  # Optional[PipSession]
+        self._session: Optional[PipSession] = None
 
     @classmethod
-    def _get_index_urls(cls, options):
-        # type: (Values) -> Optional[List[str]]
+    def _get_index_urls(cls, options: Values) -> Optional[List[str]]:
         """Return a list of index urls from user-provided options."""
         index_urls = []
         if not getattr(options, "no_index", False):
@@ -69,8 +69,7 @@ def _get_index_urls(cls, options):
         # Return None rather than an empty list
         return index_urls or None
 
-    def get_default_session(self, options):
-        # type: (Values) -> PipSession
+    def get_default_session(self, options: Values) -> PipSession:
         """Get a default-managed session."""
         if self._session is None:
             self._session = self.enter_context(self._build_session(options))
@@ -80,13 +79,16 @@ def get_default_session(self, options):
             assert self._session is not None
         return self._session
 
-    def _build_session(self, options, retries=None, timeout=None):
-        # type: (Values, Optional[int], Optional[int]) -> PipSession
+    def _build_session(
+        self,
+        options: Values,
+        retries: Optional[int] = None,
+        timeout: Optional[int] = None,
+    ) -> PipSession:
         assert not options.cache_dir or os.path.isabs(options.cache_dir)
         session = PipSession(
             cache=(
-                os.path.join(options.cache_dir, "http")
-                if options.cache_dir else None
+                os.path.join(options.cache_dir, "http") if options.cache_dir else None
             ),
             retries=retries if retries is not None else options.retries,
             trusted_hosts=options.trusted_hosts,
@@ -103,9 +105,7 @@ def _build_session(self, options, retries=None, timeout=None):
 
         # Handle timeouts
         if options.timeout or timeout:
-            session.timeout = (
-                timeout if timeout is not None else options.timeout
-            )
+            session.timeout = timeout if timeout is not None else options.timeout
 
         # Handle configured proxies
         if options.proxy:
@@ -128,24 +128,21 @@ class IndexGroupCommand(Command, SessionCommandMixin):
     This also corresponds to the commands that permit the pip version check.
     """
 
-    def handle_pip_version_check(self, options):
-        # type: (Values) -> None
+    def handle_pip_version_check(self, options: Values) -> None:
         """
         Do the pip version check if not disabled.
 
         This overrides the default behavior of not doing the check.
         """
         # Make sure the index_group options are present.
-        assert hasattr(options, 'no_index')
+        assert hasattr(options, "no_index")
 
         if options.disable_pip_version_check or options.no_index:
             return
 
         # Otherwise, check if we're using the latest version of pip available.
         session = self._build_session(
-            options,
-            retries=0,
-            timeout=min(5, options.timeout)
+            options, retries=0, timeout=min(5, options.timeout)
         )
         with session:
             pip_self_version_check(session, options)
@@ -158,18 +155,48 @@ def handle_pip_version_check(self, options):
 ]
 
 
-def with_cleanup(func):
-    # type: (Any) -> Any
+def warn_if_run_as_root() -> None:
+    """Output a warning for sudo users on Unix.
+
+    In a virtual environment, sudo pip still writes to virtualenv.
+    On Windows, users may run pip as Administrator without issues.
+    This warning only applies to Unix root users outside of virtualenv.
+    """
+    if running_under_virtualenv():
+        return
+    if not hasattr(os, "getuid"):
+        return
+    # On Windows, there are no "system managed" Python packages. Installing as
+    # Administrator via pip is the correct way of updating system environments.
+    #
+    # We choose sys.platform over utils.compat.WINDOWS here to enable Mypy platform
+    # checks: https://mypy.readthedocs.io/en/stable/common_issues.html
+    if sys.platform == "win32" or sys.platform == "cygwin":
+        return
+
+    if os.getuid() != 0:
+        return
+
+    logger.warning(
+        "Running pip as the 'root' user can result in broken permissions and "
+        "conflicting behaviour with the system package manager. "
+        "It is recommended to use a virtual environment instead: "
+        "https://pip.pypa.io/warnings/venv"
+    )
+
+
+def with_cleanup(func: Any) -> Any:
     """Decorator for common logic related to managing temporary
     directories.
     """
-    def configure_tempdir_registry(registry):
-        # type: (TempDirectoryTypeRegistry) -> None
+
+    def configure_tempdir_registry(registry: TempDirectoryTypeRegistry) -> None:
         for t in KEEPABLE_TEMPDIR_TYPES:
             registry.set_delete(t, False)
 
-    def wrapper(self, options, args):
-        # type: (RequirementCommand, Values, List[Any]) -> Optional[int]
+    def wrapper(
+        self: RequirementCommand, options: Values, args: List[Any]
+    ) -> Optional[int]:
         assert self.tempdir_registry is not None
         if options.no_clean:
             configure_tempdir_registry(self.tempdir_registry)
@@ -187,34 +214,56 @@ def wrapper(self, options, args):
 
 
 class RequirementCommand(IndexGroupCommand):
-
-    def __init__(self, *args, **kw):
-        # type: (Any, Any) -> None
+    def __init__(self, *args: Any, **kw: Any) -> None:
         super().__init__(*args, **kw)
 
         self.cmd_opts.add_option(cmdoptions.no_clean())
 
     @staticmethod
-    def determine_resolver_variant(options):
-        # type: (Values) -> str
+    def determine_resolver_variant(options: Values) -> str:
         """Determines which resolver should be used, based on the given options."""
         if "legacy-resolver" in options.deprecated_features_enabled:
             return "legacy"
 
         return "2020-resolver"
 
+    @staticmethod
+    def determine_build_failure_suppression(options: Values) -> bool:
+        """Determines whether build failures should be suppressed and backtracked on."""
+        if "backtrack-on-build-failures" not in options.deprecated_features_enabled:
+            return False
+
+        if "legacy-resolver" in options.deprecated_features_enabled:
+            raise CommandError("Cannot backtrack with legacy resolver.")
+
+        deprecated(
+            reason=(
+                "Backtracking on build failures can mask issues related to how "
+                "a package generates metadata or builds a wheel. This flag will "
+                "be removed in pip 22.2."
+            ),
+            gone_in=None,
+            replacement=(
+                "avoiding known-bad versions by explicitly telling pip to ignore them "
+                "(either directly as requirements, or via a constraints file)"
+            ),
+            feature_flag=None,
+            issue=10655,
+        )
+        return True
+
     @classmethod
     def make_requirement_preparer(
         cls,
-        temp_build_dir,           # type: TempDirectory
-        options,                  # type: Values
-        req_tracker,              # type: RequirementTracker
-        session,                  # type: PipSession
-        finder,                   # type: PackageFinder
-        use_user_site,            # type: bool
-        download_dir=None,        # type: str
-    ):
-        # type: (...) -> RequirementPreparer
+        temp_build_dir: TempDirectory,
+        options: Values,
+        build_tracker: BuildTracker,
+        session: PipSession,
+        finder: PackageFinder,
+        use_user_site: bool,
+        download_dir: Optional[str] = None,
+        verbosity: int = 0,
+    ) -> RequirementPreparer:
         """
         Create a RequirementPreparer instance for the given parameters.
         """
@@ -223,52 +272,74 @@ def make_requirement_preparer(
 
         resolver_variant = cls.determine_resolver_variant(options)
         if resolver_variant == "2020-resolver":
-            lazy_wheel = 'fast-deps' in options.features_enabled
+            lazy_wheel = "fast-deps" in options.features_enabled
             if lazy_wheel:
                 logger.warning(
-                    'pip is using lazily downloaded wheels using HTTP '
-                    'range requests to obtain dependency information. '
-                    'This experimental feature is enabled through '
-                    '--use-feature=fast-deps and it is not ready for '
-                    'production.'
+                    "pip is using lazily downloaded wheels using HTTP "
+                    "range requests to obtain dependency information. "
+                    "This experimental feature is enabled through "
+                    "--use-feature=fast-deps and it is not ready for "
+                    "production."
                 )
         else:
             lazy_wheel = False
-            if 'fast-deps' in options.features_enabled:
+            if "fast-deps" in options.features_enabled:
                 logger.warning(
-                    'fast-deps has no effect when used with the legacy resolver.'
+                    "fast-deps has no effect when used with the legacy resolver."
                 )
 
+        in_tree_build = "out-of-tree-build" not in options.deprecated_features_enabled
+        if "in-tree-build" in options.features_enabled:
+            deprecated(
+                reason="In-tree builds are now the default.",
+                replacement="to remove the --use-feature=in-tree-build flag",
+                gone_in="22.1",
+            )
+        if "out-of-tree-build" in options.deprecated_features_enabled:
+            deprecated(
+                reason="Out-of-tree builds are deprecated.",
+                replacement=None,
+                gone_in="22.1",
+            )
+
+        if options.progress_bar not in {"on", "off"}:
+            deprecated(
+                reason="Custom progress bar styles are deprecated",
+                replacement="to use the default progress bar style.",
+                gone_in="22.1",
+            )
+
         return RequirementPreparer(
             build_dir=temp_build_dir_path,
             src_dir=options.src_dir,
             download_dir=download_dir,
             build_isolation=options.build_isolation,
-            req_tracker=req_tracker,
+            build_tracker=build_tracker,
             session=session,
             progress_bar=options.progress_bar,
             finder=finder,
             require_hashes=options.require_hashes,
             use_user_site=use_user_site,
             lazy_wheel=lazy_wheel,
+            verbosity=verbosity,
+            in_tree_build=in_tree_build,
         )
 
     @classmethod
     def make_resolver(
         cls,
-        preparer,                            # type: RequirementPreparer
-        finder,                              # type: PackageFinder
-        options,                             # type: Values
-        wheel_cache=None,                    # type: Optional[WheelCache]
-        use_user_site=False,                 # type: bool
-        ignore_installed=True,               # type: bool
-        ignore_requires_python=False,        # type: bool
-        force_reinstall=False,               # type: bool
-        upgrade_strategy="to-satisfy-only",  # type: str
-        use_pep517=None,                     # type: Optional[bool]
-        py_version_info=None,                # type: Optional[Tuple[int, ...]]
-    ):
-        # type: (...) -> BaseResolver
+        preparer: RequirementPreparer,
+        finder: PackageFinder,
+        options: Values,
+        wheel_cache: Optional[WheelCache] = None,
+        use_user_site: bool = False,
+        ignore_installed: bool = True,
+        ignore_requires_python: bool = False,
+        force_reinstall: bool = False,
+        upgrade_strategy: str = "to-satisfy-only",
+        use_pep517: Optional[bool] = None,
+        py_version_info: Optional[Tuple[int, ...]] = None,
+    ) -> BaseResolver:
         """
         Create a Resolver instance for the given parameters.
         """
@@ -277,6 +348,7 @@ def make_resolver(
             isolated=options.isolated_mode,
             use_pep517=use_pep517,
         )
+        suppress_build_failures = cls.determine_build_failure_suppression(options)
         resolver_variant = cls.determine_resolver_variant(options)
         # The long import name and duplicated invocation is needed to convince
         # Mypy into correctly typechecking. Otherwise it would complain the
@@ -296,8 +368,10 @@ def make_resolver(
                 force_reinstall=force_reinstall,
                 upgrade_strategy=upgrade_strategy,
                 py_version_info=py_version_info,
+                suppress_build_failures=suppress_build_failures,
             )
         import pip._internal.resolution.legacy.resolver
+
         return pip._internal.resolution.legacy.resolver.Resolver(
             preparer=preparer,
             finder=finder,
@@ -314,21 +388,23 @@ def make_resolver(
 
     def get_requirements(
         self,
-        args,             # type: List[str]
-        options,          # type: Values
-        finder,           # type: PackageFinder
-        session,          # type: PipSession
-    ):
-        # type: (...) -> List[InstallRequirement]
+        args: List[str],
+        options: Values,
+        finder: PackageFinder,
+        session: PipSession,
+    ) -> List[InstallRequirement]:
         """
         Parse command-line arguments into the corresponding requirements.
         """
-        requirements = []  # type: List[InstallRequirement]
+        requirements: List[InstallRequirement] = []
         for filename in options.constraints:
             for parsed_req in parse_requirements(
-                    filename,
-                    constraint=True, finder=finder, options=options,
-                    session=session):
+                filename,
+                constraint=True,
+                finder=finder,
+                options=options,
+                session=session,
+            ):
                 req_to_add = install_req_from_parsed_requirement(
                     parsed_req,
                     isolated=options.isolated_mode,
@@ -338,7 +414,9 @@ def get_requirements(
 
         for req in args:
             req_to_add = install_req_from_line(
-                req, None, isolated=options.isolated_mode,
+                req,
+                None,
+                isolated=options.isolated_mode,
                 use_pep517=options.use_pep517,
                 user_supplied=True,
             )
@@ -356,8 +434,8 @@ def get_requirements(
         # NOTE: options.require_hashes may be set if --require-hashes is True
         for filename in options.requirements:
             for parsed_req in parse_requirements(
-                    filename,
-                    finder=finder, options=options, session=session):
+                filename, finder=finder, options=options, session=session
+            ):
                 req_to_add = install_req_from_parsed_requirement(
                     parsed_req,
                     isolated=options.isolated_mode,
@@ -371,22 +449,24 @@ def get_requirements(
             options.require_hashes = True
 
         if not (args or options.editables or options.requirements):
-            opts = {'name': self.name}
+            opts = {"name": self.name}
             if options.find_links:
                 raise CommandError(
-                    'You must give at least one requirement to {name} '
+                    "You must give at least one requirement to {name} "
                     '(maybe you meant "pip {name} {links}"?)'.format(
-                        **dict(opts, links=' '.join(options.find_links))))
+                        **dict(opts, links=" ".join(options.find_links))
+                    )
+                )
             else:
                 raise CommandError(
-                    'You must give at least one requirement to {name} '
-                    '(see "pip help {name}")'.format(**opts))
+                    "You must give at least one requirement to {name} "
+                    '(see "pip help {name}")'.format(**opts)
+                )
 
         return requirements
 
     @staticmethod
-    def trace_basic_info(finder):
-        # type: (PackageFinder) -> None
+    def trace_basic_info(finder: PackageFinder) -> None:
         """
         Trace basic information about the provided objects.
         """
@@ -398,12 +478,11 @@ def trace_basic_info(finder):
 
     def _build_package_finder(
         self,
-        options,               # type: Values
-        session,               # type: PipSession
-        target_python=None,    # type: Optional[TargetPython]
-        ignore_requires_python=None,  # type: Optional[bool]
-    ):
-        # type: (...) -> PackageFinder
+        options: Values,
+        session: PipSession,
+        target_python: Optional[TargetPython] = None,
+        ignore_requires_python: Optional[bool] = None,
+    ) -> PackageFinder:
         """
         Create a package finder appropriate to this requirement command.
 
@@ -423,4 +502,5 @@ def _build_package_finder(
             link_collector=link_collector,
             selection_prefs=selection_prefs,
             target_python=target_python,
+            use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
         )
diff --git a/src/pip/_internal/cli/spinners.py b/src/pip/_internal/cli/spinners.py
index 05ec2dcc765..1e313e1090a 100644
--- a/src/pip/_internal/cli/spinners.py
+++ b/src/pip/_internal/cli/spinners.py
@@ -3,34 +3,33 @@
 import logging
 import sys
 import time
+from typing import IO, Iterator
 
 from pip._vendor.progress import HIDE_CURSOR, SHOW_CURSOR
 
 from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.logging import get_indentation
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import IO, Iterator
 
 logger = logging.getLogger(__name__)
 
 
 class SpinnerInterface:
-    def spin(self):
-        # type: () -> None
+    def spin(self) -> None:
         raise NotImplementedError()
 
-    def finish(self, final_status):
-        # type: (str) -> None
+    def finish(self, final_status: str) -> None:
         raise NotImplementedError()
 
 
 class InteractiveSpinner(SpinnerInterface):
-    def __init__(self, message, file=None, spin_chars="-\\|/",
-                 # Empirically, 8 updates/second looks nice
-                 min_update_interval_seconds=0.125):
-        # type: (str, IO[str], str, float) -> None
+    def __init__(
+        self,
+        message: str,
+        file: IO[str] = None,
+        spin_chars: str = "-\\|/",
+        # Empirically, 8 updates/second looks nice
+        min_update_interval_seconds: float = 0.125,
+    ):
         self._message = message
         if file is None:
             file = sys.stdout
@@ -43,8 +42,7 @@ def __init__(self, message, file=None, spin_chars="-\\|/",
         self._file.write(" " * get_indentation() + self._message + " ... ")
         self._width = 0
 
-    def _write(self, status):
-        # type: (str) -> None
+    def _write(self, status: str) -> None:
         assert not self._finished
         # Erase what we wrote before by backspacing to the beginning, writing
         # spaces to overwrite the old text, and then backspacing again
@@ -56,16 +54,14 @@ def _write(self, status):
         self._file.flush()
         self._rate_limiter.reset()
 
-    def spin(self):
-        # type: () -> None
+    def spin(self) -> None:
         if self._finished:
             return
         if not self._rate_limiter.ready():
             return
         self._write(next(self._spin_cycle))
 
-    def finish(self, final_status):
-        # type: (str) -> None
+    def finish(self, final_status: str) -> None:
         if self._finished:
             return
         self._write(final_status)
@@ -79,63 +75,54 @@ def finish(self, final_status):
 # act as a keep-alive for systems like Travis-CI that take lack-of-output as
 # an indication that a task has frozen.
 class NonInteractiveSpinner(SpinnerInterface):
-    def __init__(self, message, min_update_interval_seconds=60):
-        # type: (str, float) -> None
+    def __init__(self, message: str, min_update_interval_seconds: float = 60.0) -> None:
         self._message = message
         self._finished = False
         self._rate_limiter = RateLimiter(min_update_interval_seconds)
         self._update("started")
 
-    def _update(self, status):
-        # type: (str) -> None
+    def _update(self, status: str) -> None:
         assert not self._finished
         self._rate_limiter.reset()
         logger.info("%s: %s", self._message, status)
 
-    def spin(self):
-        # type: () -> None
+    def spin(self) -> None:
         if self._finished:
             return
         if not self._rate_limiter.ready():
             return
         self._update("still running...")
 
-    def finish(self, final_status):
-        # type: (str) -> None
+    def finish(self, final_status: str) -> None:
         if self._finished:
             return
-        self._update(
-            "finished with status '{final_status}'".format(**locals()))
+        self._update(f"finished with status '{final_status}'")
         self._finished = True
 
 
 class RateLimiter:
-    def __init__(self, min_update_interval_seconds):
-        # type: (float) -> None
+    def __init__(self, min_update_interval_seconds: float) -> None:
         self._min_update_interval_seconds = min_update_interval_seconds
-        self._last_update = 0  # type: float
+        self._last_update: float = 0
 
-    def ready(self):
-        # type: () -> bool
+    def ready(self) -> bool:
         now = time.time()
         delta = now - self._last_update
         return delta >= self._min_update_interval_seconds
 
-    def reset(self):
-        # type: () -> None
+    def reset(self) -> None:
         self._last_update = time.time()
 
 
 @contextlib.contextmanager
-def open_spinner(message):
-    # type: (str) -> Iterator[SpinnerInterface]
+def open_spinner(message: str) -> Iterator[SpinnerInterface]:
     # Interactive spinner goes directly to sys.stdout rather than being routed
     # through the logging system, but it acts like it has level INFO,
     # i.e. it's only displayed if we're at level INFO or better.
     # Non-interactive spinner goes through the logging system, so it is always
     # in sync with logging configuration.
     if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO:
-        spinner = InteractiveSpinner(message)  # type: SpinnerInterface
+        spinner: SpinnerInterface = InteractiveSpinner(message)
     else:
         spinner = NonInteractiveSpinner(message)
     try:
@@ -152,8 +139,7 @@ def open_spinner(message):
 
 
 @contextlib.contextmanager
-def hidden_cursor(file):
-    # type: (IO[str]) -> Iterator[None]
+def hidden_cursor(file: IO[str]) -> Iterator[None]:
     # The Windows terminal does not support the hide/show cursor ANSI codes,
     # even via colorama. So don't even try.
     if WINDOWS:
diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py
index f2411201c47..c72f24f30e2 100644
--- a/src/pip/_internal/commands/__init__.py
+++ b/src/pip/_internal/commands/__init__.py
@@ -3,92 +3,105 @@
 """
 
 import importlib
-from collections import OrderedDict, namedtuple
+from collections import namedtuple
+from typing import Any, Dict, Optional
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.cli.base_command import Command
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Optional
+CommandInfo = namedtuple("CommandInfo", "module_path, class_name, summary")
 
-    from pip._internal.cli.base_command import Command
-
-
-CommandInfo = namedtuple('CommandInfo', 'module_path, class_name, summary')
-
-# The ordering matters for help display.
-#    Also, even though the module path starts with the same
-# "pip._internal.commands" prefix in each case, we include the full path
-# because it makes testing easier (specifically when modifying commands_dict
-# in test setup / teardown by adding info for a FakeCommand class defined
-# in a test-related module).
-#    Finally, we need to pass an iterable of pairs here rather than a dict
-# so that the ordering won't be lost when using Python 2.7.
-commands_dict = OrderedDict([
-    ('install', CommandInfo(
-        'pip._internal.commands.install', 'InstallCommand',
-        'Install packages.',
-    )),
-    ('download', CommandInfo(
-        'pip._internal.commands.download', 'DownloadCommand',
-        'Download packages.',
-    )),
-    ('uninstall', CommandInfo(
-        'pip._internal.commands.uninstall', 'UninstallCommand',
-        'Uninstall packages.',
-    )),
-    ('freeze', CommandInfo(
-        'pip._internal.commands.freeze', 'FreezeCommand',
-        'Output installed packages in requirements format.',
-    )),
-    ('list', CommandInfo(
-        'pip._internal.commands.list', 'ListCommand',
-        'List installed packages.',
-    )),
-    ('show', CommandInfo(
-        'pip._internal.commands.show', 'ShowCommand',
-        'Show information about installed packages.',
-    )),
-    ('check', CommandInfo(
-        'pip._internal.commands.check', 'CheckCommand',
-        'Verify installed packages have compatible dependencies.',
-    )),
-    ('config', CommandInfo(
-        'pip._internal.commands.configuration', 'ConfigurationCommand',
-        'Manage local and global configuration.',
-    )),
-    ('search', CommandInfo(
-        'pip._internal.commands.search', 'SearchCommand',
-        'Search PyPI for packages.',
-    )),
-    ('cache', CommandInfo(
-        'pip._internal.commands.cache', 'CacheCommand',
+# This dictionary does a bunch of heavy lifting for help output:
+# - Enables avoiding additional (costly) imports for presenting `--help`.
+# - The ordering matters for help display.
+#
+# Even though the module path starts with the same "pip._internal.commands"
+# prefix, the full path makes testing easier (specifically when modifying
+# `commands_dict` in test setup / teardown).
+commands_dict: Dict[str, CommandInfo] = {
+    "install": CommandInfo(
+        "pip._internal.commands.install",
+        "InstallCommand",
+        "Install packages.",
+    ),
+    "download": CommandInfo(
+        "pip._internal.commands.download",
+        "DownloadCommand",
+        "Download packages.",
+    ),
+    "uninstall": CommandInfo(
+        "pip._internal.commands.uninstall",
+        "UninstallCommand",
+        "Uninstall packages.",
+    ),
+    "freeze": CommandInfo(
+        "pip._internal.commands.freeze",
+        "FreezeCommand",
+        "Output installed packages in requirements format.",
+    ),
+    "list": CommandInfo(
+        "pip._internal.commands.list",
+        "ListCommand",
+        "List installed packages.",
+    ),
+    "show": CommandInfo(
+        "pip._internal.commands.show",
+        "ShowCommand",
+        "Show information about installed packages.",
+    ),
+    "check": CommandInfo(
+        "pip._internal.commands.check",
+        "CheckCommand",
+        "Verify installed packages have compatible dependencies.",
+    ),
+    "config": CommandInfo(
+        "pip._internal.commands.configuration",
+        "ConfigurationCommand",
+        "Manage local and global configuration.",
+    ),
+    "search": CommandInfo(
+        "pip._internal.commands.search",
+        "SearchCommand",
+        "Search PyPI for packages.",
+    ),
+    "cache": CommandInfo(
+        "pip._internal.commands.cache",
+        "CacheCommand",
         "Inspect and manage pip's wheel cache.",
-    )),
-    ('wheel', CommandInfo(
-        'pip._internal.commands.wheel', 'WheelCommand',
-        'Build wheels from your requirements.',
-    )),
-    ('hash', CommandInfo(
-        'pip._internal.commands.hash', 'HashCommand',
-        'Compute hashes of package archives.',
-    )),
-    ('completion', CommandInfo(
-        'pip._internal.commands.completion', 'CompletionCommand',
-        'A helper command used for command completion.',
-    )),
-    ('debug', CommandInfo(
-        'pip._internal.commands.debug', 'DebugCommand',
-        'Show information useful for debugging.',
-    )),
-    ('help', CommandInfo(
-        'pip._internal.commands.help', 'HelpCommand',
-        'Show help for commands.',
-    )),
-])  # type: OrderedDict[str, CommandInfo]
+    ),
+    "index": CommandInfo(
+        "pip._internal.commands.index",
+        "IndexCommand",
+        "Inspect information available from package indexes.",
+    ),
+    "wheel": CommandInfo(
+        "pip._internal.commands.wheel",
+        "WheelCommand",
+        "Build wheels from your requirements.",
+    ),
+    "hash": CommandInfo(
+        "pip._internal.commands.hash",
+        "HashCommand",
+        "Compute hashes of package archives.",
+    ),
+    "completion": CommandInfo(
+        "pip._internal.commands.completion",
+        "CompletionCommand",
+        "A helper command used for command completion.",
+    ),
+    "debug": CommandInfo(
+        "pip._internal.commands.debug",
+        "DebugCommand",
+        "Show information useful for debugging.",
+    ),
+    "help": CommandInfo(
+        "pip._internal.commands.help",
+        "HelpCommand",
+        "Show help for commands.",
+    ),
+}
 
 
-def create_command(name, **kwargs):
-    # type: (str, **Any) -> Command
+def create_command(name: str, **kwargs: Any) -> Command:
     """
     Create an instance of the Command class with the given name.
     """
@@ -100,8 +113,7 @@ def create_command(name, **kwargs):
     return command
 
 
-def get_similar_commands(name):
-    # type: (str) -> Optional[str]
+def get_similar_commands(name: str) -> Optional[str]:
     """Command name auto-correct."""
     from difflib import get_close_matches
 
diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py
index d5ac45ad738..f1a489d324f 100644
--- a/src/pip/_internal/commands/cache.py
+++ b/src/pip/_internal/commands/cache.py
@@ -1,19 +1,15 @@
-import logging
 import os
 import textwrap
+from optparse import Values
+from typing import Any, List
 
 import pip._internal.utils.filesystem as filesystem
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import ERROR, SUCCESS
 from pip._internal.exceptions import CommandError, PipError
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.logging import getLogger
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Any, List
-
-
-logger = logging.getLogger(__name__)
+logger = getLogger(__name__)
 
 
 class CacheCommand(Command):
@@ -40,22 +36,20 @@ class CacheCommand(Command):
         %prog purge
     """
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
 
         self.cmd_opts.add_option(
-            '--format',
-            action='store',
-            dest='list_format',
+            "--format",
+            action="store",
+            dest="list_format",
             default="human",
-            choices=('human', 'abspath'),
-            help="Select the output format among: human (default) or abspath"
+            choices=("human", "abspath"),
+            help="Select the output format among: human (default) or abspath",
         )
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[Any]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         handlers = {
             "dir": self.get_cache_dir,
             "info": self.get_cache_info,
@@ -65,8 +59,7 @@ def run(self, options, args):
         }
 
         if not options.cache_dir:
-            logger.error("pip cache commands can not "
-                         "function since cache is disabled.")
+            logger.error("pip cache commands can not function since cache is disabled.")
             return ERROR
 
         # Determine action
@@ -88,78 +81,77 @@ def run(self, options, args):
 
         return SUCCESS
 
-    def get_cache_dir(self, options, args):
-        # type: (Values, List[Any]) -> None
+    def get_cache_dir(self, options: Values, args: List[Any]) -> None:
         if args:
-            raise CommandError('Too many arguments')
+            raise CommandError("Too many arguments")
 
         logger.info(options.cache_dir)
 
-    def get_cache_info(self, options, args):
-        # type: (Values, List[Any]) -> None
+    def get_cache_info(self, options: Values, args: List[Any]) -> None:
         if args:
-            raise CommandError('Too many arguments')
+            raise CommandError("Too many arguments")
 
         num_http_files = len(self._find_http_files(options))
-        num_packages = len(self._find_wheels(options, '*'))
+        num_packages = len(self._find_wheels(options, "*"))
 
-        http_cache_location = self._cache_dir(options, 'http')
-        wheels_cache_location = self._cache_dir(options, 'wheels')
+        http_cache_location = self._cache_dir(options, "http")
+        wheels_cache_location = self._cache_dir(options, "wheels")
         http_cache_size = filesystem.format_directory_size(http_cache_location)
-        wheels_cache_size = filesystem.format_directory_size(
-            wheels_cache_location
+        wheels_cache_size = filesystem.format_directory_size(wheels_cache_location)
+
+        message = (
+            textwrap.dedent(
+                """
+                    Package index page cache location: {http_cache_location}
+                    Package index page cache size: {http_cache_size}
+                    Number of HTTP files: {num_http_files}
+                    Wheels location: {wheels_cache_location}
+                    Wheels size: {wheels_cache_size}
+                    Number of wheels: {package_count}
+                """
+            )
+            .format(
+                http_cache_location=http_cache_location,
+                http_cache_size=http_cache_size,
+                num_http_files=num_http_files,
+                wheels_cache_location=wheels_cache_location,
+                package_count=num_packages,
+                wheels_cache_size=wheels_cache_size,
+            )
+            .strip()
         )
 
-        message = textwrap.dedent("""
-            Package index page cache location: {http_cache_location}
-            Package index page cache size: {http_cache_size}
-            Number of HTTP files: {num_http_files}
-            Wheels location: {wheels_cache_location}
-            Wheels size: {wheels_cache_size}
-            Number of wheels: {package_count}
-        """).format(
-            http_cache_location=http_cache_location,
-            http_cache_size=http_cache_size,
-            num_http_files=num_http_files,
-            wheels_cache_location=wheels_cache_location,
-            package_count=num_packages,
-            wheels_cache_size=wheels_cache_size,
-        ).strip()
-
         logger.info(message)
 
-    def list_cache_items(self, options, args):
-        # type: (Values, List[Any]) -> None
+    def list_cache_items(self, options: Values, args: List[Any]) -> None:
         if len(args) > 1:
-            raise CommandError('Too many arguments')
+            raise CommandError("Too many arguments")
 
         if args:
             pattern = args[0]
         else:
-            pattern = '*'
+            pattern = "*"
 
         files = self._find_wheels(options, pattern)
-        if options.list_format == 'human':
+        if options.list_format == "human":
             self.format_for_human(files)
         else:
             self.format_for_abspath(files)
 
-    def format_for_human(self, files):
-        # type: (List[str]) -> None
+    def format_for_human(self, files: List[str]) -> None:
         if not files:
-            logger.info('Nothing cached.')
+            logger.info("Nothing cached.")
             return
 
         results = []
         for filename in files:
             wheel = os.path.basename(filename)
             size = filesystem.format_file_size(filename)
-            results.append(f' - {wheel} ({size})')
-        logger.info('Cache contents:\n')
-        logger.info('\n'.join(sorted(results)))
+            results.append(f" - {wheel} ({size})")
+        logger.info("Cache contents:\n")
+        logger.info("\n".join(sorted(results)))
 
-    def format_for_abspath(self, files):
-        # type: (List[str]) -> None
+    def format_for_abspath(self, files: List[str]) -> None:
         if not files:
             return
 
@@ -167,49 +159,48 @@ def format_for_abspath(self, files):
         for filename in files:
             results.append(filename)
 
-        logger.info('\n'.join(sorted(results)))
+        logger.info("\n".join(sorted(results)))
 
-    def remove_cache_items(self, options, args):
-        # type: (Values, List[Any]) -> None
+    def remove_cache_items(self, options: Values, args: List[Any]) -> None:
         if len(args) > 1:
-            raise CommandError('Too many arguments')
+            raise CommandError("Too many arguments")
 
         if not args:
-            raise CommandError('Please provide a pattern')
+            raise CommandError("Please provide a pattern")
 
         files = self._find_wheels(options, args[0])
 
-        # Only fetch http files if no specific pattern given
-        if args[0] == '*':
+        no_matching_msg = "No matching packages"
+        if args[0] == "*":
+            # Only fetch http files if no specific pattern given
             files += self._find_http_files(options)
+        else:
+            # Add the pattern to the log message
+            no_matching_msg += ' for pattern "{}"'.format(args[0])
 
         if not files:
-            raise CommandError('No matching packages')
+            logger.warning(no_matching_msg)
 
         for filename in files:
             os.unlink(filename)
-            logger.debug('Removed %s', filename)
-        logger.info('Files removed: %s', len(files))
+            logger.verbose("Removed %s", filename)
+        logger.info("Files removed: %s", len(files))
 
-    def purge_cache(self, options, args):
-        # type: (Values, List[Any]) -> None
+    def purge_cache(self, options: Values, args: List[Any]) -> None:
         if args:
-            raise CommandError('Too many arguments')
+            raise CommandError("Too many arguments")
 
-        return self.remove_cache_items(options, ['*'])
+        return self.remove_cache_items(options, ["*"])
 
-    def _cache_dir(self, options, subdir):
-        # type: (Values, str) -> str
+    def _cache_dir(self, options: Values, subdir: str) -> str:
         return os.path.join(options.cache_dir, subdir)
 
-    def _find_http_files(self, options):
-        # type: (Values) -> List[str]
-        http_dir = self._cache_dir(options, 'http')
-        return filesystem.find_files(http_dir, '*')
+    def _find_http_files(self, options: Values) -> List[str]:
+        http_dir = self._cache_dir(options, "http")
+        return filesystem.find_files(http_dir, "*")
 
-    def _find_wheels(self, options, pattern):
-        # type: (Values, str) -> List[str]
-        wheel_dir = self._cache_dir(options, 'wheels')
+    def _find_wheels(self, options: Values, pattern: str) -> List[str]:
+        wheel_dir = self._cache_dir(options, "wheels")
 
         # The wheel filename format, as specified in PEP 427, is:
         #     {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl
diff --git a/src/pip/_internal/commands/check.py b/src/pip/_internal/commands/check.py
index e066bb63c74..3864220b2b4 100644
--- a/src/pip/_internal/commands/check.py
+++ b/src/pip/_internal/commands/check.py
@@ -1,4 +1,6 @@
 import logging
+from optparse import Values
+from typing import List
 
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import ERROR, SUCCESS
@@ -7,14 +9,9 @@
     create_package_set_from_installed,
 )
 from pip._internal.utils.misc import write_output
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 logger = logging.getLogger(__name__)
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Any, List
-
 
 class CheckCommand(Command):
     """Verify installed packages have compatible dependencies."""
@@ -22,8 +19,7 @@ class CheckCommand(Command):
     usage = """
       %prog [options]"""
 
-    def run(self, options, args):
-        # type: (Values, List[Any]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
 
         package_set, parsing_probs = create_package_set_from_installed()
         missing, conflicting = check_package_set(package_set)
@@ -33,7 +29,9 @@ def run(self, options, args):
             for dependency in missing[project_name]:
                 write_output(
                     "%s %s requires %s, which is not installed.",
-                    project_name, version, dependency[0],
+                    project_name,
+                    version,
+                    dependency[0],
                 )
 
         for project_name in conflicting:
@@ -41,7 +39,11 @@ def run(self, options, args):
             for dep_name, dep_version, req in conflicting[project_name]:
                 write_output(
                     "%s %s has requirement %s, but you have %s %s.",
-                    project_name, version, req, dep_name, dep_version,
+                    project_name,
+                    version,
+                    req,
+                    dep_name,
+                    dep_version,
                 )
 
         if missing or conflicting or parsing_probs:
diff --git a/src/pip/_internal/commands/completion.py b/src/pip/_internal/commands/completion.py
index 2c19d5686d2..c0fb4caf8b2 100644
--- a/src/pip/_internal/commands/completion.py
+++ b/src/pip/_internal/commands/completion.py
@@ -1,21 +1,18 @@
 import sys
 import textwrap
+from optparse import Values
+from typing import List
 
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.utils.misc import get_prog
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
 
 BASE_COMPLETION = """
 # pip {shell} completion start{script}# pip {shell} completion end
 """
 
 COMPLETION_SCRIPTS = {
-    'bash': """
+    "bash": """
         _pip_completion()
         {{
             COMPREPLY=( $( COMP_WORDS="${{COMP_WORDS[*]}}" \\
@@ -24,7 +21,7 @@
         }}
         complete -o default -F _pip_completion {prog}
     """,
-    'zsh': """
+    "zsh": """
         function _pip_completion {{
           local words cword
           read -Ac words
@@ -35,7 +32,7 @@
         }}
         compctl -K _pip_completion {prog}
     """,
-    'fish': """
+    "fish": """
         function __fish_complete_pip
             set -lx COMP_WORDS (commandline -o) ""
             set -lx COMP_CWORD ( \\
@@ -54,43 +51,46 @@ class CompletionCommand(Command):
 
     ignore_require_venv = True
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '--bash', '-b',
-            action='store_const',
-            const='bash',
-            dest='shell',
-            help='Emit completion code for bash')
+            "--bash",
+            "-b",
+            action="store_const",
+            const="bash",
+            dest="shell",
+            help="Emit completion code for bash",
+        )
         self.cmd_opts.add_option(
-            '--zsh', '-z',
-            action='store_const',
-            const='zsh',
-            dest='shell',
-            help='Emit completion code for zsh')
+            "--zsh",
+            "-z",
+            action="store_const",
+            const="zsh",
+            dest="shell",
+            help="Emit completion code for zsh",
+        )
         self.cmd_opts.add_option(
-            '--fish', '-f',
-            action='store_const',
-            const='fish',
-            dest='shell',
-            help='Emit completion code for fish')
+            "--fish",
+            "-f",
+            action="store_const",
+            const="fish",
+            dest="shell",
+            help="Emit completion code for fish",
+        )
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        #  type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         """Prints the completion code of the given shell"""
         shells = COMPLETION_SCRIPTS.keys()
-        shell_options = ['--' + shell for shell in sorted(shells)]
+        shell_options = ["--" + shell for shell in sorted(shells)]
         if options.shell in shells:
             script = textwrap.dedent(
-                COMPLETION_SCRIPTS.get(options.shell, '').format(
-                    prog=get_prog())
+                COMPLETION_SCRIPTS.get(options.shell, "").format(prog=get_prog())
             )
             print(BASE_COMPLETION.format(script=script, shell=options.shell))
             return SUCCESS
         else:
             sys.stderr.write(
-                'ERROR: You must pass {}\n' .format(' or '.join(shell_options))
+                "ERROR: You must pass {}\n".format(" or ".join(shell_options))
             )
             return SUCCESS
diff --git a/src/pip/_internal/commands/configuration.py b/src/pip/_internal/commands/configuration.py
index a440a2b1774..941ee87b5e0 100644
--- a/src/pip/_internal/commands/configuration.py
+++ b/src/pip/_internal/commands/configuration.py
@@ -1,20 +1,20 @@
 import logging
 import os
 import subprocess
+from optparse import Values
+from typing import Any, List, Optional
 
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import ERROR, SUCCESS
-from pip._internal.configuration import Configuration, get_configuration_files, kinds
+from pip._internal.configuration import (
+    Configuration,
+    Kind,
+    get_configuration_files,
+    kinds,
+)
 from pip._internal.exceptions import PipError
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import get_prog, write_output
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Any, List, Optional
-
-    from pip._internal.configuration import Kind
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +34,7 @@ class ConfigurationCommand(Command):
 
     If none of --user, --global and --site are passed, a virtual
     environment configuration file is used if one is active and the file
-    exists. Otherwise, all modifications happen on the to the user file by
+    exists. Otherwise, all modifications happen to the user file by
     default.
     """
 
@@ -49,47 +49,45 @@ class ConfigurationCommand(Command):
         %prog [] debug
     """
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '--editor',
-            dest='editor',
-            action='store',
+            "--editor",
+            dest="editor",
+            action="store",
             default=None,
             help=(
-                'Editor to use to edit the file. Uses VISUAL or EDITOR '
-                'environment variables if not provided.'
-            )
+                "Editor to use to edit the file. Uses VISUAL or EDITOR "
+                "environment variables if not provided."
+            ),
         )
 
         self.cmd_opts.add_option(
-            '--global',
-            dest='global_file',
-            action='store_true',
+            "--global",
+            dest="global_file",
+            action="store_true",
             default=False,
-            help='Use the system-wide configuration file only'
+            help="Use the system-wide configuration file only",
         )
 
         self.cmd_opts.add_option(
-            '--user',
-            dest='user_file',
-            action='store_true',
+            "--user",
+            dest="user_file",
+            action="store_true",
             default=False,
-            help='Use the user configuration file only'
+            help="Use the user configuration file only",
         )
 
         self.cmd_opts.add_option(
-            '--site',
-            dest='site_file',
-            action='store_true',
+            "--site",
+            dest="site_file",
+            action="store_true",
             default=False,
-            help='Use the current environment configuration file only'
+            help="Use the current environment configuration file only",
         )
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         handlers = {
             "list": self.list_values,
             "edit": self.open_in_editor,
@@ -134,13 +132,16 @@ def run(self, options, args):
 
         return SUCCESS
 
-    def _determine_file(self, options, need_value):
-        # type: (Values, bool) -> Optional[Kind]
-        file_options = [key for key, value in (
-            (kinds.USER, options.user_file),
-            (kinds.GLOBAL, options.global_file),
-            (kinds.SITE, options.site_file),
-        ) if value]
+    def _determine_file(self, options: Values, need_value: bool) -> Optional[Kind]:
+        file_options = [
+            key
+            for key, value in (
+                (kinds.USER, options.user_file),
+                (kinds.GLOBAL, options.global_file),
+                (kinds.SITE, options.site_file),
+            )
+            if value
+        ]
 
         if not file_options:
             if not need_value:
@@ -161,36 +162,31 @@ def _determine_file(self, options, need_value):
             "(--user, --site, --global) to perform."
         )
 
-    def list_values(self, options, args):
-        # type: (Values, List[str]) -> None
+    def list_values(self, options: Values, args: List[str]) -> None:
         self._get_n_args(args, "list", n=0)
 
         for key, value in sorted(self.configuration.items()):
             write_output("%s=%r", key, value)
 
-    def get_name(self, options, args):
-        # type: (Values, List[str]) -> None
+    def get_name(self, options: Values, args: List[str]) -> None:
         key = self._get_n_args(args, "get [name]", n=1)
         value = self.configuration.get_value(key)
 
         write_output("%s", value)
 
-    def set_name_value(self, options, args):
-        # type: (Values, List[str]) -> None
+    def set_name_value(self, options: Values, args: List[str]) -> None:
         key, value = self._get_n_args(args, "set [name] [value]", n=2)
         self.configuration.set_value(key, value)
 
         self._save_configuration()
 
-    def unset_name(self, options, args):
-        # type: (Values, List[str]) -> None
+    def unset_name(self, options: Values, args: List[str]) -> None:
         key = self._get_n_args(args, "unset [name]", n=1)
         self.configuration.unset_value(key)
 
         self._save_configuration()
 
-    def list_config_values(self, options, args):
-        # type: (Values, List[str]) -> None
+    def list_config_values(self, options: Values, args: List[str]) -> None:
         """List config key-value pairs across different config files"""
         self._get_n_args(args, "debug", n=0)
 
@@ -202,30 +198,25 @@ def list_config_values(self, options, args):
             for fname in files:
                 with indent_log():
                     file_exists = os.path.exists(fname)
-                    write_output("%s, exists: %r",
-                                 fname, file_exists)
+                    write_output("%s, exists: %r", fname, file_exists)
                     if file_exists:
                         self.print_config_file_values(variant)
 
-    def print_config_file_values(self, variant):
-        # type: (Kind) -> None
+    def print_config_file_values(self, variant: Kind) -> None:
         """Get key-value pairs from the file of a variant"""
-        for name, value in self.configuration.\
-                get_values_in_config(variant).items():
+        for name, value in self.configuration.get_values_in_config(variant).items():
             with indent_log():
                 write_output("%s: %s", name, value)
 
-    def print_env_var_values(self):
-        # type: () -> None
+    def print_env_var_values(self) -> None:
         """Get key-values pairs present as environment variables"""
-        write_output("%s:", 'env_var')
+        write_output("%s:", "env_var")
         with indent_log():
             for key, value in sorted(self.configuration.get_environ_vars()):
-                env_var = f'PIP_{key.upper()}'
+                env_var = f"PIP_{key.upper()}"
                 write_output("%s=%r", env_var, value)
 
-    def open_in_editor(self, options, args):
-        # type: (Values, List[str]) -> None
+    def open_in_editor(self, options: Values, args: List[str]) -> None:
         editor = self._determine_editor(options)
 
         fname = self.configuration.get_file_to_edit()
@@ -234,19 +225,20 @@ def open_in_editor(self, options, args):
 
         try:
             subprocess.check_call([editor, fname])
+        except FileNotFoundError as e:
+            if not e.filename:
+                e.filename = editor
+            raise
         except subprocess.CalledProcessError as e:
             raise PipError(
-                "Editor Subprocess exited with exit code {}"
-                .format(e.returncode)
+                "Editor Subprocess exited with exit code {}".format(e.returncode)
             )
 
-    def _get_n_args(self, args, example, n):
-        # type: (List[str], str, int) -> Any
-        """Helper to make sure the command got the right number of arguments
-        """
+    def _get_n_args(self, args: List[str], example: str, n: int) -> Any:
+        """Helper to make sure the command got the right number of arguments"""
         if len(args) != n:
             msg = (
-                'Got unexpected number of arguments, expected {}. '
+                "Got unexpected number of arguments, expected {}. "
                 '(example: "{} config {}")'
             ).format(n, get_prog(), example)
             raise PipError(msg)
@@ -256,8 +248,7 @@ def _get_n_args(self, args, example, n):
         else:
             return args
 
-    def _save_configuration(self):
-        # type: () -> None
+    def _save_configuration(self) -> None:
         # We successfully ran a modifying command. Need to save the
         # configuration.
         try:
@@ -268,8 +259,7 @@ def _save_configuration(self):
             )
             raise PipError("Internal Error.")
 
-    def _determine_editor(self, options):
-        # type: (Values) -> str
+    def _determine_editor(self, options: Values) -> str:
         if options.editor is not None:
             return options.editor
         elif "VISUAL" in os.environ:
diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py
index 61df18e20cd..d3f1f28de4c 100644
--- a/src/pip/_internal/commands/debug.py
+++ b/src/pip/_internal/commands/debug.py
@@ -2,121 +2,109 @@
 import logging
 import os
 import sys
+from optparse import Values
+from types import ModuleType
+from typing import Any, Dict, List, Optional
 
 import pip._vendor
-from pip._vendor import pkg_resources
 from pip._vendor.certifi import where
+from pip._vendor.packaging.version import parse as parse_version
 
 from pip import __file__ as pip_location
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.cmdoptions import make_target_python
 from pip._internal.cli.status_codes import SUCCESS
+from pip._internal.configuration import Configuration
+from pip._internal.metadata import get_environment
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import get_pip_version
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from types import ModuleType
-    from typing import Dict, List, Optional
-
-    from pip._internal.configuration import Configuration
 
 logger = logging.getLogger(__name__)
 
 
-def show_value(name, value):
-    # type: (str, Optional[str]) -> None
-    logger.info('%s: %s', name, value)
+def show_value(name: str, value: Any) -> None:
+    logger.info("%s: %s", name, value)
 
 
-def show_sys_implementation():
-    # type: () -> None
-    logger.info('sys.implementation:')
+def show_sys_implementation() -> None:
+    logger.info("sys.implementation:")
     implementation_name = sys.implementation.name
     with indent_log():
-        show_value('name', implementation_name)
+        show_value("name", implementation_name)
 
 
-def create_vendor_txt_map():
-    # type: () -> Dict[str, str]
+def create_vendor_txt_map() -> Dict[str, str]:
     vendor_txt_path = os.path.join(
-        os.path.dirname(pip_location),
-        '_vendor',
-        'vendor.txt'
+        os.path.dirname(pip_location), "_vendor", "vendor.txt"
     )
 
     with open(vendor_txt_path) as f:
         # Purge non version specifying lines.
         # Also, remove any space prefix or suffixes (including comments).
-        lines = [line.strip().split(' ', 1)[0]
-                 for line in f.readlines() if '==' in line]
+        lines = [
+            line.strip().split(" ", 1)[0] for line in f.readlines() if "==" in line
+        ]
 
     # Transform into "module" -> version dict.
-    return dict(line.split('==', 1) for line in lines)  # type: ignore
+    return dict(line.split("==", 1) for line in lines)  # type: ignore
 
 
-def get_module_from_module_name(module_name):
-    # type: (str) -> ModuleType
+def get_module_from_module_name(module_name: str) -> ModuleType:
     # Module name can be uppercase in vendor.txt for some reason...
     module_name = module_name.lower()
     # PATCH: setuptools is actually only pkg_resources.
-    if module_name == 'setuptools':
-        module_name = 'pkg_resources'
-
-    __import__(
-        f'pip._vendor.{module_name}',
-        globals(),
-        locals(),
-        level=0
-    )
+    if module_name == "setuptools":
+        module_name = "pkg_resources"
+
+    __import__(f"pip._vendor.{module_name}", globals(), locals(), level=0)
     return getattr(pip._vendor, module_name)
 
 
-def get_vendor_version_from_module(module_name):
-    # type: (str) -> Optional[str]
+def get_vendor_version_from_module(module_name: str) -> Optional[str]:
     module = get_module_from_module_name(module_name)
-    version = getattr(module, '__version__', None)
+    version = getattr(module, "__version__", None)
 
     if not version:
-        # Try to find version in debundled module info
-        pkg_set = pkg_resources.WorkingSet([os.path.dirname(module.__file__)])
-        package = pkg_set.find(pkg_resources.Requirement.parse(module_name))
-        version = getattr(package, 'version', None)
+        # Try to find version in debundled module info.
+        env = get_environment([os.path.dirname(module.__file__)])
+        dist = env.get_distribution(module_name)
+        if dist:
+            version = str(dist.version)
 
     return version
 
 
-def show_actual_vendor_versions(vendor_txt_versions):
-    # type: (Dict[str, str]) -> None
+def show_actual_vendor_versions(vendor_txt_versions: Dict[str, str]) -> None:
     """Log the actual version and print extra info if there is
     a conflict or if the actual version could not be imported.
     """
     for module_name, expected_version in vendor_txt_versions.items():
-        extra_message = ''
+        extra_message = ""
         actual_version = get_vendor_version_from_module(module_name)
         if not actual_version:
-            extra_message = ' (Unable to locate actual module version, using'\
-                            ' vendor.txt specified version)'
+            extra_message = (
+                " (Unable to locate actual module version, using"
+                " vendor.txt specified version)"
+            )
             actual_version = expected_version
-        elif actual_version != expected_version:
-            extra_message = ' (CONFLICT: vendor.txt suggests version should'\
-                            ' be {})'.format(expected_version)
-        logger.info('%s==%s%s', module_name, actual_version, extra_message)
+        elif parse_version(actual_version) != parse_version(expected_version):
+            extra_message = (
+                " (CONFLICT: vendor.txt suggests version should"
+                " be {})".format(expected_version)
+            )
+        logger.info("%s==%s%s", module_name, actual_version, extra_message)
 
 
-def show_vendor_versions():
-    # type: () -> None
-    logger.info('vendored library versions:')
+def show_vendor_versions() -> None:
+    logger.info("vendored library versions:")
 
     vendor_txt_versions = create_vendor_txt_map()
     with indent_log():
         show_actual_vendor_versions(vendor_txt_versions)
 
 
-def show_tags(options):
-    # type: (Values) -> None
+def show_tags(options: Values) -> None:
     tag_limit = 10
 
     target_python = make_target_python(options)
@@ -124,11 +112,11 @@ def show_tags(options):
 
     # Display the target options that were explicitly provided.
     formatted_target = target_python.format_given()
-    suffix = ''
+    suffix = ""
     if formatted_target:
-        suffix = f' (target: {formatted_target})'
+        suffix = f" (target: {formatted_target})"
 
-    msg = 'Compatible tags: {}{}'.format(len(tags), suffix)
+    msg = "Compatible tags: {}{}".format(len(tags), suffix)
     logger.info(msg)
 
     if options.verbose < 1 and len(tags) > tag_limit:
@@ -143,30 +131,28 @@ def show_tags(options):
 
         if tags_limited:
             msg = (
-                '...\n'
-                '[First {tag_limit} tags shown. Pass --verbose to show all.]'
+                "...\n[First {tag_limit} tags shown. Pass --verbose to show all.]"
             ).format(tag_limit=tag_limit)
             logger.info(msg)
 
 
-def ca_bundle_info(config):
-    # type: (Configuration) -> str
+def ca_bundle_info(config: Configuration) -> str:
     levels = set()
     for key, _ in config.items():
-        levels.add(key.split('.')[0])
+        levels.add(key.split(".")[0])
 
     if not levels:
         return "Not specified"
 
-    levels_that_override_global = ['install', 'wheel', 'download']
+    levels_that_override_global = ["install", "wheel", "download"]
     global_overriding_level = [
         level for level in levels if level in levels_that_override_global
     ]
     if not global_overriding_level:
-        return 'global'
+        return "global"
 
-    if 'global' in levels:
-        levels.remove('global')
+    if "global" in levels:
+        levels.remove("global")
     return ", ".join(levels)
 
 
@@ -179,34 +165,33 @@ class DebugCommand(Command):
       %prog """
     ignore_require_venv = True
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         cmdoptions.add_target_python_options(self.cmd_opts)
         self.parser.insert_option_group(0, self.cmd_opts)
         self.parser.config.load()
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         logger.warning(
             "This command is only meant for debugging. "
             "Do not use this with automation for parsing and getting these "
             "details, since the output and options of this command may "
             "change without notice."
         )
-        show_value('pip version', get_pip_version())
-        show_value('sys.version', sys.version)
-        show_value('sys.executable', sys.executable)
-        show_value('sys.getdefaultencoding', sys.getdefaultencoding())
-        show_value('sys.getfilesystemencoding', sys.getfilesystemencoding())
+        show_value("pip version", get_pip_version())
+        show_value("sys.version", sys.version)
+        show_value("sys.executable", sys.executable)
+        show_value("sys.getdefaultencoding", sys.getdefaultencoding())
+        show_value("sys.getfilesystemencoding", sys.getfilesystemencoding())
         show_value(
-            'locale.getpreferredencoding', locale.getpreferredencoding(),
+            "locale.getpreferredencoding",
+            locale.getpreferredencoding(),
         )
-        show_value('sys.platform', sys.platform)
+        show_value("sys.platform", sys.platform)
         show_sys_implementation()
 
         show_value("'cert' config value", ca_bundle_info(self.parser.config))
-        show_value("REQUESTS_CA_BUNDLE", os.environ.get('REQUESTS_CA_BUNDLE'))
-        show_value("CURL_CA_BUNDLE", os.environ.get('CURL_CA_BUNDLE'))
+        show_value("REQUESTS_CA_BUNDLE", os.environ.get("REQUESTS_CA_BUNDLE"))
+        show_value("CURL_CA_BUNDLE", os.environ.get("CURL_CA_BUNDLE"))
         show_value("pip._vendor.certifi.where()", where())
         show_value("pip._vendor.DEBUNDLED", pip._vendor.DEBUNDLED)
 
diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py
index 0f09fcc0eed..a6d7e628f2b 100644
--- a/src/pip/_internal/commands/download.py
+++ b/src/pip/_internal/commands/download.py
@@ -1,18 +1,15 @@
 import logging
 import os
+from optparse import Values
+from typing import List
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.cmdoptions import make_target_python
 from pip._internal.cli.req_command import RequirementCommand, with_cleanup
 from pip._internal.cli.status_codes import SUCCESS
-from pip._internal.req.req_tracker import get_requirement_tracker
+from pip._internal.operations.build.build_tracker import get_build_tracker
 from pip._internal.utils.misc import ensure_dir, normalize_path, write_output
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -37,11 +34,9 @@ class DownloadCommand(RequirementCommand):
       %prog [options]  ...
       %prog [options]  ..."""
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(cmdoptions.constraints())
         self.cmd_opts.add_option(cmdoptions.requirements())
-        self.cmd_opts.add_option(cmdoptions.build_dir())
         self.cmd_opts.add_option(cmdoptions.no_deps())
         self.cmd_opts.add_option(cmdoptions.global_options())
         self.cmd_opts.add_option(cmdoptions.no_binary())
@@ -57,11 +52,14 @@ def add_options(self):
         self.cmd_opts.add_option(cmdoptions.ignore_requires_python())
 
         self.cmd_opts.add_option(
-            '-d', '--dest', '--destination-dir', '--destination-directory',
-            dest='download_dir',
-            metavar='dir',
+            "-d",
+            "--dest",
+            "--destination-dir",
+            "--destination-directory",
+            dest="download_dir",
+            metavar="dir",
             default=os.curdir,
-            help=("Download packages into ."),
+            help="Download packages into .",
         )
 
         cmdoptions.add_target_python_options(self.cmd_opts)
@@ -75,8 +73,7 @@ def add_options(self):
         self.parser.insert_option_group(0, self.cmd_opts)
 
     @with_cleanup
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
 
         options.ignore_installed = True
         # editable doesn't really make sense for `pip download`, but the bowels
@@ -98,7 +95,7 @@ def run(self, options, args):
             ignore_requires_python=options.ignore_requires_python,
         )
 
-        req_tracker = self.enter_context(get_requirement_tracker())
+        build_tracker = self.enter_context(get_build_tracker())
 
         directory = TempDirectory(
             delete=not options.no_clean,
@@ -111,11 +108,12 @@ def run(self, options, args):
         preparer = self.make_requirement_preparer(
             temp_build_dir=directory,
             options=options,
-            req_tracker=req_tracker,
+            build_tracker=build_tracker,
             session=session,
             finder=finder,
             download_dir=options.download_dir,
             use_user_site=False,
+            verbosity=self.verbosity,
         )
 
         resolver = self.make_resolver(
@@ -128,17 +126,15 @@ def run(self, options, args):
 
         self.trace_basic_info(finder)
 
-        requirement_set = resolver.resolve(
-            reqs, check_supported_wheels=True
-        )
+        requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
 
-        downloaded = []  # type: List[str]
+        downloaded: List[str] = []
         for req in requirement_set.requirements.values():
             if req.satisfied_by is None:
                 assert req.name is not None
                 preparer.save_linked_requirement(req)
                 downloaded.append(req.name)
         if downloaded:
-            write_output('Successfully downloaded %s', ' '.join(downloaded))
+            write_output("Successfully downloaded %s", " ".join(downloaded))
 
         return SUCCESS
diff --git a/src/pip/_internal/commands/freeze.py b/src/pip/_internal/commands/freeze.py
index bf20db6c370..5fa6d39b2c7 100644
--- a/src/pip/_internal/commands/freeze.py
+++ b/src/pip/_internal/commands/freeze.py
@@ -1,18 +1,14 @@
 import sys
+from optparse import Values
+from typing import List
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.operations.freeze import freeze
 from pip._internal.utils.compat import stdlib_pkgs
-from pip._internal.utils.deprecation import deprecated
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'}
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
+DEV_PKGS = {"pip", "setuptools", "distribute", "wheel"}
 
 
 class FreezeCommand(Command):
@@ -26,56 +22,59 @@ class FreezeCommand(Command):
       %prog [options]"""
     log_streams = ("ext://sys.stderr", "ext://sys.stderr")
 
-    def add_options(self):
-        # type: () -> None
-        self.cmd_opts.add_option(
-            '-r', '--requirement',
-            dest='requirements',
-            action='append',
-            default=[],
-            metavar='file',
-            help="Use the order in the given requirements file and its "
-                 "comments when generating output. This option can be "
-                 "used multiple times.")
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '-f', '--find-links',
-            dest='find_links',
-            action='append',
+            "-r",
+            "--requirement",
+            dest="requirements",
+            action="append",
             default=[],
-            metavar='URL',
-            help='URL for finding packages, which will be added to the '
-                 'output.')
+            metavar="file",
+            help=(
+                "Use the order in the given requirements file and its "
+                "comments when generating output. This option can be "
+                "used multiple times."
+            ),
+        )
         self.cmd_opts.add_option(
-            '-l', '--local',
-            dest='local',
-            action='store_true',
+            "-l",
+            "--local",
+            dest="local",
+            action="store_true",
             default=False,
-            help='If in a virtualenv that has global access, do not output '
-                 'globally-installed packages.')
+            help=(
+                "If in a virtualenv that has global access, do not output "
+                "globally-installed packages."
+            ),
+        )
         self.cmd_opts.add_option(
-            '--user',
-            dest='user',
-            action='store_true',
+            "--user",
+            dest="user",
+            action="store_true",
             default=False,
-            help='Only output packages installed in user-site.')
+            help="Only output packages installed in user-site.",
+        )
         self.cmd_opts.add_option(cmdoptions.list_path())
         self.cmd_opts.add_option(
-            '--all',
-            dest='freeze_all',
-            action='store_true',
-            help='Do not skip these packages in the output:'
-                 ' {}'.format(', '.join(DEV_PKGS)))
+            "--all",
+            dest="freeze_all",
+            action="store_true",
+            help=(
+                "Do not skip these packages in the output:"
+                " {}".format(", ".join(DEV_PKGS))
+            ),
+        )
         self.cmd_opts.add_option(
-            '--exclude-editable',
-            dest='exclude_editable',
-            action='store_true',
-            help='Exclude editable package from output.')
+            "--exclude-editable",
+            dest="exclude_editable",
+            action="store_true",
+            help="Exclude editable package from output.",
+        )
         self.cmd_opts.add_option(cmdoptions.list_exclude())
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         skip = set(stdlib_pkgs)
         if not options.freeze_all:
             skip.update(DEV_PKGS)
@@ -85,17 +84,8 @@ def run(self, options, args):
 
         cmdoptions.check_list_path_option(options)
 
-        if options.find_links:
-            deprecated(
-                "--find-links option in pip freeze is deprecated.",
-                replacement=None,
-                gone_in="21.2",
-                issue=9069,
-            )
-
         for line in freeze(
             requirement=options.requirements,
-            find_links=options.find_links,
             local_only=options.local,
             user_only=options.user,
             paths=options.path,
@@ -103,5 +93,5 @@ def run(self, options, args):
             skip=skip,
             exclude_editable=options.exclude_editable,
         ):
-            sys.stdout.write(line + '\n')
+            sys.stdout.write(line + "\n")
         return SUCCESS
diff --git a/src/pip/_internal/commands/hash.py b/src/pip/_internal/commands/hash.py
index db68f6ce7bc..042dac813e7 100644
--- a/src/pip/_internal/commands/hash.py
+++ b/src/pip/_internal/commands/hash.py
@@ -1,16 +1,13 @@
 import hashlib
 import logging
 import sys
+from optparse import Values
+from typing import List
 
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import ERROR, SUCCESS
 from pip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES
 from pip._internal.utils.misc import read_chunks, write_output
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -23,38 +20,39 @@ class HashCommand(Command):
     installs.
     """
 
-    usage = '%prog [options]  ...'
+    usage = "%prog [options]  ..."
     ignore_require_venv = True
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '-a', '--algorithm',
-            dest='algorithm',
+            "-a",
+            "--algorithm",
+            dest="algorithm",
             choices=STRONG_HASHES,
-            action='store',
+            action="store",
             default=FAVORITE_HASH,
-            help='The hash algorithm to use: one of {}'.format(
-                 ', '.join(STRONG_HASHES)))
+            help="The hash algorithm to use: one of {}".format(
+                ", ".join(STRONG_HASHES)
+            ),
+        )
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         if not args:
             self.parser.print_usage(sys.stderr)
             return ERROR
 
         algorithm = options.algorithm
         for path in args:
-            write_output('%s:\n--hash=%s:%s',
-                         path, algorithm, _hash_of_file(path, algorithm))
+            write_output(
+                "%s:\n--hash=%s:%s", path, algorithm, _hash_of_file(path, algorithm)
+            )
         return SUCCESS
 
 
-def _hash_of_file(path, algorithm):
-    # type: (str, str) -> str
+def _hash_of_file(path: str, algorithm: str) -> str:
     """Return the hash digest of a file."""
-    with open(path, 'rb') as archive:
+    with open(path, "rb") as archive:
         hash = hashlib.new(algorithm)
         for chunk in read_chunks(archive):
             hash.update(chunk)
diff --git a/src/pip/_internal/commands/help.py b/src/pip/_internal/commands/help.py
index 8372ac615dc..62066318b74 100644
--- a/src/pip/_internal/commands/help.py
+++ b/src/pip/_internal/commands/help.py
@@ -1,11 +1,9 @@
+from optparse import Values
+from typing import List
+
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.exceptions import CommandError
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
 
 
 class HelpCommand(Command):
@@ -15,8 +13,7 @@ class HelpCommand(Command):
       %prog """
     ignore_require_venv = True
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         from pip._internal.commands import (
             commands_dict,
             create_command,
@@ -36,7 +33,7 @@ def run(self, options, args):
             if guess:
                 msg.append(f'maybe you meant "{guess}"')
 
-            raise CommandError(' - '.join(msg))
+            raise CommandError(" - ".join(msg))
 
         command = create_command(cmd_name)
         command.parser.print_help()
diff --git a/src/pip/_internal/commands/index.py b/src/pip/_internal/commands/index.py
new file mode 100644
index 00000000000..9d8aae3b542
--- /dev/null
+++ b/src/pip/_internal/commands/index.py
@@ -0,0 +1,139 @@
+import logging
+from optparse import Values
+from typing import Any, Iterable, List, Optional, Union
+
+from pip._vendor.packaging.version import LegacyVersion, Version
+
+from pip._internal.cli import cmdoptions
+from pip._internal.cli.req_command import IndexGroupCommand
+from pip._internal.cli.status_codes import ERROR, SUCCESS
+from pip._internal.commands.search import print_dist_installation_info
+from pip._internal.exceptions import CommandError, DistributionNotFound, PipError
+from pip._internal.index.collector import LinkCollector
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.models.selection_prefs import SelectionPreferences
+from pip._internal.models.target_python import TargetPython
+from pip._internal.network.session import PipSession
+from pip._internal.utils.misc import write_output
+
+logger = logging.getLogger(__name__)
+
+
+class IndexCommand(IndexGroupCommand):
+    """
+    Inspect information available from package indexes.
+    """
+
+    usage = """
+        %prog versions 
+    """
+
+    def add_options(self) -> None:
+        cmdoptions.add_target_python_options(self.cmd_opts)
+
+        self.cmd_opts.add_option(cmdoptions.ignore_requires_python())
+        self.cmd_opts.add_option(cmdoptions.pre())
+        self.cmd_opts.add_option(cmdoptions.no_binary())
+        self.cmd_opts.add_option(cmdoptions.only_binary())
+
+        index_opts = cmdoptions.make_option_group(
+            cmdoptions.index_group,
+            self.parser,
+        )
+
+        self.parser.insert_option_group(0, index_opts)
+        self.parser.insert_option_group(0, self.cmd_opts)
+
+    def run(self, options: Values, args: List[str]) -> int:
+        handlers = {
+            "versions": self.get_available_package_versions,
+        }
+
+        logger.warning(
+            "pip index is currently an experimental command. "
+            "It may be removed/changed in a future release "
+            "without prior warning."
+        )
+
+        # Determine action
+        if not args or args[0] not in handlers:
+            logger.error(
+                "Need an action (%s) to perform.",
+                ", ".join(sorted(handlers)),
+            )
+            return ERROR
+
+        action = args[0]
+
+        # Error handling happens here, not in the action-handlers.
+        try:
+            handlers[action](options, args[1:])
+        except PipError as e:
+            logger.error(e.args[0])
+            return ERROR
+
+        return SUCCESS
+
+    def _build_package_finder(
+        self,
+        options: Values,
+        session: PipSession,
+        target_python: Optional[TargetPython] = None,
+        ignore_requires_python: Optional[bool] = None,
+    ) -> PackageFinder:
+        """
+        Create a package finder appropriate to the index command.
+        """
+        link_collector = LinkCollector.create(session, options=options)
+
+        # Pass allow_yanked=False to ignore yanked versions.
+        selection_prefs = SelectionPreferences(
+            allow_yanked=False,
+            allow_all_prereleases=options.pre,
+            ignore_requires_python=ignore_requires_python,
+        )
+
+        return PackageFinder.create(
+            link_collector=link_collector,
+            selection_prefs=selection_prefs,
+            target_python=target_python,
+            use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
+        )
+
+    def get_available_package_versions(self, options: Values, args: List[Any]) -> None:
+        if len(args) != 1:
+            raise CommandError("You need to specify exactly one argument")
+
+        target_python = cmdoptions.make_target_python(options)
+        query = args[0]
+
+        with self._build_session(options) as session:
+            finder = self._build_package_finder(
+                options=options,
+                session=session,
+                target_python=target_python,
+                ignore_requires_python=options.ignore_requires_python,
+            )
+
+            versions: Iterable[Union[LegacyVersion, Version]] = (
+                candidate.version for candidate in finder.find_all_candidates(query)
+            )
+
+            if not options.pre:
+                # Remove prereleases
+                versions = (
+                    version for version in versions if not version.is_prerelease
+                )
+            versions = set(versions)
+
+            if not versions:
+                raise DistributionNotFound(
+                    "No matching distribution found for {}".format(query)
+                )
+
+            formatted_versions = [str(ver) for ver in sorted(versions, reverse=True)]
+            latest = formatted_versions[0]
+
+        write_output("{} ({})".format(query, latest))
+        write_output("Available versions: {}".format(", ".join(formatted_versions)))
+        print_dist_installation_info(query, latest)
diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py
index e303adf86fc..db41b9a1b03 100644
--- a/src/pip/_internal/commands/install.py
+++ b/src/pip/_internal/commands/install.py
@@ -1,56 +1,57 @@
 import errno
-import logging
 import operator
 import os
 import shutil
 import site
-from optparse import SUPPRESS_HELP
+from optparse import SUPPRESS_HELP, Values
+from typing import Iterable, List, Optional
 
-from pip._vendor import pkg_resources
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cache import WheelCache
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.cmdoptions import make_target_python
-from pip._internal.cli.req_command import RequirementCommand, with_cleanup
+from pip._internal.cli.req_command import (
+    RequirementCommand,
+    warn_if_run_as_root,
+    with_cleanup,
+)
 from pip._internal.cli.status_codes import ERROR, SUCCESS
 from pip._internal.exceptions import CommandError, InstallationError
-from pip._internal.locations import distutils_scheme
-from pip._internal.operations.check import check_install_conflicts
+from pip._internal.locations import get_scheme
+from pip._internal.metadata import get_environment
+from pip._internal.models.format_control import FormatControl
+from pip._internal.operations.build.build_tracker import get_build_tracker
+from pip._internal.operations.check import ConflictDetails, check_install_conflicts
 from pip._internal.req import install_given_reqs
-from pip._internal.req.req_tracker import get_requirement_tracker
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.distutils_args import parse_distutils_args
 from pip._internal.utils.filesystem import test_writable_dir
+from pip._internal.utils.logging import getLogger
 from pip._internal.utils.misc import (
     ensure_dir,
-    get_installed_version,
     get_pip_version,
     protect_pip_from_modification_on_windows,
     write_output,
 )
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-from pip._internal.utils.virtualenv import virtualenv_no_global
-from pip._internal.wheel_builder import build, should_build_for_install_command
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Iterable, List, Optional
-
-    from pip._internal.models.format_control import FormatControl
-    from pip._internal.operations.check import ConflictDetails
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._internal.wheel_builder import BinaryAllowedPredicate
-
+from pip._internal.utils.virtualenv import (
+    running_under_virtualenv,
+    virtualenv_no_global,
+)
+from pip._internal.wheel_builder import (
+    BinaryAllowedPredicate,
+    build,
+    should_build_for_install_command,
+)
 
-logger = logging.getLogger(__name__)
+logger = getLogger(__name__)
 
 
-def get_check_binary_allowed(format_control):
-    # type: (FormatControl) -> BinaryAllowedPredicate
-    def check_binary_allowed(req):
-        # type: (InstallRequirement) -> bool
-        canonical_name = canonicalize_name(req.name)
+def get_check_binary_allowed(format_control: FormatControl) -> BinaryAllowedPredicate:
+    def check_binary_allowed(req: InstallRequirement) -> bool:
+        canonical_name = canonicalize_name(req.name or "")
         allowed_formats = format_control.get_allowed_formats(canonical_name)
         return "binary" in allowed_formats
 
@@ -77,8 +78,7 @@ class InstallCommand(RequirementCommand):
       %prog [options] [-e]  ...
       %prog [options]  ..."""
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(cmdoptions.requirements())
         self.cmd_opts.add_option(cmdoptions.constraints())
         self.cmd_opts.add_option(cmdoptions.no_deps())
@@ -86,87 +86,103 @@ def add_options(self):
 
         self.cmd_opts.add_option(cmdoptions.editable())
         self.cmd_opts.add_option(
-            '-t', '--target',
-            dest='target_dir',
-            metavar='dir',
+            "-t",
+            "--target",
+            dest="target_dir",
+            metavar="dir",
             default=None,
-            help='Install packages into . '
-                 'By default this will not replace existing files/folders in '
-                 '. Use --upgrade to replace existing packages in  '
-                 'with new versions.'
+            help=(
+                "Install packages into . "
+                "By default this will not replace existing files/folders in "
+                ". Use --upgrade to replace existing packages in  "
+                "with new versions."
+            ),
         )
         cmdoptions.add_target_python_options(self.cmd_opts)
 
         self.cmd_opts.add_option(
-            '--user',
-            dest='use_user_site',
-            action='store_true',
-            help="Install to the Python user install directory for your "
-                 "platform. Typically ~/.local/, or %APPDATA%\\Python on "
-                 "Windows. (See the Python documentation for site.USER_BASE "
-                 "for full details.)")
+            "--user",
+            dest="use_user_site",
+            action="store_true",
+            help=(
+                "Install to the Python user install directory for your "
+                "platform. Typically ~/.local/, or %APPDATA%\\Python on "
+                "Windows. (See the Python documentation for site.USER_BASE "
+                "for full details.)"
+            ),
+        )
         self.cmd_opts.add_option(
-            '--no-user',
-            dest='use_user_site',
-            action='store_false',
-            help=SUPPRESS_HELP)
+            "--no-user",
+            dest="use_user_site",
+            action="store_false",
+            help=SUPPRESS_HELP,
+        )
         self.cmd_opts.add_option(
-            '--root',
-            dest='root_path',
-            metavar='dir',
+            "--root",
+            dest="root_path",
+            metavar="dir",
             default=None,
-            help="Install everything relative to this alternate root "
-                 "directory.")
+            help="Install everything relative to this alternate root directory.",
+        )
         self.cmd_opts.add_option(
-            '--prefix',
-            dest='prefix_path',
-            metavar='dir',
+            "--prefix",
+            dest="prefix_path",
+            metavar="dir",
             default=None,
-            help="Installation prefix where lib, bin and other top-level "
-                 "folders are placed")
-
-        self.cmd_opts.add_option(cmdoptions.build_dir())
+            help=(
+                "Installation prefix where lib, bin and other top-level "
+                "folders are placed"
+            ),
+        )
 
         self.cmd_opts.add_option(cmdoptions.src())
 
         self.cmd_opts.add_option(
-            '-U', '--upgrade',
-            dest='upgrade',
-            action='store_true',
-            help='Upgrade all specified packages to the newest available '
-                 'version. The handling of dependencies depends on the '
-                 'upgrade-strategy used.'
+            "-U",
+            "--upgrade",
+            dest="upgrade",
+            action="store_true",
+            help=(
+                "Upgrade all specified packages to the newest available "
+                "version. The handling of dependencies depends on the "
+                "upgrade-strategy used."
+            ),
         )
 
         self.cmd_opts.add_option(
-            '--upgrade-strategy',
-            dest='upgrade_strategy',
-            default='only-if-needed',
-            choices=['only-if-needed', 'eager'],
-            help='Determines how dependency upgrading should be handled '
-                 '[default: %default]. '
-                 '"eager" - dependencies are upgraded regardless of '
-                 'whether the currently installed version satisfies the '
-                 'requirements of the upgraded package(s). '
-                 '"only-if-needed" -  are upgraded only when they do not '
-                 'satisfy the requirements of the upgraded package(s).'
+            "--upgrade-strategy",
+            dest="upgrade_strategy",
+            default="only-if-needed",
+            choices=["only-if-needed", "eager"],
+            help=(
+                "Determines how dependency upgrading should be handled "
+                "[default: %default]. "
+                '"eager" - dependencies are upgraded regardless of '
+                "whether the currently installed version satisfies the "
+                "requirements of the upgraded package(s). "
+                '"only-if-needed" -  are upgraded only when they do not '
+                "satisfy the requirements of the upgraded package(s)."
+            ),
         )
 
         self.cmd_opts.add_option(
-            '--force-reinstall',
-            dest='force_reinstall',
-            action='store_true',
-            help='Reinstall all packages even if they are already '
-                 'up-to-date.')
+            "--force-reinstall",
+            dest="force_reinstall",
+            action="store_true",
+            help="Reinstall all packages even if they are already up-to-date.",
+        )
 
         self.cmd_opts.add_option(
-            '-I', '--ignore-installed',
-            dest='ignore_installed',
-            action='store_true',
-            help='Ignore the installed packages, overwriting them. '
-                 'This can break your system if the existing package '
-                 'is of a different version or was installed '
-                 'with a different package manager!'
+            "-I",
+            "--ignore-installed",
+            dest="ignore_installed",
+            action="store_true",
+            help=(
+                "Ignore the installed packages, overwriting them. "
+                "This can break your system if the existing package "
+                "is of a different version or was installed "
+                "with a different package manager!"
+            ),
         )
 
         self.cmd_opts.add_option(cmdoptions.ignore_requires_python())
@@ -222,8 +238,7 @@ def add_options(self):
         self.parser.insert_option_group(0, self.cmd_opts)
 
     @with_cleanup
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         if options.use_user_site and options.target_dir is not None:
             raise CommandError("Can not combine '--user' and '--target'")
 
@@ -236,7 +251,7 @@ def run(self, options, args):
 
         install_options = options.install_options or []
 
-        logger.debug("Using %s", get_pip_version())
+        logger.verbose("Using %s", get_pip_version())
         options.use_user_site = decide_user_install(
             options.use_user_site,
             prefix_path=options.prefix_path,
@@ -245,16 +260,19 @@ def run(self, options, args):
             isolated_mode=options.isolated_mode,
         )
 
-        target_temp_dir = None  # type: Optional[TempDirectory]
-        target_temp_dir_path = None  # type: Optional[str]
+        target_temp_dir: Optional[TempDirectory] = None
+        target_temp_dir_path: Optional[str] = None
         if options.target_dir:
             options.ignore_installed = True
             options.target_dir = os.path.abspath(options.target_dir)
-            if (os.path.exists(options.target_dir) and not
-                    os.path.isdir(options.target_dir)):
+            if (
+                # fmt: off
+                os.path.exists(options.target_dir) and
+                not os.path.isdir(options.target_dir)
+                # fmt: on
+            ):
                 raise CommandError(
-                    "Target path exists but is not a directory, will not "
-                    "continue."
+                    "Target path exists but is not a directory, will not continue."
                 )
 
             # Create a target directory for using with the target option
@@ -275,7 +293,7 @@ def run(self, options, args):
         )
         wheel_cache = WheelCache(options.cache_dir, options.format_control)
 
-        req_tracker = self.enter_context(get_requirement_tracker())
+        build_tracker = self.enter_context(get_build_tracker())
 
         directory = TempDirectory(
             delete=not options.no_clean,
@@ -286,17 +304,22 @@ def run(self, options, args):
         try:
             reqs = self.get_requirements(args, options, finder, session)
 
-            reject_location_related_install_options(
-                reqs, options.install_options
-            )
+            # Only when installing is it permitted to use PEP 660.
+            # In other circumstances (pip wheel, pip download) we generate
+            # regular (i.e. non editable) metadata and wheels.
+            for req in reqs:
+                req.permit_editable_wheels = True
+
+            reject_location_related_install_options(reqs, options.install_options)
 
             preparer = self.make_requirement_preparer(
                 temp_build_dir=directory,
                 options=options,
-                req_tracker=req_tracker,
+                build_tracker=build_tracker,
                 session=session,
                 finder=finder,
                 use_user_site=options.use_user_site,
+                verbosity=self.verbosity,
             )
             resolver = self.make_resolver(
                 preparer=preparer,
@@ -325,19 +348,14 @@ def run(self, options, args):
                 # If we're not replacing an already installed pip,
                 # we're not modifying it.
                 modifying_pip = pip_req.satisfied_by is None
-            protect_pip_from_modification_on_windows(
-                modifying_pip=modifying_pip
-            )
+            protect_pip_from_modification_on_windows(modifying_pip=modifying_pip)
 
-            check_binary_allowed = get_check_binary_allowed(
-                finder.format_control
-            )
+            check_binary_allowed = get_check_binary_allowed(finder.format_control)
 
             reqs_to_build = [
-                r for r in requirement_set.requirements.values()
-                if should_build_for_install_command(
-                    r, check_binary_allowed
-                )
+                r
+                for r in requirement_set.requirements.values()
+                if should_build_for_install_command(r, check_binary_allowed)
             ]
 
             _, build_failures = build(
@@ -348,44 +366,40 @@ def run(self, options, args):
                 global_options=[],
             )
 
-            # If we're using PEP 517, we cannot do a direct install
+            # If we're using PEP 517, we cannot do a legacy setup.py install
             # so we fail here.
-            pep517_build_failure_names = [
-                r.name   # type: ignore
-                for r in build_failures if r.use_pep517
-            ]  # type: List[str]
+            pep517_build_failure_names: List[str] = [
+                r.name for r in build_failures if r.use_pep517  # type: ignore
+            ]
             if pep517_build_failure_names:
                 raise InstallationError(
-                    "Could not build wheels for {} which use"
-                    " PEP 517 and cannot be installed directly".format(
+                    "Could not build wheels for {}, which is required to "
+                    "install pyproject.toml-based projects".format(
                         ", ".join(pep517_build_failure_names)
                     )
                 )
 
             # For now, we just warn about failures building legacy
-            # requirements, as we'll fall through to a direct
-            # install for those.
+            # requirements, as we'll fall through to a setup.py install for
+            # those.
             for r in build_failures:
                 if not r.use_pep517:
                     r.legacy_install_reason = 8368
 
-            to_install = resolver.get_installation_order(
-                requirement_set
-            )
+            to_install = resolver.get_installation_order(requirement_set)
 
             # Check for conflicts in the package set we're installing.
-            conflicts = None  # type: Optional[ConflictDetails]
+            conflicts: Optional[ConflictDetails] = None
             should_warn_about_conflicts = (
-                not options.ignore_dependencies and
-                options.warn_about_conflicts
+                not options.ignore_dependencies and options.warn_about_conflicts
             )
             if should_warn_about_conflicts:
                 conflicts = self._determine_conflicts(to_install)
 
             # Don't warn about script install locations if
-            # --target has been specified
+            # --target or --prefix has been specified
             warn_script_location = options.warn_script_location
-            if options.target_dir:
+            if options.target_dir or options.prefix_path:
                 warn_script_location = False
 
             installed = install_given_reqs(
@@ -407,18 +421,16 @@ def run(self, options, args):
                 prefix=options.prefix_path,
                 isolated=options.isolated_mode,
             )
-            working_set = pkg_resources.WorkingSet(lib_locations)
+            env = get_environment(lib_locations)
 
-            installed.sort(key=operator.attrgetter('name'))
+            installed.sort(key=operator.attrgetter("name"))
             items = []
             for result in installed:
                 item = result.name
                 try:
-                    installed_version = get_installed_version(
-                        result.name, working_set=working_set
-                    )
-                    if installed_version:
-                        item += '-' + installed_version
+                    installed_dist = env.get_distribution(item)
+                    if installed_dist is not None:
+                        item = f"{item}-{installed_dist.version}"
                 except Exception:
                     pass
                 items.append(item)
@@ -429,16 +441,19 @@ def run(self, options, args):
                     resolver_variant=self.determine_resolver_variant(options),
                 )
 
-            installed_desc = ' '.join(items)
+            installed_desc = " ".join(items)
             if installed_desc:
                 write_output(
-                    'Successfully installed %s', installed_desc,
+                    "Successfully installed %s",
+                    installed_desc,
                 )
         except OSError as error:
-            show_traceback = (self.verbosity >= 1)
+            show_traceback = self.verbosity >= 1
 
             message = create_os_error_message(
-                error, show_traceback, options.use_user_site,
+                error,
+                show_traceback,
+                options.use_user_site,
             )
             logger.error(message, exc_info=show_traceback)  # noqa
 
@@ -450,10 +465,12 @@ def run(self, options, args):
                 options.target_dir, target_temp_dir, options.upgrade
             )
 
+        warn_if_run_as_root()
         return SUCCESS
 
-    def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
-        # type: (str, TempDirectory, bool) -> None
+    def _handle_target_dir(
+        self, target_dir: str, target_temp_dir: TempDirectory, upgrade: bool
+    ) -> None:
         ensure_dir(target_dir)
 
         # Checking both purelib and platlib directories for installed
@@ -462,10 +479,10 @@ def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
 
         # Checking both purelib and platlib directories for installed
         # packages to be moved to target directory
-        scheme = distutils_scheme('', home=target_temp_dir.path)
-        purelib_dir = scheme['purelib']
-        platlib_dir = scheme['platlib']
-        data_dir = scheme['data']
+        scheme = get_scheme("", home=target_temp_dir.path)
+        purelib_dir = scheme.purelib
+        platlib_dir = scheme.platlib
+        data_dir = scheme.data
 
         if os.path.exists(purelib_dir):
             lib_dir_list.append(purelib_dir)
@@ -484,18 +501,18 @@ def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
                 if os.path.exists(target_item_dir):
                     if not upgrade:
                         logger.warning(
-                            'Target directory %s already exists. Specify '
-                            '--upgrade to force replacement.',
-                            target_item_dir
+                            "Target directory %s already exists. Specify "
+                            "--upgrade to force replacement.",
+                            target_item_dir,
                         )
                         continue
                     if os.path.islink(target_item_dir):
                         logger.warning(
-                            'Target directory %s already exists and is '
-                            'a link. pip will not automatically replace '
-                            'links, please remove if replacement is '
-                            'desired.',
-                            target_item_dir
+                            "Target directory %s already exists and is "
+                            "a link. pip will not automatically replace "
+                            "links, please remove if replacement is "
+                            "desired.",
+                            target_item_dir,
                         )
                         continue
                     if os.path.isdir(target_item_dir):
@@ -503,13 +520,11 @@ def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
                     else:
                         os.remove(target_item_dir)
 
-                shutil.move(
-                    os.path.join(lib_dir, item),
-                    target_item_dir
-                )
+                shutil.move(os.path.join(lib_dir, item), target_item_dir)
 
-    def _determine_conflicts(self, to_install):
-        # type: (List[InstallRequirement]) -> Optional[ConflictDetails]
+    def _determine_conflicts(
+        self, to_install: List[InstallRequirement]
+    ) -> Optional[ConflictDetails]:
         try:
             return check_install_conflicts(to_install)
         except Exception:
@@ -519,13 +534,14 @@ def _determine_conflicts(self, to_install):
             )
             return None
 
-    def _warn_about_conflicts(self, conflict_details, resolver_variant):
-        # type: (ConflictDetails, str) -> None
+    def _warn_about_conflicts(
+        self, conflict_details: ConflictDetails, resolver_variant: str
+    ) -> None:
         package_set, (missing, conflicting) = conflict_details
         if not missing and not conflicting:
             return
 
-        parts = []  # type: List[str]
+        parts: List[str] = []
         if resolver_variant == "legacy":
             parts.append(
                 "pip's legacy dependency resolver does not consider dependency "
@@ -566,7 +582,7 @@ def _warn_about_conflicts(self, conflict_details, resolver_variant):
                     requirement=req,
                     dep_name=dep_name,
                     dep_version=dep_version,
-                    you=("you" if resolver_variant == "2020-resolver" else "you'll")
+                    you=("you" if resolver_variant == "2020-resolver" else "you'll"),
                 )
                 parts.append(message)
 
@@ -574,34 +590,37 @@ def _warn_about_conflicts(self, conflict_details, resolver_variant):
 
 
 def get_lib_location_guesses(
-        user=False,  # type: bool
-        home=None,  # type: Optional[str]
-        root=None,  # type: Optional[str]
-        isolated=False,  # type: bool
-        prefix=None  # type: Optional[str]
-):
-    # type:(...) -> List[str]
-    scheme = distutils_scheme('', user=user, home=home, root=root,
-                              isolated=isolated, prefix=prefix)
-    return [scheme['purelib'], scheme['platlib']]
-
-
-def site_packages_writable(root, isolated):
-    # type: (Optional[str], bool) -> bool
+    user: bool = False,
+    home: Optional[str] = None,
+    root: Optional[str] = None,
+    isolated: bool = False,
+    prefix: Optional[str] = None,
+) -> List[str]:
+    scheme = get_scheme(
+        "",
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+    return [scheme.purelib, scheme.platlib]
+
+
+def site_packages_writable(root: Optional[str], isolated: bool) -> bool:
     return all(
-        test_writable_dir(d) for d in set(
-            get_lib_location_guesses(root=root, isolated=isolated))
+        test_writable_dir(d)
+        for d in set(get_lib_location_guesses(root=root, isolated=isolated))
     )
 
 
 def decide_user_install(
-    use_user_site,  # type: Optional[bool]
-    prefix_path=None,  # type: Optional[str]
-    target_dir=None,  # type: Optional[str]
-    root_path=None,  # type: Optional[str]
-    isolated_mode=False,  # type: bool
-):
-    # type: (...) -> bool
+    use_user_site: Optional[bool],
+    prefix_path: Optional[str] = None,
+    target_dir: Optional[str] = None,
+    root_path: Optional[str] = None,
+    isolated_mode: bool = False,
+) -> bool:
     """Determine whether to do a user install based on the input options.
 
     If use_user_site is False, no additional checks are done.
@@ -649,18 +668,21 @@ def decide_user_install(
         logger.debug("Non-user install because site-packages writeable")
         return False
 
-    logger.info("Defaulting to user installation because normal site-packages "
-                "is not writeable")
+    logger.info(
+        "Defaulting to user installation because normal site-packages "
+        "is not writeable"
+    )
     return True
 
 
-def reject_location_related_install_options(requirements, options):
-    # type: (List[InstallRequirement], Optional[List[str]]) -> None
+def reject_location_related_install_options(
+    requirements: List[InstallRequirement], options: Optional[List[str]]
+) -> None:
     """If any location-changing --install-option arguments were passed for
     requirements or on the command-line, then show a deprecation warning.
     """
-    def format_options(option_names):
-        # type: (Iterable[str]) -> List[str]
+
+    def format_options(option_names: Iterable[str]) -> List[str]:
         return ["--{}".format(name.replace("_", "-")) for name in option_names]
 
     offenders = []
@@ -679,9 +701,7 @@ def format_options(option_names):
         location_options = parse_distutils_args(options)
         if location_options:
             offenders.append(
-                "{!r} from command line".format(
-                    format_options(location_options.keys())
-                )
+                "{!r} from command line".format(format_options(location_options.keys()))
             )
 
     if not offenders:
@@ -690,14 +710,13 @@ def format_options(option_names):
     raise CommandError(
         "Location-changing options found in --install-option: {}."
         " This is unsupported, use pip-level options like --user,"
-        " --prefix, --root, and --target instead.".format(
-            "; ".join(offenders)
-        )
+        " --prefix, --root, and --target instead.".format("; ".join(offenders))
     )
 
 
-def create_os_error_message(error, show_traceback, using_user_site):
-    # type: (OSError, bool, bool) -> str
+def create_os_error_message(
+    error: OSError, show_traceback: bool, using_user_site: bool
+) -> str:
     """Format an error message for an OSError
 
     It may occur anytime during the execution of the install command.
@@ -721,13 +740,32 @@ def create_os_error_message(error, show_traceback, using_user_site):
         user_option_part = "Consider using the `--user` option"
         permissions_part = "Check the permissions"
 
-        if not using_user_site:
-            parts.extend([
-                user_option_part, " or ",
-                permissions_part.lower(),
-            ])
+        if not running_under_virtualenv() and not using_user_site:
+            parts.extend(
+                [
+                    user_option_part,
+                    " or ",
+                    permissions_part.lower(),
+                ]
+            )
         else:
             parts.append(permissions_part)
         parts.append(".\n")
 
+    # Suggest the user to enable Long Paths if path length is
+    # more than 260
+    if (
+        WINDOWS
+        and error.errno == errno.ENOENT
+        and error.filename
+        and len(error.filename) > 260
+    ):
+        parts.append(
+            "HINT: This error might have occurred since "
+            "this system does not have Windows Long Path "
+            "support enabled. You can find information on "
+            "how to enable this at "
+            "https://pip.pypa.io/warnings/enable-long-paths\n"
+        )
+
     return "".join(parts).strip() + "\n"
diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py
index 89cfb625e59..57f05e00829 100644
--- a/src/pip/_internal/commands/list.py
+++ b/src/pip/_internal/commands/list.py
@@ -1,5 +1,9 @@
 import json
 import logging
+from optparse import Values
+from typing import TYPE_CHECKING, Iterator, List, Optional, Sequence, Tuple, cast
+
+from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import IndexGroupCommand
@@ -7,25 +11,27 @@
 from pip._internal.exceptions import CommandError
 from pip._internal.index.collector import LinkCollector
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution, get_environment
 from pip._internal.models.selection_prefs import SelectionPreferences
+from pip._internal.network.session import PipSession
 from pip._internal.utils.compat import stdlib_pkgs
-from pip._internal.utils.misc import (
-    dist_is_editable,
-    get_installed_distributions,
-    tabulate,
-    write_output,
-)
-from pip._internal.utils.packaging import get_installer
-from pip._internal.utils.parallel import map_multithread
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.misc import tabulate, write_output
+
+if TYPE_CHECKING:
+    from pip._internal.metadata.base import DistributionVersion
+
+    class _DistWithLatestInfo(BaseDistribution):
+        """Give the distribution object a couple of extra fields.
+
+        These will be populated during ``get_outdated()``. This is dirty but
+        makes the rest of the code much cleaner.
+        """
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Iterator, List, Set, Tuple
+        latest_version: DistributionVersion
+        latest_filetype: str
 
-    from pip._vendor.pkg_resources import Distribution
+    _ProcessedDists = Sequence[_DistWithLatestInfo]
 
-    from pip._internal.network.session import PipSession
 
 logger = logging.getLogger(__name__)
 
@@ -41,86 +47,94 @@ class ListCommand(IndexGroupCommand):
     usage = """
       %prog [options]"""
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '-o', '--outdated',
-            action='store_true',
+            "-o",
+            "--outdated",
+            action="store_true",
             default=False,
-            help='List outdated packages')
+            help="List outdated packages",
+        )
         self.cmd_opts.add_option(
-            '-u', '--uptodate',
-            action='store_true',
+            "-u",
+            "--uptodate",
+            action="store_true",
             default=False,
-            help='List uptodate packages')
+            help="List uptodate packages",
+        )
         self.cmd_opts.add_option(
-            '-e', '--editable',
-            action='store_true',
+            "-e",
+            "--editable",
+            action="store_true",
             default=False,
-            help='List editable projects.')
+            help="List editable projects.",
+        )
         self.cmd_opts.add_option(
-            '-l', '--local',
-            action='store_true',
+            "-l",
+            "--local",
+            action="store_true",
             default=False,
-            help=('If in a virtualenv that has global access, do not list '
-                  'globally-installed packages.'),
+            help=(
+                "If in a virtualenv that has global access, do not list "
+                "globally-installed packages."
+            ),
         )
         self.cmd_opts.add_option(
-            '--user',
-            dest='user',
-            action='store_true',
+            "--user",
+            dest="user",
+            action="store_true",
             default=False,
-            help='Only output packages installed in user-site.')
+            help="Only output packages installed in user-site.",
+        )
         self.cmd_opts.add_option(cmdoptions.list_path())
         self.cmd_opts.add_option(
-            '--pre',
-            action='store_true',
+            "--pre",
+            action="store_true",
             default=False,
-            help=("Include pre-release and development versions. By default, "
-                  "pip only finds stable versions."),
+            help=(
+                "Include pre-release and development versions. By default, "
+                "pip only finds stable versions."
+            ),
         )
 
         self.cmd_opts.add_option(
-            '--format',
-            action='store',
-            dest='list_format',
+            "--format",
+            action="store",
+            dest="list_format",
             default="columns",
-            choices=('columns', 'freeze', 'json'),
-            help="Select the output format among: columns (default), freeze, "
-                 "or json",
+            choices=("columns", "freeze", "json"),
+            help="Select the output format among: columns (default), freeze, or json",
         )
 
         self.cmd_opts.add_option(
-            '--not-required',
-            action='store_true',
-            dest='not_required',
-            help="List packages that are not dependencies of "
-                 "installed packages.",
+            "--not-required",
+            action="store_true",
+            dest="not_required",
+            help="List packages that are not dependencies of installed packages.",
         )
 
         self.cmd_opts.add_option(
-            '--exclude-editable',
-            action='store_false',
-            dest='include_editable',
-            help='Exclude editable package from output.',
+            "--exclude-editable",
+            action="store_false",
+            dest="include_editable",
+            help="Exclude editable package from output.",
         )
         self.cmd_opts.add_option(
-            '--include-editable',
-            action='store_true',
-            dest='include_editable',
-            help='Include editable package from output.',
+            "--include-editable",
+            action="store_true",
+            dest="include_editable",
+            help="Include editable package from output.",
             default=True,
         )
         self.cmd_opts.add_option(cmdoptions.list_exclude())
-        index_opts = cmdoptions.make_option_group(
-            cmdoptions.index_group, self.parser
-        )
+        index_opts = cmdoptions.make_option_group(cmdoptions.index_group, self.parser)
 
         self.parser.insert_option_group(0, index_opts)
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def _build_package_finder(self, options, session):
-        # type: (Values, PipSession) -> PackageFinder
+    def _build_package_finder(
+        self, options: Values, session: PipSession
+    ) -> PackageFinder:
         """
         Create a package finder appropriate to this list command.
         """
@@ -135,28 +149,29 @@ def _build_package_finder(self, options, session):
         return PackageFinder.create(
             link_collector=link_collector,
             selection_prefs=selection_prefs,
+            use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
         )
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         if options.outdated and options.uptodate:
-            raise CommandError(
-                "Options --outdated and --uptodate cannot be combined.")
+            raise CommandError("Options --outdated and --uptodate cannot be combined.")
 
         cmdoptions.check_list_path_option(options)
 
         skip = set(stdlib_pkgs)
         if options.excludes:
-            skip.update(options.excludes)
-
-        packages = get_installed_distributions(
-            local_only=options.local,
-            user_only=options.user,
-            editables_only=options.editable,
-            include_editables=options.include_editable,
-            paths=options.path,
-            skip=skip,
-        )
+            skip.update(canonicalize_name(n) for n in options.excludes)
+
+        packages: "_ProcessedDists" = [
+            cast("_DistWithLatestInfo", d)
+            for d in get_environment(options.path).iter_installed_distributions(
+                local_only=options.local,
+                user_only=options.user,
+                editables_only=options.editable,
+                include_editables=options.include_editable,
+                skip=skip,
+            )
+        ]
 
         # get_not_required must be called firstly in order to find and
         # filter out all dependencies correctly. Otherwise a package
@@ -173,46 +188,58 @@ def run(self, options, args):
         self.output_package_listing(packages, options)
         return SUCCESS
 
-    def get_outdated(self, packages, options):
-        # type: (List[Distribution], Values) -> List[Distribution]
+    def get_outdated(
+        self, packages: "_ProcessedDists", options: Values
+    ) -> "_ProcessedDists":
         return [
-            dist for dist in self.iter_packages_latest_infos(packages, options)
-            if dist.latest_version > dist.parsed_version
+            dist
+            for dist in self.iter_packages_latest_infos(packages, options)
+            if dist.latest_version > dist.version
         ]
 
-    def get_uptodate(self, packages, options):
-        # type: (List[Distribution], Values) -> List[Distribution]
+    def get_uptodate(
+        self, packages: "_ProcessedDists", options: Values
+    ) -> "_ProcessedDists":
         return [
-            dist for dist in self.iter_packages_latest_infos(packages, options)
-            if dist.latest_version == dist.parsed_version
+            dist
+            for dist in self.iter_packages_latest_infos(packages, options)
+            if dist.latest_version == dist.version
         ]
 
-    def get_not_required(self, packages, options):
-        # type: (List[Distribution], Values) -> List[Distribution]
-        dep_keys = set()  # type: Set[Distribution]
-        for dist in packages:
-            dep_keys.update(requirement.key for requirement in dist.requires())
+    def get_not_required(
+        self, packages: "_ProcessedDists", options: Values
+    ) -> "_ProcessedDists":
+        dep_keys = {
+            canonicalize_name(dep.name)
+            for dist in packages
+            for dep in (dist.iter_dependencies() or ())
+        }
 
         # Create a set to remove duplicate packages, and cast it to a list
         # to keep the return type consistent with get_outdated and
         # get_uptodate
-        return list({pkg for pkg in packages if pkg.key not in dep_keys})
+        return list({pkg for pkg in packages if pkg.canonical_name not in dep_keys})
 
-    def iter_packages_latest_infos(self, packages, options):
-        # type: (List[Distribution], Values) -> Iterator[Distribution]
+    def iter_packages_latest_infos(
+        self, packages: "_ProcessedDists", options: Values
+    ) -> Iterator["_DistWithLatestInfo"]:
         with self._build_session(options) as session:
             finder = self._build_package_finder(options, session)
 
-            def latest_info(dist):
-                # type: (Distribution) -> Distribution
-                all_candidates = finder.find_all_candidates(dist.key)
+            def latest_info(
+                dist: "_DistWithLatestInfo",
+            ) -> Optional["_DistWithLatestInfo"]:
+                all_candidates = finder.find_all_candidates(dist.canonical_name)
                 if not options.pre:
                     # Remove prereleases
-                    all_candidates = [candidate for candidate in all_candidates
-                                      if not candidate.version.is_prerelease]
+                    all_candidates = [
+                        candidate
+                        for candidate in all_candidates
+                        if not candidate.version.is_prerelease
+                    ]
 
                 evaluator = finder.make_candidate_evaluator(
-                    project_name=dist.project_name,
+                    project_name=dist.canonical_name,
                 )
                 best_candidate = evaluator.sort_best_candidate(all_candidates)
                 if best_candidate is None:
@@ -220,39 +247,41 @@ def latest_info(dist):
 
                 remote_version = best_candidate.version
                 if best_candidate.link.is_wheel:
-                    typ = 'wheel'
+                    typ = "wheel"
                 else:
-                    typ = 'sdist'
-                # This is dirty but makes the rest of the code much cleaner
+                    typ = "sdist"
                 dist.latest_version = remote_version
                 dist.latest_filetype = typ
                 return dist
 
-            for dist in map_multithread(latest_info, packages):
+            for dist in map(latest_info, packages):
                 if dist is not None:
                     yield dist
 
-    def output_package_listing(self, packages, options):
-        # type: (List[Distribution], Values) -> None
+    def output_package_listing(
+        self, packages: "_ProcessedDists", options: Values
+    ) -> None:
         packages = sorted(
             packages,
-            key=lambda dist: dist.project_name.lower(),
+            key=lambda dist: dist.canonical_name,
         )
-        if options.list_format == 'columns' and packages:
+        if options.list_format == "columns" and packages:
             data, header = format_for_columns(packages, options)
             self.output_package_listing_columns(data, header)
-        elif options.list_format == 'freeze':
+        elif options.list_format == "freeze":
             for dist in packages:
                 if options.verbose >= 1:
-                    write_output("%s==%s (%s)", dist.project_name,
-                                 dist.version, dist.location)
+                    write_output(
+                        "%s==%s (%s)", dist.raw_name, dist.version, dist.location
+                    )
                 else:
-                    write_output("%s==%s", dist.project_name, dist.version)
-        elif options.list_format == 'json':
+                    write_output("%s==%s", dist.raw_name, dist.version)
+        elif options.list_format == "json":
             write_output(format_for_json(packages, options))
 
-    def output_package_listing_columns(self, data, header):
-        # type: (List[List[str]], List[str]) -> None
+    def output_package_listing_columns(
+        self, data: List[List[str]], header: List[str]
+    ) -> None:
         # insert the header first: we need to know the size of column names
         if len(data) > 0:
             data.insert(0, header)
@@ -261,63 +290,72 @@ def output_package_listing_columns(self, data, header):
 
         # Create and add a separator.
         if len(data) > 0:
-            pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes)))
+            pkg_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes)))
 
         for val in pkg_strings:
             write_output(val)
 
 
-def format_for_columns(pkgs, options):
-    # type: (List[Distribution], Values) -> Tuple[List[List[str]], List[str]]
+def format_for_columns(
+    pkgs: "_ProcessedDists", options: Values
+) -> Tuple[List[List[str]], List[str]]:
     """
     Convert the package data into something usable
     by output_package_listing_columns.
     """
+    header = ["Package", "Version"]
+
     running_outdated = options.outdated
-    # Adjust the header for the `pip list --outdated` case.
     if running_outdated:
-        header = ["Package", "Version", "Latest", "Type"]
-    else:
-        header = ["Package", "Version"]
+        header.extend(["Latest", "Type"])
 
-    data = []
-    if options.verbose >= 1 or any(dist_is_editable(x) for x in pkgs):
+    has_editables = any(x.editable for x in pkgs)
+    if has_editables:
+        header.append("Editable project location")
+
+    if options.verbose >= 1:
         header.append("Location")
     if options.verbose >= 1:
         header.append("Installer")
 
+    data = []
     for proj in pkgs:
         # if we're working on the 'outdated' list, separate out the
         # latest_version and type
-        row = [proj.project_name, proj.version]
+        row = [proj.raw_name, str(proj.version)]
 
         if running_outdated:
-            row.append(proj.latest_version)
+            row.append(str(proj.latest_version))
             row.append(proj.latest_filetype)
 
-        if options.verbose >= 1 or dist_is_editable(proj):
-            row.append(proj.location)
+        if has_editables:
+            row.append(proj.editable_project_location or "")
+
+        if options.verbose >= 1:
+            row.append(proj.location or "")
         if options.verbose >= 1:
-            row.append(get_installer(proj))
+            row.append(proj.installer)
 
         data.append(row)
 
     return data, header
 
 
-def format_for_json(packages, options):
-    # type: (List[Distribution], Values) -> str
+def format_for_json(packages: "_ProcessedDists", options: Values) -> str:
     data = []
     for dist in packages:
         info = {
-            'name': dist.project_name,
-            'version': str(dist.version),
+            "name": dist.raw_name,
+            "version": str(dist.version),
         }
         if options.verbose >= 1:
-            info['location'] = dist.location
-            info['installer'] = get_installer(dist)
+            info["location"] = dist.location or ""
+            info["installer"] = dist.installer
         if options.outdated:
-            info['latest_version'] = str(dist.latest_version)
-            info['latest_filetype'] = dist.latest_filetype
+            info["latest_version"] = str(dist.latest_version)
+            info["latest_filetype"] = dist.latest_filetype
+        editable_project_location = dist.editable_project_location
+        if editable_project_location:
+            info["editable_project_location"] = editable_project_location
         data.append(info)
     return json.dumps(data)
diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py
index b3143b72fba..03ed925b246 100644
--- a/src/pip/_internal/commands/search.py
+++ b/src/pip/_internal/commands/search.py
@@ -2,34 +2,31 @@
 import shutil
 import sys
 import textwrap
+import xmlrpc.client
 from collections import OrderedDict
+from optparse import Values
+from typing import TYPE_CHECKING, Dict, List, Optional
 
-from pip._vendor import pkg_resources
 from pip._vendor.packaging.version import parse as parse_version
 
-# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
-#       why we ignore the type on this import
-from pip._vendor.six.moves import xmlrpc_client  # type: ignore
-
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.req_command import SessionCommandMixin
 from pip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS
 from pip._internal.exceptions import CommandError
+from pip._internal.metadata import get_default_environment
 from pip._internal.models.index import PyPI
 from pip._internal.network.xmlrpc import PipXmlrpcTransport
 from pip._internal.utils.logging import indent_log
-from pip._internal.utils.misc import get_distribution, write_output
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.misc import write_output
+
+if TYPE_CHECKING:
+    from typing import TypedDict
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Dict, List, Optional
+    class TransformedHit(TypedDict):
+        name: str
+        summary: str
+        versions: List[str]
 
-    from typing_extensions import TypedDict
-    TransformedHit = TypedDict(
-        'TransformedHit',
-        {'name': str, 'summary': str, 'versions': List[str]},
-    )
 
 logger = logging.getLogger(__name__)
 
@@ -41,21 +38,21 @@ class SearchCommand(Command, SessionCommandMixin):
       %prog [options] """
     ignore_require_venv = True
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '-i', '--index',
-            dest='index',
-            metavar='URL',
+            "-i",
+            "--index",
+            dest="index",
+            metavar="URL",
             default=PyPI.pypi_url,
-            help='Base URL of Python Package Index (default %default)')
+            help="Base URL of Python Package Index (default %default)",
+        )
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         if not args:
-            raise CommandError('Missing required argument (search query).')
+            raise CommandError("Missing required argument (search query).")
         query = args
         pypi_hits = self.search(query, options)
         hits = transform_hits(pypi_hits)
@@ -69,99 +66,109 @@ def run(self, options, args):
             return SUCCESS
         return NO_MATCHES_FOUND
 
-    def search(self, query, options):
-        # type: (List[str], Values) -> List[Dict[str, str]]
+    def search(self, query: List[str], options: Values) -> List[Dict[str, str]]:
         index_url = options.index
 
         session = self.get_default_session(options)
 
         transport = PipXmlrpcTransport(index_url, session)
-        pypi = xmlrpc_client.ServerProxy(index_url, transport)
+        pypi = xmlrpc.client.ServerProxy(index_url, transport)
         try:
-            hits = pypi.search({'name': query, 'summary': query}, 'or')
-        except xmlrpc_client.Fault as fault:
+            hits = pypi.search({"name": query, "summary": query}, "or")
+        except xmlrpc.client.Fault as fault:
             message = "XMLRPC request failed [code: {code}]\n{string}".format(
                 code=fault.faultCode,
                 string=fault.faultString,
             )
             raise CommandError(message)
+        assert isinstance(hits, list)
         return hits
 
 
-def transform_hits(hits):
-    # type: (List[Dict[str, str]]) -> List[TransformedHit]
+def transform_hits(hits: List[Dict[str, str]]) -> List["TransformedHit"]:
     """
     The list from pypi is really a list of versions. We want a list of
     packages with the list of versions stored inline. This converts the
     list from pypi into one we can use.
     """
-    packages = OrderedDict()  # type: OrderedDict[str, TransformedHit]
+    packages: Dict[str, "TransformedHit"] = OrderedDict()
     for hit in hits:
-        name = hit['name']
-        summary = hit['summary']
-        version = hit['version']
+        name = hit["name"]
+        summary = hit["summary"]
+        version = hit["version"]
 
         if name not in packages.keys():
             packages[name] = {
-                'name': name,
-                'summary': summary,
-                'versions': [version],
+                "name": name,
+                "summary": summary,
+                "versions": [version],
             }
         else:
-            packages[name]['versions'].append(version)
+            packages[name]["versions"].append(version)
 
             # if this is the highest version, replace summary and score
-            if version == highest_version(packages[name]['versions']):
-                packages[name]['summary'] = summary
+            if version == highest_version(packages[name]["versions"]):
+                packages[name]["summary"] = summary
 
     return list(packages.values())
 
 
-def print_results(hits, name_column_width=None, terminal_width=None):
-    # type: (List[TransformedHit], Optional[int], Optional[int]) -> None
+def print_dist_installation_info(name: str, latest: str) -> None:
+    env = get_default_environment()
+    dist = env.get_distribution(name)
+    if dist is not None:
+        with indent_log():
+            if dist.version == latest:
+                write_output("INSTALLED: %s (latest)", dist.version)
+            else:
+                write_output("INSTALLED: %s", dist.version)
+                if parse_version(latest).pre:
+                    write_output(
+                        "LATEST:    %s (pre-release; install"
+                        " with `pip install --pre`)",
+                        latest,
+                    )
+                else:
+                    write_output("LATEST:    %s", latest)
+
+
+def print_results(
+    hits: List["TransformedHit"],
+    name_column_width: Optional[int] = None,
+    terminal_width: Optional[int] = None,
+) -> None:
     if not hits:
         return
     if name_column_width is None:
-        name_column_width = max([
-            len(hit['name']) + len(highest_version(hit.get('versions', ['-'])))
-            for hit in hits
-        ]) + 4
+        name_column_width = (
+            max(
+                [
+                    len(hit["name"]) + len(highest_version(hit.get("versions", ["-"])))
+                    for hit in hits
+                ]
+            )
+            + 4
+        )
 
-    installed_packages = [p.project_name for p in pkg_resources.working_set]
     for hit in hits:
-        name = hit['name']
-        summary = hit['summary'] or ''
-        latest = highest_version(hit.get('versions', ['-']))
+        name = hit["name"]
+        summary = hit["summary"] or ""
+        latest = highest_version(hit.get("versions", ["-"]))
         if terminal_width is not None:
             target_width = terminal_width - name_column_width - 5
             if target_width > 10:
                 # wrap and indent summary to fit terminal
                 summary_lines = textwrap.wrap(summary, target_width)
-                summary = ('\n' + ' ' * (name_column_width + 3)).join(
-                    summary_lines)
+                summary = ("\n" + " " * (name_column_width + 3)).join(summary_lines)
 
-        line = '{name_latest:{name_column_width}} - {summary}'.format(
-            name_latest='{name} ({latest})'.format(**locals()),
-            **locals())
+        name_latest = f"{name} ({latest})"
+        line = f"{name_latest:{name_column_width}} - {summary}"
         try:
             write_output(line)
-            if name in installed_packages:
-                dist = get_distribution(name)
-                assert dist is not None
-                with indent_log():
-                    if dist.version == latest:
-                        write_output('INSTALLED: %s (latest)', dist.version)
-                    else:
-                        write_output('INSTALLED: %s', dist.version)
-                        if parse_version(latest).pre:
-                            write_output('LATEST:    %s (pre-release; install'
-                                         ' with "pip install --pre")', latest)
-                        else:
-                            write_output('LATEST:    %s', latest)
+            print_dist_installation_info(name, latest)
         except UnicodeEncodeError:
             pass
 
 
-def highest_version(versions):
-    # type: (List[str]) -> str
+def highest_version(versions: List[str]) -> str:
     return max(versions, key=parse_version)
diff --git a/src/pip/_internal/commands/show.py b/src/pip/_internal/commands/show.py
index a6363cfd0a2..d5540d68b78 100644
--- a/src/pip/_internal/commands/show.py
+++ b/src/pip/_internal/commands/show.py
@@ -1,18 +1,13 @@
 import logging
-import os
-from email.parser import FeedParser
+from optparse import Values
+from typing import Iterator, List, NamedTuple, Optional
 
-from pip._vendor import pkg_resources
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.status_codes import ERROR, SUCCESS
+from pip._internal.metadata import BaseDistribution, get_default_environment
 from pip._internal.utils.misc import write_output
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import Dict, Iterator, List
 
 logger = logging.getLogger(__name__)
 
@@ -28,123 +23,122 @@ class ShowCommand(Command):
       %prog [options]  ..."""
     ignore_require_venv = True
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '-f', '--files',
-            dest='files',
-            action='store_true',
+            "-f",
+            "--files",
+            dest="files",
+            action="store_true",
             default=False,
-            help='Show the full list of installed files for each package.')
+            help="Show the full list of installed files for each package.",
+        )
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         if not args:
-            logger.warning('ERROR: Please provide a package name or names.')
+            logger.warning("ERROR: Please provide a package name or names.")
             return ERROR
         query = args
 
         results = search_packages_info(query)
         if not print_results(
-                results, list_files=options.files, verbose=options.verbose):
+            results, list_files=options.files, verbose=options.verbose
+        ):
             return ERROR
         return SUCCESS
 
 
-def search_packages_info(query):
-    # type: (List[str]) -> Iterator[Dict[str, str]]
+class _PackageInfo(NamedTuple):
+    name: str
+    version: str
+    location: str
+    requires: List[str]
+    required_by: List[str]
+    installer: str
+    metadata_version: str
+    classifiers: List[str]
+    summary: str
+    homepage: str
+    author: str
+    author_email: str
+    license: str
+    entry_points: List[str]
+    files: Optional[List[str]]
+
+
+def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]:
     """
     Gather details from installed distributions. Print distribution name,
     version, location, and installed files. Installed files requires a
     pip generated 'installed-files.txt' in the distributions '.egg-info'
     directory.
     """
-    installed = {}
-    for p in pkg_resources.working_set:
-        installed[canonicalize_name(p.project_name)] = p
+    env = get_default_environment()
 
+    installed = {dist.canonical_name: dist for dist in env.iter_distributions()}
     query_names = [canonicalize_name(name) for name in query]
     missing = sorted(
         [name for name, pkg in zip(query, query_names) if pkg not in installed]
     )
     if missing:
-        logger.warning('Package(s) not found: %s', ', '.join(missing))
-
-    def get_requiring_packages(package_name):
-        # type: (str) -> List[str]
-        canonical_name = canonicalize_name(package_name)
-        return [
-            pkg.project_name for pkg in pkg_resources.working_set
-            if canonical_name in
-               [canonicalize_name(required.name) for required in
-                pkg.requires()]
-        ]
-
-    for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
-        package = {
-            'name': dist.project_name,
-            'version': dist.version,
-            'location': dist.location,
-            'requires': [dep.project_name for dep in dist.requires()],
-            'required_by': get_requiring_packages(dist.project_name)
-        }
-        file_list = None
-        metadata = ''
-        if isinstance(dist, pkg_resources.DistInfoDistribution):
-            # RECORDs should be part of .dist-info metadatas
-            if dist.has_metadata('RECORD'):
-                lines = dist.get_metadata_lines('RECORD')
-                paths = [line.split(',')[0] for line in lines]
-                paths = [os.path.join(dist.location, p) for p in paths]
-                file_list = [os.path.relpath(p, dist.location) for p in paths]
-
-            if dist.has_metadata('METADATA'):
-                metadata = dist.get_metadata('METADATA')
+        logger.warning("Package(s) not found: %s", ", ".join(missing))
+
+    def _get_requiring_packages(current_dist: BaseDistribution) -> Iterator[str]:
+        return (
+            dist.metadata["Name"] or "UNKNOWN"
+            for dist in installed.values()
+            if current_dist.canonical_name
+            in {canonicalize_name(d.name) for d in dist.iter_dependencies()}
+        )
+
+    for query_name in query_names:
+        try:
+            dist = installed[query_name]
+        except KeyError:
+            continue
+
+        requires = sorted((req.name for req in dist.iter_dependencies()), key=str.lower)
+        required_by = sorted(_get_requiring_packages(dist), key=str.lower)
+
+        try:
+            entry_points_text = dist.read_text("entry_points.txt")
+            entry_points = entry_points_text.splitlines(keepends=False)
+        except FileNotFoundError:
+            entry_points = []
+
+        files_iter = dist.iter_declared_entries()
+        if files_iter is None:
+            files: Optional[List[str]] = None
         else:
-            # Otherwise use pip's log for .egg-info's
-            if dist.has_metadata('installed-files.txt'):
-                paths = dist.get_metadata_lines('installed-files.txt')
-                paths = [os.path.join(dist.egg_info, p) for p in paths]
-                file_list = [os.path.relpath(p, dist.location) for p in paths]
-
-            if dist.has_metadata('PKG-INFO'):
-                metadata = dist.get_metadata('PKG-INFO')
-
-        if dist.has_metadata('entry_points.txt'):
-            entry_points = dist.get_metadata_lines('entry_points.txt')
-            package['entry_points'] = entry_points
-
-        if dist.has_metadata('INSTALLER'):
-            for line in dist.get_metadata_lines('INSTALLER'):
-                if line.strip():
-                    package['installer'] = line.strip()
-                    break
-
-        # @todo: Should pkg_resources.Distribution have a
-        # `get_pkg_info` method?
-        feed_parser = FeedParser()
-        feed_parser.feed(metadata)
-        pkg_info_dict = feed_parser.close()
-        for key in ('metadata-version', 'summary',
-                    'home-page', 'author', 'author-email', 'license'):
-            package[key] = pkg_info_dict.get(key)
-
-        # It looks like FeedParser cannot deal with repeated headers
-        classifiers = []
-        for line in metadata.splitlines():
-            if line.startswith('Classifier: '):
-                classifiers.append(line[len('Classifier: '):])
-        package['classifiers'] = classifiers
-
-        if file_list:
-            package['files'] = sorted(file_list)
-        yield package
-
-
-def print_results(distributions, list_files=False, verbose=False):
-    # type: (Iterator[Dict[str, str]], bool, bool) -> bool
+            files = sorted(files_iter)
+
+        metadata = dist.metadata
+
+        yield _PackageInfo(
+            name=dist.raw_name,
+            version=str(dist.version),
+            location=dist.location or "",
+            requires=requires,
+            required_by=required_by,
+            installer=dist.installer,
+            metadata_version=dist.metadata_version or "",
+            classifiers=metadata.get_all("Classifier", []),
+            summary=metadata.get("Summary", ""),
+            homepage=metadata.get("Home-page", ""),
+            author=metadata.get("Author", ""),
+            author_email=metadata.get("Author-email", ""),
+            license=metadata.get("License", ""),
+            entry_points=entry_points,
+            files=files,
+        )
+
+
+def print_results(
+    distributions: Iterator[_PackageInfo],
+    list_files: bool,
+    verbose: bool,
+) -> bool:
     """
     Print the information from installed distributions found.
     """
@@ -154,31 +148,31 @@ def print_results(distributions, list_files=False, verbose=False):
         if i > 0:
             write_output("---")
 
-        write_output("Name: %s", dist.get('name', ''))
-        write_output("Version: %s", dist.get('version', ''))
-        write_output("Summary: %s", dist.get('summary', ''))
-        write_output("Home-page: %s", dist.get('home-page', ''))
-        write_output("Author: %s", dist.get('author', ''))
-        write_output("Author-email: %s", dist.get('author-email', ''))
-        write_output("License: %s", dist.get('license', ''))
-        write_output("Location: %s", dist.get('location', ''))
-        write_output("Requires: %s", ', '.join(dist.get('requires', [])))
-        write_output("Required-by: %s", ', '.join(dist.get('required_by', [])))
+        write_output("Name: %s", dist.name)
+        write_output("Version: %s", dist.version)
+        write_output("Summary: %s", dist.summary)
+        write_output("Home-page: %s", dist.homepage)
+        write_output("Author: %s", dist.author)
+        write_output("Author-email: %s", dist.author_email)
+        write_output("License: %s", dist.license)
+        write_output("Location: %s", dist.location)
+        write_output("Requires: %s", ", ".join(dist.requires))
+        write_output("Required-by: %s", ", ".join(dist.required_by))
 
         if verbose:
-            write_output("Metadata-Version: %s",
-                         dist.get('metadata-version', ''))
-            write_output("Installer: %s", dist.get('installer', ''))
+            write_output("Metadata-Version: %s", dist.metadata_version)
+            write_output("Installer: %s", dist.installer)
             write_output("Classifiers:")
-            for classifier in dist.get('classifiers', []):
+            for classifier in dist.classifiers:
                 write_output("  %s", classifier)
             write_output("Entry-points:")
-            for entry in dist.get('entry_points', []):
+            for entry in dist.entry_points:
                 write_output("  %s", entry.strip())
         if list_files:
             write_output("Files:")
-            for line in dist.get('files', []):
-                write_output("  %s", line.strip())
-            if "files" not in dist:
-                write_output("Cannot locate installed-files.txt")
+            if dist.files is None:
+                write_output("Cannot locate RECORD or installed-files.txt")
+            else:
+                for line in dist.files:
+                    write_output("  %s", line.strip())
     return results_printed
diff --git a/src/pip/_internal/commands/uninstall.py b/src/pip/_internal/commands/uninstall.py
index 6dc96c3d630..bb9e8e6a380 100644
--- a/src/pip/_internal/commands/uninstall.py
+++ b/src/pip/_internal/commands/uninstall.py
@@ -1,7 +1,11 @@
+import logging
+from optparse import Values
+from typing import List
+
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cli.base_command import Command
-from pip._internal.cli.req_command import SessionCommandMixin
+from pip._internal.cli.req_command import SessionCommandMixin, warn_if_run_as_root
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.exceptions import InstallationError
 from pip._internal.req import parse_requirements
@@ -10,11 +14,8 @@
     install_req_from_parsed_requirement,
 )
 from pip._internal.utils.misc import protect_pip_from_modification_on_windows
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
+logger = logging.getLogger(__name__)
 
 
 class UninstallCommand(Command, SessionCommandMixin):
@@ -32,51 +33,60 @@ class UninstallCommand(Command, SessionCommandMixin):
       %prog [options]  ...
       %prog [options] -r  ..."""
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
         self.cmd_opts.add_option(
-            '-r', '--requirement',
-            dest='requirements',
-            action='append',
+            "-r",
+            "--requirement",
+            dest="requirements",
+            action="append",
             default=[],
-            metavar='file',
-            help='Uninstall all the packages listed in the given requirements '
-                 'file.  This option can be used multiple times.',
+            metavar="file",
+            help=(
+                "Uninstall all the packages listed in the given requirements "
+                "file.  This option can be used multiple times."
+            ),
         )
         self.cmd_opts.add_option(
-            '-y', '--yes',
-            dest='yes',
-            action='store_true',
-            help="Don't ask for confirmation of uninstall deletions.")
+            "-y",
+            "--yes",
+            dest="yes",
+            action="store_true",
+            help="Don't ask for confirmation of uninstall deletions.",
+        )
 
         self.parser.insert_option_group(0, self.cmd_opts)
 
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         session = self.get_default_session(options)
 
         reqs_to_uninstall = {}
         for name in args:
             req = install_req_from_line(
-                name, isolated=options.isolated_mode,
+                name,
+                isolated=options.isolated_mode,
             )
             if req.name:
                 reqs_to_uninstall[canonicalize_name(req.name)] = req
+            else:
+                logger.warning(
+                    "Invalid requirement: %r ignored -"
+                    " the uninstall command expects named"
+                    " requirements.",
+                    name,
+                )
         for filename in options.requirements:
             for parsed_req in parse_requirements(
-                    filename,
-                    options=options,
-                    session=session):
+                filename, options=options, session=session
+            ):
                 req = install_req_from_parsed_requirement(
-                    parsed_req,
-                    isolated=options.isolated_mode
+                    parsed_req, isolated=options.isolated_mode
                 )
                 if req.name:
                     reqs_to_uninstall[canonicalize_name(req.name)] = req
         if not reqs_to_uninstall:
             raise InstallationError(
-                'You must give at least one requirement to {self.name} (see '
-                '"pip help {self.name}")'.format(**locals())
+                f"You must give at least one requirement to {self.name} (see "
+                f'"pip help {self.name}")'
             )
 
         protect_pip_from_modification_on_windows(
@@ -85,9 +95,11 @@ def run(self, options, args):
 
         for req in reqs_to_uninstall.values():
             uninstall_pathset = req.uninstall(
-                auto_confirm=options.yes, verbose=self.verbosity > 0,
+                auto_confirm=options.yes,
+                verbose=self.verbosity > 0,
             )
             if uninstall_pathset:
                 uninstall_pathset.commit()
 
+        warn_if_run_as_root()
         return SUCCESS
diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py
index 28918fa748a..4fefe8111dd 100644
--- a/src/pip/_internal/commands/wheel.py
+++ b/src/pip/_internal/commands/wheel.py
@@ -1,24 +1,20 @@
 import logging
 import os
 import shutil
+from optparse import Values
+from typing import List
 
 from pip._internal.cache import WheelCache
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.req_command import RequirementCommand, with_cleanup
 from pip._internal.cli.status_codes import SUCCESS
 from pip._internal.exceptions import CommandError
-from pip._internal.req.req_tracker import get_requirement_tracker
+from pip._internal.operations.build.build_tracker import get_build_tracker
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.misc import ensure_dir, normalize_path
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.wheel_builder import build, should_build_for_wheel_command
 
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import List
-
-    from pip._internal.req.req_install import InstallRequirement
-
 logger = logging.getLogger(__name__)
 
 
@@ -30,10 +26,8 @@ class WheelCommand(RequirementCommand):
     recompiling your software during every install. For more details, see the
     wheel docs: https://wheel.readthedocs.io/en/latest/
 
-    Requirements: setuptools>=0.8, and wheel.
-
-    'pip wheel' uses the bdist_wheel setuptools extension from the wheel
-    package to build individual wheels.
+    'pip wheel' uses the build system interface as described here:
+    https://pip.pypa.io/en/stable/reference/build-system/
 
     """
 
@@ -44,27 +38,22 @@ class WheelCommand(RequirementCommand):
       %prog [options] [-e]  ...
       %prog [options]  ..."""
 
-    def add_options(self):
-        # type: () -> None
+    def add_options(self) -> None:
 
         self.cmd_opts.add_option(
-            '-w', '--wheel-dir',
-            dest='wheel_dir',
-            metavar='dir',
+            "-w",
+            "--wheel-dir",
+            dest="wheel_dir",
+            metavar="dir",
             default=os.curdir,
-            help=("Build wheels into , where the default is the "
-                  "current working directory."),
+            help=(
+                "Build wheels into , where the default is the "
+                "current working directory."
+            ),
         )
         self.cmd_opts.add_option(cmdoptions.no_binary())
         self.cmd_opts.add_option(cmdoptions.only_binary())
         self.cmd_opts.add_option(cmdoptions.prefer_binary())
-        self.cmd_opts.add_option(
-            '--build-option',
-            dest='build_options',
-            metavar='options',
-            action='append',
-            help="Extra arguments to be supplied to 'setup.py bdist_wheel'.",
-        )
         self.cmd_opts.add_option(cmdoptions.no_build_isolation())
         self.cmd_opts.add_option(cmdoptions.use_pep517())
         self.cmd_opts.add_option(cmdoptions.no_use_pep517())
@@ -74,31 +63,27 @@ def add_options(self):
         self.cmd_opts.add_option(cmdoptions.src())
         self.cmd_opts.add_option(cmdoptions.ignore_requires_python())
         self.cmd_opts.add_option(cmdoptions.no_deps())
-        self.cmd_opts.add_option(cmdoptions.build_dir())
         self.cmd_opts.add_option(cmdoptions.progress_bar())
 
         self.cmd_opts.add_option(
-            '--no-verify',
-            dest='no_verify',
-            action='store_true',
+            "--no-verify",
+            dest="no_verify",
+            action="store_true",
             default=False,
             help="Don't verify if built wheel is valid.",
         )
 
-        self.cmd_opts.add_option(
-            '--global-option',
-            dest='global_options',
-            action='append',
-            metavar='options',
-            help="Extra global options to be supplied to the setup.py "
-            "call before the 'bdist_wheel' command.")
+        self.cmd_opts.add_option(cmdoptions.build_options())
+        self.cmd_opts.add_option(cmdoptions.global_options())
 
         self.cmd_opts.add_option(
-            '--pre',
-            action='store_true',
+            "--pre",
+            action="store_true",
             default=False,
-            help=("Include pre-release and development versions. By default, "
-                  "pip only finds stable versions."),
+            help=(
+                "Include pre-release and development versions. By default, "
+                "pip only finds stable versions."
+            ),
         )
 
         self.cmd_opts.add_option(cmdoptions.require_hashes())
@@ -112,8 +97,7 @@ def add_options(self):
         self.parser.insert_option_group(0, self.cmd_opts)
 
     @with_cleanup
-    def run(self, options, args):
-        # type: (Values, List[str]) -> int
+    def run(self, options: Values, args: List[str]) -> int:
         cmdoptions.check_install_build_global(options)
 
         session = self.get_default_session(options)
@@ -124,7 +108,7 @@ def run(self, options, args):
         options.wheel_dir = normalize_path(options.wheel_dir)
         ensure_dir(options.wheel_dir)
 
-        req_tracker = self.enter_context(get_requirement_tracker())
+        build_tracker = self.enter_context(get_build_tracker())
 
         directory = TempDirectory(
             delete=not options.no_clean,
@@ -137,11 +121,12 @@ def run(self, options, args):
         preparer = self.make_requirement_preparer(
             temp_build_dir=directory,
             options=options,
-            req_tracker=req_tracker,
+            build_tracker=build_tracker,
             session=session,
             finder=finder,
             download_dir=options.wheel_dir,
             use_user_site=False,
+            verbosity=self.verbosity,
         )
 
         resolver = self.make_resolver(
@@ -155,11 +140,9 @@ def run(self, options, args):
 
         self.trace_basic_info(finder)
 
-        requirement_set = resolver.resolve(
-            reqs, check_supported_wheels=True
-        )
+        requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
 
-        reqs_to_build = []  # type: List[InstallRequirement]
+        reqs_to_build: List[InstallRequirement] = []
         for req in requirement_set.requirements.values():
             if req.is_wheel:
                 preparer.save_linked_requirement(req)
@@ -183,12 +166,11 @@ def run(self, options, args):
             except OSError as e:
                 logger.warning(
                     "Building wheel for %s failed: %s",
-                    req.name, e,
+                    req.name,
+                    e,
                 )
                 build_failures.append(req)
         if len(build_failures) != 0:
-            raise CommandError(
-                "Failed to build one or more wheels"
-            )
+            raise CommandError("Failed to build one or more wheels")
 
         return SUCCESS
diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py
index 9d9a36e8077..a8092d1ae06 100644
--- a/src/pip/_internal/configuration.py
+++ b/src/pip/_internal/configuration.py
@@ -13,9 +13,9 @@
 
 import configparser
 import locale
-import logging
 import os
 import sys
+from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
 
 from pip._internal.exceptions import (
     ConfigurationError,
@@ -23,45 +23,39 @@
 )
 from pip._internal.utils import appdirs
 from pip._internal.utils.compat import WINDOWS
+from pip._internal.utils.logging import getLogger
 from pip._internal.utils.misc import ensure_dir, enum
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
+RawConfigParser = configparser.RawConfigParser  # Shorthand
+Kind = NewType("Kind", str)
 
-    RawConfigParser = configparser.RawConfigParser  # Shorthand
-    Kind = NewType("Kind", str)
-
-CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf'
+CONFIG_BASENAME = "pip.ini" if WINDOWS else "pip.conf"
 ENV_NAMES_IGNORED = "version", "help"
 
 # The kinds of configurations there are.
 kinds = enum(
-    USER="user",        # User Specific
-    GLOBAL="global",    # System Wide
-    SITE="site",        # [Virtual] Environment Specific
-    ENV="env",          # from PIP_CONFIG_FILE
+    USER="user",  # User Specific
+    GLOBAL="global",  # System Wide
+    SITE="site",  # [Virtual] Environment Specific
+    ENV="env",  # from PIP_CONFIG_FILE
     ENV_VAR="env-var",  # from Environment Variables
 )
 OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
 VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE
 
-logger = logging.getLogger(__name__)
+logger = getLogger(__name__)
 
 
 # NOTE: Maybe use the optionx attribute to normalize keynames.
-def _normalize_name(name):
-    # type: (str) -> str
-    """Make a name consistent regardless of source (environment or file)
-    """
-    name = name.lower().replace('_', '-')
-    if name.startswith('--'):
+def _normalize_name(name: str) -> str:
+    """Make a name consistent regardless of source (environment or file)"""
+    name = name.lower().replace("_", "-")
+    if name.startswith("--"):
         name = name[2:]  # only prefer long opts
     return name
 
 
-def _disassemble_key(name):
-    # type: (str) -> List[str]
+def _disassemble_key(name: str) -> List[str]:
     if "." not in name:
         error_message = (
             "Key does not contain dot separated section and key. "
@@ -71,22 +65,18 @@ def _disassemble_key(name):
     return name.split(".", 1)
 
 
-def get_configuration_files():
-    # type: () -> Dict[Kind, List[str]]
+def get_configuration_files() -> Dict[Kind, List[str]]:
     global_config_files = [
-        os.path.join(path, CONFIG_BASENAME)
-        for path in appdirs.site_config_dirs('pip')
+        os.path.join(path, CONFIG_BASENAME) for path in appdirs.site_config_dirs("pip")
     ]
 
     site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
     legacy_config_file = os.path.join(
-        os.path.expanduser('~'),
-        'pip' if WINDOWS else '.pip',
+        os.path.expanduser("~"),
+        "pip" if WINDOWS else ".pip",
         CONFIG_BASENAME,
     )
-    new_config_file = os.path.join(
-        appdirs.user_config_dir("pip"), CONFIG_BASENAME
-    )
+    new_config_file = os.path.join(appdirs.user_config_dir("pip"), CONFIG_BASENAME)
     return {
         kinds.GLOBAL: global_config_files,
         kinds.SITE: [site_config_file],
@@ -108,8 +98,7 @@ class Configuration:
     and the data stored is also nice.
     """
 
-    def __init__(self, isolated, load_only=None):
-        # type: (bool, Optional[Kind]) -> None
+    def __init__(self, isolated: bool, load_only: Optional[Kind] = None) -> None:
         super().__init__()
 
         if load_only is not None and load_only not in VALID_LOAD_ONLY:
@@ -122,54 +111,44 @@ def __init__(self, isolated, load_only=None):
         self.load_only = load_only
 
         # Because we keep track of where we got the data from
-        self._parsers = {
+        self._parsers: Dict[Kind, List[Tuple[str, RawConfigParser]]] = {
             variant: [] for variant in OVERRIDE_ORDER
-        }  # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
-        self._config = {
+        }
+        self._config: Dict[Kind, Dict[str, Any]] = {
             variant: {} for variant in OVERRIDE_ORDER
-        }  # type: Dict[Kind, Dict[str, Any]]
-        self._modified_parsers = []  # type: List[Tuple[str, RawConfigParser]]
+        }
+        self._modified_parsers: List[Tuple[str, RawConfigParser]] = []
 
-    def load(self):
-        # type: () -> None
-        """Loads configuration from configuration files and environment
-        """
+    def load(self) -> None:
+        """Loads configuration from configuration files and environment"""
         self._load_config_files()
         if not self.isolated:
             self._load_environment_vars()
 
-    def get_file_to_edit(self):
-        # type: () -> Optional[str]
-        """Returns the file with highest priority in configuration
-        """
-        assert self.load_only is not None, \
-            "Need to be specified a file to be editing"
+    def get_file_to_edit(self) -> Optional[str]:
+        """Returns the file with highest priority in configuration"""
+        assert self.load_only is not None, "Need to be specified a file to be editing"
 
         try:
             return self._get_parser_to_modify()[0]
         except IndexError:
             return None
 
-    def items(self):
-        # type: () -> Iterable[Tuple[str, Any]]
+    def items(self) -> Iterable[Tuple[str, Any]]:
         """Returns key-value pairs like dict.items() representing the loaded
         configuration
         """
         return self._dictionary.items()
 
-    def get_value(self, key):
-        # type: (str) -> Any
-        """Get a value from the configuration.
-        """
+    def get_value(self, key: str) -> Any:
+        """Get a value from the configuration."""
         try:
             return self._dictionary[key]
         except KeyError:
             raise ConfigurationError(f"No such key - {key}")
 
-    def set_value(self, key, value):
-        # type: (str, Any) -> None
-        """Modify a value in the configuration.
-        """
+    def set_value(self, key: str, value: Any) -> None:
+        """Modify a value in the configuration."""
         self._ensure_have_load_only()
 
         assert self.load_only
@@ -186,8 +165,7 @@ def set_value(self, key, value):
         self._config[self.load_only][key] = value
         self._mark_as_modified(fname, parser)
 
-    def unset_value(self, key):
-        # type: (str) -> None
+    def unset_value(self, key: str) -> None:
         """Unset a value in the configuration."""
         self._ensure_have_load_only()
 
@@ -199,8 +177,9 @@ def unset_value(self, key):
 
         if parser is not None:
             section, name = _disassemble_key(key)
-            if not (parser.has_section(section)
-                    and parser.remove_option(section, name)):
+            if not (
+                parser.has_section(section) and parser.remove_option(section, name)
+            ):
                 # The option was not removed.
                 raise ConfigurationError(
                     "Fatal Internal error [id=1]. Please report as a bug."
@@ -213,10 +192,8 @@ def unset_value(self, key):
 
         del self._config[self.load_only][key]
 
-    def save(self):
-        # type: () -> None
-        """Save the current in-memory state.
-        """
+    def save(self) -> None:
+        """Save the current in-memory state."""
         self._ensure_have_load_only()
 
         for fname, parser in self._modified_parsers:
@@ -232,17 +209,14 @@ def save(self):
     # Private routines
     #
 
-    def _ensure_have_load_only(self):
-        # type: () -> None
+    def _ensure_have_load_only(self) -> None:
         if self.load_only is None:
             raise ConfigurationError("Needed a specific file to be modifying.")
         logger.debug("Will be working with %s variant only", self.load_only)
 
     @property
-    def _dictionary(self):
-        # type: () -> Dict[str, Any]
-        """A dictionary representing the loaded configuration.
-        """
+    def _dictionary(self) -> Dict[str, Any]:
+        """A dictionary representing the loaded configuration."""
         # NOTE: Dictionaries are not populated if not loaded. So, conditionals
         #       are not needed here.
         retval = {}
@@ -252,10 +226,8 @@ def _dictionary(self):
 
         return retval
 
-    def _load_config_files(self):
-        # type: () -> None
-        """Loads configuration from configuration files
-        """
+    def _load_config_files(self) -> None:
+        """Loads configuration from configuration files"""
         config_files = dict(self.iter_config_files())
         if config_files[kinds.ENV][0:1] == [os.devnull]:
             logger.debug(
@@ -269,9 +241,7 @@ def _load_config_files(self):
                 # If there's specific variant set in `load_only`, load only
                 # that variant, not the others.
                 if self.load_only is not None and variant != self.load_only:
-                    logger.debug(
-                        "Skipping file '%s' (variant: %s)", fname, variant
-                    )
+                    logger.debug("Skipping file '%s' (variant: %s)", fname, variant)
                     continue
 
                 parser = self._load_file(variant, fname)
@@ -279,9 +249,8 @@ def _load_config_files(self):
                 # Keeping track of the parsers used
                 self._parsers[variant].append((fname, parser))
 
-    def _load_file(self, variant, fname):
-        # type: (Kind, str) -> RawConfigParser
-        logger.debug("For variant '%s', will try loading '%s'", variant, fname)
+    def _load_file(self, variant: Kind, fname: str) -> RawConfigParser:
+        logger.verbose("For variant '%s', will try loading '%s'", variant, fname)
         parser = self._construct_parser(fname)
 
         for section in parser.sections():
@@ -290,22 +259,20 @@ def _load_file(self, variant, fname):
 
         return parser
 
-    def _construct_parser(self, fname):
-        # type: (str) -> RawConfigParser
+    def _construct_parser(self, fname: str) -> RawConfigParser:
         parser = configparser.RawConfigParser()
         # If there is no such file, don't bother reading it but create the
         # parser anyway, to hold the data.
         # Doing this is useful when modifying and saving files, where we don't
         # need to construct a parser.
         if os.path.exists(fname):
+            locale_encoding = locale.getpreferredencoding(False)
             try:
-                parser.read(fname)
+                parser.read(fname, encoding=locale_encoding)
             except UnicodeDecodeError:
                 # See https://github.com/pypa/pip/issues/4963
                 raise ConfigurationFileCouldNotBeLoaded(
-                    reason="contains invalid {} characters".format(
-                        locale.getpreferredencoding(False)
-                    ),
+                    reason=f"contains invalid {locale_encoding} characters",
                     fname=fname,
                 )
             except configparser.Error as error:
@@ -313,16 +280,15 @@ def _construct_parser(self, fname):
                 raise ConfigurationFileCouldNotBeLoaded(error=error)
         return parser
 
-    def _load_environment_vars(self):
-        # type: () -> None
-        """Loads configuration from environment variables
-        """
+    def _load_environment_vars(self) -> None:
+        """Loads configuration from environment variables"""
         self._config[kinds.ENV_VAR].update(
             self._normalized_keys(":env:", self.get_environ_vars())
         )
 
-    def _normalized_keys(self, section, items):
-        # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
+    def _normalized_keys(
+        self, section: str, items: Iterable[Tuple[str, Any]]
+    ) -> Dict[str, Any]:
         """Normalizes items to construct a dictionary with normalized keys.
 
         This routine is where the names become keys and are made the same
@@ -334,8 +300,7 @@ def _normalized_keys(self, section, items):
             normalized[key] = val
         return normalized
 
-    def get_environ_vars(self):
-        # type: () -> Iterable[Tuple[str, str]]
+    def get_environ_vars(self) -> Iterable[Tuple[str, str]]:
         """Returns a generator with all environmental vars with prefix PIP_"""
         for key, val in os.environ.items():
             if key.startswith("PIP_"):
@@ -344,8 +309,7 @@ def get_environ_vars(self):
                     yield name, val
 
     # XXX: This is patched in the tests.
-    def iter_config_files(self):
-        # type: () -> Iterable[Tuple[Kind, List[str]]]
+    def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]:
         """Yields variant and configuration files associated with it.
 
         This should be treated like items of a dictionary.
@@ -353,7 +317,7 @@ def iter_config_files(self):
         # SMELL: Move the conditions out of this function
 
         # environment variables have the lowest priority
-        config_file = os.environ.get('PIP_CONFIG_FILE', None)
+        config_file = os.environ.get("PIP_CONFIG_FILE", None)
         if config_file is not None:
             yield kinds.ENV, [config_file]
         else:
@@ -375,13 +339,11 @@ def iter_config_files(self):
         # finally virtualenv configuration first trumping others
         yield kinds.SITE, config_files[kinds.SITE]
 
-    def get_values_in_config(self, variant):
-        # type: (Kind) -> Dict[str, Any]
+    def get_values_in_config(self, variant: Kind) -> Dict[str, Any]:
         """Get values present in a config file"""
         return self._config[variant]
 
-    def _get_parser_to_modify(self):
-        # type: () -> Tuple[str, RawConfigParser]
+    def _get_parser_to_modify(self) -> Tuple[str, RawConfigParser]:
         # Determine which parser to modify
         assert self.load_only
         parsers = self._parsers[self.load_only]
@@ -395,12 +357,10 @@ def _get_parser_to_modify(self):
         return parsers[-1]
 
     # XXX: This is patched in the tests.
-    def _mark_as_modified(self, fname, parser):
-        # type: (str, RawConfigParser) -> None
+    def _mark_as_modified(self, fname: str, parser: RawConfigParser) -> None:
         file_parser_tuple = (fname, parser)
         if file_parser_tuple not in self._modified_parsers:
             self._modified_parsers.append(file_parser_tuple)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return f"{self.__class__.__name__}({self._dictionary!r})"
diff --git a/src/pip/_internal/distributions/__init__.py b/src/pip/_internal/distributions/__init__.py
index d5c1afc5bc1..9a89a838b9a 100644
--- a/src/pip/_internal/distributions/__init__.py
+++ b/src/pip/_internal/distributions/__init__.py
@@ -1,16 +1,13 @@
+from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.distributions.sdist import SourceDistribution
 from pip._internal.distributions.wheel import WheelDistribution
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.req.req_install import InstallRequirement
 
-if MYPY_CHECK_RUNNING:
-    from pip._internal.distributions.base import AbstractDistribution
-    from pip._internal.req.req_install import InstallRequirement
 
-
-def make_distribution_for_install_requirement(install_req):
-    # type: (InstallRequirement) -> AbstractDistribution
-    """Returns a Distribution for the given InstallRequirement
-    """
+def make_distribution_for_install_requirement(
+    install_req: InstallRequirement,
+) -> AbstractDistribution:
+    """Returns a Distribution for the given InstallRequirement"""
     # Editable requirements will always be source distributions. They use the
     # legacy logic until we create a modern standard for them.
     if install_req.editable:
diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py
index 37db810b351..149fff55dab 100644
--- a/src/pip/_internal/distributions/base.py
+++ b/src/pip/_internal/distributions/base.py
@@ -1,14 +1,8 @@
 import abc
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Optional
-
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.index.package_finder import PackageFinder
-    from pip._internal.req import InstallRequirement
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata.base import BaseDistribution
+from pip._internal.req import InstallRequirement
 
 
 class AbstractDistribution(metaclass=abc.ABCMeta):
@@ -26,17 +20,17 @@ class AbstractDistribution(metaclass=abc.ABCMeta):
      - we must be able to create a Distribution object exposing the
        above metadata.
     """
-    def __init__(self, req):
-        # type: (InstallRequirement) -> None
+
+    def __init__(self, req: InstallRequirement) -> None:
         super().__init__()
         self.req = req
 
     @abc.abstractmethod
-    def get_pkg_resources_distribution(self):
-        # type: () -> Optional[Distribution]
+    def get_metadata_distribution(self) -> BaseDistribution:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def prepare_distribution_metadata(self, finder, build_isolation):
-        # type: (PackageFinder, bool) -> None
+    def prepare_distribution_metadata(
+        self, finder: PackageFinder, build_isolation: bool
+    ) -> None:
         raise NotImplementedError()
diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py
index a813b211fe6..be5962f9800 100644
--- a/src/pip/_internal/distributions/installed.py
+++ b/src/pip/_internal/distributions/installed.py
@@ -1,12 +1,6 @@
 from pip._internal.distributions.base import AbstractDistribution
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Optional
-
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.index.package_finder import PackageFinder
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
 
 
 class InstalledDistribution(AbstractDistribution):
@@ -16,10 +10,11 @@ class InstalledDistribution(AbstractDistribution):
     been computed.
     """
 
-    def get_pkg_resources_distribution(self):
-        # type: () -> Optional[Distribution]
+    def get_metadata_distribution(self) -> BaseDistribution:
+        assert self.req.satisfied_by is not None, "not actually installed"
         return self.req.satisfied_by
 
-    def prepare_distribution_metadata(self, finder, build_isolation):
-        # type: (PackageFinder, bool) -> None
+    def prepare_distribution_metadata(
+        self, finder: PackageFinder, build_isolation: bool
+    ) -> None:
         pass
diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py
index 9b708fdd83c..463c409d473 100644
--- a/src/pip/_internal/distributions/sdist.py
+++ b/src/pip/_internal/distributions/sdist.py
@@ -1,18 +1,12 @@
 import logging
+from typing import Iterable, Set, Tuple
 
 from pip._internal.build_env import BuildEnvironment
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.exceptions import InstallationError
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
 from pip._internal.utils.subprocess import runner_with_spinner_message
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Set, Tuple
-
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.index.package_finder import PackageFinder
-
 
 logger = logging.getLogger(__name__)
 
@@ -24,40 +18,44 @@ class SourceDistribution(AbstractDistribution):
     generated, either using PEP 517 or using the legacy `setup.py egg_info`.
     """
 
-    def get_pkg_resources_distribution(self):
-        # type: () -> Distribution
+    def get_metadata_distribution(self) -> BaseDistribution:
         return self.req.get_dist()
 
-    def prepare_distribution_metadata(self, finder, build_isolation):
-        # type: (PackageFinder, bool) -> None
+    def prepare_distribution_metadata(
+        self, finder: PackageFinder, build_isolation: bool
+    ) -> None:
         # Load pyproject.toml, to determine whether PEP 517 is to be used
         self.req.load_pyproject_toml()
 
         # Set up the build isolation, if this requirement should be isolated
         should_isolate = self.req.use_pep517 and build_isolation
         if should_isolate:
-            self._setup_isolation(finder)
-
-        self.req.prepare_metadata()
-
-    def _setup_isolation(self, finder):
-        # type: (PackageFinder) -> None
-        def _raise_conflicts(conflicting_with, conflicting_reqs):
-            # type: (str, Set[Tuple[str, str]]) -> None
-            format_string = (
-                "Some build dependencies for {requirement} "
-                "conflict with {conflicting_with}: {description}."
+            # Setup an isolated environment and install the build backend static
+            # requirements in it.
+            self._prepare_build_backend(finder)
+            # Check that if the requirement is editable, it either supports PEP 660 or
+            # has a setup.py or a setup.cfg. This cannot be done earlier because we need
+            # to setup the build backend to verify it supports build_editable, nor can
+            # it be done later, because we want to avoid installing build requirements
+            # needlessly. Doing it here also works around setuptools generating
+            # UNKNOWN.egg-info when running get_requires_for_build_wheel on a directory
+            # without setup.py nor setup.cfg.
+            self.req.isolated_editable_sanity_check()
+            # Install the dynamic build requirements.
+            self._install_build_reqs(finder)
+        elif self.req.use_pep517:
+            pyproject_requires = self.req.pyproject_requires
+            assert pyproject_requires is not None
+            conflicting, missing = self.req.build_env.check_requirements(
+                pyproject_requires
             )
-            error_message = format_string.format(
-                requirement=self.req,
-                conflicting_with=conflicting_with,
-                description=', '.join(
-                    f'{installed} is incompatible with {wanted}'
-                    for installed, wanted in sorted(conflicting)
-                )
-            )
-            raise InstallationError(error_message)
+            if conflicting:
+                self._raise_conflicts("the backend dependencies", conflicting)
+            if missing:
+                self._raise_missing_reqs(missing)
+        self.req.prepare_metadata()
 
+    def _prepare_build_backend(self, finder: PackageFinder) -> None:
         # Isolate in a BuildEnvironment and install the build-time
         # requirements.
         pyproject_requires = self.req.pyproject_requires
@@ -65,15 +63,13 @@ def _raise_conflicts(conflicting_with, conflicting_reqs):
 
         self.req.build_env = BuildEnvironment()
         self.req.build_env.install_requirements(
-            finder, pyproject_requires, 'overlay',
-            "Installing build dependencies"
+            finder, pyproject_requires, "overlay", kind="build dependencies"
         )
         conflicting, missing = self.req.build_env.check_requirements(
             self.req.requirements_to_check
         )
         if conflicting:
-            _raise_conflicts("PEP 517/518 supported requirements",
-                             conflicting)
+            self._raise_conflicts("PEP 517/518 supported requirements", conflicting)
         if missing:
             logger.warning(
                 "Missing build requirements in pyproject.toml for %s.",
@@ -82,24 +78,68 @@ def _raise_conflicts(conflicting_with, conflicting_reqs):
             logger.warning(
                 "The project does not specify a build backend, and "
                 "pip cannot fall back to setuptools without %s.",
-                " and ".join(map(repr, sorted(missing)))
+                " and ".join(map(repr, sorted(missing))),
             )
-        # Install any extra build dependencies that the backend requests.
-        # This must be done in a second pass, as the pyproject.toml
-        # dependencies must be installed before we can call the backend.
+
+    def _get_build_requires_wheel(self) -> Iterable[str]:
+        with self.req.build_env:
+            runner = runner_with_spinner_message("Getting requirements to build wheel")
+            backend = self.req.pep517_backend
+            assert backend is not None
+            with backend.subprocess_runner(runner):
+                return backend.get_requires_for_build_wheel()
+
+    def _get_build_requires_editable(self) -> Iterable[str]:
         with self.req.build_env:
             runner = runner_with_spinner_message(
-                "Getting requirements to build wheel"
+                "Getting requirements to build editable"
             )
             backend = self.req.pep517_backend
             assert backend is not None
             with backend.subprocess_runner(runner):
-                reqs = backend.get_requires_for_build_wheel()
+                return backend.get_requires_for_build_editable()
 
-        conflicting, missing = self.req.build_env.check_requirements(reqs)
+    def _install_build_reqs(self, finder: PackageFinder) -> None:
+        # Install any extra build dependencies that the backend requests.
+        # This must be done in a second pass, as the pyproject.toml
+        # dependencies must be installed before we can call the backend.
+        if (
+            self.req.editable
+            and self.req.permit_editable_wheels
+            and self.req.supports_pyproject_editable()
+        ):
+            build_reqs = self._get_build_requires_editable()
+        else:
+            build_reqs = self._get_build_requires_wheel()
+        conflicting, missing = self.req.build_env.check_requirements(build_reqs)
         if conflicting:
-            _raise_conflicts("the backend dependencies", conflicting)
+            self._raise_conflicts("the backend dependencies", conflicting)
         self.req.build_env.install_requirements(
-            finder, missing, 'normal',
-            "Installing backend dependencies"
+            finder, missing, "normal", kind="backend dependencies"
+        )
+
+    def _raise_conflicts(
+        self, conflicting_with: str, conflicting_reqs: Set[Tuple[str, str]]
+    ) -> None:
+        format_string = (
+            "Some build dependencies for {requirement} "
+            "conflict with {conflicting_with}: {description}."
+        )
+        error_message = format_string.format(
+            requirement=self.req,
+            conflicting_with=conflicting_with,
+            description=", ".join(
+                f"{installed} is incompatible with {wanted}"
+                for installed, wanted in sorted(conflicting_reqs)
+            ),
+        )
+        raise InstallationError(error_message)
+
+    def _raise_missing_reqs(self, missing: Set[str]) -> None:
+        format_string = (
+            "Some build dependencies for {requirement} are missing: {missing}."
+        )
+        error_message = format_string.format(
+            requirement=self.req, missing=", ".join(map(repr, sorted(missing)))
         )
+        raise InstallationError(error_message)
diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py
index 2adc2286271..340b0f3c5c7 100644
--- a/src/pip/_internal/distributions/wheel.py
+++ b/src/pip/_internal/distributions/wheel.py
@@ -1,13 +1,12 @@
-from zipfile import ZipFile
+from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.distributions.base import AbstractDistribution
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
-
-if MYPY_CHECK_RUNNING:
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.index.package_finder import PackageFinder
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import (
+    BaseDistribution,
+    FilesystemWheel,
+    get_wheel_distribution,
+)
 
 
 class WheelDistribution(AbstractDistribution):
@@ -16,22 +15,17 @@ class WheelDistribution(AbstractDistribution):
     This does not need any preparation as wheels can be directly unpacked.
     """
 
-    def get_pkg_resources_distribution(self):
-        # type: () -> Distribution
+    def get_metadata_distribution(self) -> BaseDistribution:
         """Loads the metadata from the wheel file into memory and returns a
         Distribution that uses it, not relying on the wheel file or
         requirement.
         """
-        # Set as part of preparation during download.
-        assert self.req.local_file_path
-        # Wheels are never unnamed.
-        assert self.req.name
-
-        with ZipFile(self.req.local_file_path, allowZip64=True) as z:
-            return pkg_resources_distribution_for_wheel(
-                z, self.req.name, self.req.local_file_path
-            )
-
-    def prepare_distribution_metadata(self, finder, build_isolation):
-        # type: (PackageFinder, bool) -> None
+        assert self.req.local_file_path, "Set as part of preparation during download"
+        assert self.req.name, "Wheels are never unnamed"
+        wheel = FilesystemWheel(self.req.local_file_path)
+        return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+
+    def prepare_distribution_metadata(
+        self, finder: PackageFinder, build_isolation: bool
+    ) -> None:
         pass
diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py
index 43d083205a0..97b9612a187 100644
--- a/src/pip/_internal/exceptions.py
+++ b/src/pip/_internal/exceptions.py
@@ -1,24 +1,174 @@
-"""Exceptions used throughout package"""
+"""Exceptions used throughout package.
 
+This module MUST NOT try to import from anything within `pip._internal` to
+operate. This is expected to be importable from any/all files within the
+subpackage and, thus, should not depend on them.
+"""
+
+import configparser
+import re
 from itertools import chain, groupby, repeat
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._vendor.requests.models import Request, Response
+from pip._vendor.rich.console import Console, ConsoleOptions, RenderResult
+from pip._vendor.rich.markup import escape
+from pip._vendor.rich.text import Text
 
-if MYPY_CHECK_RUNNING:
-    import configparser
+if TYPE_CHECKING:
     from hashlib import _Hash
-    from typing import Any, Dict, List, Optional
-
-    from pip._vendor.pkg_resources import Distribution
-    from pip._vendor.requests.models import Request, Response
+    from typing import Literal
 
+    from pip._internal.metadata import BaseDistribution
     from pip._internal.req.req_install import InstallRequirement
 
 
+#
+# Scaffolding
+#
+def _is_kebab_case(s: str) -> bool:
+    return re.match(r"^[a-z]+(-[a-z]+)*$", s) is not None
+
+
+def _prefix_with_indent(
+    s: Union[Text, str],
+    console: Console,
+    *,
+    prefix: str,
+    indent: str,
+) -> Text:
+    if isinstance(s, Text):
+        text = s
+    else:
+        text = console.render_str(s)
+
+    return console.render_str(prefix, overflow="ignore") + console.render_str(
+        f"\n{indent}", overflow="ignore"
+    ).join(text.split(allow_blank=True))
+
+
 class PipError(Exception):
-    """Base pip exception"""
+    """The base pip error."""
+
+
+class DiagnosticPipError(PipError):
+    """An error, that presents diagnostic information to the user.
+
+    This contains a bunch of logic, to enable pretty presentation of our error
+    messages. Each error gets a unique reference. Each error can also include
+    additional context, a hint and/or a note -- which are presented with the
+    main error message in a consistent style.
+
+    This is adapted from the error output styling in `sphinx-theme-builder`.
+    """
+
+    reference: str
+
+    def __init__(
+        self,
+        *,
+        kind: 'Literal["error", "warning"]' = "error",
+        reference: Optional[str] = None,
+        message: Union[str, Text],
+        context: Optional[Union[str, Text]],
+        hint_stmt: Optional[Union[str, Text]],
+        note_stmt: Optional[Union[str, Text]] = None,
+        link: Optional[str] = None,
+    ) -> None:
+        # Ensure a proper reference is provided.
+        if reference is None:
+            assert hasattr(self, "reference"), "error reference not provided!"
+            reference = self.reference
+        assert _is_kebab_case(reference), "error reference must be kebab-case!"
+
+        self.kind = kind
+        self.reference = reference
+
+        self.message = message
+        self.context = context
+
+        self.note_stmt = note_stmt
+        self.hint_stmt = hint_stmt
+
+        self.link = link
+
+        super().__init__(f"<{self.__class__.__name__}: {self.reference}>")
+
+    def __repr__(self) -> str:
+        return (
+            f"<{self.__class__.__name__}("
+            f"reference={self.reference!r}, "
+            f"message={self.message!r}, "
+            f"context={self.context!r}, "
+            f"note_stmt={self.note_stmt!r}, "
+            f"hint_stmt={self.hint_stmt!r}"
+            ")>"
+        )
+
+    def __rich_console__(
+        self,
+        console: Console,
+        options: ConsoleOptions,
+    ) -> RenderResult:
+        colour = "red" if self.kind == "error" else "yellow"
+
+        yield f"[{colour} bold]{self.kind}[/]: [bold]{self.reference}[/]"
+        yield ""
+
+        if not options.ascii_only:
+            # Present the main message, with relevant context indented.
+            if self.context is not None:
+                yield _prefix_with_indent(
+                    self.message,
+                    console,
+                    prefix=f"[{colour}]×[/] ",
+                    indent=f"[{colour}]│[/] ",
+                )
+                yield _prefix_with_indent(
+                    self.context,
+                    console,
+                    prefix=f"[{colour}]╰─>[/] ",
+                    indent=f"[{colour}]   [/] ",
+                )
+            else:
+                yield _prefix_with_indent(
+                    self.message,
+                    console,
+                    prefix="[red]×[/] ",
+                    indent="  ",
+                )
+        else:
+            yield self.message
+            if self.context is not None:
+                yield ""
+                yield self.context
+
+        if self.note_stmt is not None or self.hint_stmt is not None:
+            yield ""
+
+        if self.note_stmt is not None:
+            yield _prefix_with_indent(
+                self.note_stmt,
+                console,
+                prefix="[magenta bold]note[/]: ",
+                indent="      ",
+            )
+        if self.hint_stmt is not None:
+            yield _prefix_with_indent(
+                self.hint_stmt,
+                console,
+                prefix="[cyan bold]hint[/]: ",
+                indent="      ",
+            )
+
+        if self.link is not None:
+            yield ""
+            yield f"Link: {self.link}"
 
 
+#
+# Actual Errors
+#
 class ConfigurationError(PipError):
     """General exception in configuration"""
 
@@ -31,17 +181,54 @@ class UninstallationError(PipError):
     """General exception during uninstallation"""
 
 
+class MissingPyProjectBuildRequires(DiagnosticPipError):
+    """Raised when pyproject.toml has `build-system`, but no `build-system.requires`."""
+
+    reference = "missing-pyproject-build-system-requires"
+
+    def __init__(self, *, package: str) -> None:
+        super().__init__(
+            message=f"Can not process {escape(package)}",
+            context=Text(
+                "This package has an invalid pyproject.toml file.\n"
+                "The [build-system] table is missing the mandatory `requires` key."
+            ),
+            note_stmt="This is an issue with the package mentioned above, not pip.",
+            hint_stmt=Text("See PEP 518 for the detailed specification."),
+        )
+
+
+class InvalidPyProjectBuildRequires(DiagnosticPipError):
+    """Raised when pyproject.toml an invalid `build-system.requires`."""
+
+    reference = "invalid-pyproject-build-system-requires"
+
+    def __init__(self, *, package: str, reason: str) -> None:
+        super().__init__(
+            message=f"Can not process {escape(package)}",
+            context=Text(
+                "This package has an invalid `build-system.requires` key in "
+                f"pyproject.toml.\n{reason}"
+            ),
+            note_stmt="This is an issue with the package mentioned above, not pip.",
+            hint_stmt=Text("See PEP 518 for the detailed specification."),
+        )
+
+
 class NoneMetadataError(PipError):
-    """
-    Raised when accessing "METADATA" or "PKG-INFO" metadata for a
-    pip._vendor.pkg_resources.Distribution object and
-    `dist.has_metadata('METADATA')` returns True but
-    `dist.get_metadata('METADATA')` returns None (and similarly for
-    "PKG-INFO").
+    """Raised when accessing a Distribution's "METADATA" or "PKG-INFO".
+
+    This signifies an inconsistency, when the Distribution claims to have
+    the metadata file (if not, raise ``FileNotFoundError`` instead), but is
+    not actually able to produce its content. This may be due to permission
+    errors.
     """
 
-    def __init__(self, dist, metadata_name):
-        # type: (Distribution, str) -> None
+    def __init__(
+        self,
+        dist: "BaseDistribution",
+        metadata_name: str,
+    ) -> None:
         """
         :param dist: A Distribution object.
         :param metadata_name: The name of the metadata being accessed
@@ -50,17 +237,28 @@ def __init__(self, dist, metadata_name):
         self.dist = dist
         self.metadata_name = metadata_name
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         # Use `dist` in the error message because its stringification
         # includes more information, like the version and location.
-        return (
-            'None {} metadata found for distribution: {}'.format(
-                self.metadata_name, self.dist,
-            )
+        return "None {} metadata found for distribution: {}".format(
+            self.metadata_name,
+            self.dist,
         )
 
 
+class UserInstallationInvalid(InstallationError):
+    """A --user install is requested on an environment without user site."""
+
+    def __str__(self) -> str:
+        return "User base directory is not specified"
+
+
+class InvalidSchemeCombination(InstallationError):
+    def __str__(self) -> str:
+        before = ", ".join(str(a) for a in self.args[:-1])
+        return f"Cannot set {before} and {self.args[-1]} together"
+
+
 class DistributionNotFound(InstallationError):
     """Raised when a distribution cannot be found to satisfy a requirement"""
 
@@ -89,8 +287,9 @@ class PreviousBuildDirError(PipError):
 class NetworkConnectionError(PipError):
     """HTTP connection error"""
 
-    def __init__(self, error_msg, response=None, request=None):
-        # type: (str, Response, Request) -> None
+    def __init__(
+        self, error_msg: str, response: Response = None, request: Request = None
+    ) -> None:
         """
         Initialize NetworkConnectionError with  `request` and `response`
         objects.
@@ -98,13 +297,15 @@ def __init__(self, error_msg, response=None, request=None):
         self.response = response
         self.request = request
         self.error_msg = error_msg
-        if (self.response is not None and not self.request and
-                hasattr(response, 'request')):
+        if (
+            self.response is not None
+            and not self.request
+            and hasattr(response, "request")
+        ):
             self.request = self.response.request
         super().__init__(error_msg, response, request)
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return str(self.error_msg)
 
 
@@ -116,6 +317,17 @@ class UnsupportedWheel(InstallationError):
     """Unsupported wheel."""
 
 
+class InvalidWheel(InstallationError):
+    """Invalid (e.g. corrupt) wheel."""
+
+    def __init__(self, location: str, name: str):
+        self.location = location
+        self.name = name
+
+    def __str__(self) -> str:
+        return f"Wheel '{self.name}' located at {self.location} is invalid."
+
+
 class MetadataInconsistent(InstallationError):
     """Built metadata contains inconsistent information.
 
@@ -123,64 +335,119 @@ class MetadataInconsistent(InstallationError):
     that do not match the information previously obtained from sdist filename
     or user-supplied ``#egg=`` value.
     """
-    def __init__(self, ireq, field, built):
-        # type: (InstallRequirement, str, Any) -> None
+
+    def __init__(
+        self, ireq: "InstallRequirement", field: str, f_val: str, m_val: str
+    ) -> None:
         self.ireq = ireq
         self.field = field
-        self.built = built
+        self.f_val = f_val
+        self.m_val = m_val
 
-    def __str__(self):
-        # type: () -> str
-        return "Requested {} has different {} in metadata: {!r}".format(
-            self.ireq, self.field, self.built,
+    def __str__(self) -> str:
+        template = (
+            "Requested {} has inconsistent {}: "
+            "filename has {!r}, but metadata has {!r}"
         )
+        return template.format(self.ireq, self.field, self.f_val, self.m_val)
 
 
-class InstallationSubprocessError(InstallationError):
-    """A subprocess call failed during installation."""
-    def __init__(self, returncode, description):
-        # type: (int, str) -> None
-        self.returncode = returncode
-        self.description = description
+class LegacyInstallFailure(DiagnosticPipError):
+    """Error occurred while executing `setup.py install`"""
 
-    def __str__(self):
-        # type: () -> str
-        return (
-            "Command errored out with exit status {}: {} "
-            "Check the logs for full command output."
-        ).format(self.returncode, self.description)
+    reference = "legacy-install-failure"
+
+    def __init__(self, package_details: str) -> None:
+        super().__init__(
+            message="Encountered error while trying to install package.",
+            context=package_details,
+            hint_stmt="See above for output from the failure.",
+            note_stmt="This is an issue with the package mentioned above, not pip.",
+        )
+
+
+class InstallationSubprocessError(DiagnosticPipError, InstallationError):
+    """A subprocess call failed."""
+
+    reference = "subprocess-exited-with-error"
+
+    def __init__(
+        self,
+        *,
+        command_description: str,
+        exit_code: int,
+        output_lines: Optional[List[str]],
+    ) -> None:
+        if output_lines is None:
+            output_prompt = Text("See above for output.")
+        else:
+            output_prompt = (
+                Text.from_markup(f"[red][{len(output_lines)} lines of output][/]\n")
+                + Text("".join(output_lines))
+                + Text.from_markup(R"[red]\[end of output][/]")
+            )
+
+        super().__init__(
+            message=(
+                f"[green]{escape(command_description)}[/] did not run successfully.\n"
+                f"exit code: {exit_code}"
+            ),
+            context=output_prompt,
+            hint_stmt=None,
+            note_stmt=(
+                "This error originates from a subprocess, and is likely not a "
+                "problem with pip."
+            ),
+        )
+
+        self.command_description = command_description
+        self.exit_code = exit_code
+
+    def __str__(self) -> str:
+        return f"{self.command_description} exited with {self.exit_code}"
+
+
+class MetadataGenerationFailed(InstallationSubprocessError, InstallationError):
+    reference = "metadata-generation-failed"
+
+    def __init__(
+        self,
+        *,
+        package_details: str,
+    ) -> None:
+        super(InstallationSubprocessError, self).__init__(
+            message="Encountered error while generating package metadata.",
+            context=escape(package_details),
+            hint_stmt="See above for details.",
+            note_stmt="This is an issue with the package mentioned above, not pip.",
+        )
+
+    def __str__(self) -> str:
+        return "metadata generation failed"
 
 
 class HashErrors(InstallationError):
     """Multiple HashError instances rolled into one for reporting"""
 
-    def __init__(self):
-        # type: () -> None
-        self.errors = []  # type: List[HashError]
+    def __init__(self) -> None:
+        self.errors: List["HashError"] = []
 
-    def append(self, error):
-        # type: (HashError) -> None
+    def append(self, error: "HashError") -> None:
         self.errors.append(error)
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         lines = []
         self.errors.sort(key=lambda e: e.order)
         for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__):
             lines.append(cls.head)
             lines.extend(e.body() for e in errors_of_cls)
         if lines:
-            return '\n'.join(lines)
-        return ''
+            return "\n".join(lines)
+        return ""
 
-    def __nonzero__(self):
-        # type: () -> bool
+    def __bool__(self) -> bool:
         return bool(self.errors)
 
-    def __bool__(self):
-        # type: () -> bool
-        return self.__nonzero__()
-
 
 class HashError(InstallationError):
     """
@@ -198,12 +465,12 @@ class HashError(InstallationError):
         typically available earlier.
 
     """
-    req = None  # type: Optional[InstallRequirement]
-    head = ''
-    order = -1  # type: int
 
-    def body(self):
-        # type: () -> str
+    req: Optional["InstallRequirement"] = None
+    head = ""
+    order: int = -1
+
+    def body(self) -> str:
         """Return a summary of me for display under the heading.
 
         This default implementation simply prints a description of the
@@ -213,21 +480,19 @@ def body(self):
             its link already populated by the resolver's _populate_link().
 
         """
-        return f'    {self._requirement_name()}'
+        return f"    {self._requirement_name()}"
 
-    def __str__(self):
-        # type: () -> str
-        return f'{self.head}\n{self.body()}'
+    def __str__(self) -> str:
+        return f"{self.head}\n{self.body()}"
 
-    def _requirement_name(self):
-        # type: () -> str
+    def _requirement_name(self) -> str:
         """Return a description of the requirement that triggered me.
 
         This default implementation returns long description of the req, with
         line numbers
 
         """
-        return str(self.req) if self.req else 'unknown package'
+        return str(self.req) if self.req else "unknown package"
 
 
 class VcsHashUnsupported(HashError):
@@ -235,8 +500,10 @@ class VcsHashUnsupported(HashError):
     we don't have a method for hashing those."""
 
     order = 0
-    head = ("Can't verify hashes for these requirements because we don't "
-            "have a way to hash version control repositories:")
+    head = (
+        "Can't verify hashes for these requirements because we don't "
+        "have a way to hash version control repositories:"
+    )
 
 
 class DirectoryUrlHashUnsupported(HashError):
@@ -244,32 +511,34 @@ class DirectoryUrlHashUnsupported(HashError):
     we don't have a method for hashing those."""
 
     order = 1
-    head = ("Can't verify hashes for these file:// requirements because they "
-            "point to directories:")
+    head = (
+        "Can't verify hashes for these file:// requirements because they "
+        "point to directories:"
+    )
 
 
 class HashMissing(HashError):
     """A hash was needed for a requirement but is absent."""
 
     order = 2
-    head = ('Hashes are required in --require-hashes mode, but they are '
-            'missing from some requirements. Here is a list of those '
-            'requirements along with the hashes their downloaded archives '
-            'actually had. Add lines like these to your requirements files to '
-            'prevent tampering. (If you did not enable --require-hashes '
-            'manually, note that it turns on automatically when any package '
-            'has a hash.)')
-
-    def __init__(self, gotten_hash):
-        # type: (str) -> None
+    head = (
+        "Hashes are required in --require-hashes mode, but they are "
+        "missing from some requirements. Here is a list of those "
+        "requirements along with the hashes their downloaded archives "
+        "actually had. Add lines like these to your requirements files to "
+        "prevent tampering. (If you did not enable --require-hashes "
+        "manually, note that it turns on automatically when any package "
+        "has a hash.)"
+    )
+
+    def __init__(self, gotten_hash: str) -> None:
         """
         :param gotten_hash: The hash of the (possibly malicious) archive we
             just downloaded
         """
         self.gotten_hash = gotten_hash
 
-    def body(self):
-        # type: () -> str
+    def body(self) -> str:
         # Dodge circular import.
         from pip._internal.utils.hashes import FAVORITE_HASH
 
@@ -278,13 +547,16 @@ def body(self):
             # In the case of URL-based requirements, display the original URL
             # seen in the requirements file rather than the package name,
             # so the output can be directly copied into the requirements file.
-            package = (self.req.original_link if self.req.original_link
-                       # In case someone feeds something downright stupid
-                       # to InstallRequirement's constructor.
-                       else getattr(self.req, 'req', None))
-        return '    {} --hash={}:{}'.format(package or 'unknown package',
-                                            FAVORITE_HASH,
-                                            self.gotten_hash)
+            package = (
+                self.req.original_link
+                if self.req.original_link
+                # In case someone feeds something downright stupid
+                # to InstallRequirement's constructor.
+                else getattr(self.req, "req", None)
+            )
+        return "    {} --hash={}:{}".format(
+            package or "unknown package", FAVORITE_HASH, self.gotten_hash
+        )
 
 
 class HashUnpinned(HashError):
@@ -292,8 +564,10 @@ class HashUnpinned(HashError):
     version."""
 
     order = 3
-    head = ('In --require-hashes mode, all requirements must have their '
-            'versions pinned with ==. These do not:')
+    head = (
+        "In --require-hashes mode, all requirements must have their "
+        "versions pinned with ==. These do not:"
+    )
 
 
 class HashMismatch(HashError):
@@ -305,14 +579,16 @@ class HashMismatch(HashError):
         improve its error message.
 
     """
-    order = 4
-    head = ('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS '
-            'FILE. If you have updated the package versions, please update '
-            'the hashes. Otherwise, examine the package contents carefully; '
-            'someone may have tampered with them.')
 
-    def __init__(self, allowed, gots):
-        # type: (Dict[str, List[str]], Dict[str, _Hash]) -> None
+    order = 4
+    head = (
+        "THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS "
+        "FILE. If you have updated the package versions, please update "
+        "the hashes. Otherwise, examine the package contents carefully; "
+        "someone may have tampered with them."
+    )
+
+    def __init__(self, allowed: Dict[str, List[str]], gots: Dict[str, "_Hash"]) -> None:
         """
         :param allowed: A dict of algorithm names pointing to lists of allowed
             hex digests
@@ -322,13 +598,10 @@ def __init__(self, allowed, gots):
         self.allowed = allowed
         self.gots = gots
 
-    def body(self):
-        # type: () -> str
-        return '    {}:\n{}'.format(self._requirement_name(),
-                                    self._hash_comparison())
+    def body(self) -> str:
+        return "    {}:\n{}".format(self._requirement_name(), self._hash_comparison())
 
-    def _hash_comparison(self):
-        # type: () -> str
+    def _hash_comparison(self) -> str:
         """
         Return a comparison of actual and expected hash values.
 
@@ -339,20 +612,22 @@ def _hash_comparison(self):
                     Got        bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef
 
         """
-        def hash_then_or(hash_name):
-            # type: (str) -> chain[str]
+
+        def hash_then_or(hash_name: str) -> "chain[str]":
             # For now, all the decent hashes have 6-char names, so we can get
             # away with hard-coding space literals.
-            return chain([hash_name], repeat('    or'))
+            return chain([hash_name], repeat("    or"))
 
-        lines = []  # type: List[str]
+        lines: List[str] = []
         for hash_name, expecteds in self.allowed.items():
             prefix = hash_then_or(hash_name)
-            lines.extend(('        Expected {} {}'.format(next(prefix), e))
-                         for e in expecteds)
-            lines.append('             Got        {}\n'.format(
-                         self.gots[hash_name].hexdigest()))
-        return '\n'.join(lines)
+            lines.extend(
+                ("        Expected {} {}".format(next(prefix), e)) for e in expecteds
+            )
+            lines.append(
+                "             Got        {}\n".format(self.gots[hash_name].hexdigest())
+            )
+        return "\n".join(lines)
 
 
 class UnsupportedPythonVersion(InstallationError):
@@ -361,18 +636,20 @@ class UnsupportedPythonVersion(InstallationError):
 
 
 class ConfigurationFileCouldNotBeLoaded(ConfigurationError):
-    """When there are errors while loading a configuration file
-    """
-
-    def __init__(self, reason="could not be loaded", fname=None, error=None):
-        # type: (str, Optional[str], Optional[configparser.Error]) -> None
+    """When there are errors while loading a configuration file"""
+
+    def __init__(
+        self,
+        reason: str = "could not be loaded",
+        fname: Optional[str] = None,
+        error: Optional[configparser.Error] = None,
+    ) -> None:
         super().__init__(error)
         self.reason = reason
         self.fname = fname
         self.error = error
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         if self.fname is not None:
             message_part = f" in {self.fname}."
         else:
diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py
index ee4eb719992..e6e9469af1a 100644
--- a/src/pip/_internal/index/collector.py
+++ b/src/pip/_internal/index/collector.py
@@ -1,79 +1,78 @@
 """
-The main purpose of this module is to expose LinkCollector.collect_links().
+The main purpose of this module is to expose LinkCollector.collect_sources().
 """
 
 import cgi
+import collections
 import functools
 import itertools
 import logging
-import mimetypes
 import os
 import re
 import urllib.parse
 import urllib.request
-from collections import OrderedDict
+import xml.etree.ElementTree
+from html.parser import HTMLParser
+from optparse import Values
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    MutableMapping,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 from pip._vendor import html5lib, requests
-from pip._vendor.distlib.compat import unescape
+from pip._vendor.requests import Response
 from pip._vendor.requests.exceptions import RetryError, SSLError
 
 from pip._internal.exceptions import NetworkConnectionError
 from pip._internal.models.link import Link
 from pip._internal.models.search_scope import SearchScope
+from pip._internal.network.session import PipSession
 from pip._internal.network.utils import raise_for_status
 from pip._internal.utils.filetypes import is_archive_file
 from pip._internal.utils.misc import pairwise, redact_auth_from_url
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-from pip._internal.utils.urls import path_to_url, url_to_path
-from pip._internal.vcs import is_url, vcs
-
-if MYPY_CHECK_RUNNING:
-    import xml.etree.ElementTree
-    from optparse import Values
-    from typing import (
-        Callable,
-        Iterable,
-        List,
-        MutableMapping,
-        Optional,
-        Sequence,
-        Tuple,
-        Union,
-    )
-
-    from pip._vendor.requests import Response
-
-    from pip._internal.network.session import PipSession
+from pip._internal.vcs import vcs
 
-    HTMLElement = xml.etree.ElementTree.Element
-    ResponseHeaders = MutableMapping[str, str]
+from .sources import CandidatesFromPage, LinkSource, build_source
 
+if TYPE_CHECKING:
+    from typing import Protocol
+else:
+    Protocol = object
 
 logger = logging.getLogger(__name__)
 
+HTMLElement = xml.etree.ElementTree.Element
+ResponseHeaders = MutableMapping[str, str]
 
-def _match_vcs_scheme(url):
-    # type: (str) -> Optional[str]
+
+def _match_vcs_scheme(url: str) -> Optional[str]:
     """Look for VCS schemes in the URL.
 
     Returns the matched VCS scheme, or None if there's no match.
     """
     for scheme in vcs.schemes:
-        if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
+        if url.lower().startswith(scheme) and url[len(scheme)] in "+:":
             return scheme
     return None
 
 
 class _NotHTML(Exception):
-    def __init__(self, content_type, request_desc):
-        # type: (str, str) -> None
+    def __init__(self, content_type: str, request_desc: str) -> None:
         super().__init__(content_type, request_desc)
         self.content_type = content_type
         self.request_desc = request_desc
 
 
-def _ensure_html_header(response):
-    # type: (Response) -> None
+def _ensure_html_header(response: Response) -> None:
     """Check the Content-Type header to ensure the response contains HTML.
 
     Raises `_NotHTML` if the content type is not text/html.
@@ -87,15 +86,14 @@ class _NotHTTP(Exception):
     pass
 
 
-def _ensure_html_response(url, session):
-    # type: (str, PipSession) -> None
+def _ensure_html_response(url: str, session: PipSession) -> None:
     """Send a HEAD request to the URL, and ensure the response contains HTML.
 
     Raises `_NotHTTP` if the URL is not available for a HEAD request, or
     `_NotHTML` if the content type is not text/html.
     """
     scheme, netloc, path, query, fragment = urllib.parse.urlsplit(url)
-    if scheme not in {'http', 'https'}:
+    if scheme not in {"http", "https"}:
         raise _NotHTTP()
 
     resp = session.head(url, allow_redirects=True)
@@ -104,8 +102,7 @@ def _ensure_html_response(url, session):
     _ensure_html_header(resp)
 
 
-def _get_html_response(url, session):
-    # type: (str, PipSession) -> Response
+def _get_html_response(url: str, session: PipSession) -> Response:
     """Access an HTML page with GET, and return the response.
 
     This consists of three parts:
@@ -121,7 +118,7 @@ def _get_html_response(url, session):
     if is_archive_file(Link(url).filename):
         _ensure_html_response(url, session=session)
 
-    logger.debug('Getting page %s', redact_auth_from_url(url))
+    logger.debug("Getting page %s", redact_auth_from_url(url))
 
     resp = session.get(
         url,
@@ -155,19 +152,16 @@ def _get_html_response(url, session):
     return resp
 
 
-def _get_encoding_from_headers(headers):
-    # type: (ResponseHeaders) -> Optional[str]
-    """Determine if we have any encoding information in our headers.
-    """
+def _get_encoding_from_headers(headers: ResponseHeaders) -> Optional[str]:
+    """Determine if we have any encoding information in our headers."""
     if headers and "Content-Type" in headers:
         content_type, params = cgi.parse_header(headers["Content-Type"])
         if "charset" in params:
-            return params['charset']
+            return params["charset"]
     return None
 
 
-def _determine_base_url(document, page_url):
-    # type: (HTMLElement, str) -> str
+def _determine_base_url(document: HTMLElement, page_url: str) -> str:
     """Determine the HTML document's base URL.
 
     This looks for a ```` tag in the HTML document. If present, its href
@@ -178,6 +172,8 @@ def _determine_base_url(document, page_url):
     :param document: An HTML document representation. The current
         implementation expects the result of ``html5lib.parse()``.
     :param page_url: The URL of the HTML document.
+
+    TODO: Remove when `html5lib` is dropped.
     """
     for base in document.findall(".//base"):
         href = base.get("href")
@@ -186,8 +182,7 @@ def _determine_base_url(document, page_url):
     return page_url
 
 
-def _clean_url_path_part(part):
-    # type: (str) -> str
+def _clean_url_path_part(part: str) -> str:
     """
     Clean a "part" of a URL path (i.e. after splitting on "@" characters).
     """
@@ -195,8 +190,7 @@ def _clean_url_path_part(part):
     return urllib.parse.quote(urllib.parse.unquote(part))
 
 
-def _clean_file_url_path(part):
-    # type: (str) -> str
+def _clean_file_url_path(part: str) -> str:
     """
     Clean the first part of a URL path that corresponds to a local
     filesystem path (i.e. the first part after splitting on "@" characters).
@@ -210,11 +204,10 @@ def _clean_file_url_path(part):
 
 
 # percent-encoded:                   /
-_reserved_chars_re = re.compile('(@|%2F)', re.IGNORECASE)
+_reserved_chars_re = re.compile("(@|%2F)", re.IGNORECASE)
 
 
-def _clean_url_path(path, is_local_path):
-    # type: (str, bool) -> str
+def _clean_url_path(path: str, is_local_path: bool) -> str:
     """
     Clean the path portion of a URL.
     """
@@ -228,16 +221,15 @@ def _clean_url_path(path, is_local_path):
     parts = _reserved_chars_re.split(path)
 
     cleaned_parts = []
-    for to_clean, reserved in pairwise(itertools.chain(parts, [''])):
+    for to_clean, reserved in pairwise(itertools.chain(parts, [""])):
         cleaned_parts.append(clean_func(to_clean))
         # Normalize %xx escapes (e.g. %2f -> %2F)
         cleaned_parts.append(reserved.upper())
 
-    return ''.join(cleaned_parts)
+    return "".join(cleaned_parts)
 
 
-def _clean_link(url):
-    # type: (str) -> str
+def _clean_link(url: str) -> str:
     """
     Make sure a link is fully quoted.
     For example, if ' ' occurs in the URL, it will be replaced with "%20",
@@ -253,26 +245,20 @@ def _clean_link(url):
 
 
 def _create_link_from_element(
-    anchor,    # type: HTMLElement
-    page_url,  # type: str
-    base_url,  # type: str
-):
-    # type: (...) -> Optional[Link]
+    element_attribs: Dict[str, Optional[str]],
+    page_url: str,
+    base_url: str,
+) -> Optional[Link]:
     """
-    Convert an anchor element in a simple repository page to a Link.
+    Convert an anchor element's attributes in a simple repository page to a Link.
     """
-    href = anchor.get("href")
+    href = element_attribs.get("href")
     if not href:
         return None
 
     url = _clean_link(urllib.parse.urljoin(base_url, href))
-    pyrequire = anchor.get('data-requires-python')
-    pyrequire = unescape(pyrequire) if pyrequire else None
-
-    yanked_reason = anchor.get('data-yanked')
-    if yanked_reason:
-        # This is a unicode string in Python 2 (and 3).
-        yanked_reason = unescape(yanked_reason)
+    pyrequire = element_attribs.get("data-requires-python")
+    yanked_reason = element_attribs.get("data-yanked")
 
     link = Link(
         url,
@@ -285,25 +271,25 @@ def _create_link_from_element(
 
 
 class CacheablePageContent:
-    def __init__(self, page):
-        # type: (HTMLPage) -> None
+    def __init__(self, page: "HTMLPage") -> None:
         assert page.cache_link_parsing
         self.page = page
 
-    def __eq__(self, other):
-        # type: (object) -> bool
-        return (isinstance(other, type(self)) and
-                self.page.url == other.page.url)
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, type(self)) and self.page.url == other.page.url
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(self.page.url)
 
 
-def with_cached_html_pages(
-    fn,    # type: Callable[[HTMLPage], Iterable[Link]]
-):
-    # type: (...) -> Callable[[HTMLPage], List[Link]]
+class ParseLinks(Protocol):
+    def __call__(
+        self, page: "HTMLPage", use_deprecated_html5lib: bool
+    ) -> Iterable[Link]:
+        ...
+
+
+def with_cached_html_pages(fn: ParseLinks) -> ParseLinks:
     """
     Given a function that parses an Iterable[Link] from an HTMLPage, cache the
     function's result (keyed by CacheablePageContent), unless the HTMLPage
@@ -311,25 +297,25 @@ def with_cached_html_pages(
     """
 
     @functools.lru_cache(maxsize=None)
-    def wrapper(cacheable_page):
-        # type: (CacheablePageContent) -> List[Link]
-        return list(fn(cacheable_page.page))
+    def wrapper(
+        cacheable_page: CacheablePageContent, use_deprecated_html5lib: bool
+    ) -> List[Link]:
+        return list(fn(cacheable_page.page, use_deprecated_html5lib))
 
     @functools.wraps(fn)
-    def wrapper_wrapper(page):
-        # type: (HTMLPage) -> List[Link]
+    def wrapper_wrapper(page: "HTMLPage", use_deprecated_html5lib: bool) -> List[Link]:
         if page.cache_link_parsing:
-            return wrapper(CacheablePageContent(page))
-        return list(fn(page))
+            return wrapper(CacheablePageContent(page), use_deprecated_html5lib)
+        return list(fn(page, use_deprecated_html5lib))
 
     return wrapper_wrapper
 
 
-@with_cached_html_pages
-def parse_links(page):
-    # type: (HTMLPage) -> Iterable[Link]
+def _parse_links_html5lib(page: "HTMLPage") -> Iterable[Link]:
     """
     Parse an HTML document, and yield its anchor elements as Link objects.
+
+    TODO: Remove when `html5lib` is dropped.
     """
     document = html5lib.parse(
         page.content,
@@ -340,6 +326,33 @@ def parse_links(page):
     url = page.url
     base_url = _determine_base_url(document, url)
     for anchor in document.findall(".//a"):
+        link = _create_link_from_element(
+            anchor.attrib,
+            page_url=url,
+            base_url=base_url,
+        )
+        if link is None:
+            continue
+        yield link
+
+
+@with_cached_html_pages
+def parse_links(page: "HTMLPage", use_deprecated_html5lib: bool) -> Iterable[Link]:
+    """
+    Parse an HTML document, and yield its anchor elements as Link objects.
+    """
+
+    if use_deprecated_html5lib:
+        yield from _parse_links_html5lib(page)
+        return
+
+    parser = HTMLLinkParser(page.url)
+    encoding = page.encoding or "utf-8"
+    parser.feed(page.content.decode(encoding))
+
+    url = page.url
+    base_url = parser.base_url or url
+    for anchor in parser.anchors:
         link = _create_link_from_element(
             anchor,
             page_url=url,
@@ -355,12 +368,11 @@ class HTMLPage:
 
     def __init__(
         self,
-        content,                  # type: bytes
-        encoding,                 # type: Optional[str]
-        url,                      # type: str
-        cache_link_parsing=True,  # type: bool
-    ):
-        # type: (...) -> None
+        content: bytes,
+        encoding: Optional[str],
+        url: str,
+        cache_link_parsing: bool = True,
+    ) -> None:
         """
         :param encoding: the encoding to decode the given content.
         :param url: the URL from which the HTML was downloaded.
@@ -373,70 +385,103 @@ def __init__(
         self.url = url
         self.cache_link_parsing = cache_link_parsing
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return redact_auth_from_url(self.url)
 
 
+class HTMLLinkParser(HTMLParser):
+    """
+    HTMLParser that keeps the first base HREF and a list of all anchor
+    elements' attributes.
+    """
+
+    def __init__(self, url: str) -> None:
+        super().__init__(convert_charrefs=True)
+
+        self.url: str = url
+        self.base_url: Optional[str] = None
+        self.anchors: List[Dict[str, Optional[str]]] = []
+
+    def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
+        if tag == "base" and self.base_url is None:
+            href = self.get_href(attrs)
+            if href is not None:
+                self.base_url = href
+        elif tag == "a":
+            self.anchors.append(dict(attrs))
+
+    def get_href(self, attrs: List[Tuple[str, Optional[str]]]) -> Optional[str]:
+        for name, value in attrs:
+            if name == "href":
+                return value
+        return None
+
+
 def _handle_get_page_fail(
-    link,  # type: Link
-    reason,  # type: Union[str, Exception]
-    meth=None  # type: Optional[Callable[..., None]]
-):
-    # type: (...) -> None
+    link: Link,
+    reason: Union[str, Exception],
+    meth: Optional[Callable[..., None]] = None,
+) -> None:
     if meth is None:
         meth = logger.debug
     meth("Could not fetch URL %s: %s - skipping", link, reason)
 
 
-def _make_html_page(response, cache_link_parsing=True):
-    # type: (Response, bool) -> HTMLPage
+def _make_html_page(response: Response, cache_link_parsing: bool = True) -> HTMLPage:
     encoding = _get_encoding_from_headers(response.headers)
     return HTMLPage(
         response.content,
         encoding=encoding,
         url=response.url,
-        cache_link_parsing=cache_link_parsing)
+        cache_link_parsing=cache_link_parsing,
+    )
 
 
-def _get_html_page(link, session=None):
-    # type: (Link, Optional[PipSession]) -> Optional[HTMLPage]
+def _get_html_page(
+    link: Link, session: Optional[PipSession] = None
+) -> Optional["HTMLPage"]:
     if session is None:
         raise TypeError(
             "_get_html_page() missing 1 required keyword argument: 'session'"
         )
 
-    url = link.url.split('#', 1)[0]
+    url = link.url.split("#", 1)[0]
 
     # Check for VCS schemes that do not support lookup as web pages.
     vcs_scheme = _match_vcs_scheme(url)
     if vcs_scheme:
-        logger.warning('Cannot look at %s URL %s because it does not support '
-                       'lookup as web pages.', vcs_scheme, link)
+        logger.warning(
+            "Cannot look at %s URL %s because it does not support lookup as web pages.",
+            vcs_scheme,
+            link,
+        )
         return None
 
     # Tack index.html onto file:// URLs that point to directories
     scheme, _, path, _, _, _ = urllib.parse.urlparse(url)
-    if (scheme == 'file' and os.path.isdir(urllib.request.url2pathname(path))):
+    if scheme == "file" and os.path.isdir(urllib.request.url2pathname(path)):
         # add trailing slash if not present so urljoin doesn't trim
         # final segment
-        if not url.endswith('/'):
-            url += '/'
-        url = urllib.parse.urljoin(url, 'index.html')
-        logger.debug(' file: URL is directory, getting %s', url)
+        if not url.endswith("/"):
+            url += "/"
+        url = urllib.parse.urljoin(url, "index.html")
+        logger.debug(" file: URL is directory, getting %s", url)
 
     try:
         resp = _get_html_response(url, session=session)
     except _NotHTTP:
         logger.warning(
-            'Skipping page %s because it looks like an archive, and cannot '
-            'be checked by a HTTP HEAD request.', link,
+            "Skipping page %s because it looks like an archive, and cannot "
+            "be checked by a HTTP HEAD request.",
+            link,
         )
     except _NotHTML as exc:
         logger.warning(
-            'Skipping page %s because the %s request got Content-Type: %s.'
-            'The only supported Content-Type is text/html',
-            link, exc.request_desc, exc.content_type,
+            "Skipping page %s because the %s request got Content-Type: %s."
+            "The only supported Content-Type is text/html",
+            link,
+            exc.request_desc,
+            exc.content_type,
         )
     except NetworkConnectionError as exc:
         _handle_get_page_fail(link, exc)
@@ -451,112 +496,13 @@ def _get_html_page(link, session=None):
     except requests.Timeout:
         _handle_get_page_fail(link, "timed out")
     else:
-        return _make_html_page(resp,
-                               cache_link_parsing=link.cache_link_parsing)
+        return _make_html_page(resp, cache_link_parsing=link.cache_link_parsing)
     return None
 
 
-def _remove_duplicate_links(links):
-    # type: (Iterable[Link]) -> List[Link]
-    """
-    Return a list of links, with duplicates removed and ordering preserved.
-    """
-    # We preserve the ordering when removing duplicates because we can.
-    return list(OrderedDict.fromkeys(links))
-
-
-def group_locations(locations, expand_dir=False):
-    # type: (Sequence[str], bool) -> Tuple[List[str], List[str]]
-    """
-    Divide a list of locations into two groups: "files" (archives) and "urls."
-
-    :return: A pair of lists (files, urls).
-    """
-    files = []
-    urls = []
-
-    # puts the url for the given file path into the appropriate list
-    def sort_path(path):
-        # type: (str) -> None
-        url = path_to_url(path)
-        if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
-            urls.append(url)
-        else:
-            files.append(url)
-
-    for url in locations:
-
-        is_local_path = os.path.exists(url)
-        is_file_url = url.startswith('file:')
-
-        if is_local_path or is_file_url:
-            if is_local_path:
-                path = url
-            else:
-                path = url_to_path(url)
-            if os.path.isdir(path):
-                if expand_dir:
-                    path = os.path.realpath(path)
-                    for item in os.listdir(path):
-                        sort_path(os.path.join(path, item))
-                elif is_file_url:
-                    urls.append(url)
-                else:
-                    logger.warning(
-                        "Path '%s' is ignored: it is a directory.", path,
-                    )
-            elif os.path.isfile(path):
-                sort_path(path)
-            else:
-                logger.warning(
-                    "Url '%s' is ignored: it is neither a file "
-                    "nor a directory.", url,
-                )
-        elif is_url(url):
-            # Only add url with clear scheme
-            urls.append(url)
-        else:
-            logger.warning(
-                "Url '%s' is ignored. It is either a non-existing "
-                "path or lacks a specific scheme.", url,
-            )
-
-    return files, urls
-
-
-class CollectedLinks:
-
-    """
-    Encapsulates the return value of a call to LinkCollector.collect_links().
-
-    The return value includes both URLs to project pages containing package
-    links, as well as individual package Link objects collected from other
-    sources.
-
-    This info is stored separately as:
-
-    (1) links from the configured file locations,
-    (2) links from the configured find_links, and
-    (3) urls to HTML project pages, as described by the PEP 503 simple
-        repository API.
-    """
-
-    def __init__(
-        self,
-        files,         # type: List[Link]
-        find_links,    # type: List[Link]
-        project_urls,  # type: List[Link]
-    ):
-        # type: (...) -> None
-        """
-        :param files: Links from file locations.
-        :param find_links: Links from find_links.
-        :param project_urls: URLs to HTML project pages, as described by
-            the PEP 503 simple repository API.
-        """
-        self.files = files
-        self.find_links = find_links
-        self.project_urls = project_urls
+class CollectedSources(NamedTuple):
+    find_links: Sequence[Optional[LinkSource]]
+    index_urls: Sequence[Optional[LinkSource]]
 
 
 class LinkCollector:
@@ -565,21 +511,24 @@ class LinkCollector:
     Responsible for collecting Link objects from all configured locations,
     making network requests as needed.
 
-    The class's main method is its collect_links() method.
+    The class's main method is its collect_sources() method.
     """
 
     def __init__(
         self,
-        session,       # type: PipSession
-        search_scope,  # type: SearchScope
-    ):
-        # type: (...) -> None
+        session: PipSession,
+        search_scope: SearchScope,
+    ) -> None:
         self.search_scope = search_scope
         self.session = session
 
     @classmethod
-    def create(cls, session, options, suppress_no_index=False):
-        # type: (PipSession, Values, bool) -> LinkCollector
+    def create(
+        cls,
+        session: PipSession,
+        options: Values,
+        suppress_no_index: bool = False,
+    ) -> "LinkCollector":
         """
         :param session: The Session to use to make requests.
         :param suppress_no_index: Whether to ignore the --no-index option
@@ -588,8 +537,8 @@ def create(cls, session, options, suppress_no_index=False):
         index_urls = [options.index_url] + options.extra_index_urls
         if options.no_index and not suppress_no_index:
             logger.debug(
-                'Ignoring indexes: %s',
-                ','.join(redact_auth_from_url(url) for url in index_urls),
+                "Ignoring indexes: %s",
+                ",".join(redact_auth_from_url(url) for url in index_urls),
             )
             index_urls = []
 
@@ -597,70 +546,65 @@ def create(cls, session, options, suppress_no_index=False):
         find_links = options.find_links or []
 
         search_scope = SearchScope.create(
-            find_links=find_links, index_urls=index_urls,
+            find_links=find_links,
+            index_urls=index_urls,
         )
         link_collector = LinkCollector(
-            session=session, search_scope=search_scope,
+            session=session,
+            search_scope=search_scope,
         )
         return link_collector
 
     @property
-    def find_links(self):
-        # type: () -> List[str]
+    def find_links(self) -> List[str]:
         return self.search_scope.find_links
 
-    def fetch_page(self, location):
-        # type: (Link) -> Optional[HTMLPage]
+    def fetch_page(self, location: Link) -> Optional[HTMLPage]:
         """
         Fetch an HTML page containing package links.
         """
         return _get_html_page(location, session=self.session)
 
-    def collect_links(self, project_name):
-        # type: (str) -> CollectedLinks
-        """Find all available links for the given project name.
-
-        :return: All the Link objects (unfiltered), as a CollectedLinks object.
-        """
-        search_scope = self.search_scope
-        index_locations = search_scope.get_index_urls_locations(project_name)
-        index_file_loc, index_url_loc = group_locations(index_locations)
-        fl_file_loc, fl_url_loc = group_locations(
-            self.find_links, expand_dir=True,
-        )
-
-        file_links = [
-            Link(url) for url in itertools.chain(index_file_loc, fl_file_loc)
-        ]
-
-        # We trust every directly linked archive in find_links
-        find_link_links = [Link(url, '-f') for url in self.find_links]
-
-        # We trust every url that the user has given us whether it was given
-        # via --index-url or --find-links.
-        # We want to filter out anything that does not have a secure origin.
-        url_locations = [
-            link for link in itertools.chain(
-                # Mark PyPI indices as "cache_link_parsing == False" -- this
-                # will avoid caching the result of parsing the page for links.
-                (Link(url, cache_link_parsing=False) for url in index_url_loc),
-                (Link(url) for url in fl_url_loc),
+    def collect_sources(
+        self,
+        project_name: str,
+        candidates_from_page: CandidatesFromPage,
+    ) -> CollectedSources:
+        # The OrderedDict calls deduplicate sources by URL.
+        index_url_sources = collections.OrderedDict(
+            build_source(
+                loc,
+                candidates_from_page=candidates_from_page,
+                page_validator=self.session.is_secure_origin,
+                expand_dir=False,
+                cache_link_parsing=False,
+            )
+            for loc in self.search_scope.get_index_urls_locations(project_name)
+        ).values()
+        find_links_sources = collections.OrderedDict(
+            build_source(
+                loc,
+                candidates_from_page=candidates_from_page,
+                page_validator=self.session.is_secure_origin,
+                expand_dir=True,
+                cache_link_parsing=True,
             )
-            if self.session.is_secure_origin(link)
-        ]
-
-        url_locations = _remove_duplicate_links(url_locations)
-        lines = [
-            '{} location(s) to search for versions of {}:'.format(
-                len(url_locations), project_name,
-            ),
-        ]
-        for link in url_locations:
-            lines.append(f'* {link}')
-        logger.debug('\n'.join(lines))
-
-        return CollectedLinks(
-            files=file_links,
-            find_links=find_link_links,
-            project_urls=url_locations,
+            for loc in self.find_links
+        ).values()
+
+        if logger.isEnabledFor(logging.DEBUG):
+            lines = [
+                f"* {s.link}"
+                for s in itertools.chain(find_links_sources, index_url_sources)
+                if s is not None and s.link is not None
+            ]
+            lines = [
+                f"{len(lines)} location(s) to search "
+                f"for versions of {project_name}:"
+            ] + lines
+            logger.debug("\n".join(lines))
+
+        return CollectedSources(
+            find_links=list(find_links_sources),
+            index_urls=list(index_url_sources),
         )
diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py
index 731188926fd..f70f74b17c6 100644
--- a/src/pip/_internal/index/package_finder.py
+++ b/src/pip/_internal/index/package_finder.py
@@ -3,12 +3,17 @@
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
 
+import enum
 import functools
+import itertools
 import logging
 import re
+from typing import FrozenSet, Iterable, List, Optional, Set, Tuple, Union
 
 from pip._vendor.packaging import specifiers
+from pip._vendor.packaging.tags import Tag
 from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.version import _BaseVersion
 from pip._vendor.packaging.version import parse as parse_version
 
 from pip._internal.exceptions import (
@@ -17,50 +22,37 @@
     InvalidWheelFilename,
     UnsupportedWheel,
 )
-from pip._internal.index.collector import parse_links
+from pip._internal.index.collector import LinkCollector, parse_links
 from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.models.format_control import FormatControl
 from pip._internal.models.link import Link
+from pip._internal.models.search_scope import SearchScope
 from pip._internal.models.selection_prefs import SelectionPreferences
 from pip._internal.models.target_python import TargetPython
 from pip._internal.models.wheel import Wheel
+from pip._internal.req import InstallRequirement
+from pip._internal.utils._log import getLogger
 from pip._internal.utils.filetypes import WHEEL_EXTENSION
+from pip._internal.utils.hashes import Hashes
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import build_netloc
 from pip._internal.utils.packaging import check_requires_python
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS
-from pip._internal.utils.urls import url_to_path
 
-if MYPY_CHECK_RUNNING:
-    from typing import FrozenSet, Iterable, List, Optional, Set, Tuple, Union
+__all__ = ["FormatControl", "BestCandidateResult", "PackageFinder"]
 
-    from pip._vendor.packaging.tags import Tag
-    from pip._vendor.packaging.version import _BaseVersion
 
-    from pip._internal.index.collector import LinkCollector
-    from pip._internal.models.search_scope import SearchScope
-    from pip._internal.req import InstallRequirement
-    from pip._internal.utils.hashes import Hashes
+logger = getLogger(__name__)
 
-    BuildTag = Union[Tuple[()], Tuple[int, str]]
-    CandidateSortingKey = (
-        Tuple[int, int, int, _BaseVersion, BuildTag, Optional[int]]
-    )
-
-
-__all__ = ['FormatControl', 'BestCandidateResult', 'PackageFinder']
-
-
-logger = logging.getLogger(__name__)
+BuildTag = Union[Tuple[()], Tuple[int, str]]
+CandidateSortingKey = Tuple[int, int, int, _BaseVersion, Optional[int], BuildTag]
 
 
 def _check_link_requires_python(
-    link,  # type: Link
-    version_info,  # type: Tuple[int, int, int]
-    ignore_requires_python=False,  # type: bool
-):
-    # type: (...) -> bool
+    link: Link,
+    version_info: Tuple[int, int, int],
+    ignore_requires_python: bool = False,
+) -> bool:
     """
     Return whether the given Python version is compatible with a link's
     "Requires-Python" value.
@@ -72,39 +64,54 @@ def _check_link_requires_python(
     """
     try:
         is_compatible = check_requires_python(
-            link.requires_python, version_info=version_info,
+            link.requires_python,
+            version_info=version_info,
         )
     except specifiers.InvalidSpecifier:
         logger.debug(
             "Ignoring invalid Requires-Python (%r) for link: %s",
-            link.requires_python, link,
+            link.requires_python,
+            link,
         )
     else:
         if not is_compatible:
-            version = '.'.join(map(str, version_info))
+            version = ".".join(map(str, version_info))
             if not ignore_requires_python:
-                logger.debug(
-                    'Link requires a different Python (%s not in: %r): %s',
-                    version, link.requires_python, link,
+                logger.verbose(
+                    "Link requires a different Python (%s not in: %r): %s",
+                    version,
+                    link.requires_python,
+                    link,
                 )
                 return False
 
             logger.debug(
-                'Ignoring failed Requires-Python check (%s not in: %r) '
-                'for link: %s',
-                version, link.requires_python, link,
+                "Ignoring failed Requires-Python check (%s not in: %r) for link: %s",
+                version,
+                link.requires_python,
+                link,
             )
 
     return True
 
 
+class LinkType(enum.Enum):
+    candidate = enum.auto()
+    different_project = enum.auto()
+    yanked = enum.auto()
+    format_unsupported = enum.auto()
+    format_invalid = enum.auto()
+    platform_mismatch = enum.auto()
+    requires_python_mismatch = enum.auto()
+
+
 class LinkEvaluator:
 
     """
     Responsible for evaluating links for a particular project.
     """
 
-    _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')
+    _py_version_re = re.compile(r"-py([123]\.?[0-9]?)$")
 
     # Don't include an allow_yanked default value to make sure each call
     # site considers whether yanked releases are allowed. This also causes
@@ -112,14 +119,13 @@ class LinkEvaluator:
     # people when reading the code.
     def __init__(
         self,
-        project_name,    # type: str
-        canonical_name,  # type: str
-        formats,         # type: FrozenSet[str]
-        target_python,   # type: TargetPython
-        allow_yanked,    # type: bool
-        ignore_requires_python=None,  # type: Optional[bool]
-    ):
-        # type: (...) -> None
+        project_name: str,
+        canonical_name: str,
+        formats: FrozenSet[str],
+        target_python: TargetPython,
+        allow_yanked: bool,
+        ignore_requires_python: Optional[bool] = None,
+    ) -> None:
         """
         :param project_name: The user supplied package name.
         :param canonical_name: The canonical package name.
@@ -148,20 +154,20 @@ def __init__(
 
         self.project_name = project_name
 
-    def evaluate_link(self, link):
-        # type: (Link) -> Tuple[bool, Optional[str]]
+    def evaluate_link(self, link: Link) -> Tuple[LinkType, str]:
         """
         Determine whether a link is a candidate for installation.
 
-        :return: A tuple (is_candidate, result), where `result` is (1) a
-            version string if `is_candidate` is True, and (2) if
-            `is_candidate` is False, an optional string to log the reason
-            the link fails to qualify.
+        :return: A tuple (result, detail), where *result* is an enum
+            representing whether the evaluation found a candidate, or the reason
+            why one is not found. If a candidate is found, *detail* will be the
+            candidate's version string; if one is not found, it contains the
+            reason the link fails to qualify.
         """
         version = None
         if link.is_yanked and not self._allow_yanked:
-            reason = link.yanked_reason or ''
-            return (False, f'yanked for reason: {reason}')
+            reason = link.yanked_reason or ""
+            return (LinkType.yanked, f"yanked for reason: {reason}")
 
         if link.egg_fragment:
             egg_info = link.egg_fragment
@@ -169,79 +175,85 @@ def evaluate_link(self, link):
         else:
             egg_info, ext = link.splitext()
             if not ext:
-                return (False, 'not a file')
+                return (LinkType.format_unsupported, "not a file")
             if ext not in SUPPORTED_EXTENSIONS:
-                return (False, f'unsupported archive format: {ext}')
+                return (
+                    LinkType.format_unsupported,
+                    f"unsupported archive format: {ext}",
+                )
             if "binary" not in self._formats and ext == WHEEL_EXTENSION:
-                reason = 'No binaries permitted for {}'.format(
-                    self.project_name)
-                return (False, reason)
-            if "macosx10" in link.path and ext == '.zip':
-                return (False, 'macosx10 one')
+                reason = f"No binaries permitted for {self.project_name}"
+                return (LinkType.format_unsupported, reason)
+            if "macosx10" in link.path and ext == ".zip":
+                return (LinkType.format_unsupported, "macosx10 one")
             if ext == WHEEL_EXTENSION:
                 try:
                     wheel = Wheel(link.filename)
                 except InvalidWheelFilename:
-                    return (False, 'invalid wheel filename')
+                    return (
+                        LinkType.format_invalid,
+                        "invalid wheel filename",
+                    )
                 if canonicalize_name(wheel.name) != self._canonical_name:
-                    reason = 'wrong project name (not {})'.format(
-                        self.project_name)
-                    return (False, reason)
+                    reason = f"wrong project name (not {self.project_name})"
+                    return (LinkType.different_project, reason)
 
                 supported_tags = self._target_python.get_tags()
                 if not wheel.supported(supported_tags):
                     # Include the wheel's tags in the reason string to
                     # simplify troubleshooting compatibility issues.
-                    file_tags = wheel.get_formatted_file_tags()
+                    file_tags = ", ".join(wheel.get_formatted_file_tags())
                     reason = (
-                        "none of the wheel's tags match: {}".format(
-                            ', '.join(file_tags)
-                        )
+                        f"none of the wheel's tags ({file_tags}) are compatible "
+                        f"(run pip debug --verbose to show compatible tags)"
                     )
-                    return (False, reason)
+                    return (LinkType.platform_mismatch, reason)
 
                 version = wheel.version
 
         # This should be up by the self.ok_binary check, but see issue 2700.
         if "source" not in self._formats and ext != WHEEL_EXTENSION:
-            reason = f'No sources permitted for {self.project_name}'
-            return (False, reason)
+            reason = f"No sources permitted for {self.project_name}"
+            return (LinkType.format_unsupported, reason)
 
         if not version:
             version = _extract_version_from_fragment(
-                egg_info, self._canonical_name,
+                egg_info,
+                self._canonical_name,
             )
         if not version:
-            reason = f'Missing project version for {self.project_name}'
-            return (False, reason)
+            reason = f"Missing project version for {self.project_name}"
+            return (LinkType.format_invalid, reason)
 
         match = self._py_version_re.search(version)
         if match:
-            version = version[:match.start()]
+            version = version[: match.start()]
             py_version = match.group(1)
             if py_version != self._target_python.py_version:
-                return (False, 'Python version is incorrect')
+                return (
+                    LinkType.platform_mismatch,
+                    "Python version is incorrect",
+                )
 
         supports_python = _check_link_requires_python(
-            link, version_info=self._target_python.py_version_info,
+            link,
+            version_info=self._target_python.py_version_info,
             ignore_requires_python=self._ignore_requires_python,
         )
         if not supports_python:
-            # Return None for the reason text to suppress calling
-            # _log_skipped_link().
-            return (False, None)
+            reason = f"{version} Requires-Python {link.requires_python}"
+            return (LinkType.requires_python_mismatch, reason)
 
-        logger.debug('Found link %s, version: %s', link, version)
+        logger.debug("Found link %s, version: %s", link, version)
 
-        return (True, version)
+        return (LinkType.candidate, version)
 
 
 def filter_unallowed_hashes(
-    candidates,    # type: List[InstallationCandidate]
-    hashes,        # type: Hashes
-    project_name,  # type: str
-):
-    # type: (...) -> List[InstallationCandidate]
+    candidates: List[InstallationCandidate],
+    hashes: Hashes,
+    project_name: str,
+) -> List[InstallationCandidate]:
     """
     Filter out candidates whose hashes aren't allowed, and return a new
     list of candidates.
@@ -259,8 +271,8 @@ def filter_unallowed_hashes(
     """
     if not hashes:
         logger.debug(
-            'Given no hashes to check %s links for project %r: '
-            'discarding no candidates',
+            "Given no hashes to check %s links for project %r: "
+            "discarding no candidates",
             len(candidates),
             project_name,
         )
@@ -290,22 +302,22 @@ def filter_unallowed_hashes(
         filtered = list(candidates)
 
     if len(filtered) == len(candidates):
-        discard_message = 'discarding no candidates'
+        discard_message = "discarding no candidates"
     else:
-        discard_message = 'discarding {} non-matches:\n  {}'.format(
+        discard_message = "discarding {} non-matches:\n  {}".format(
             len(non_matches),
-            '\n  '.join(str(candidate.link) for candidate in non_matches)
+            "\n  ".join(str(candidate.link) for candidate in non_matches),
         )
 
     logger.debug(
-        'Checked %s links for project %r against %s hashes '
-        '(%s matches, %s no digest): %s',
+        "Checked %s links for project %r against %s hashes "
+        "(%s matches, %s no digest): %s",
         len(candidates),
         project_name,
         hashes.digest_count,
         match_count,
         len(matches_or_no_digest) - match_count,
-        discard_message
+        discard_message,
     )
 
     return filtered
@@ -320,10 +332,9 @@ class CandidatePreferences:
 
     def __init__(
         self,
-        prefer_binary=False,  # type: bool
-        allow_all_prereleases=False,  # type: bool
-    ):
-        # type: (...) -> None
+        prefer_binary: bool = False,
+        allow_all_prereleases: bool = False,
+    ) -> None:
         """
         :param allow_all_prereleases: Whether to allow all pre-releases.
         """
@@ -340,11 +351,10 @@ class BestCandidateResult:
 
     def __init__(
         self,
-        candidates,             # type: List[InstallationCandidate]
-        applicable_candidates,  # type: List[InstallationCandidate]
-        best_candidate,         # type: Optional[InstallationCandidate]
-    ):
-        # type: (...) -> None
+        candidates: List[InstallationCandidate],
+        applicable_candidates: List[InstallationCandidate],
+        best_candidate: Optional[InstallationCandidate],
+    ) -> None:
         """
         :param candidates: A sequence of all available candidates found.
         :param applicable_candidates: The applicable candidates.
@@ -363,16 +373,12 @@ def __init__(
 
         self.best_candidate = best_candidate
 
-    def iter_all(self):
-        # type: () -> Iterable[InstallationCandidate]
-        """Iterate through all candidates.
-        """
+    def iter_all(self) -> Iterable[InstallationCandidate]:
+        """Iterate through all candidates."""
         return iter(self._candidates)
 
-    def iter_applicable(self):
-        # type: () -> Iterable[InstallationCandidate]
-        """Iterate through the applicable candidates.
-        """
+    def iter_applicable(self) -> Iterable[InstallationCandidate]:
+        """Iterate through the applicable candidates."""
         return iter(self._applicable_candidates)
 
 
@@ -386,14 +392,13 @@ class CandidateEvaluator:
     @classmethod
     def create(
         cls,
-        project_name,         # type: str
-        target_python=None,   # type: Optional[TargetPython]
-        prefer_binary=False,  # type: bool
-        allow_all_prereleases=False,  # type: bool
-        specifier=None,       # type: Optional[specifiers.BaseSpecifier]
-        hashes=None,          # type: Optional[Hashes]
-    ):
-        # type: (...) -> CandidateEvaluator
+        project_name: str,
+        target_python: Optional[TargetPython] = None,
+        prefer_binary: bool = False,
+        allow_all_prereleases: bool = False,
+        specifier: Optional[specifiers.BaseSpecifier] = None,
+        hashes: Optional[Hashes] = None,
+    ) -> "CandidateEvaluator":
         """Create a CandidateEvaluator object.
 
         :param target_python: The target Python interpreter to use when
@@ -422,14 +427,13 @@ def create(
 
     def __init__(
         self,
-        project_name,         # type: str
-        supported_tags,       # type: List[Tag]
-        specifier,            # type: specifiers.BaseSpecifier
-        prefer_binary=False,  # type: bool
-        allow_all_prereleases=False,  # type: bool
-        hashes=None,                  # type: Optional[Hashes]
-    ):
-        # type: (...) -> None
+        project_name: str,
+        supported_tags: List[Tag],
+        specifier: specifiers.BaseSpecifier,
+        prefer_binary: bool = False,
+        allow_all_prereleases: bool = False,
+        hashes: Optional[Hashes] = None,
+    ) -> None:
         """
         :param supported_tags: The PEP 425 tags supported by the target
             Python in order of preference (most preferred first).
@@ -440,12 +444,17 @@ def __init__(
         self._project_name = project_name
         self._specifier = specifier
         self._supported_tags = supported_tags
+        # Since the index of the tag in the _supported_tags list is used
+        # as a priority, precompute a map from tag to index/priority to be
+        # used in wheel.find_most_preferred_tag.
+        self._wheel_tag_preferences = {
+            tag: idx for idx, tag in enumerate(supported_tags)
+        }
 
     def get_applicable_candidates(
         self,
-        candidates,  # type: List[InstallationCandidate]
-    ):
-        # type: (...) -> List[InstallationCandidate]
+        candidates: List[InstallationCandidate],
+    ) -> List[InstallationCandidate]:
         """
         Return the applicable candidates from a list of candidates.
         """
@@ -453,7 +462,8 @@ def get_applicable_candidates(
         allow_prereleases = self._allow_all_prereleases or None
         specifier = self._specifier
         versions = {
-            str(v) for v in specifier.filter(
+            str(v)
+            for v in specifier.filter(
                 # We turn the version object into a str here because otherwise
                 # when we're debundled but setuptools isn't, Python will see
                 # packaging.version.Version and
@@ -467,9 +477,7 @@ def get_applicable_candidates(
         }
 
         # Again, converting version to str to deal with debundling.
-        applicable_candidates = [
-            c for c in candidates if str(c.version) in versions
-        ]
+        applicable_candidates = [c for c in candidates if str(c.version) in versions]
 
         filtered_applicable_candidates = filter_unallowed_hashes(
             candidates=applicable_candidates,
@@ -479,8 +487,7 @@ def get_applicable_candidates(
 
         return sorted(filtered_applicable_candidates, key=self._sort_key)
 
-    def _sort_key(self, candidate):
-        # type: (InstallationCandidate) -> CandidateSortingKey
+    def _sort_key(self, candidate: InstallationCandidate) -> CandidateSortingKey:
         """
         Function to pass as the `key` argument to a call to sorted() to sort
         InstallationCandidates by preference.
@@ -512,22 +519,27 @@ def _sort_key(self, candidate):
         """
         valid_tags = self._supported_tags
         support_num = len(valid_tags)
-        build_tag = ()  # type: BuildTag
+        build_tag: BuildTag = ()
         binary_preference = 0
         link = candidate.link
         if link.is_wheel:
             # can raise InvalidWheelFilename
             wheel = Wheel(link.filename)
-            if not wheel.supported(valid_tags):
+            try:
+                pri = -(
+                    wheel.find_most_preferred_tag(
+                        valid_tags, self._wheel_tag_preferences
+                    )
+                )
+            except ValueError:
                 raise UnsupportedWheel(
                     "{} is not a supported wheel for this platform. It "
                     "can't be sorted.".format(wheel.filename)
                 )
             if self._prefer_binary:
                 binary_preference = 1
-            pri = -(wheel.support_index_min(valid_tags))
             if wheel.build_tag is not None:
-                match = re.match(r'^(\d+)(.*)$', wheel.build_tag)
+                match = re.match(r"^(\d+)(.*)$", wheel.build_tag)
                 build_tag_groups = match.groups()
                 build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
         else:  # sdist
@@ -535,15 +547,18 @@ def _sort_key(self, candidate):
         has_allowed_hash = int(link.is_hash_allowed(self._hashes))
         yank_value = -1 * int(link.is_yanked)  # -1 for yanked.
         return (
-            has_allowed_hash, yank_value, binary_preference, candidate.version,
-            build_tag, pri,
+            has_allowed_hash,
+            yank_value,
+            binary_preference,
+            candidate.version,
+            pri,
+            build_tag,
         )
 
     def sort_best_candidate(
         self,
-        candidates,    # type: List[InstallationCandidate]
-    ):
-        # type: (...) -> Optional[InstallationCandidate]
+        candidates: List[InstallationCandidate],
+    ) -> Optional[InstallationCandidate]:
         """
         Return the best candidate per the instance's sort order, or None if
         no candidate is acceptable.
@@ -555,9 +570,8 @@ def sort_best_candidate(
 
     def compute_best_candidate(
         self,
-        candidates,      # type: List[InstallationCandidate]
-    ):
-        # type: (...) -> BestCandidateResult
+        candidates: List[InstallationCandidate],
+    ) -> BestCandidateResult:
         """
         Compute and return a `BestCandidateResult` instance.
         """
@@ -581,14 +595,14 @@ class PackageFinder:
 
     def __init__(
         self,
-        link_collector,       # type: LinkCollector
-        target_python,        # type: TargetPython
-        allow_yanked,         # type: bool
-        format_control=None,  # type: Optional[FormatControl]
-        candidate_prefs=None,         # type: CandidatePreferences
-        ignore_requires_python=None,  # type: Optional[bool]
-    ):
-        # type: (...) -> None
+        link_collector: LinkCollector,
+        target_python: TargetPython,
+        allow_yanked: bool,
+        use_deprecated_html5lib: bool,
+        format_control: Optional[FormatControl] = None,
+        candidate_prefs: Optional[CandidatePreferences] = None,
+        ignore_requires_python: Optional[bool] = None,
+    ) -> None:
         """
         This constructor is primarily meant to be used by the create() class
         method and from tests.
@@ -609,11 +623,12 @@ def __init__(
         self._ignore_requires_python = ignore_requires_python
         self._link_collector = link_collector
         self._target_python = target_python
+        self._use_deprecated_html5lib = use_deprecated_html5lib
 
         self.format_control = format_control
 
         # These are boring links that have already been logged somehow.
-        self._logged_links = set()  # type: Set[Link]
+        self._logged_links: Set[Tuple[Link, LinkType, str]] = set()
 
     # Don't include an allow_yanked default value to make sure each call
     # site considers whether yanked releases are allowed. This also causes
@@ -622,11 +637,12 @@ def __init__(
     @classmethod
     def create(
         cls,
-        link_collector,      # type: LinkCollector
-        selection_prefs,     # type: SelectionPreferences
-        target_python=None,  # type: Optional[TargetPython]
-    ):
-        # type: (...) -> PackageFinder
+        link_collector: LinkCollector,
+        selection_prefs: SelectionPreferences,
+        target_python: Optional[TargetPython] = None,
+        *,
+        use_deprecated_html5lib: bool,
+    ) -> "PackageFinder":
         """Create a PackageFinder.
 
         :param selection_prefs: The candidate selection preferences, as a
@@ -650,59 +666,57 @@ def create(
             allow_yanked=selection_prefs.allow_yanked,
             format_control=selection_prefs.format_control,
             ignore_requires_python=selection_prefs.ignore_requires_python,
+            use_deprecated_html5lib=use_deprecated_html5lib,
         )
 
     @property
-    def target_python(self):
-        # type: () -> TargetPython
+    def target_python(self) -> TargetPython:
         return self._target_python
 
     @property
-    def search_scope(self):
-        # type: () -> SearchScope
+    def search_scope(self) -> SearchScope:
         return self._link_collector.search_scope
 
     @search_scope.setter
-    def search_scope(self, search_scope):
-        # type: (SearchScope) -> None
+    def search_scope(self, search_scope: SearchScope) -> None:
         self._link_collector.search_scope = search_scope
 
     @property
-    def find_links(self):
-        # type: () -> List[str]
+    def find_links(self) -> List[str]:
         return self._link_collector.find_links
 
     @property
-    def index_urls(self):
-        # type: () -> List[str]
+    def index_urls(self) -> List[str]:
         return self.search_scope.index_urls
 
     @property
-    def trusted_hosts(self):
-        # type: () -> Iterable[str]
+    def trusted_hosts(self) -> Iterable[str]:
         for host_port in self._link_collector.session.pip_trusted_origins:
             yield build_netloc(*host_port)
 
     @property
-    def allow_all_prereleases(self):
-        # type: () -> bool
+    def allow_all_prereleases(self) -> bool:
         return self._candidate_prefs.allow_all_prereleases
 
-    def set_allow_all_prereleases(self):
-        # type: () -> None
+    def set_allow_all_prereleases(self) -> None:
         self._candidate_prefs.allow_all_prereleases = True
 
     @property
-    def prefer_binary(self):
-        # type: () -> bool
+    def prefer_binary(self) -> bool:
         return self._candidate_prefs.prefer_binary
 
-    def set_prefer_binary(self):
-        # type: () -> None
+    def set_prefer_binary(self) -> None:
         self._candidate_prefs.prefer_binary = True
 
-    def make_link_evaluator(self, project_name):
-        # type: (str) -> LinkEvaluator
+    def requires_python_skipped_reasons(self) -> List[str]:
+        reasons = {
+            detail
+            for _, result, detail in self._logged_links
+            if result == LinkType.requires_python_mismatch
+        }
+        return sorted(reasons)
+
+    def make_link_evaluator(self, project_name: str) -> LinkEvaluator:
         canonical_name = canonicalize_name(project_name)
         formats = self.format_control.get_allowed_formats(canonical_name)
 
@@ -715,14 +729,13 @@ def make_link_evaluator(self, project_name):
             ignore_requires_python=self._ignore_requires_python,
         )
 
-    def _sort_links(self, links):
-        # type: (Iterable[Link]) -> List[Link]
+    def _sort_links(self, links: Iterable[Link]) -> List[Link]:
         """
         Returns elements of links in order, non-egg links first, egg links
         second, while eliminating duplicates
         """
         eggs, no_eggs = [], []
-        seen = set()  # type: Set[Link]
+        seen: Set[Link] = set()
         for link in links:
             if link not in seen:
                 seen.add(link)
@@ -732,34 +745,35 @@ def _sort_links(self, links):
                     no_eggs.append(link)
         return no_eggs + eggs
 
-    def _log_skipped_link(self, link, reason):
-        # type: (Link, str) -> None
-        if link not in self._logged_links:
+    def _log_skipped_link(self, link: Link, result: LinkType, detail: str) -> None:
+        entry = (link, result, detail)
+        if entry not in self._logged_links:
             # Put the link at the end so the reason is more visible and because
             # the link string is usually very long.
-            logger.debug('Skipping link: %s: %s', reason, link)
-            self._logged_links.add(link)
+            logger.debug("Skipping link: %s: %s", detail, link)
+            self._logged_links.add(entry)
 
-    def get_install_candidate(self, link_evaluator, link):
-        # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate]
+    def get_install_candidate(
+        self, link_evaluator: LinkEvaluator, link: Link
+    ) -> Optional[InstallationCandidate]:
         """
         If the link is a candidate for install, convert it to an
         InstallationCandidate and return it. Otherwise, return None.
         """
-        is_candidate, result = link_evaluator.evaluate_link(link)
-        if not is_candidate:
-            if result:
-                self._log_skipped_link(link, reason=result)
+        result, detail = link_evaluator.evaluate_link(link)
+        if result != LinkType.candidate:
+            self._log_skipped_link(link, result, detail)
             return None
 
         return InstallationCandidate(
             name=link_evaluator.project_name,
             link=link,
-            version=result,
+            version=detail,
         )
 
-    def evaluate_links(self, link_evaluator, links):
-        # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate]
+    def evaluate_links(
+        self, link_evaluator: LinkEvaluator, links: Iterable[Link]
+    ) -> List[InstallationCandidate]:
         """
         Convert links that are candidates to InstallationCandidate objects.
         """
@@ -771,16 +785,18 @@ def evaluate_links(self, link_evaluator, links):
 
         return candidates
 
-    def process_project_url(self, project_url, link_evaluator):
-        # type: (Link, LinkEvaluator) -> List[InstallationCandidate]
+    def process_project_url(
+        self, project_url: Link, link_evaluator: LinkEvaluator
+    ) -> List[InstallationCandidate]:
         logger.debug(
-            'Fetching project page and analyzing links: %s', project_url,
+            "Fetching project page and analyzing links: %s",
+            project_url,
         )
         html_page = self._link_collector.fetch_page(project_url)
         if html_page is None:
             return []
 
-        page_links = list(parse_links(html_page))
+        page_links = list(parse_links(html_page, self._use_deprecated_html5lib))
 
         with indent_log():
             package_links = self.evaluate_links(
@@ -791,8 +807,7 @@ def process_project_url(self, project_url, link_evaluator):
         return package_links
 
     @functools.lru_cache(maxsize=None)
-    def find_all_candidates(self, project_name):
-        # type: (str) -> List[InstallationCandidate]
+    def find_all_candidates(self, project_name: str) -> List[InstallationCandidate]:
         """Find all available InstallationCandidate for project_name
 
         This checks index_urls and find_links.
@@ -801,48 +816,56 @@ def find_all_candidates(self, project_name):
         See LinkEvaluator.evaluate_link() for details on which files
         are accepted.
         """
-        collected_links = self._link_collector.collect_links(project_name)
-
         link_evaluator = self.make_link_evaluator(project_name)
 
-        find_links_versions = self.evaluate_links(
-            link_evaluator,
-            links=collected_links.find_links,
+        collected_sources = self._link_collector.collect_sources(
+            project_name=project_name,
+            candidates_from_page=functools.partial(
+                self.process_project_url,
+                link_evaluator=link_evaluator,
+            ),
         )
 
-        page_versions = []
-        for project_url in collected_links.project_urls:
-            package_links = self.process_project_url(
-                project_url, link_evaluator=link_evaluator,
-            )
-            page_versions.extend(package_links)
+        page_candidates_it = itertools.chain.from_iterable(
+            source.page_candidates()
+            for sources in collected_sources
+            for source in sources
+            if source is not None
+        )
+        page_candidates = list(page_candidates_it)
 
-        file_versions = self.evaluate_links(
+        file_links_it = itertools.chain.from_iterable(
+            source.file_links()
+            for sources in collected_sources
+            for source in sources
+            if source is not None
+        )
+        file_candidates = self.evaluate_links(
             link_evaluator,
-            links=collected_links.files,
+            sorted(file_links_it, reverse=True),
         )
-        if file_versions:
-            file_versions.sort(reverse=True)
-            logger.debug(
-                'Local files found: %s',
-                ', '.join([
-                    url_to_path(candidate.link.url)
-                    for candidate in file_versions
-                ])
-            )
+
+        if logger.isEnabledFor(logging.DEBUG) and file_candidates:
+            paths = []
+            for candidate in file_candidates:
+                assert candidate.link.url  # we need to have a URL
+                try:
+                    paths.append(candidate.link.file_path)
+                except Exception:
+                    paths.append(candidate.link.url)  # it's not a local file
+
+            logger.debug("Local files found: %s", ", ".join(paths))
 
         # This is an intentional priority ordering
-        return file_versions + find_links_versions + page_versions
+        return file_candidates + page_candidates
 
     def make_candidate_evaluator(
         self,
-        project_name,    # type: str
-        specifier=None,  # type: Optional[specifiers.BaseSpecifier]
-        hashes=None,     # type: Optional[Hashes]
-    ):
-        # type: (...) -> CandidateEvaluator
-        """Create a CandidateEvaluator object to use.
-        """
+        project_name: str,
+        specifier: Optional[specifiers.BaseSpecifier] = None,
+        hashes: Optional[Hashes] = None,
+    ) -> CandidateEvaluator:
+        """Create a CandidateEvaluator object to use."""
         candidate_prefs = self._candidate_prefs
         return CandidateEvaluator.create(
             project_name=project_name,
@@ -856,11 +879,10 @@ def make_candidate_evaluator(
     @functools.lru_cache(maxsize=None)
     def find_best_candidate(
         self,
-        project_name,       # type: str
-        specifier=None,     # type: Optional[specifiers.BaseSpecifier]
-        hashes=None,        # type: Optional[Hashes]
-    ):
-        # type: (...) -> BestCandidateResult
+        project_name: str,
+        specifier: Optional[specifiers.BaseSpecifier] = None,
+        hashes: Optional[Hashes] = None,
+    ) -> BestCandidateResult:
         """Find matches for the given project and specifier.
 
         :param specifier: An optional object implementing `filter`
@@ -877,8 +899,9 @@ def find_best_candidate(
         )
         return candidate_evaluator.compute_best_candidate(candidates)
 
-    def find_requirement(self, req, upgrade):
-        # type: (InstallRequirement, bool) -> Optional[InstallationCandidate]
+    def find_requirement(
+        self, req: InstallRequirement, upgrade: bool
+    ) -> Optional[InstallationCandidate]:
         """Try to find a Link matching req
 
         Expects req, an InstallRequirement and upgrade, a boolean
@@ -887,55 +910,60 @@ def find_requirement(self, req, upgrade):
         """
         hashes = req.hashes(trust_internet=False)
         best_candidate_result = self.find_best_candidate(
-            req.name, specifier=req.specifier, hashes=hashes,
+            req.name,
+            specifier=req.specifier,
+            hashes=hashes,
         )
         best_candidate = best_candidate_result.best_candidate
 
-        installed_version = None    # type: Optional[_BaseVersion]
+        installed_version: Optional[_BaseVersion] = None
         if req.satisfied_by is not None:
-            installed_version = parse_version(req.satisfied_by.version)
+            installed_version = req.satisfied_by.version
 
-        def _format_versions(cand_iter):
-            # type: (Iterable[InstallationCandidate]) -> str
+        def _format_versions(cand_iter: Iterable[InstallationCandidate]) -> str:
             # This repeated parse_version and str() conversion is needed to
             # handle different vendoring sources from pip and pkg_resources.
             # If we stop using the pkg_resources provided specifier and start
             # using our own, we can drop the cast to str().
-            return ", ".join(sorted(
-                {str(c.version) for c in cand_iter},
-                key=parse_version,
-            )) or "none"
+            return (
+                ", ".join(
+                    sorted(
+                        {str(c.version) for c in cand_iter},
+                        key=parse_version,
+                    )
+                )
+                or "none"
+            )
 
         if installed_version is None and best_candidate is None:
             logger.critical(
-                'Could not find a version that satisfies the requirement %s '
-                '(from versions: %s)',
+                "Could not find a version that satisfies the requirement %s "
+                "(from versions: %s)",
                 req,
                 _format_versions(best_candidate_result.iter_all()),
             )
 
             raise DistributionNotFound(
-                'No matching distribution found for {}'.format(
-                    req)
+                "No matching distribution found for {}".format(req)
             )
 
         best_installed = False
         if installed_version and (
-                best_candidate is None or
-                best_candidate.version <= installed_version):
+            best_candidate is None or best_candidate.version <= installed_version
+        ):
             best_installed = True
 
         if not upgrade and installed_version is not None:
             if best_installed:
                 logger.debug(
-                    'Existing installed version (%s) is most up-to-date and '
-                    'satisfies requirement',
+                    "Existing installed version (%s) is most up-to-date and "
+                    "satisfies requirement",
                     installed_version,
                 )
             else:
                 logger.debug(
-                    'Existing installed version (%s) satisfies requirement '
-                    '(most up-to-date version is %s)',
+                    "Existing installed version (%s) satisfies requirement "
+                    "(most up-to-date version is %s)",
                     installed_version,
                     best_candidate.version,
                 )
@@ -944,23 +972,21 @@ def _format_versions(cand_iter):
         if best_installed:
             # We have an existing version, and its the best version
             logger.debug(
-                'Installed version (%s) is most up-to-date (past versions: '
-                '%s)',
+                "Installed version (%s) is most up-to-date (past versions: %s)",
                 installed_version,
                 _format_versions(best_candidate_result.iter_applicable()),
             )
             raise BestVersionAlreadyInstalled
 
         logger.debug(
-            'Using version %s (newest of versions: %s)',
+            "Using version %s (newest of versions: %s)",
             best_candidate.version,
             _format_versions(best_candidate_result.iter_applicable()),
         )
         return best_candidate
 
 
-def _find_name_version_sep(fragment, canonical_name):
-    # type: (str, str) -> int
+def _find_name_version_sep(fragment: str, canonical_name: str) -> int:
     """Find the separator's index based on the package's canonical name.
 
     :param fragment: A + filename "fragment" (stem) or
@@ -986,8 +1012,7 @@ def _find_name_version_sep(fragment, canonical_name):
     raise ValueError(f"{fragment} does not match {canonical_name}")
 
 
-def _extract_version_from_fragment(fragment, canonical_name):
-    # type: (str, str) -> Optional[str]
+def _extract_version_from_fragment(fragment: str, canonical_name: str) -> Optional[str]:
     """Parse the version string from a + filename
     "fragment" (stem) or egg fragment.
 
diff --git a/src/pip/_internal/index/sources.py b/src/pip/_internal/index/sources.py
new file mode 100644
index 00000000000..eec3f12f7e3
--- /dev/null
+++ b/src/pip/_internal/index/sources.py
@@ -0,0 +1,224 @@
+import logging
+import mimetypes
+import os
+import pathlib
+from typing import Callable, Iterable, Optional, Tuple
+
+from pip._internal.models.candidate import InstallationCandidate
+from pip._internal.models.link import Link
+from pip._internal.utils.urls import path_to_url, url_to_path
+from pip._internal.vcs import is_url
+
+logger = logging.getLogger(__name__)
+
+FoundCandidates = Iterable[InstallationCandidate]
+FoundLinks = Iterable[Link]
+CandidatesFromPage = Callable[[Link], Iterable[InstallationCandidate]]
+PageValidator = Callable[[Link], bool]
+
+
+class LinkSource:
+    @property
+    def link(self) -> Optional[Link]:
+        """Returns the underlying link, if there's one."""
+        raise NotImplementedError()
+
+    def page_candidates(self) -> FoundCandidates:
+        """Candidates found by parsing an archive listing HTML file."""
+        raise NotImplementedError()
+
+    def file_links(self) -> FoundLinks:
+        """Links found by specifying archives directly."""
+        raise NotImplementedError()
+
+
+def _is_html_file(file_url: str) -> bool:
+    return mimetypes.guess_type(file_url, strict=False)[0] == "text/html"
+
+
+class _FlatDirectorySource(LinkSource):
+    """Link source specified by ``--find-links=``.
+
+    This looks the content of the directory, and returns:
+
+    * ``page_candidates``: Links listed on each HTML file in the directory.
+    * ``file_candidates``: Archives in the directory.
+    """
+
+    def __init__(
+        self,
+        candidates_from_page: CandidatesFromPage,
+        path: str,
+    ) -> None:
+        self._candidates_from_page = candidates_from_page
+        self._path = pathlib.Path(os.path.realpath(path))
+
+    @property
+    def link(self) -> Optional[Link]:
+        return None
+
+    def page_candidates(self) -> FoundCandidates:
+        for path in self._path.iterdir():
+            url = path_to_url(str(path))
+            if not _is_html_file(url):
+                continue
+            yield from self._candidates_from_page(Link(url))
+
+    def file_links(self) -> FoundLinks:
+        for path in self._path.iterdir():
+            url = path_to_url(str(path))
+            if _is_html_file(url):
+                continue
+            yield Link(url)
+
+
+class _LocalFileSource(LinkSource):
+    """``--find-links=`` or ``--[extra-]index-url=``.
+
+    If a URL is supplied, it must be a ``file:`` URL. If a path is supplied to
+    the option, it is converted to a URL first. This returns:
+
+    * ``page_candidates``: Links listed on an HTML file.
+    * ``file_candidates``: The non-HTML file.
+    """
+
+    def __init__(
+        self,
+        candidates_from_page: CandidatesFromPage,
+        link: Link,
+    ) -> None:
+        self._candidates_from_page = candidates_from_page
+        self._link = link
+
+    @property
+    def link(self) -> Optional[Link]:
+        return self._link
+
+    def page_candidates(self) -> FoundCandidates:
+        if not _is_html_file(self._link.url):
+            return
+        yield from self._candidates_from_page(self._link)
+
+    def file_links(self) -> FoundLinks:
+        if _is_html_file(self._link.url):
+            return
+        yield self._link
+
+
+class _RemoteFileSource(LinkSource):
+    """``--find-links=`` or ``--[extra-]index-url=``.
+
+    This returns:
+
+    * ``page_candidates``: Links listed on an HTML file.
+    * ``file_candidates``: The non-HTML file.
+    """
+
+    def __init__(
+        self,
+        candidates_from_page: CandidatesFromPage,
+        page_validator: PageValidator,
+        link: Link,
+    ) -> None:
+        self._candidates_from_page = candidates_from_page
+        self._page_validator = page_validator
+        self._link = link
+
+    @property
+    def link(self) -> Optional[Link]:
+        return self._link
+
+    def page_candidates(self) -> FoundCandidates:
+        if not self._page_validator(self._link):
+            return
+        yield from self._candidates_from_page(self._link)
+
+    def file_links(self) -> FoundLinks:
+        yield self._link
+
+
+class _IndexDirectorySource(LinkSource):
+    """``--[extra-]index-url=``.
+
+    This is treated like a remote URL; ``candidates_from_page`` contains logic
+    for this by appending ``index.html`` to the link.
+    """
+
+    def __init__(
+        self,
+        candidates_from_page: CandidatesFromPage,
+        link: Link,
+    ) -> None:
+        self._candidates_from_page = candidates_from_page
+        self._link = link
+
+    @property
+    def link(self) -> Optional[Link]:
+        return self._link
+
+    def page_candidates(self) -> FoundCandidates:
+        yield from self._candidates_from_page(self._link)
+
+    def file_links(self) -> FoundLinks:
+        return ()
+
+
+def build_source(
+    location: str,
+    *,
+    candidates_from_page: CandidatesFromPage,
+    page_validator: PageValidator,
+    expand_dir: bool,
+    cache_link_parsing: bool,
+) -> Tuple[Optional[str], Optional[LinkSource]]:
+
+    path: Optional[str] = None
+    url: Optional[str] = None
+    if os.path.exists(location):  # Is a local path.
+        url = path_to_url(location)
+        path = location
+    elif location.startswith("file:"):  # A file: URL.
+        url = location
+        path = url_to_path(location)
+    elif is_url(location):
+        url = location
+
+    if url is None:
+        msg = (
+            "Location '%s' is ignored: "
+            "it is either a non-existing path or lacks a specific scheme."
+        )
+        logger.warning(msg, location)
+        return (None, None)
+
+    if path is None:
+        source: LinkSource = _RemoteFileSource(
+            candidates_from_page=candidates_from_page,
+            page_validator=page_validator,
+            link=Link(url, cache_link_parsing=cache_link_parsing),
+        )
+        return (url, source)
+
+    if os.path.isdir(path):
+        if expand_dir:
+            source = _FlatDirectorySource(
+                candidates_from_page=candidates_from_page,
+                path=path,
+            )
+        else:
+            source = _IndexDirectorySource(
+                candidates_from_page=candidates_from_page,
+                link=Link(url, cache_link_parsing=cache_link_parsing),
+            )
+        return (url, source)
+    elif os.path.isfile(path):
+        source = _LocalFileSource(
+            candidates_from_page=candidates_from_page,
+            link=Link(url, cache_link_parsing=cache_link_parsing),
+        )
+        return (url, source)
+    logger.warning(
+        "Location '%s' is ignored: it is neither a file nor a directory.",
+        location,
+    )
+    return (url, None)
diff --git a/src/pip/_internal/locations.py b/src/pip/_internal/locations.py
deleted file mode 100644
index 2a903297733..00000000000
--- a/src/pip/_internal/locations.py
+++ /dev/null
@@ -1,184 +0,0 @@
-"""Locations where we look for configs, install stuff, etc"""
-
-# The following comment should be removed at some point in the future.
-# mypy: strict-optional=False
-
-import os
-import os.path
-import site
-import sys
-import sysconfig
-from distutils.command.install import SCHEME_KEYS  # type: ignore
-from distutils.command.install import install as distutils_install_command
-
-from pip._internal.models.scheme import Scheme
-from pip._internal.utils import appdirs
-from pip._internal.utils.compat import WINDOWS
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast
-from pip._internal.utils.virtualenv import running_under_virtualenv
-
-if MYPY_CHECK_RUNNING:
-    from distutils.cmd import Command as DistutilsCommand
-    from typing import Dict, List, Optional, Union
-
-
-# Application Directories
-USER_CACHE_DIR = appdirs.user_cache_dir("pip")
-
-
-def get_major_minor_version():
-    # type: () -> str
-    """
-    Return the major-minor version of the current Python as a string, e.g.
-    "3.7" or "3.10".
-    """
-    return '{}.{}'.format(*sys.version_info)
-
-
-def get_src_prefix():
-    # type: () -> str
-    if running_under_virtualenv():
-        src_prefix = os.path.join(sys.prefix, 'src')
-    else:
-        # FIXME: keep src in cwd for now (it is not a temporary folder)
-        try:
-            src_prefix = os.path.join(os.getcwd(), 'src')
-        except OSError:
-            # In case the current working directory has been renamed or deleted
-            sys.exit(
-                "The folder you are executing pip from can no longer be found."
-            )
-
-    # under macOS + virtualenv sys.prefix is not properly resolved
-    # it is something like /path/to/python/bin/..
-    return os.path.abspath(src_prefix)
-
-
-# FIXME doesn't account for venv linked to global site-packages
-
-site_packages = sysconfig.get_path("purelib")  # type: Optional[str]
-
-try:
-    # Use getusersitepackages if this is present, as it ensures that the
-    # value is initialised properly.
-    user_site = site.getusersitepackages()
-except AttributeError:
-    user_site = site.USER_SITE
-
-if WINDOWS:
-    bin_py = os.path.join(sys.prefix, 'Scripts')
-    bin_user = os.path.join(user_site, 'Scripts')
-    # buildout uses 'bin' on Windows too?
-    if not os.path.exists(bin_py):
-        bin_py = os.path.join(sys.prefix, 'bin')
-        bin_user = os.path.join(user_site, 'bin')
-else:
-    bin_py = os.path.join(sys.prefix, 'bin')
-    bin_user = os.path.join(user_site, 'bin')
-
-    # Forcing to use /usr/local/bin for standard macOS framework installs
-    # Also log to ~/Library/Logs/ for use with the Console.app log viewer
-    if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/':
-        bin_py = '/usr/local/bin'
-
-
-def distutils_scheme(
-    dist_name, user=False, home=None, root=None, isolated=False, prefix=None
-):
-    # type:(str, bool, str, str, bool, str) -> Dict[str, str]
-    """
-    Return a distutils install scheme
-    """
-    from distutils.dist import Distribution
-
-    dist_args = {'name': dist_name}  # type: Dict[str, Union[str, List[str]]]
-    if isolated:
-        dist_args["script_args"] = ["--no-user-cfg"]
-
-    d = Distribution(dist_args)
-    d.parse_config_files()
-    obj = None  # type: Optional[DistutilsCommand]
-    obj = d.get_command_obj('install', create=True)
-    assert obj is not None
-    i = cast(distutils_install_command, obj)
-    # NOTE: setting user or home has the side-effect of creating the home dir
-    # or user base for installations during finalize_options()
-    # ideally, we'd prefer a scheme class that has no side-effects.
-    assert not (user and prefix), f"user={user} prefix={prefix}"
-    assert not (home and prefix), f"home={home} prefix={prefix}"
-    i.user = user or i.user
-    if user or home:
-        i.prefix = ""
-    i.prefix = prefix or i.prefix
-    i.home = home or i.home
-    i.root = root or i.root
-    i.finalize_options()
-
-    scheme = {}
-    for key in SCHEME_KEYS:
-        scheme[key] = getattr(i, 'install_' + key)
-
-    # install_lib specified in setup.cfg should install *everything*
-    # into there (i.e. it takes precedence over both purelib and
-    # platlib).  Note, i.install_lib is *always* set after
-    # finalize_options(); we only want to override here if the user
-    # has explicitly requested it hence going back to the config
-    if 'install_lib' in d.get_option_dict('install'):
-        scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))
-
-    if running_under_virtualenv():
-        scheme['headers'] = os.path.join(
-            i.prefix,
-            'include',
-            'site',
-            f'python{get_major_minor_version()}',
-            dist_name,
-        )
-
-        if root is not None:
-            path_no_drive = os.path.splitdrive(
-                os.path.abspath(scheme["headers"]))[1]
-            scheme["headers"] = os.path.join(
-                root,
-                path_no_drive[1:],
-            )
-
-    return scheme
-
-
-def get_scheme(
-    dist_name,  # type: str
-    user=False,  # type: bool
-    home=None,  # type: Optional[str]
-    root=None,  # type: Optional[str]
-    isolated=False,  # type: bool
-    prefix=None,  # type: Optional[str]
-):
-    # type: (...) -> Scheme
-    """
-    Get the "scheme" corresponding to the input parameters. The distutils
-    documentation provides the context for the available schemes:
-    https://docs.python.org/3/install/index.html#alternate-installation
-
-    :param dist_name: the name of the package to retrieve the scheme for, used
-        in the headers scheme path
-    :param user: indicates to use the "user" scheme
-    :param home: indicates to use the "home" scheme and provides the base
-        directory for the same
-    :param root: root under which other directories are re-based
-    :param isolated: equivalent to --no-user-cfg, i.e. do not consider
-        ~/.pydistutils.cfg (posix) or ~/pydistutils.cfg (non-posix) for
-        scheme paths
-    :param prefix: indicates to use the "prefix" scheme and provides the
-        base directory for the same
-    """
-    scheme = distutils_scheme(
-        dist_name, user, home, root, isolated, prefix
-    )
-    return Scheme(
-        platlib=scheme["platlib"],
-        purelib=scheme["purelib"],
-        headers=scheme["headers"],
-        scripts=scheme["scripts"],
-        data=scheme["data"],
-    )
diff --git a/src/pip/_internal/locations/__init__.py b/src/pip/_internal/locations/__init__.py
new file mode 100644
index 00000000000..ac0c166e519
--- /dev/null
+++ b/src/pip/_internal/locations/__init__.py
@@ -0,0 +1,520 @@
+import functools
+import logging
+import os
+import pathlib
+import sys
+import sysconfig
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from pip._internal.models.scheme import SCHEME_KEYS, Scheme
+from pip._internal.utils.compat import WINDOWS
+from pip._internal.utils.deprecation import deprecated
+from pip._internal.utils.virtualenv import running_under_virtualenv
+
+from . import _distutils, _sysconfig
+from .base import (
+    USER_CACHE_DIR,
+    get_major_minor_version,
+    get_src_prefix,
+    is_osx_framework,
+    site_packages,
+    user_site,
+)
+
+__all__ = [
+    "USER_CACHE_DIR",
+    "get_bin_prefix",
+    "get_bin_user",
+    "get_major_minor_version",
+    "get_platlib",
+    "get_prefixed_libs",
+    "get_purelib",
+    "get_scheme",
+    "get_src_prefix",
+    "site_packages",
+    "user_site",
+]
+
+
+logger = logging.getLogger(__name__)
+
+
+_PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
+
+_USE_SYSCONFIG_DEFAULT = sys.version_info >= (3, 10)
+
+
+def _should_use_sysconfig() -> bool:
+    """This function determines the value of _USE_SYSCONFIG.
+
+    By default, pip uses sysconfig on Python 3.10+.
+    But Python distributors can override this decision by setting:
+        sysconfig._PIP_USE_SYSCONFIG = True / False
+    Rationale in https://github.com/pypa/pip/issues/10647
+
+    This is a function for testability, but should be constant during any one
+    run.
+    """
+    return bool(getattr(sysconfig, "_PIP_USE_SYSCONFIG", _USE_SYSCONFIG_DEFAULT))
+
+
+_USE_SYSCONFIG = _should_use_sysconfig()
+
+# Be noisy about incompatibilities if this platforms "should" be using
+# sysconfig, but is explicitly opting out and using distutils instead.
+if _USE_SYSCONFIG_DEFAULT and not _USE_SYSCONFIG:
+    _MISMATCH_LEVEL = logging.WARNING
+else:
+    _MISMATCH_LEVEL = logging.DEBUG
+
+
+def _looks_like_bpo_44860() -> bool:
+    """The resolution to bpo-44860 will change this incorrect platlib.
+
+    See .
+    """
+    from distutils.command.install import INSTALL_SCHEMES  # type: ignore
+
+    try:
+        unix_user_platlib = INSTALL_SCHEMES["unix_user"]["platlib"]
+    except KeyError:
+        return False
+    return unix_user_platlib == "$usersite"
+
+
+def _looks_like_red_hat_patched_platlib_purelib(scheme: Dict[str, str]) -> bool:
+    platlib = scheme["platlib"]
+    if "/$platlibdir/" in platlib:
+        platlib = platlib.replace("/$platlibdir/", f"/{_PLATLIBDIR}/")
+    if "/lib64/" not in platlib:
+        return False
+    unpatched = platlib.replace("/lib64/", "/lib/")
+    return unpatched.replace("$platbase/", "$base/") == scheme["purelib"]
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_red_hat_lib() -> bool:
+    """Red Hat patches platlib in unix_prefix and unix_home, but not purelib.
+
+    This is the only way I can see to tell a Red Hat-patched Python.
+    """
+    from distutils.command.install import INSTALL_SCHEMES  # type: ignore
+
+    return all(
+        k in INSTALL_SCHEMES
+        and _looks_like_red_hat_patched_platlib_purelib(INSTALL_SCHEMES[k])
+        for k in ("unix_prefix", "unix_home")
+    )
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_debian_scheme() -> bool:
+    """Debian adds two additional schemes."""
+    from distutils.command.install import INSTALL_SCHEMES  # type: ignore
+
+    return "deb_system" in INSTALL_SCHEMES and "unix_local" in INSTALL_SCHEMES
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_red_hat_scheme() -> bool:
+    """Red Hat patches ``sys.prefix`` and ``sys.exec_prefix``.
+
+    Red Hat's ``00251-change-user-install-location.patch`` changes the install
+    command's ``prefix`` and ``exec_prefix`` to append ``"/local"``. This is
+    (fortunately?) done quite unconditionally, so we create a default command
+    object without any configuration to detect this.
+    """
+    from distutils.command.install import install
+    from distutils.dist import Distribution
+
+    cmd: Any = install(Distribution())
+    cmd.finalize_options()
+    return (
+        cmd.exec_prefix == f"{os.path.normpath(sys.exec_prefix)}/local"
+        and cmd.prefix == f"{os.path.normpath(sys.prefix)}/local"
+    )
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_slackware_scheme() -> bool:
+    """Slackware patches sysconfig but fails to patch distutils and site.
+
+    Slackware changes sysconfig's user scheme to use ``"lib64"`` for the lib
+    path, but does not do the same to the site module.
+    """
+    if user_site is None:  # User-site not available.
+        return False
+    try:
+        paths = sysconfig.get_paths(scheme="posix_user", expand=False)
+    except KeyError:  # User-site not available.
+        return False
+    return "/lib64/" in paths["purelib"] and "/lib64/" not in user_site
+
+
+@functools.lru_cache(maxsize=None)
+def _looks_like_msys2_mingw_scheme() -> bool:
+    """MSYS2 patches distutils and sysconfig to use a UNIX-like scheme.
+
+    However, MSYS2 incorrectly patches sysconfig ``nt`` scheme. The fix is
+    likely going to be included in their 3.10 release, so we ignore the warning.
+    See msys2/MINGW-packages#9319.
+
+    MSYS2 MINGW's patch uses lowercase ``"lib"`` instead of the usual uppercase,
+    and is missing the final ``"site-packages"``.
+    """
+    paths = sysconfig.get_paths("nt", expand=False)
+    return all(
+        "Lib" not in p and "lib" in p and not p.endswith("site-packages")
+        for p in (paths[key] for key in ("platlib", "purelib"))
+    )
+
+
+def _fix_abiflags(parts: Tuple[str]) -> Iterator[str]:
+    ldversion = sysconfig.get_config_var("LDVERSION")
+    abiflags: str = getattr(sys, "abiflags", None)
+
+    # LDVERSION does not end with sys.abiflags. Just return the path unchanged.
+    if not ldversion or not abiflags or not ldversion.endswith(abiflags):
+        yield from parts
+        return
+
+    # Strip sys.abiflags from LDVERSION-based path components.
+    for part in parts:
+        if part.endswith(ldversion):
+            part = part[: (0 - len(abiflags))]
+        yield part
+
+
+@functools.lru_cache(maxsize=None)
+def _warn_mismatched(old: pathlib.Path, new: pathlib.Path, *, key: str) -> None:
+    issue_url = "https://github.com/pypa/pip/issues/10151"
+    message = (
+        "Value for %s does not match. Please report this to <%s>"
+        "\ndistutils: %s"
+        "\nsysconfig: %s"
+    )
+    logger.log(_MISMATCH_LEVEL, message, key, issue_url, old, new)
+
+
+def _warn_if_mismatch(old: pathlib.Path, new: pathlib.Path, *, key: str) -> bool:
+    if old == new:
+        return False
+    _warn_mismatched(old, new, key=key)
+    return True
+
+
+@functools.lru_cache(maxsize=None)
+def _log_context(
+    *,
+    user: bool = False,
+    home: Optional[str] = None,
+    root: Optional[str] = None,
+    prefix: Optional[str] = None,
+) -> None:
+    parts = [
+        "Additional context:",
+        "user = %r",
+        "home = %r",
+        "root = %r",
+        "prefix = %r",
+    ]
+
+    logger.log(_MISMATCH_LEVEL, "\n".join(parts), user, home, root, prefix)
+
+
+def get_scheme(
+    dist_name: str,
+    user: bool = False,
+    home: Optional[str] = None,
+    root: Optional[str] = None,
+    isolated: bool = False,
+    prefix: Optional[str] = None,
+) -> Scheme:
+    new = _sysconfig.get_scheme(
+        dist_name,
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_scheme(
+        dist_name,
+        user=user,
+        home=home,
+        root=root,
+        isolated=isolated,
+        prefix=prefix,
+    )
+
+    warning_contexts = []
+    for k in SCHEME_KEYS:
+        old_v = pathlib.Path(getattr(old, k))
+        new_v = pathlib.Path(getattr(new, k))
+
+        if old_v == new_v:
+            continue
+
+        # distutils incorrectly put PyPy packages under ``site-packages/python``
+        # in the ``posix_home`` scheme, but PyPy devs said they expect the
+        # directory name to be ``pypy`` instead. So we treat this as a bug fix
+        # and not warn about it. See bpo-43307 and python/cpython#24628.
+        skip_pypy_special_case = (
+            sys.implementation.name == "pypy"
+            and home is not None
+            and k in ("platlib", "purelib")
+            and old_v.parent == new_v.parent
+            and old_v.name.startswith("python")
+            and new_v.name.startswith("pypy")
+        )
+        if skip_pypy_special_case:
+            continue
+
+        # sysconfig's ``osx_framework_user`` does not include ``pythonX.Y`` in
+        # the ``include`` value, but distutils's ``headers`` does. We'll let
+        # CPython decide whether this is a bug or feature. See bpo-43948.
+        skip_osx_framework_user_special_case = (
+            user
+            and is_osx_framework()
+            and k == "headers"
+            and old_v.parent.parent == new_v.parent
+            and old_v.parent.name.startswith("python")
+        )
+        if skip_osx_framework_user_special_case:
+            continue
+
+        # On Red Hat and derived Linux distributions, distutils is patched to
+        # use "lib64" instead of "lib" for platlib.
+        if k == "platlib" and _looks_like_red_hat_lib():
+            continue
+
+        # On Python 3.9+, sysconfig's posix_user scheme sets platlib against
+        # sys.platlibdir, but distutils's unix_user incorrectly coninutes
+        # using the same $usersite for both platlib and purelib. This creates a
+        # mismatch when sys.platlibdir is not "lib".
+        skip_bpo_44860 = (
+            user
+            and k == "platlib"
+            and not WINDOWS
+            and sys.version_info >= (3, 9)
+            and _PLATLIBDIR != "lib"
+            and _looks_like_bpo_44860()
+        )
+        if skip_bpo_44860:
+            continue
+
+        # Slackware incorrectly patches posix_user to use lib64 instead of lib,
+        # but not usersite to match the location.
+        skip_slackware_user_scheme = (
+            user
+            and k in ("platlib", "purelib")
+            and not WINDOWS
+            and _looks_like_slackware_scheme()
+        )
+        if skip_slackware_user_scheme:
+            continue
+
+        # Both Debian and Red Hat patch Python to place the system site under
+        # /usr/local instead of /usr. Debian also places lib in dist-packages
+        # instead of site-packages, but the /usr/local check should cover it.
+        skip_linux_system_special_case = (
+            not (user or home or prefix or running_under_virtualenv())
+            and old_v.parts[1:3] == ("usr", "local")
+            and len(new_v.parts) > 1
+            and new_v.parts[1] == "usr"
+            and (len(new_v.parts) < 3 or new_v.parts[2] != "local")
+            and (_looks_like_red_hat_scheme() or _looks_like_debian_scheme())
+        )
+        if skip_linux_system_special_case:
+            continue
+
+        # On Python 3.7 and earlier, sysconfig does not include sys.abiflags in
+        # the "pythonX.Y" part of the path, but distutils does.
+        skip_sysconfig_abiflag_bug = (
+            sys.version_info < (3, 8)
+            and not WINDOWS
+            and k in ("headers", "platlib", "purelib")
+            and tuple(_fix_abiflags(old_v.parts)) == new_v.parts
+        )
+        if skip_sysconfig_abiflag_bug:
+            continue
+
+        # MSYS2 MINGW's sysconfig patch does not include the "site-packages"
+        # part of the path. This is incorrect and will be fixed in MSYS.
+        skip_msys2_mingw_bug = (
+            WINDOWS and k in ("platlib", "purelib") and _looks_like_msys2_mingw_scheme()
+        )
+        if skip_msys2_mingw_bug:
+            continue
+
+        # CPython's POSIX install script invokes pip (via ensurepip) against the
+        # interpreter located in the source tree, not the install site. This
+        # triggers special logic in sysconfig that's not present in distutils.
+        # https://github.com/python/cpython/blob/8c21941ddaf/Lib/sysconfig.py#L178-L194
+        skip_cpython_build = (
+            sysconfig.is_python_build(check_home=True)
+            and not WINDOWS
+            and k in ("headers", "include", "platinclude")
+        )
+        if skip_cpython_build:
+            continue
+
+        warning_contexts.append((old_v, new_v, f"scheme.{k}"))
+
+    if not warning_contexts:
+        return old
+
+    # Check if this path mismatch is caused by distutils config files. Those
+    # files will no longer work once we switch to sysconfig, so this raises a
+    # deprecation message for them.
+    default_old = _distutils.distutils_scheme(
+        dist_name,
+        user,
+        home,
+        root,
+        isolated,
+        prefix,
+        ignore_config_files=True,
+    )
+    if any(default_old[k] != getattr(old, k) for k in SCHEME_KEYS):
+        deprecated(
+            reason=(
+                "Configuring installation scheme with distutils config files "
+                "is deprecated and will no longer work in the near future. If you "
+                "are using a Homebrew or Linuxbrew Python, please see discussion "
+                "at https://github.com/Homebrew/homebrew-core/issues/76621"
+            ),
+            replacement=None,
+            gone_in=None,
+        )
+        return old
+
+    # Post warnings about this mismatch so user can report them back.
+    for old_v, new_v, key in warning_contexts:
+        _warn_mismatched(old_v, new_v, key=key)
+    _log_context(user=user, home=home, root=root, prefix=prefix)
+
+    return old
+
+
+def get_bin_prefix() -> str:
+    new = _sysconfig.get_bin_prefix()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_bin_prefix()
+    if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"):
+        _log_context()
+    return old
+
+
+def get_bin_user() -> str:
+    return _sysconfig.get_scheme("", user=True).scripts
+
+
+def _looks_like_deb_system_dist_packages(value: str) -> bool:
+    """Check if the value is Debian's APT-controlled dist-packages.
+
+    Debian's ``distutils.sysconfig.get_python_lib()`` implementation returns the
+    default package path controlled by APT, but does not patch ``sysconfig`` to
+    do the same. This is similar to the bug worked around in ``get_scheme()``,
+    but here the default is ``deb_system`` instead of ``unix_local``. Ultimately
+    we can't do anything about this Debian bug, and this detection allows us to
+    skip the warning when needed.
+    """
+    if not _looks_like_debian_scheme():
+        return False
+    if value == "/usr/lib/python3/dist-packages":
+        return True
+    return False
+
+
+def get_purelib() -> str:
+    """Return the default pure-Python lib location."""
+    new = _sysconfig.get_purelib()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_purelib()
+    if _looks_like_deb_system_dist_packages(old):
+        return old
+    if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"):
+        _log_context()
+    return old
+
+
+def get_platlib() -> str:
+    """Return the default platform-shared lib location."""
+    new = _sysconfig.get_platlib()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_platlib()
+    if _looks_like_deb_system_dist_packages(old):
+        return old
+    if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"):
+        _log_context()
+    return old
+
+
+def _deduplicated(v1: str, v2: str) -> List[str]:
+    """Deduplicate values from a list."""
+    if v1 == v2:
+        return [v1]
+    return [v1, v2]
+
+
+def _looks_like_apple_library(path: str) -> bool:
+    """Apple patches sysconfig to *always* look under */Library/Python*."""
+    if sys.platform[:6] != "darwin":
+        return False
+    return path == f"/Library/Python/{get_major_minor_version()}/site-packages"
+
+
+def get_prefixed_libs(prefix: str) -> List[str]:
+    """Return the lib locations under ``prefix``."""
+    new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
+    if _USE_SYSCONFIG:
+        return _deduplicated(new_pure, new_plat)
+
+    old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
+    old_lib_paths = _deduplicated(old_pure, old_plat)
+
+    # Apple's Python (shipped with Xcode and Command Line Tools) hard-code
+    # platlib and purelib to '/Library/Python/X.Y/site-packages'. This will
+    # cause serious build isolation bugs when Apple starts shipping 3.10 because
+    # pip will install build backends to the wrong location. This tells users
+    # who is at fault so Apple may notice it and fix the issue in time.
+    if all(_looks_like_apple_library(p) for p in old_lib_paths):
+        deprecated(
+            reason=(
+                "Python distributed by Apple's Command Line Tools incorrectly "
+                "patches sysconfig to always point to '/Library/Python'. This "
+                "will cause build isolation to operate incorrectly on Python "
+                "3.10 or later. Please help report this to Apple so they can "
+                "fix this. https://developer.apple.com/bug-reporting/"
+            ),
+            replacement=None,
+            gone_in=None,
+        )
+        return old_lib_paths
+
+    warned = [
+        _warn_if_mismatch(
+            pathlib.Path(old_pure),
+            pathlib.Path(new_pure),
+            key="prefixed-purelib",
+        ),
+        _warn_if_mismatch(
+            pathlib.Path(old_plat),
+            pathlib.Path(new_plat),
+            key="prefixed-platlib",
+        ),
+    ]
+    if any(warned):
+        _log_context(prefix=prefix)
+
+    return old_lib_paths
diff --git a/src/pip/_internal/locations/_distutils.py b/src/pip/_internal/locations/_distutils.py
new file mode 100644
index 00000000000..2ec79e65bea
--- /dev/null
+++ b/src/pip/_internal/locations/_distutils.py
@@ -0,0 +1,169 @@
+"""Locations where we look for configs, install stuff, etc"""
+
+# The following comment should be removed at some point in the future.
+# mypy: strict-optional=False
+
+import logging
+import os
+import sys
+from distutils.cmd import Command as DistutilsCommand
+from distutils.command.install import SCHEME_KEYS
+from distutils.command.install import install as distutils_install_command
+from distutils.sysconfig import get_python_lib
+from typing import Dict, List, Optional, Tuple, Union, cast
+
+from pip._internal.models.scheme import Scheme
+from pip._internal.utils.compat import WINDOWS
+from pip._internal.utils.virtualenv import running_under_virtualenv
+
+from .base import get_major_minor_version
+
+logger = logging.getLogger(__name__)
+
+
+def distutils_scheme(
+    dist_name: str,
+    user: bool = False,
+    home: str = None,
+    root: str = None,
+    isolated: bool = False,
+    prefix: str = None,
+    *,
+    ignore_config_files: bool = False,
+) -> Dict[str, str]:
+    """
+    Return a distutils install scheme
+    """
+    from distutils.dist import Distribution
+
+    dist_args: Dict[str, Union[str, List[str]]] = {"name": dist_name}
+    if isolated:
+        dist_args["script_args"] = ["--no-user-cfg"]
+
+    d = Distribution(dist_args)
+    if not ignore_config_files:
+        try:
+            d.parse_config_files()
+        except UnicodeDecodeError:
+            # Typeshed does not include find_config_files() for some reason.
+            paths = d.find_config_files()  # type: ignore
+            logger.warning(
+                "Ignore distutils configs in %s due to encoding errors.",
+                ", ".join(os.path.basename(p) for p in paths),
+            )
+    obj: Optional[DistutilsCommand] = None
+    obj = d.get_command_obj("install", create=True)
+    assert obj is not None
+    i = cast(distutils_install_command, obj)
+    # NOTE: setting user or home has the side-effect of creating the home dir
+    # or user base for installations during finalize_options()
+    # ideally, we'd prefer a scheme class that has no side-effects.
+    assert not (user and prefix), f"user={user} prefix={prefix}"
+    assert not (home and prefix), f"home={home} prefix={prefix}"
+    i.user = user or i.user
+    if user or home:
+        i.prefix = ""
+    i.prefix = prefix or i.prefix
+    i.home = home or i.home
+    i.root = root or i.root
+    i.finalize_options()
+
+    scheme = {}
+    for key in SCHEME_KEYS:
+        scheme[key] = getattr(i, "install_" + key)
+
+    # install_lib specified in setup.cfg should install *everything*
+    # into there (i.e. it takes precedence over both purelib and
+    # platlib).  Note, i.install_lib is *always* set after
+    # finalize_options(); we only want to override here if the user
+    # has explicitly requested it hence going back to the config
+    if "install_lib" in d.get_option_dict("install"):
+        scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib))
+
+    if running_under_virtualenv():
+        if home:
+            prefix = home
+        elif user:
+            prefix = i.install_userbase  # type: ignore
+        else:
+            prefix = i.prefix
+        scheme["headers"] = os.path.join(
+            prefix,
+            "include",
+            "site",
+            f"python{get_major_minor_version()}",
+            dist_name,
+        )
+
+        if root is not None:
+            path_no_drive = os.path.splitdrive(os.path.abspath(scheme["headers"]))[1]
+            scheme["headers"] = os.path.join(root, path_no_drive[1:])
+
+    return scheme
+
+
+def get_scheme(
+    dist_name: str,
+    user: bool = False,
+    home: Optional[str] = None,
+    root: Optional[str] = None,
+    isolated: bool = False,
+    prefix: Optional[str] = None,
+) -> Scheme:
+    """
+    Get the "scheme" corresponding to the input parameters. The distutils
+    documentation provides the context for the available schemes:
+    https://docs.python.org/3/install/index.html#alternate-installation
+
+    :param dist_name: the name of the package to retrieve the scheme for, used
+        in the headers scheme path
+    :param user: indicates to use the "user" scheme
+    :param home: indicates to use the "home" scheme and provides the base
+        directory for the same
+    :param root: root under which other directories are re-based
+    :param isolated: equivalent to --no-user-cfg, i.e. do not consider
+        ~/.pydistutils.cfg (posix) or ~/pydistutils.cfg (non-posix) for
+        scheme paths
+    :param prefix: indicates to use the "prefix" scheme and provides the
+        base directory for the same
+    """
+    scheme = distutils_scheme(dist_name, user, home, root, isolated, prefix)
+    return Scheme(
+        platlib=scheme["platlib"],
+        purelib=scheme["purelib"],
+        headers=scheme["headers"],
+        scripts=scheme["scripts"],
+        data=scheme["data"],
+    )
+
+
+def get_bin_prefix() -> str:
+    # XXX: In old virtualenv versions, sys.prefix can contain '..' components,
+    # so we need to call normpath to eliminate them.
+    prefix = os.path.normpath(sys.prefix)
+    if WINDOWS:
+        bin_py = os.path.join(prefix, "Scripts")
+        # buildout uses 'bin' on Windows too?
+        if not os.path.exists(bin_py):
+            bin_py = os.path.join(prefix, "bin")
+        return bin_py
+    # Forcing to use /usr/local/bin for standard macOS framework installs
+    # Also log to ~/Library/Logs/ for use with the Console.app log viewer
+    if sys.platform[:6] == "darwin" and prefix[:16] == "/System/Library/":
+        return "/usr/local/bin"
+    return os.path.join(prefix, "bin")
+
+
+def get_purelib() -> str:
+    return get_python_lib(plat_specific=False)
+
+
+def get_platlib() -> str:
+    return get_python_lib(plat_specific=True)
+
+
+def get_prefixed_libs(prefix: str) -> Tuple[str, str]:
+    return (
+        get_python_lib(plat_specific=False, prefix=prefix),
+        get_python_lib(plat_specific=True, prefix=prefix),
+    )
diff --git a/src/pip/_internal/locations/_sysconfig.py b/src/pip/_internal/locations/_sysconfig.py
new file mode 100644
index 00000000000..5e141aa1be7
--- /dev/null
+++ b/src/pip/_internal/locations/_sysconfig.py
@@ -0,0 +1,219 @@
+import distutils.util  # FIXME: For change_root.
+import logging
+import os
+import sys
+import sysconfig
+import typing
+
+from pip._internal.exceptions import InvalidSchemeCombination, UserInstallationInvalid
+from pip._internal.models.scheme import SCHEME_KEYS, Scheme
+from pip._internal.utils.virtualenv import running_under_virtualenv
+
+from .base import get_major_minor_version, is_osx_framework
+
+logger = logging.getLogger(__name__)
+
+
+# Notes on _infer_* functions.
+# Unfortunately ``get_default_scheme()`` didn't exist before 3.10, so there's no
+# way to ask things like "what is the '_prefix' scheme on this platform". These
+# functions try to answer that with some heuristics while accounting for ad-hoc
+# platforms not covered by CPython's default sysconfig implementation. If the
+# ad-hoc implementation does not fully implement sysconfig, we'll fall back to
+# a POSIX scheme.
+
+_AVAILABLE_SCHEMES = set(sysconfig.get_scheme_names())
+
+_PREFERRED_SCHEME_API = getattr(sysconfig, "get_preferred_scheme", None)
+
+
+def _should_use_osx_framework_prefix() -> bool:
+    """Check for Apple's ``osx_framework_library`` scheme.
+
+    Python distributed by Apple's Command Line Tools has this special scheme
+    that's used when:
+
+    * This is a framework build.
+    * We are installing into the system prefix.
+
+    This does not account for ``pip install --prefix`` (also means we're not
+    installing to the system prefix), which should use ``posix_prefix``, but
+    logic here means ``_infer_prefix()`` outputs ``osx_framework_library``. But
+    since ``prefix`` is not available for ``sysconfig.get_default_scheme()``,
+    which is the stdlib replacement for ``_infer_prefix()``, presumably Apple
+    wouldn't be able to magically switch between ``osx_framework_library`` and
+    ``posix_prefix``. ``_infer_prefix()`` returning ``osx_framework_library``
+    means its behavior is consistent whether we use the stdlib implementation
+    or our own, and we deal with this special case in ``get_scheme()`` instead.
+    """
+    return (
+        "osx_framework_library" in _AVAILABLE_SCHEMES
+        and not running_under_virtualenv()
+        and is_osx_framework()
+    )
+
+
+def _infer_prefix() -> str:
+    """Try to find a prefix scheme for the current platform.
+
+    This tries:
+
+    * A special ``osx_framework_library`` for Python distributed by Apple's
+      Command Line Tools, when not running in a virtual environment.
+    * Implementation + OS, used by PyPy on Windows (``pypy_nt``).
+    * Implementation without OS, used by PyPy on POSIX (``pypy``).
+    * OS + "prefix", used by CPython on POSIX (``posix_prefix``).
+    * Just the OS name, used by CPython on Windows (``nt``).
+
+    If none of the above works, fall back to ``posix_prefix``.
+    """
+    if _PREFERRED_SCHEME_API:
+        return _PREFERRED_SCHEME_API("prefix")
+    if _should_use_osx_framework_prefix():
+        return "osx_framework_library"
+    implementation_suffixed = f"{sys.implementation.name}_{os.name}"
+    if implementation_suffixed in _AVAILABLE_SCHEMES:
+        return implementation_suffixed
+    if sys.implementation.name in _AVAILABLE_SCHEMES:
+        return sys.implementation.name
+    suffixed = f"{os.name}_prefix"
+    if suffixed in _AVAILABLE_SCHEMES:
+        return suffixed
+    if os.name in _AVAILABLE_SCHEMES:  # On Windows, prefx is just called "nt".
+        return os.name
+    return "posix_prefix"
+
+
+def _infer_user() -> str:
+    """Try to find a user scheme for the current platform."""
+    if _PREFERRED_SCHEME_API:
+        return _PREFERRED_SCHEME_API("user")
+    if is_osx_framework() and not running_under_virtualenv():
+        suffixed = "osx_framework_user"
+    else:
+        suffixed = f"{os.name}_user"
+    if suffixed in _AVAILABLE_SCHEMES:
+        return suffixed
+    if "posix_user" not in _AVAILABLE_SCHEMES:  # User scheme unavailable.
+        raise UserInstallationInvalid()
+    return "posix_user"
+
+
+def _infer_home() -> str:
+    """Try to find a home for the current platform."""
+    if _PREFERRED_SCHEME_API:
+        return _PREFERRED_SCHEME_API("home")
+    suffixed = f"{os.name}_home"
+    if suffixed in _AVAILABLE_SCHEMES:
+        return suffixed
+    return "posix_home"
+
+
+# Update these keys if the user sets a custom home.
+_HOME_KEYS = [
+    "installed_base",
+    "base",
+    "installed_platbase",
+    "platbase",
+    "prefix",
+    "exec_prefix",
+]
+if sysconfig.get_config_var("userbase") is not None:
+    _HOME_KEYS.append("userbase")
+
+
+def get_scheme(
+    dist_name: str,
+    user: bool = False,
+    home: typing.Optional[str] = None,
+    root: typing.Optional[str] = None,
+    isolated: bool = False,
+    prefix: typing.Optional[str] = None,
+) -> Scheme:
+    """
+    Get the "scheme" corresponding to the input parameters.
+
+    :param dist_name: the name of the package to retrieve the scheme for, used
+        in the headers scheme path
+    :param user: indicates to use the "user" scheme
+    :param home: indicates to use the "home" scheme
+    :param root: root under which other directories are re-based
+    :param isolated: ignored, but kept for distutils compatibility (where
+        this controls whether the user-site pydistutils.cfg is honored)
+    :param prefix: indicates to use the "prefix" scheme and provides the
+        base directory for the same
+    """
+    if user and prefix:
+        raise InvalidSchemeCombination("--user", "--prefix")
+    if home and prefix:
+        raise InvalidSchemeCombination("--home", "--prefix")
+
+    if home is not None:
+        scheme_name = _infer_home()
+    elif user:
+        scheme_name = _infer_user()
+    else:
+        scheme_name = _infer_prefix()
+
+    # Special case: When installing into a custom prefix, use posix_prefix
+    # instead of osx_framework_library. See _should_use_osx_framework_prefix()
+    # docstring for details.
+    if prefix is not None and scheme_name == "osx_framework_library":
+        scheme_name = "posix_prefix"
+
+    if home is not None:
+        variables = {k: home for k in _HOME_KEYS}
+    elif prefix is not None:
+        variables = {k: prefix for k in _HOME_KEYS}
+    else:
+        variables = {}
+
+    paths = sysconfig.get_paths(scheme=scheme_name, vars=variables)
+
+    # Logic here is very arbitrary, we're doing it for compatibility, don't ask.
+    # 1. Pip historically uses a special header path in virtual environments.
+    # 2. If the distribution name is not known, distutils uses 'UNKNOWN'. We
+    #    only do the same when not running in a virtual environment because
+    #    pip's historical header path logic (see point 1) did not do this.
+    if running_under_virtualenv():
+        if user:
+            base = variables.get("userbase", sys.prefix)
+        else:
+            base = variables.get("base", sys.prefix)
+        python_xy = f"python{get_major_minor_version()}"
+        paths["include"] = os.path.join(base, "include", "site", python_xy)
+    elif not dist_name:
+        dist_name = "UNKNOWN"
+
+    scheme = Scheme(
+        platlib=paths["platlib"],
+        purelib=paths["purelib"],
+        headers=os.path.join(paths["include"], dist_name),
+        scripts=paths["scripts"],
+        data=paths["data"],
+    )
+    if root is not None:
+        for key in SCHEME_KEYS:
+            value = distutils.util.change_root(root, getattr(scheme, key))
+            setattr(scheme, key, value)
+    return scheme
+
+
+def get_bin_prefix() -> str:
+    # Forcing to use /usr/local/bin for standard macOS framework installs.
+    if sys.platform[:6] == "darwin" and sys.prefix[:16] == "/System/Library/":
+        return "/usr/local/bin"
+    return sysconfig.get_paths()["scripts"]
+
+
+def get_purelib() -> str:
+    return sysconfig.get_paths()["purelib"]
+
+
+def get_platlib() -> str:
+    return sysconfig.get_paths()["platlib"]
+
+
+def get_prefixed_libs(prefix: str) -> typing.Tuple[str, str]:
+    paths = sysconfig.get_paths(vars={"base": prefix, "platbase": prefix})
+    return (paths["purelib"], paths["platlib"])
diff --git a/src/pip/_internal/locations/base.py b/src/pip/_internal/locations/base.py
new file mode 100644
index 00000000000..86dad4a3a84
--- /dev/null
+++ b/src/pip/_internal/locations/base.py
@@ -0,0 +1,52 @@
+import functools
+import os
+import site
+import sys
+import sysconfig
+import typing
+
+from pip._internal.utils import appdirs
+from pip._internal.utils.virtualenv import running_under_virtualenv
+
+# Application Directories
+USER_CACHE_DIR = appdirs.user_cache_dir("pip")
+
+# FIXME doesn't account for venv linked to global site-packages
+site_packages: typing.Optional[str] = sysconfig.get_path("purelib")
+
+
+def get_major_minor_version() -> str:
+    """
+    Return the major-minor version of the current Python as a string, e.g.
+    "3.7" or "3.10".
+    """
+    return "{}.{}".format(*sys.version_info)
+
+
+def get_src_prefix() -> str:
+    if running_under_virtualenv():
+        src_prefix = os.path.join(sys.prefix, "src")
+    else:
+        # FIXME: keep src in cwd for now (it is not a temporary folder)
+        try:
+            src_prefix = os.path.join(os.getcwd(), "src")
+        except OSError:
+            # In case the current working directory has been renamed or deleted
+            sys.exit("The folder you are executing pip from can no longer be found.")
+
+    # under macOS + virtualenv sys.prefix is not properly resolved
+    # it is something like /path/to/python/bin/..
+    return os.path.abspath(src_prefix)
+
+
+try:
+    # Use getusersitepackages if this is present, as it ensures that the
+    # value is initialised properly.
+    user_site: typing.Optional[str] = site.getusersitepackages()
+except AttributeError:
+    user_site = site.USER_SITE
+
+
+@functools.lru_cache(maxsize=None)
+def is_osx_framework() -> bool:
+    return bool(sysconfig.get_config_var("PYTHONFRAMEWORK"))
diff --git a/src/pip/_internal/main.py b/src/pip/_internal/main.py
index 1c99c49a1f1..33c6d24cd85 100644
--- a/src/pip/_internal/main.py
+++ b/src/pip/_internal/main.py
@@ -1,11 +1,7 @@
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from typing import List, Optional
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
 
-
-def main(args=None):
-    # type: (Optional[List[str]]) -> int
+def main(args: Optional[List[str]] = None) -> int:
     """This is preserved for old console scripts that may still be referencing
     it.
 
diff --git a/src/pip/_internal/metadata/__init__.py b/src/pip/_internal/metadata/__init__.py
new file mode 100644
index 00000000000..cc037c14f08
--- /dev/null
+++ b/src/pip/_internal/metadata/__init__.py
@@ -0,0 +1,62 @@
+from typing import List, Optional
+
+from .base import BaseDistribution, BaseEnvironment, FilesystemWheel, MemoryWheel, Wheel
+
+__all__ = [
+    "BaseDistribution",
+    "BaseEnvironment",
+    "FilesystemWheel",
+    "MemoryWheel",
+    "Wheel",
+    "get_default_environment",
+    "get_environment",
+    "get_wheel_distribution",
+]
+
+
+def get_default_environment() -> BaseEnvironment:
+    """Get the default representation for the current environment.
+
+    This returns an Environment instance from the chosen backend. The default
+    Environment instance should be built from ``sys.path`` and may use caching
+    to share instance state accorss calls.
+    """
+    from .pkg_resources import Environment
+
+    return Environment.default()
+
+
+def get_environment(paths: Optional[List[str]]) -> BaseEnvironment:
+    """Get a representation of the environment specified by ``paths``.
+
+    This returns an Environment instance from the chosen backend based on the
+    given import paths. The backend must build a fresh instance representing
+    the state of installed distributions when this function is called.
+    """
+    from .pkg_resources import Environment
+
+    return Environment.from_paths(paths)
+
+
+def get_directory_distribution(directory: str) -> BaseDistribution:
+    """Get the distribution metadata representation in the specified directory.
+
+    This returns a Distribution instance from the chosen backend based on
+    the given on-disk ``.dist-info`` directory.
+    """
+    from .pkg_resources import Distribution
+
+    return Distribution.from_directory(directory)
+
+
+def get_wheel_distribution(wheel: Wheel, canonical_name: str) -> BaseDistribution:
+    """Get the representation of the specified wheel's distribution metadata.
+
+    This returns a Distribution instance from the chosen backend based on
+    the given wheel's ``.dist-info`` directory.
+
+    :param canonical_name: Normalized project name of the given wheel.
+    """
+    from .pkg_resources import Distribution
+
+    return Distribution.from_wheel(wheel, canonical_name)
diff --git a/src/pip/_internal/metadata/base.py b/src/pip/_internal/metadata/base.py
new file mode 100644
index 00000000000..1a5a781cb3e
--- /dev/null
+++ b/src/pip/_internal/metadata/base.py
@@ -0,0 +1,546 @@
+import csv
+import email.message
+import json
+import logging
+import pathlib
+import re
+import zipfile
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Collection,
+    Container,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
+from pip._vendor.packaging.utils import NormalizedName
+from pip._vendor.packaging.version import LegacyVersion, Version
+
+from pip._internal.exceptions import NoneMetadataError
+from pip._internal.locations import site_packages, user_site
+from pip._internal.models.direct_url import (
+    DIRECT_URL_METADATA_NAME,
+    DirectUrl,
+    DirectUrlValidationError,
+)
+from pip._internal.utils.compat import stdlib_pkgs  # TODO: Move definition here.
+from pip._internal.utils.egg_link import (
+    egg_link_path_from_location,
+    egg_link_path_from_sys_path,
+)
+from pip._internal.utils.misc import is_local, normalize_path
+from pip._internal.utils.urls import url_to_path
+
+if TYPE_CHECKING:
+    from typing import Protocol
+else:
+    Protocol = object
+
+DistributionVersion = Union[LegacyVersion, Version]
+
+InfoPath = Union[str, pathlib.PurePosixPath]
+
+logger = logging.getLogger(__name__)
+
+
+class BaseEntryPoint(Protocol):
+    @property
+    def name(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def value(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def group(self) -> str:
+        raise NotImplementedError()
+
+
+def _convert_installed_files_path(
+    entry: Tuple[str, ...],
+    info: Tuple[str, ...],
+) -> str:
+    """Convert a legacy installed-files.txt path into modern RECORD path.
+
+    The legacy format stores paths relative to the info directory, while the
+    modern format stores paths relative to the package root, e.g. the
+    site-packages directory.
+
+    :param entry: Path parts of the installed-files.txt entry.
+    :param info: Path parts of the egg-info directory relative to package root.
+    :returns: The converted entry.
+
+    For best compatibility with symlinks, this does not use ``abspath()`` or
+    ``Path.resolve()``, but tries to work with path parts:
+
+    1. While ``entry`` starts with ``..``, remove the equal amounts of parts
+       from ``info``; if ``info`` is empty, start appending ``..`` instead.
+    2. Join the two directly.
+    """
+    while entry and entry[0] == "..":
+        if not info or info[-1] == "..":
+            info += ("..",)
+        else:
+            info = info[:-1]
+        entry = entry[1:]
+    return str(pathlib.Path(*info, *entry))
+
+
+class BaseDistribution(Protocol):
+    def __repr__(self) -> str:
+        return f"{self.raw_name} {self.version} ({self.location})"
+
+    def __str__(self) -> str:
+        return f"{self.raw_name} {self.version}"
+
+    @property
+    def location(self) -> Optional[str]:
+        """Where the distribution is loaded from.
+
+        A string value is not necessarily a filesystem path, since distributions
+        can be loaded from other sources, e.g. arbitrary zip archives. ``None``
+        means the distribution is created in-memory.
+
+        Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
+        this is a symbolic link, we want to preserve the relative path between
+        it and files in the distribution.
+        """
+        raise NotImplementedError()
+
+    @property
+    def editable_project_location(self) -> Optional[str]:
+        """The project location for editable distributions.
+
+        This is the directory where pyproject.toml or setup.py is located.
+        None if the distribution is not installed in editable mode.
+        """
+        # TODO: this property is relatively costly to compute, memoize it ?
+        direct_url = self.direct_url
+        if direct_url:
+            if direct_url.is_local_editable():
+                return url_to_path(direct_url.url)
+        else:
+            # Search for an .egg-link file by walking sys.path, as it was
+            # done before by dist_is_editable().
+            egg_link_path = egg_link_path_from_sys_path(self.raw_name)
+            if egg_link_path:
+                # TODO: get project location from second line of egg_link file
+                #       (https://github.com/pypa/pip/issues/10243)
+                return self.location
+        return None
+
+    @property
+    def installed_location(self) -> Optional[str]:
+        """The distribution's "installed" location.
+
+        This should generally be a ``site-packages`` directory. This is
+        usually ``dist.location``, except for legacy develop-installed packages,
+        where ``dist.location`` is the source code location, and this is where
+        the ``.egg-link`` file is.
+
+        The returned location is normalized (in particular, with symlinks removed).
+        """
+        egg_link = egg_link_path_from_location(self.raw_name)
+        if egg_link:
+            location = egg_link
+        elif self.location:
+            location = self.location
+        else:
+            return None
+        return normalize_path(location)
+
+    @property
+    def info_location(self) -> Optional[str]:
+        """Location of the .[egg|dist]-info directory or file.
+
+        Similarly to ``location``, a string value is not necessarily a
+        filesystem path. ``None`` means the distribution is created in-memory.
+
+        For a modern .dist-info installation on disk, this should be something
+        like ``{location}/{raw_name}-{version}.dist-info``.
+
+        Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
+        this is a symbolic link, we want to preserve the relative path between
+        it and other files in the distribution.
+        """
+        raise NotImplementedError()
+
+    @property
+    def installed_by_distutils(self) -> bool:
+        """Whether this distribution is installed with legacy distutils format.
+
+        A distribution installed with "raw" distutils not patched by setuptools
+        uses one single file at ``info_location`` to store metadata. We need to
+        treat this specially on uninstallation.
+        """
+        info_location = self.info_location
+        if not info_location:
+            return False
+        return pathlib.Path(info_location).is_file()
+
+    @property
+    def installed_as_egg(self) -> bool:
+        """Whether this distribution is installed as an egg.
+
+        This usually indicates the distribution was installed by (older versions
+        of) easy_install.
+        """
+        location = self.location
+        if not location:
+            return False
+        return location.endswith(".egg")
+
+    @property
+    def installed_with_setuptools_egg_info(self) -> bool:
+        """Whether this distribution is installed with the ``.egg-info`` format.
+
+        This usually indicates the distribution was installed with setuptools
+        with an old pip version or with ``single-version-externally-managed``.
+
+        Note that this ensure the metadata store is a directory. distutils can
+        also installs an ``.egg-info``, but as a file, not a directory. This
+        property is *False* for that case. Also see ``installed_by_distutils``.
+        """
+        info_location = self.info_location
+        if not info_location:
+            return False
+        if not info_location.endswith(".egg-info"):
+            return False
+        return pathlib.Path(info_location).is_dir()
+
+    @property
+    def installed_with_dist_info(self) -> bool:
+        """Whether this distribution is installed with the "modern format".
+
+        This indicates a "modern" installation, e.g. storing metadata in the
+        ``.dist-info`` directory. This applies to installations made by
+        setuptools (but through pip, not directly), or anything using the
+        standardized build backend interface (PEP 517).
+        """
+        info_location = self.info_location
+        if not info_location:
+            return False
+        if not info_location.endswith(".dist-info"):
+            return False
+        return pathlib.Path(info_location).is_dir()
+
+    @property
+    def canonical_name(self) -> NormalizedName:
+        raise NotImplementedError()
+
+    @property
+    def version(self) -> DistributionVersion:
+        raise NotImplementedError()
+
+    @property
+    def setuptools_filename(self) -> str:
+        """Convert a project name to its setuptools-compatible filename.
+
+        This is a copy of ``pkg_resources.to_filename()`` for compatibility.
+        """
+        return self.raw_name.replace("-", "_")
+
+    @property
+    def direct_url(self) -> Optional[DirectUrl]:
+        """Obtain a DirectUrl from this distribution.
+
+        Returns None if the distribution has no `direct_url.json` metadata,
+        or if `direct_url.json` is invalid.
+        """
+        try:
+            content = self.read_text(DIRECT_URL_METADATA_NAME)
+        except FileNotFoundError:
+            return None
+        try:
+            return DirectUrl.from_json(content)
+        except (
+            UnicodeDecodeError,
+            json.JSONDecodeError,
+            DirectUrlValidationError,
+        ) as e:
+            logger.warning(
+                "Error parsing %s for %s: %s",
+                DIRECT_URL_METADATA_NAME,
+                self.canonical_name,
+                e,
+            )
+            return None
+
+    @property
+    def installer(self) -> str:
+        try:
+            installer_text = self.read_text("INSTALLER")
+        except (OSError, ValueError, NoneMetadataError):
+            return ""  # Fail silently if the installer file cannot be read.
+        for line in installer_text.splitlines():
+            cleaned_line = line.strip()
+            if cleaned_line:
+                return cleaned_line
+        return ""
+
+    @property
+    def editable(self) -> bool:
+        return bool(self.editable_project_location)
+
+    @property
+    def local(self) -> bool:
+        """If distribution is installed in the current virtual environment.
+
+        Always True if we're not in a virtualenv.
+        """
+        if self.installed_location is None:
+            return False
+        return is_local(self.installed_location)
+
+    @property
+    def in_usersite(self) -> bool:
+        if self.installed_location is None or user_site is None:
+            return False
+        return self.installed_location.startswith(normalize_path(user_site))
+
+    @property
+    def in_site_packages(self) -> bool:
+        if self.installed_location is None or site_packages is None:
+            return False
+        return self.installed_location.startswith(normalize_path(site_packages))
+
+    def is_file(self, path: InfoPath) -> bool:
+        """Check whether an entry in the info directory is a file."""
+        raise NotImplementedError()
+
+    def iterdir(self, path: InfoPath) -> Iterator[pathlib.PurePosixPath]:
+        """Iterate through a directory in the info directory.
+
+        Each item yielded would be a path relative to the info directory.
+
+        :raise FileNotFoundError: If ``name`` does not exist in the directory.
+        :raise NotADirectoryError: If ``name`` does not point to a directory.
+        """
+        raise NotImplementedError()
+
+    def read_text(self, path: InfoPath) -> str:
+        """Read a file in the info directory.
+
+        :raise FileNotFoundError: If ``name`` does not exist in the directory.
+        :raise NoneMetadataError: If ``name`` exists in the info directory, but
+            cannot be read.
+        """
+        raise NotImplementedError()
+
+    def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
+        raise NotImplementedError()
+
+    @property
+    def metadata(self) -> email.message.Message:
+        """Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
+
+        This should return an empty message if the metadata file is unavailable.
+
+        :raises NoneMetadataError: If the metadata file is available, but does
+            not contain valid metadata.
+        """
+        raise NotImplementedError()
+
+    @property
+    def metadata_version(self) -> Optional[str]:
+        """Value of "Metadata-Version:" in distribution metadata, if available."""
+        return self.metadata.get("Metadata-Version")
+
+    @property
+    def raw_name(self) -> str:
+        """Value of "Name:" in distribution metadata."""
+        # The metadata should NEVER be missing the Name: key, but if it somehow
+        # does, fall back to the known canonical name.
+        return self.metadata.get("Name", self.canonical_name)
+
+    @property
+    def requires_python(self) -> SpecifierSet:
+        """Value of "Requires-Python:" in distribution metadata.
+
+        If the key does not exist or contains an invalid value, an empty
+        SpecifierSet should be returned.
+        """
+        value = self.metadata.get("Requires-Python")
+        if value is None:
+            return SpecifierSet()
+        try:
+            # Convert to str to satisfy the type checker; this can be a Header object.
+            spec = SpecifierSet(str(value))
+        except InvalidSpecifier as e:
+            message = "Package %r has an invalid Requires-Python: %s"
+            logger.warning(message, self.raw_name, e)
+            return SpecifierSet()
+        return spec
+
+    def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
+        """Dependencies of this distribution.
+
+        For modern .dist-info distributions, this is the collection of
+        "Requires-Dist:" entries in distribution metadata.
+        """
+        raise NotImplementedError()
+
+    def iter_provided_extras(self) -> Iterable[str]:
+        """Extras provided by this distribution.
+
+        For modern .dist-info distributions, this is the collection of
+        "Provides-Extra:" entries in distribution metadata.
+        """
+        raise NotImplementedError()
+
+    def _iter_declared_entries_from_record(self) -> Optional[Iterator[str]]:
+        try:
+            text = self.read_text("RECORD")
+        except FileNotFoundError:
+            return None
+        # This extra Path-str cast normalizes entries.
+        return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
+
+    def _iter_declared_entries_from_legacy(self) -> Optional[Iterator[str]]:
+        try:
+            text = self.read_text("installed-files.txt")
+        except FileNotFoundError:
+            return None
+        paths = (p for p in text.splitlines(keepends=False) if p)
+        root = self.location
+        info = self.info_location
+        if root is None or info is None:
+            return paths
+        try:
+            info_rel = pathlib.Path(info).relative_to(root)
+        except ValueError:  # info is not relative to root.
+            return paths
+        if not info_rel.parts:  # info *is* root.
+            return paths
+        return (
+            _convert_installed_files_path(pathlib.Path(p).parts, info_rel.parts)
+            for p in paths
+        )
+
+    def iter_declared_entries(self) -> Optional[Iterator[str]]:
+        """Iterate through file entires declared in this distribution.
+
+        For modern .dist-info distributions, this is the files listed in the
+        ``RECORD`` metadata file. For legacy setuptools distributions, this
+        comes from ``installed-files.txt``, with entries normalized to be
+        compatible with the format used by ``RECORD``.
+
+        :return: An iterator for listed entries, or None if the distribution
+            contains neither ``RECORD`` nor ``installed-files.txt``.
+        """
+        return (
+            self._iter_declared_entries_from_record()
+            or self._iter_declared_entries_from_legacy()
+        )
+
+
+class BaseEnvironment:
+    """An environment containing distributions to introspect."""
+
+    @classmethod
+    def default(cls) -> "BaseEnvironment":
+        raise NotImplementedError()
+
+    @classmethod
+    def from_paths(cls, paths: Optional[List[str]]) -> "BaseEnvironment":
+        raise NotImplementedError()
+
+    def get_distribution(self, name: str) -> Optional["BaseDistribution"]:
+        """Given a requirement name, return the installed distributions.
+
+        The name may not be normalized. The implementation must canonicalize
+        it for lookup.
+        """
+        raise NotImplementedError()
+
+    def _iter_distributions(self) -> Iterator["BaseDistribution"]:
+        """Iterate through installed distributions.
+
+        This function should be implemented by subclass, but never called
+        directly. Use the public ``iter_distribution()`` instead, which
+        implements additional logic to make sure the distributions are valid.
+        """
+        raise NotImplementedError()
+
+    def iter_distributions(self) -> Iterator["BaseDistribution"]:
+        """Iterate through installed distributions."""
+        for dist in self._iter_distributions():
+            # Make sure the distribution actually comes from a valid Python
+            # packaging distribution. Pip's AdjacentTempDirectory leaves folders
+            # e.g. ``~atplotlib.dist-info`` if cleanup was interrupted. The
+            # valid project name pattern is taken from PEP 508.
+            project_name_valid = re.match(
+                r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$",
+                dist.canonical_name,
+                flags=re.IGNORECASE,
+            )
+            if not project_name_valid:
+                logger.warning(
+                    "Ignoring invalid distribution %s (%s)",
+                    dist.canonical_name,
+                    dist.location,
+                )
+                continue
+            yield dist
+
+    def iter_installed_distributions(
+        self,
+        local_only: bool = True,
+        skip: Container[str] = stdlib_pkgs,
+        include_editables: bool = True,
+        editables_only: bool = False,
+        user_only: bool = False,
+    ) -> Iterator[BaseDistribution]:
+        """Return a list of installed distributions.
+
+        :param local_only: If True (default), only return installations
+        local to the current virtualenv, if in a virtualenv.
+        :param skip: An iterable of canonicalized project names to ignore;
+            defaults to ``stdlib_pkgs``.
+        :param include_editables: If False, don't report editables.
+        :param editables_only: If True, only report editables.
+        :param user_only: If True, only report installations in the user
+        site directory.
+        """
+        it = self.iter_distributions()
+        if local_only:
+            it = (d for d in it if d.local)
+        if not include_editables:
+            it = (d for d in it if not d.editable)
+        if editables_only:
+            it = (d for d in it if d.editable)
+        if user_only:
+            it = (d for d in it if d.in_usersite)
+        return (d for d in it if d.canonical_name not in skip)
+
+
+class Wheel(Protocol):
+    location: str
+
+    def as_zipfile(self) -> zipfile.ZipFile:
+        raise NotImplementedError()
+
+
+class FilesystemWheel(Wheel):
+    def __init__(self, location: str) -> None:
+        self.location = location
+
+    def as_zipfile(self) -> zipfile.ZipFile:
+        return zipfile.ZipFile(self.location, allowZip64=True)
+
+
+class MemoryWheel(Wheel):
+    def __init__(self, location: str, stream: IO[bytes]) -> None:
+        self.location = location
+        self.stream = stream
+
+    def as_zipfile(self) -> zipfile.ZipFile:
+        return zipfile.ZipFile(self.stream, allowZip64=True)
diff --git a/src/pip/_internal/metadata/pkg_resources.py b/src/pip/_internal/metadata/pkg_resources.py
new file mode 100644
index 00000000000..d39f0ba31da
--- /dev/null
+++ b/src/pip/_internal/metadata/pkg_resources.py
@@ -0,0 +1,256 @@
+import email.message
+import email.parser
+import logging
+import os
+import pathlib
+import zipfile
+from typing import Collection, Iterable, Iterator, List, Mapping, NamedTuple, Optional
+
+from pip._vendor import pkg_resources
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
+from pip._vendor.packaging.version import parse as parse_version
+
+from pip._internal.exceptions import InvalidWheel, NoneMetadataError, UnsupportedWheel
+from pip._internal.utils.misc import display_path
+from pip._internal.utils.wheel import parse_wheel, read_wheel_metadata_file
+
+from .base import (
+    BaseDistribution,
+    BaseEntryPoint,
+    BaseEnvironment,
+    DistributionVersion,
+    InfoPath,
+    Wheel,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class EntryPoint(NamedTuple):
+    name: str
+    value: str
+    group: str
+
+
+class WheelMetadata:
+    """IMetadataProvider that reads metadata files from a dictionary.
+
+    This also maps metadata decoding exceptions to our internal exception type.
+    """
+
+    def __init__(self, metadata: Mapping[str, bytes], wheel_name: str) -> None:
+        self._metadata = metadata
+        self._wheel_name = wheel_name
+
+    def has_metadata(self, name: str) -> bool:
+        return name in self._metadata
+
+    def get_metadata(self, name: str) -> str:
+        try:
+            return self._metadata[name].decode()
+        except UnicodeDecodeError as e:
+            # Augment the default error with the origin of the file.
+            raise UnsupportedWheel(
+                f"Error decoding metadata for {self._wheel_name}: {e} in {name} file"
+            )
+
+    def get_metadata_lines(self, name: str) -> Iterable[str]:
+        return pkg_resources.yield_lines(self.get_metadata(name))
+
+    def metadata_isdir(self, name: str) -> bool:
+        return False
+
+    def metadata_listdir(self, name: str) -> List[str]:
+        return []
+
+    def run_script(self, script_name: str, namespace: str) -> None:
+        pass
+
+
+class Distribution(BaseDistribution):
+    def __init__(self, dist: pkg_resources.Distribution) -> None:
+        self._dist = dist
+
+    @classmethod
+    def from_directory(cls, directory: str) -> "Distribution":
+        dist_dir = directory.rstrip(os.sep)
+
+        # Build a PathMetadata object, from path to metadata. :wink:
+        base_dir, dist_dir_name = os.path.split(dist_dir)
+        metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
+
+        # Determine the correct Distribution object type.
+        if dist_dir.endswith(".egg-info"):
+            dist_cls = pkg_resources.Distribution
+            dist_name = os.path.splitext(dist_dir_name)[0]
+        else:
+            assert dist_dir.endswith(".dist-info")
+            dist_cls = pkg_resources.DistInfoDistribution
+            dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
+
+        dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
+        return cls(dist)
+
+    @classmethod
+    def from_wheel(cls, wheel: Wheel, name: str) -> "Distribution":
+        """Load the distribution from a given wheel.
+
+        :raises InvalidWheel: Whenever loading of the wheel causes a
+            :py:exc:`zipfile.BadZipFile` exception to be thrown.
+        :raises UnsupportedWheel: If the wheel is a valid zip, but malformed
+            internally.
+        """
+        try:
+            with wheel.as_zipfile() as zf:
+                info_dir, _ = parse_wheel(zf, name)
+                metadata_text = {
+                    path.split("/", 1)[-1]: read_wheel_metadata_file(zf, path)
+                    for path in zf.namelist()
+                    if path.startswith(f"{info_dir}/")
+                }
+        except zipfile.BadZipFile as e:
+            raise InvalidWheel(wheel.location, name) from e
+        except UnsupportedWheel as e:
+            raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
+        dist = pkg_resources.DistInfoDistribution(
+            location=wheel.location,
+            metadata=WheelMetadata(metadata_text, wheel.location),
+            project_name=name,
+        )
+        return cls(dist)
+
+    @property
+    def location(self) -> Optional[str]:
+        return self._dist.location
+
+    @property
+    def info_location(self) -> Optional[str]:
+        return self._dist.egg_info
+
+    @property
+    def installed_by_distutils(self) -> bool:
+        # A distutils-installed distribution is provided by FileMetadata. This
+        # provider has a "path" attribute not present anywhere else. Not the
+        # best introspection logic, but pip has been doing this for a long time.
+        try:
+            return bool(self._dist._provider.path)
+        except AttributeError:
+            return False
+
+    @property
+    def canonical_name(self) -> NormalizedName:
+        return canonicalize_name(self._dist.project_name)
+
+    @property
+    def version(self) -> DistributionVersion:
+        return parse_version(self._dist.version)
+
+    def is_file(self, path: InfoPath) -> bool:
+        return self._dist.has_metadata(str(path))
+
+    def iterdir(self, path: InfoPath) -> Iterator[pathlib.PurePosixPath]:
+        name = str(path)
+        if not self._dist.has_metadata(name):
+            raise FileNotFoundError(name)
+        if not self._dist.isdir(name):
+            raise NotADirectoryError(name)
+        for child in self._dist.metadata_listdir(name):
+            yield pathlib.PurePosixPath(path, child)
+
+    def read_text(self, path: InfoPath) -> str:
+        name = str(path)
+        if not self._dist.has_metadata(name):
+            raise FileNotFoundError(name)
+        content = self._dist.get_metadata(name)
+        if content is None:
+            raise NoneMetadataError(self, name)
+        return content
+
+    def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
+        for group, entries in self._dist.get_entry_map().items():
+            for name, entry_point in entries.items():
+                name, _, value = str(entry_point).partition("=")
+                yield EntryPoint(name=name.strip(), value=value.strip(), group=group)
+
+    @property
+    def metadata(self) -> email.message.Message:
+        """
+        :raises NoneMetadataError: if the distribution reports `has_metadata()`
+            True but `get_metadata()` returns None.
+        """
+        if isinstance(self._dist, pkg_resources.DistInfoDistribution):
+            metadata_name = "METADATA"
+        else:
+            metadata_name = "PKG-INFO"
+        try:
+            metadata = self.read_text(metadata_name)
+        except FileNotFoundError:
+            if self.location:
+                displaying_path = display_path(self.location)
+            else:
+                displaying_path = repr(self.location)
+            logger.warning("No metadata found in %s", displaying_path)
+            metadata = ""
+        feed_parser = email.parser.FeedParser()
+        feed_parser.feed(metadata)
+        return feed_parser.close()
+
+    def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
+        if extras:  # pkg_resources raises on invalid extras, so we sanitize.
+            extras = frozenset(extras).intersection(self._dist.extras)
+        return self._dist.requires(extras)
+
+    def iter_provided_extras(self) -> Iterable[str]:
+        return self._dist.extras
+
+
+class Environment(BaseEnvironment):
+    def __init__(self, ws: pkg_resources.WorkingSet) -> None:
+        self._ws = ws
+
+    @classmethod
+    def default(cls) -> BaseEnvironment:
+        return cls(pkg_resources.working_set)
+
+    @classmethod
+    def from_paths(cls, paths: Optional[List[str]]) -> BaseEnvironment:
+        return cls(pkg_resources.WorkingSet(paths))
+
+    def _search_distribution(self, name: str) -> Optional[BaseDistribution]:
+        """Find a distribution matching the ``name`` in the environment.
+
+        This searches from *all* distributions available in the environment, to
+        match the behavior of ``pkg_resources.get_distribution()``.
+        """
+        canonical_name = canonicalize_name(name)
+        for dist in self.iter_distributions():
+            if dist.canonical_name == canonical_name:
+                return dist
+        return None
+
+    def get_distribution(self, name: str) -> Optional[BaseDistribution]:
+        # Search the distribution by looking through the working set.
+        dist = self._search_distribution(name)
+        if dist:
+            return dist
+
+        # If distribution could not be found, call working_set.require to
+        # update the working set, and try to find the distribution again.
+        # This might happen for e.g. when you install a package twice, once
+        # using setup.py develop and again using setup.py install. Now when
+        # running pip uninstall twice, the package gets removed from the
+        # working set in the first uninstall, so we have to populate the
+        # working set again so that pip knows about it and the packages gets
+        # picked up and is successfully uninstalled the second time too.
+        try:
+            # We didn't pass in any version specifiers, so this can never
+            # raise pkg_resources.VersionConflict.
+            self._ws.require(name)
+        except pkg_resources.DistributionNotFound:
+            return None
+        return self._search_distribution(name)
+
+    def _iter_distributions(self) -> Iterator[BaseDistribution]:
+        for dist in self._ws:
+            yield Distribution(dist)
diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py
index d8a8d42ebcf..a4963aec638 100644
--- a/src/pip/_internal/models/candidate.py
+++ b/src/pip/_internal/models/candidate.py
@@ -1,39 +1,34 @@
 from pip._vendor.packaging.version import parse as parse_version
 
+from pip._internal.models.link import Link
 from pip._internal.utils.models import KeyBasedCompareMixin
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from pip._vendor.packaging.version import _BaseVersion
-
-    from pip._internal.models.link import Link
 
 
 class InstallationCandidate(KeyBasedCompareMixin):
-    """Represents a potential "candidate" for installation.
-    """
+    """Represents a potential "candidate" for installation."""
 
     __slots__ = ["name", "version", "link"]
 
-    def __init__(self, name, version, link):
-        # type: (str, str, Link) -> None
+    def __init__(self, name: str, version: str, link: Link) -> None:
         self.name = name
-        self.version = parse_version(version)  # type: _BaseVersion
+        self.version = parse_version(version)
         self.link = link
 
         super().__init__(
             key=(self.name, self.version, self.link),
-            defining_class=InstallationCandidate
+            defining_class=InstallationCandidate,
         )
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "".format(
-            self.name, self.version, self.link,
+            self.name,
+            self.version,
+            self.link,
         )
 
-    def __str__(self):
-        # type: () -> str
-        return '{!r} candidate (version {} at {})'.format(
-            self.name, self.version, self.link,
+    def __str__(self) -> str:
+        return "{!r} candidate (version {} at {})".format(
+            self.name,
+            self.version,
+            self.link,
         )
diff --git a/src/pip/_internal/models/direct_url.py b/src/pip/_internal/models/direct_url.py
index a8869bd0442..92060d45db8 100644
--- a/src/pip/_internal/models/direct_url.py
+++ b/src/pip/_internal/models/direct_url.py
@@ -2,17 +2,7 @@
 import json
 import re
 import urllib.parse
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Iterable, Optional, Type, TypeVar, Union
-
-    T = TypeVar("T")
-
-
-DIRECT_URL_METADATA_NAME = "direct_url.json"
-ENV_VAR_RE = re.compile(r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$")
+from typing import Any, Dict, Iterable, Optional, Type, TypeVar, Union
 
 __all__ = [
     "DirectUrl",
@@ -22,13 +12,19 @@
     "VcsInfo",
 ]
 
+T = TypeVar("T")
+
+DIRECT_URL_METADATA_NAME = "direct_url.json"
+ENV_VAR_RE = re.compile(r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$")
+
 
 class DirectUrlValidationError(Exception):
     pass
 
 
-def _get(d, expected_type, key, default=None):
-    # type: (Dict[str, Any], Type[T], str, Optional[T]) -> Optional[T]
+def _get(
+    d: Dict[str, Any], expected_type: Type[T], key: str, default: Optional[T] = None
+) -> Optional[T]:
     """Get value from dictionary and verify expected type."""
     if key not in d:
         return default
@@ -42,16 +38,16 @@ def _get(d, expected_type, key, default=None):
     return value
 
 
-def _get_required(d, expected_type, key, default=None):
-    # type: (Dict[str, Any], Type[T], str, Optional[T]) -> T
+def _get_required(
+    d: Dict[str, Any], expected_type: Type[T], key: str, default: Optional[T] = None
+) -> T:
     value = _get(d, expected_type, key, default)
     if value is None:
         raise DirectUrlValidationError(f"{key} must have a value")
     return value
 
 
-def _exactly_one_of(infos):
-    # type: (Iterable[Optional[InfoType]]) -> InfoType
+def _exactly_one_of(infos: Iterable[Optional["InfoType"]]) -> "InfoType":
     infos = [info for info in infos if info is not None]
     if not infos:
         raise DirectUrlValidationError(
@@ -65,8 +61,7 @@ def _exactly_one_of(infos):
     return infos[0]
 
 
-def _filter_none(**kwargs):
-    # type: (Any) -> Dict[str, Any]
+def _filter_none(**kwargs: Any) -> Dict[str, Any]:
     """Make dict excluding None values."""
     return {k: v for k, v in kwargs.items() if v is not None}
 
@@ -76,12 +71,12 @@ class VcsInfo:
 
     def __init__(
         self,
-        vcs,  # type: str
-        commit_id,  # type: str
-        requested_revision=None,  # type: Optional[str]
-        resolved_revision=None,  # type: Optional[str]
-        resolved_revision_type=None,  # type: Optional[str]
-    ):
+        vcs: str,
+        commit_id: str,
+        requested_revision: Optional[str] = None,
+        resolved_revision: Optional[str] = None,
+        resolved_revision_type: Optional[str] = None,
+    ) -> None:
         self.vcs = vcs
         self.requested_revision = requested_revision
         self.commit_id = commit_id
@@ -89,8 +84,7 @@ def __init__(
         self.resolved_revision_type = resolved_revision_type
 
     @classmethod
-    def _from_dict(cls, d):
-        # type: (Optional[Dict[str, Any]]) -> Optional[VcsInfo]
+    def _from_dict(cls, d: Optional[Dict[str, Any]]) -> Optional["VcsInfo"]:
         if d is None:
             return None
         return cls(
@@ -101,8 +95,7 @@ def _from_dict(cls, d):
             resolved_revision_type=_get(d, str, "resolved_revision_type"),
         )
 
-    def _to_dict(self):
-        # type: () -> Dict[str, Any]
+    def _to_dict(self) -> Dict[str, Any]:
         return _filter_none(
             vcs=self.vcs,
             requested_revision=self.requested_revision,
@@ -117,19 +110,17 @@ class ArchiveInfo:
 
     def __init__(
         self,
-        hash=None,  # type: Optional[str]
-    ):
+        hash: Optional[str] = None,
+    ) -> None:
         self.hash = hash
 
     @classmethod
-    def _from_dict(cls, d):
-        # type: (Optional[Dict[str, Any]]) -> Optional[ArchiveInfo]
+    def _from_dict(cls, d: Optional[Dict[str, Any]]) -> Optional["ArchiveInfo"]:
         if d is None:
             return None
         return cls(hash=_get(d, str, "hash"))
 
-    def _to_dict(self):
-        # type: () -> Dict[str, Any]
+    def _to_dict(self) -> Dict[str, Any]:
         return _filter_none(hash=self.hash)
 
 
@@ -138,49 +129,42 @@ class DirInfo:
 
     def __init__(
         self,
-        editable=False,  # type: bool
-    ):
+        editable: bool = False,
+    ) -> None:
         self.editable = editable
 
     @classmethod
-    def _from_dict(cls, d):
-        # type: (Optional[Dict[str, Any]]) -> Optional[DirInfo]
+    def _from_dict(cls, d: Optional[Dict[str, Any]]) -> Optional["DirInfo"]:
         if d is None:
             return None
-        return cls(
-            editable=_get_required(d, bool, "editable", default=False)
-        )
+        return cls(editable=_get_required(d, bool, "editable", default=False))
 
-    def _to_dict(self):
-        # type: () -> Dict[str, Any]
+    def _to_dict(self) -> Dict[str, Any]:
         return _filter_none(editable=self.editable or None)
 
 
-if MYPY_CHECK_RUNNING:
-    InfoType = Union[ArchiveInfo, DirInfo, VcsInfo]
+InfoType = Union[ArchiveInfo, DirInfo, VcsInfo]
 
 
 class DirectUrl:
-
     def __init__(
         self,
-        url,  # type: str
-        info,  # type: InfoType
-        subdirectory=None,  # type: Optional[str]
-    ):
+        url: str,
+        info: InfoType,
+        subdirectory: Optional[str] = None,
+    ) -> None:
         self.url = url
         self.info = info
         self.subdirectory = subdirectory
 
-    def _remove_auth_from_netloc(self, netloc):
-        # type: (str) -> str
+    def _remove_auth_from_netloc(self, netloc: str) -> str:
         if "@" not in netloc:
             return netloc
         user_pass, netloc_no_user_pass = netloc.split("@", 1)
         if (
-            isinstance(self.info, VcsInfo) and
-            self.info.vcs == "git" and
-            user_pass == "git"
+            isinstance(self.info, VcsInfo)
+            and self.info.vcs == "git"
+            and user_pass == "git"
         ):
             return netloc
         if ENV_VAR_RE.match(user_pass):
@@ -188,8 +172,7 @@ def _remove_auth_from_netloc(self, netloc):
         return netloc_no_user_pass
 
     @property
-    def redacted_url(self):
-        # type: () -> str
+    def redacted_url(self) -> str:
         """url with user:password part removed unless it is formed with
         environment variables as specified in PEP 610, or it is ``git``
         in the case of a git URL.
@@ -201,13 +184,11 @@ def redacted_url(self):
         )
         return surl
 
-    def validate(self):
-        # type: () -> None
+    def validate(self) -> None:
         self.from_dict(self.to_dict())
 
     @classmethod
-    def from_dict(cls, d):
-        # type: (Dict[str, Any]) -> DirectUrl
+    def from_dict(cls, d: Dict[str, Any]) -> "DirectUrl":
         return DirectUrl(
             url=_get_required(d, str, "url"),
             subdirectory=_get(d, str, "subdirectory"),
@@ -220,8 +201,7 @@ def from_dict(cls, d):
             ),
         )
 
-    def to_dict(self):
-        # type: () -> Dict[str, Any]
+    def to_dict(self) -> Dict[str, Any]:
         res = _filter_none(
             url=self.redacted_url,
             subdirectory=self.subdirectory,
@@ -230,10 +210,11 @@ def to_dict(self):
         return res
 
     @classmethod
-    def from_json(cls, s):
-        # type: (str) -> DirectUrl
+    def from_json(cls, s: str) -> "DirectUrl":
         return cls.from_dict(json.loads(s))
 
-    def to_json(self):
-        # type: () -> str
+    def to_json(self) -> str:
         return json.dumps(self.to_dict(), sort_keys=True)
+
+    def is_local_editable(self) -> bool:
+        return isinstance(self.info, DirInfo) and self.info.editable
diff --git a/src/pip/_internal/models/format_control.py b/src/pip/_internal/models/format_control.py
index eb46f25359a..db3995eac9f 100644
--- a/src/pip/_internal/models/format_control.py
+++ b/src/pip/_internal/models/format_control.py
@@ -1,20 +1,20 @@
+from typing import FrozenSet, Optional, Set
+
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.exceptions import CommandError
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import FrozenSet, Optional, Set
 
 
 class FormatControl:
-    """Helper for managing formats from which a package can be installed.
-    """
+    """Helper for managing formats from which a package can be installed."""
 
     __slots__ = ["no_binary", "only_binary"]
 
-    def __init__(self, no_binary=None, only_binary=None):
-        # type: (Optional[Set[str]], Optional[Set[str]]) -> None
+    def __init__(
+        self,
+        no_binary: Optional[Set[str]] = None,
+        only_binary: Optional[Set[str]] = None,
+    ) -> None:
         if no_binary is None:
             no_binary = set()
         if only_binary is None:
@@ -23,66 +23,58 @@ def __init__(self, no_binary=None, only_binary=None):
         self.no_binary = no_binary
         self.only_binary = only_binary
 
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, self.__class__):
             return NotImplemented
 
         if self.__slots__ != other.__slots__:
             return False
 
-        return all(
-            getattr(self, k) == getattr(other, k)
-            for k in self.__slots__
-        )
+        return all(getattr(self, k) == getattr(other, k) for k in self.__slots__)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{}({}, {})".format(
-            self.__class__.__name__,
-            self.no_binary,
-            self.only_binary
+            self.__class__.__name__, self.no_binary, self.only_binary
         )
 
     @staticmethod
-    def handle_mutual_excludes(value, target, other):
-        # type: (str, Set[str], Set[str]) -> None
-        if value.startswith('-'):
+    def handle_mutual_excludes(value: str, target: Set[str], other: Set[str]) -> None:
+        if value.startswith("-"):
             raise CommandError(
                 "--no-binary / --only-binary option requires 1 argument."
             )
-        new = value.split(',')
-        while ':all:' in new:
+        new = value.split(",")
+        while ":all:" in new:
             other.clear()
             target.clear()
-            target.add(':all:')
-            del new[:new.index(':all:') + 1]
+            target.add(":all:")
+            del new[: new.index(":all:") + 1]
             # Without a none, we want to discard everything as :all: covers it
-            if ':none:' not in new:
+            if ":none:" not in new:
                 return
         for name in new:
-            if name == ':none:':
+            if name == ":none:":
                 target.clear()
                 continue
             name = canonicalize_name(name)
             other.discard(name)
             target.add(name)
 
-    def get_allowed_formats(self, canonical_name):
-        # type: (str) -> FrozenSet[str]
+    def get_allowed_formats(self, canonical_name: str) -> FrozenSet[str]:
         result = {"binary", "source"}
         if canonical_name in self.only_binary:
-            result.discard('source')
+            result.discard("source")
         elif canonical_name in self.no_binary:
-            result.discard('binary')
-        elif ':all:' in self.only_binary:
-            result.discard('source')
-        elif ':all:' in self.no_binary:
-            result.discard('binary')
+            result.discard("binary")
+        elif ":all:" in self.only_binary:
+            result.discard("source")
+        elif ":all:" in self.no_binary:
+            result.discard("binary")
         return frozenset(result)
 
-    def disallow_binaries(self):
-        # type: () -> None
+    def disallow_binaries(self) -> None:
         self.handle_mutual_excludes(
-            ':all:', self.no_binary, self.only_binary,
+            ":all:",
+            self.no_binary,
+            self.only_binary,
         )
diff --git a/src/pip/_internal/models/index.py b/src/pip/_internal/models/index.py
index b148abb4250..b94c32511f0 100644
--- a/src/pip/_internal/models/index.py
+++ b/src/pip/_internal/models/index.py
@@ -2,33 +2,27 @@
 
 
 class PackageIndex:
-    """Represents a Package Index and provides easier access to endpoints
-    """
+    """Represents a Package Index and provides easier access to endpoints"""
 
-    __slots__ = ['url', 'netloc', 'simple_url', 'pypi_url',
-                 'file_storage_domain']
+    __slots__ = ["url", "netloc", "simple_url", "pypi_url", "file_storage_domain"]
 
-    def __init__(self, url, file_storage_domain):
-        # type: (str, str) -> None
+    def __init__(self, url: str, file_storage_domain: str) -> None:
         super().__init__()
         self.url = url
         self.netloc = urllib.parse.urlsplit(url).netloc
-        self.simple_url = self._url_for_path('simple')
-        self.pypi_url = self._url_for_path('pypi')
+        self.simple_url = self._url_for_path("simple")
+        self.pypi_url = self._url_for_path("pypi")
 
         # This is part of a temporary hack used to block installs of PyPI
         # packages which depend on external urls only necessary until PyPI can
         # block such packages themselves
         self.file_storage_domain = file_storage_domain
 
-    def _url_for_path(self, path):
-        # type: (str) -> str
+    def _url_for_path(self, path: str) -> str:
         return urllib.parse.urljoin(self.url, path)
 
 
-PyPI = PackageIndex(
-    'https://pypi.org/', file_storage_domain='files.pythonhosted.org'
-)
+PyPI = PackageIndex("https://pypi.org/", file_storage_domain="files.pythonhosted.org")
 TestPyPI = PackageIndex(
-    'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org'
+    "https://test.pypi.org/", file_storage_domain="test-files.pythonhosted.org"
 )
diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py
index 06a7ceb3fce..6069b278b9b 100644
--- a/src/pip/_internal/models/link.py
+++ b/src/pip/_internal/models/link.py
@@ -1,28 +1,32 @@
+import functools
+import logging
 import os
 import posixpath
 import re
 import urllib.parse
+from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
 
 from pip._internal.utils.filetypes import WHEEL_EXTENSION
+from pip._internal.utils.hashes import Hashes
 from pip._internal.utils.misc import (
     redact_auth_from_url,
     split_auth_from_netloc,
     splitext,
 )
 from pip._internal.utils.models import KeyBasedCompareMixin
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import path_to_url, url_to_path
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, Tuple, Union
-
+if TYPE_CHECKING:
     from pip._internal.index.collector import HTMLPage
-    from pip._internal.utils.hashes import Hashes
+
+logger = logging.getLogger(__name__)
+
+
+_SUPPORTED_HASHES = ("sha1", "sha224", "sha384", "sha256", "sha512", "md5")
 
 
 class Link(KeyBasedCompareMixin):
-    """Represents a parsed link from a Package Index's simple URL
-    """
+    """Represents a parsed link from a Package Index's simple URL"""
 
     __slots__ = [
         "_parsed_url",
@@ -35,13 +39,12 @@ class Link(KeyBasedCompareMixin):
 
     def __init__(
         self,
-        url,                   # type: str
-        comes_from=None,       # type: Optional[Union[str, HTMLPage]]
-        requires_python=None,  # type: Optional[str]
-        yanked_reason=None,    # type: Optional[str]
-        cache_link_parsing=True,  # type: bool
-    ):
-        # type: (...) -> None
+        url: str,
+        comes_from: Optional[Union[str, "HTMLPage"]] = None,
+        requires_python: Optional[str] = None,
+        yanked_reason: Optional[str] = None,
+        cache_link_parsing: bool = True,
+    ) -> None:
         """
         :param url: url of the resource pointed to (href of the link)
         :param comes_from: instance of HTMLPage where the link was found,
@@ -64,7 +67,7 @@ def __init__(
         """
 
         # url can be a UNC windows share
-        if url.startswith('\\\\'):
+        if url.startswith("\\\\"):
             url = path_to_url(url)
 
         self._parsed_url = urllib.parse.urlsplit(url)
@@ -80,31 +83,28 @@ def __init__(
 
         self.cache_link_parsing = cache_link_parsing
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         if self.requires_python:
-            rp = f' (requires-python:{self.requires_python})'
+            rp = f" (requires-python:{self.requires_python})"
         else:
-            rp = ''
+            rp = ""
         if self.comes_from:
-            return '{} (from {}){}'.format(
-                redact_auth_from_url(self._url), self.comes_from, rp)
+            return "{} (from {}){}".format(
+                redact_auth_from_url(self._url), self.comes_from, rp
+            )
         else:
             return redact_auth_from_url(str(self._url))
 
-    def __repr__(self):
-        # type: () -> str
-        return f''
+    def __repr__(self) -> str:
+        return f""
 
     @property
-    def url(self):
-        # type: () -> str
+    def url(self) -> str:
         return self._url
 
     @property
-    def filename(self):
-        # type: () -> str
-        path = self.path.rstrip('/')
+    def filename(self) -> str:
+        path = self.path.rstrip("/")
         name = posixpath.basename(path)
         if not name:
             # Make sure we don't leak auth information if the netloc
@@ -113,126 +113,106 @@ def filename(self):
             return netloc
 
         name = urllib.parse.unquote(name)
-        assert name, (
-            'URL {self._url!r} produced no filename'.format(**locals()))
+        assert name, f"URL {self._url!r} produced no filename"
         return name
 
     @property
-    def file_path(self):
-        # type: () -> str
+    def file_path(self) -> str:
         return url_to_path(self.url)
 
     @property
-    def scheme(self):
-        # type: () -> str
+    def scheme(self) -> str:
         return self._parsed_url.scheme
 
     @property
-    def netloc(self):
-        # type: () -> str
+    def netloc(self) -> str:
         """
         This can contain auth information.
         """
         return self._parsed_url.netloc
 
     @property
-    def path(self):
-        # type: () -> str
+    def path(self) -> str:
         return urllib.parse.unquote(self._parsed_url.path)
 
-    def splitext(self):
-        # type: () -> Tuple[str, str]
-        return splitext(posixpath.basename(self.path.rstrip('/')))
+    def splitext(self) -> Tuple[str, str]:
+        return splitext(posixpath.basename(self.path.rstrip("/")))
 
     @property
-    def ext(self):
-        # type: () -> str
+    def ext(self) -> str:
         return self.splitext()[1]
 
     @property
-    def url_without_fragment(self):
-        # type: () -> str
+    def url_without_fragment(self) -> str:
         scheme, netloc, path, query, fragment = self._parsed_url
-        return urllib.parse.urlunsplit((scheme, netloc, path, query, None))
+        return urllib.parse.urlunsplit((scheme, netloc, path, query, ""))
 
-    _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)')
+    _egg_fragment_re = re.compile(r"[#&]egg=([^&]*)")
 
     @property
-    def egg_fragment(self):
-        # type: () -> Optional[str]
+    def egg_fragment(self) -> Optional[str]:
         match = self._egg_fragment_re.search(self._url)
         if not match:
             return None
         return match.group(1)
 
-    _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)')
+    _subdirectory_fragment_re = re.compile(r"[#&]subdirectory=([^&]*)")
 
     @property
-    def subdirectory_fragment(self):
-        # type: () -> Optional[str]
+    def subdirectory_fragment(self) -> Optional[str]:
         match = self._subdirectory_fragment_re.search(self._url)
         if not match:
             return None
         return match.group(1)
 
     _hash_re = re.compile(
-        r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)'
+        r"({choices})=([a-f0-9]+)".format(choices="|".join(_SUPPORTED_HASHES))
     )
 
     @property
-    def hash(self):
-        # type: () -> Optional[str]
+    def hash(self) -> Optional[str]:
         match = self._hash_re.search(self._url)
         if match:
             return match.group(2)
         return None
 
     @property
-    def hash_name(self):
-        # type: () -> Optional[str]
+    def hash_name(self) -> Optional[str]:
         match = self._hash_re.search(self._url)
         if match:
             return match.group(1)
         return None
 
     @property
-    def show_url(self):
-        # type: () -> str
-        return posixpath.basename(self._url.split('#', 1)[0].split('?', 1)[0])
+    def show_url(self) -> str:
+        return posixpath.basename(self._url.split("#", 1)[0].split("?", 1)[0])
 
     @property
-    def is_file(self):
-        # type: () -> bool
-        return self.scheme == 'file'
+    def is_file(self) -> bool:
+        return self.scheme == "file"
 
-    def is_existing_dir(self):
-        # type: () -> bool
+    def is_existing_dir(self) -> bool:
         return self.is_file and os.path.isdir(self.file_path)
 
     @property
-    def is_wheel(self):
-        # type: () -> bool
+    def is_wheel(self) -> bool:
         return self.ext == WHEEL_EXTENSION
 
     @property
-    def is_vcs(self):
-        # type: () -> bool
+    def is_vcs(self) -> bool:
         from pip._internal.vcs import vcs
 
         return self.scheme in vcs.all_schemes
 
     @property
-    def is_yanked(self):
-        # type: () -> bool
+    def is_yanked(self) -> bool:
         return self.yanked_reason is not None
 
     @property
-    def has_hash(self):
-        # type: () -> bool
+    def has_hash(self) -> bool:
         return self.hash_name is not None
 
-    def is_hash_allowed(self, hashes):
-        # type: (Optional[Hashes]) -> bool
+    def is_hash_allowed(self, hashes: Optional[Hashes]) -> bool:
         """
         Return True if the link has a hash and it is allowed.
         """
@@ -243,3 +223,66 @@ def is_hash_allowed(self, hashes):
         assert self.hash is not None
 
         return hashes.is_hash_allowed(self.hash_name, hex_digest=self.hash)
+
+
+class _CleanResult(NamedTuple):
+    """Convert link for equivalency check.
+
+    This is used in the resolver to check whether two URL-specified requirements
+    likely point to the same distribution and can be considered equivalent. This
+    equivalency logic avoids comparing URLs literally, which can be too strict
+    (e.g. "a=1&b=2" vs "b=2&a=1") and produce conflicts unexpecting to users.
+
+    Currently this does three things:
+
+    1. Drop the basic auth part. This is technically wrong since a server can
+       serve different content based on auth, but if it does that, it is even
+       impossible to guarantee two URLs without auth are equivalent, since
+       the user can input different auth information when prompted. So the
+       practical solution is to assume the auth doesn't affect the response.
+    2. Parse the query to avoid the ordering issue. Note that ordering under the
+       same key in the query are NOT cleaned; i.e. "a=1&a=2" and "a=2&a=1" are
+       still considered different.
+    3. Explicitly drop most of the fragment part, except ``subdirectory=`` and
+       hash values, since it should have no impact the downloaded content. Note
+       that this drops the "egg=" part historically used to denote the requested
+       project (and extras), which is wrong in the strictest sense, but too many
+       people are supplying it inconsistently to cause superfluous resolution
+       conflicts, so we choose to also ignore them.
+    """
+
+    parsed: urllib.parse.SplitResult
+    query: Dict[str, List[str]]
+    subdirectory: str
+    hashes: Dict[str, str]
+
+
+def _clean_link(link: Link) -> _CleanResult:
+    parsed = link._parsed_url
+    netloc = parsed.netloc.rsplit("@", 1)[-1]
+    # According to RFC 8089, an empty host in file: means localhost.
+    if parsed.scheme == "file" and not netloc:
+        netloc = "localhost"
+    fragment = urllib.parse.parse_qs(parsed.fragment)
+    if "egg" in fragment:
+        logger.debug("Ignoring egg= fragment in %s", link)
+    try:
+        # If there are multiple subdirectory values, use the first one.
+        # This matches the behavior of Link.subdirectory_fragment.
+        subdirectory = fragment["subdirectory"][0]
+    except (IndexError, KeyError):
+        subdirectory = ""
+    # If there are multiple hash values under the same algorithm, use the
+    # first one. This matches the behavior of Link.hash_value.
+    hashes = {k: fragment[k][0] for k in _SUPPORTED_HASHES if k in fragment}
+    return _CleanResult(
+        parsed=parsed._replace(netloc=netloc, query="", fragment=""),
+        query=urllib.parse.parse_qs(parsed.query),
+        subdirectory=subdirectory,
+        hashes=hashes,
+    )
+
+
+@functools.lru_cache(maxsize=None)
+def links_equivalent(link1: Link, link2: Link) -> bool:
+    return _clean_link(link1) == _clean_link(link2)
diff --git a/src/pip/_internal/models/scheme.py b/src/pip/_internal/models/scheme.py
index 697cd19b478..f51190ac603 100644
--- a/src/pip/_internal/models/scheme.py
+++ b/src/pip/_internal/models/scheme.py
@@ -6,7 +6,7 @@
 """
 
 
-SCHEME_KEYS = ['platlib', 'purelib', 'headers', 'scripts', 'data']
+SCHEME_KEYS = ["platlib", "purelib", "headers", "scripts", "data"]
 
 
 class Scheme:
@@ -18,12 +18,12 @@ class Scheme:
 
     def __init__(
         self,
-        platlib,  # type: str
-        purelib,  # type: str
-        headers,  # type: str
-        scripts,  # type: str
-        data,  # type: str
-    ):
+        platlib: str,
+        purelib: str,
+        headers: str,
+        scripts: str,
+        data: str,
+    ) -> None:
         self.platlib = platlib
         self.purelib = purelib
         self.headers = headers
diff --git a/src/pip/_internal/models/search_scope.py b/src/pip/_internal/models/search_scope.py
index c972f1d1704..e4e54c2f4c6 100644
--- a/src/pip/_internal/models/search_scope.py
+++ b/src/pip/_internal/models/search_scope.py
@@ -3,17 +3,13 @@
 import os
 import posixpath
 import urllib.parse
+from typing import List
 
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.models.index import PyPI
 from pip._internal.utils.compat import has_tls
 from pip._internal.utils.misc import normalize_path, redact_auth_from_url
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List
-
 
 logger = logging.getLogger(__name__)
 
@@ -29,10 +25,9 @@ class SearchScope:
     @classmethod
     def create(
         cls,
-        find_links,  # type: List[str]
-        index_urls,  # type: List[str]
-    ):
-        # type: (...) -> SearchScope
+        find_links: List[str],
+        index_urls: List[str],
+    ) -> "SearchScope":
         """
         Create a SearchScope object after normalizing the `find_links`.
         """
@@ -41,9 +36,9 @@ def create(
         # it and if it exists, use the normalized version.
         # This is deliberately conservative - it might be fine just to
         # blindly normalize anything starting with a ~...
-        built_find_links = []  # type: List[str]
+        built_find_links: List[str] = []
         for link in find_links:
-            if link.startswith('~'):
+            if link.startswith("~"):
                 new_link = normalize_path(link)
                 if os.path.exists(new_link):
                     link = new_link
@@ -54,11 +49,11 @@ def create(
         if not has_tls():
             for link in itertools.chain(index_urls, built_find_links):
                 parsed = urllib.parse.urlparse(link)
-                if parsed.scheme == 'https':
+                if parsed.scheme == "https":
                     logger.warning(
-                        'pip is configured with locations that require '
-                        'TLS/SSL, however the ssl module in Python is not '
-                        'available.'
+                        "pip is configured with locations that require "
+                        "TLS/SSL, however the ssl module in Python is not "
+                        "available."
                     )
                     break
 
@@ -69,15 +64,13 @@ def create(
 
     def __init__(
         self,
-        find_links,  # type: List[str]
-        index_urls,  # type: List[str]
-    ):
-        # type: (...) -> None
+        find_links: List[str],
+        index_urls: List[str],
+    ) -> None:
         self.find_links = find_links
         self.index_urls = index_urls
 
-    def get_formatted_locations(self):
-        # type: () -> str
+    def get_formatted_locations(self) -> str:
         lines = []
         redacted_index_urls = []
         if self.index_urls and self.index_urls != [PyPI.simple_url]:
@@ -95,41 +88,42 @@ def get_formatted_locations(self):
                 # exceptions for malformed URLs
                 if not purl.scheme and not purl.netloc:
                     logger.warning(
-                        'The index url "%s" seems invalid, '
-                        'please provide a scheme.', redacted_index_url)
+                        'The index url "%s" seems invalid, please provide a scheme.',
+                        redacted_index_url,
+                    )
 
                 redacted_index_urls.append(redacted_index_url)
 
-            lines.append('Looking in indexes: {}'.format(
-                ', '.join(redacted_index_urls)))
+            lines.append(
+                "Looking in indexes: {}".format(", ".join(redacted_index_urls))
+            )
 
         if self.find_links:
             lines.append(
-                'Looking in links: {}'.format(', '.join(
-                    redact_auth_from_url(url) for url in self.find_links))
+                "Looking in links: {}".format(
+                    ", ".join(redact_auth_from_url(url) for url in self.find_links)
+                )
             )
-        return '\n'.join(lines)
+        return "\n".join(lines)
 
-    def get_index_urls_locations(self, project_name):
-        # type: (str) -> List[str]
+    def get_index_urls_locations(self, project_name: str) -> List[str]:
         """Returns the locations found via self.index_urls
 
         Checks the url_name on the main (first in the list) index and
         use this url_name to produce all locations
         """
 
-        def mkurl_pypi_url(url):
-            # type: (str) -> str
+        def mkurl_pypi_url(url: str) -> str:
             loc = posixpath.join(
-                url,
-                urllib.parse.quote(canonicalize_name(project_name)))
+                url, urllib.parse.quote(canonicalize_name(project_name))
+            )
             # For maximum compatibility with easy_install, ensure the path
             # ends in a trailing slash.  Although this isn't in the spec
             # (and PyPI can handle it without the slash) some other index
             # implementations might break if they relied on easy_install's
             # behavior.
-            if not loc.endswith('/'):
-                loc = loc + '/'
+            if not loc.endswith("/"):
+                loc = loc + "/"
             return loc
 
         return [mkurl_pypi_url(url) for url in self.index_urls]
diff --git a/src/pip/_internal/models/selection_prefs.py b/src/pip/_internal/models/selection_prefs.py
index 4d5822268b7..977bc4caa75 100644
--- a/src/pip/_internal/models/selection_prefs.py
+++ b/src/pip/_internal/models/selection_prefs.py
@@ -1,9 +1,6 @@
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from typing import Optional
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional
-
-    from pip._internal.models.format_control import FormatControl
+from pip._internal.models.format_control import FormatControl
 
 
 class SelectionPreferences:
@@ -12,8 +9,13 @@ class SelectionPreferences:
     and installing files.
     """
 
-    __slots__ = ['allow_yanked', 'allow_all_prereleases', 'format_control',
-                 'prefer_binary', 'ignore_requires_python']
+    __slots__ = [
+        "allow_yanked",
+        "allow_all_prereleases",
+        "format_control",
+        "prefer_binary",
+        "ignore_requires_python",
+    ]
 
     # Don't include an allow_yanked default value to make sure each call
     # site considers whether yanked releases are allowed. This also causes
@@ -21,13 +23,12 @@ class SelectionPreferences:
     # people when reading the code.
     def __init__(
         self,
-        allow_yanked,  # type: bool
-        allow_all_prereleases=False,  # type: bool
-        format_control=None,          # type: Optional[FormatControl]
-        prefer_binary=False,          # type: bool
-        ignore_requires_python=None,  # type: Optional[bool]
-    ):
-        # type: (...) -> None
+        allow_yanked: bool,
+        allow_all_prereleases: bool = False,
+        format_control: Optional[FormatControl] = None,
+        prefer_binary: bool = False,
+        ignore_requires_python: Optional[bool] = None,
+    ) -> None:
         """Create a SelectionPreferences object.
 
         :param allow_yanked: Whether files marked as yanked (in the sense
diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py
index 6e6e8b52eee..744bd7ef58b 100644
--- a/src/pip/_internal/models/target_python.py
+++ b/src/pip/_internal/models/target_python.py
@@ -1,13 +1,10 @@
 import sys
+from typing import List, Optional, Tuple
+
+from pip._vendor.packaging.tags import Tag
 
 from pip._internal.utils.compatibility_tags import get_supported, version_info_to_nodot
 from pip._internal.utils.misc import normalize_version_info
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional, Tuple
-
-    from pip._vendor.packaging.tags import Tag
 
 
 class TargetPython:
@@ -29,12 +26,11 @@ class TargetPython:
 
     def __init__(
         self,
-        platforms=None,  # type: Optional[List[str]]
-        py_version_info=None,  # type: Optional[Tuple[int, ...]]
-        abis=None,  # type: Optional[List[str]]
-        implementation=None,  # type: Optional[str]
-    ):
-        # type: (...) -> None
+        platforms: Optional[List[str]] = None,
+        py_version_info: Optional[Tuple[int, ...]] = None,
+        abis: Optional[List[str]] = None,
+        implementation: Optional[str] = None,
+    ) -> None:
         """
         :param platforms: A list of strings or None. If None, searches for
             packages that are supported by the current system. Otherwise, will
@@ -57,7 +53,7 @@ def __init__(
         else:
             py_version_info = normalize_version_info(py_version_info)
 
-        py_version = '.'.join(map(str, py_version_info[:2]))
+        py_version = ".".join(map(str, py_version_info[:2]))
 
         self.abis = abis
         self.implementation = implementation
@@ -66,32 +62,29 @@ def __init__(
         self.py_version_info = py_version_info
 
         # This is used to cache the return value of get_tags().
-        self._valid_tags = None  # type: Optional[List[Tag]]
+        self._valid_tags: Optional[List[Tag]] = None
 
-    def format_given(self):
-        # type: () -> str
+    def format_given(self) -> str:
         """
         Format the given, non-None attributes for display.
         """
         display_version = None
         if self._given_py_version_info is not None:
-            display_version = '.'.join(
+            display_version = ".".join(
                 str(part) for part in self._given_py_version_info
             )
 
         key_values = [
-            ('platforms', self.platforms),
-            ('version_info', display_version),
-            ('abis', self.abis),
-            ('implementation', self.implementation),
+            ("platforms", self.platforms),
+            ("version_info", display_version),
+            ("abis", self.abis),
+            ("implementation", self.implementation),
         ]
-        return ' '.join(
-            f'{key}={value!r}' for key, value in key_values
-            if value is not None
+        return " ".join(
+            f"{key}={value!r}" for key, value in key_values if value is not None
         )
 
-    def get_tags(self):
-        # type: () -> List[Tag]
+    def get_tags(self) -> List[Tag]:
         """
         Return the supported PEP 425 tags to check wheel candidates against.
 
diff --git a/src/pip/_internal/models/wheel.py b/src/pip/_internal/models/wheel.py
index 5e03f9ff83b..e0916122763 100644
--- a/src/pip/_internal/models/wheel.py
+++ b/src/pip/_internal/models/wheel.py
@@ -2,14 +2,11 @@
 name that have meaning.
 """
 import re
+from typing import Dict, Iterable, List
 
 from pip._vendor.packaging.tags import Tag
 
 from pip._internal.exceptions import InvalidWheelFilename
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List
 
 
 class Wheel:
@@ -19,42 +16,36 @@ class Wheel:
         r"""^(?P(?P.+?)-(?P.*?))
         ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?)
         \.whl|\.dist-info)$""",
-        re.VERBOSE
+        re.VERBOSE,
     )
 
-    def __init__(self, filename):
-        # type: (str) -> None
+    def __init__(self, filename: str) -> None:
         """
         :raises InvalidWheelFilename: when the filename is invalid for a wheel
         """
         wheel_info = self.wheel_file_re.match(filename)
         if not wheel_info:
-            raise InvalidWheelFilename(
-                f"{filename} is not a valid wheel filename."
-            )
+            raise InvalidWheelFilename(f"{filename} is not a valid wheel filename.")
         self.filename = filename
-        self.name = wheel_info.group('name').replace('_', '-')
+        self.name = wheel_info.group("name").replace("_", "-")
         # we'll assume "_" means "-" due to wheel naming scheme
         # (https://github.com/pypa/pip/issues/1150)
-        self.version = wheel_info.group('ver').replace('_', '-')
-        self.build_tag = wheel_info.group('build')
-        self.pyversions = wheel_info.group('pyver').split('.')
-        self.abis = wheel_info.group('abi').split('.')
-        self.plats = wheel_info.group('plat').split('.')
+        self.version = wheel_info.group("ver").replace("_", "-")
+        self.build_tag = wheel_info.group("build")
+        self.pyversions = wheel_info.group("pyver").split(".")
+        self.abis = wheel_info.group("abi").split(".")
+        self.plats = wheel_info.group("plat").split(".")
 
         # All the tag combinations from this file
         self.file_tags = {
-            Tag(x, y, z) for x in self.pyversions
-            for y in self.abis for z in self.plats
+            Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats
         }
 
-    def get_formatted_file_tags(self):
-        # type: () -> List[str]
+    def get_formatted_file_tags(self) -> List[str]:
         """Return the wheel's tags as a sorted list of strings."""
         return sorted(str(tag) for tag in self.file_tags)
 
-    def support_index_min(self, tags):
-        # type: (List[Tag]) -> int
+    def support_index_min(self, tags: List[Tag]) -> int:
         """Return the lowest index that one of the wheel's file_tag combinations
         achieves in the given list of supported tags.
 
@@ -69,8 +60,28 @@ def support_index_min(self, tags):
         """
         return min(tags.index(tag) for tag in self.file_tags if tag in tags)
 
-    def supported(self, tags):
-        # type: (List[Tag]) -> bool
+    def find_most_preferred_tag(
+        self, tags: List[Tag], tag_to_priority: Dict[Tag, int]
+    ) -> int:
+        """Return the priority of the most preferred tag that one of the wheel's file
+        tag combinations achieves in the given list of supported tags using the given
+        tag_to_priority mapping, where lower priorities are more-preferred.
+
+        This is used in place of support_index_min in some cases in order to avoid
+        an expensive linear scan of a large list of tags.
+
+        :param tags: the PEP 425 tags to check the wheel against.
+        :param tag_to_priority: a mapping from tag to priority of that tag, where
+            lower is more preferred.
+
+        :raises ValueError: If none of the wheel's file tags match one of
+            the supported tags.
+        """
+        return min(
+            tag_to_priority[tag] for tag in self.file_tags if tag in tag_to_priority
+        )
+
+    def supported(self, tags: Iterable[Tag]) -> bool:
         """Return whether the wheel is compatible with one of the given tags.
 
         :param tags: the PEP 425 tags to check the wheel against.
diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py
index 174eb834875..ca42798bd95 100644
--- a/src/pip/_internal/network/auth.py
+++ b/src/pip/_internal/network/auth.py
@@ -4,12 +4,14 @@
 providing credentials in the context of network requests.
 """
 
-import logging
 import urllib.parse
+from typing import Any, Dict, List, Optional, Tuple
 
 from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
+from pip._vendor.requests.models import Request, Response
 from pip._vendor.requests.utils import get_netrc_auth
 
+from pip._internal.utils.logging import getLogger
 from pip._internal.utils.misc import (
     ask,
     ask_input,
@@ -17,32 +19,25 @@
     remove_auth_from_url,
     split_auth_netloc_from_url,
 )
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.vcs.versioncontrol import AuthInfo
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, List, Optional, Tuple
+logger = getLogger(__name__)
 
-    from pip._vendor.requests.models import Request, Response
-
-    from pip._internal.vcs.versioncontrol import AuthInfo
-
-    Credentials = Tuple[str, str, str]
-
-logger = logging.getLogger(__name__)
+Credentials = Tuple[str, str, str]
 
 try:
     import keyring
 except ImportError:
-    keyring = None
+    keyring = None  # type: ignore[assignment]
 except Exception as exc:
     logger.warning(
-        "Keyring is skipped due to an exception: %s", str(exc),
+        "Keyring is skipped due to an exception: %s",
+        str(exc),
     )
-    keyring = None
+    keyring = None  # type: ignore[assignment]
 
 
-def get_keyring_auth(url, username):
-    # type: (str, str) -> Optional[AuthInfo]
+def get_keyring_auth(url: Optional[str], username: Optional[str]) -> Optional[AuthInfo]:
     """Return the tuple auth for a given url from keyring."""
     global keyring
     if not url or not keyring:
@@ -68,28 +63,28 @@ def get_keyring_auth(url, username):
 
     except Exception as exc:
         logger.warning(
-            "Keyring is skipped due to an exception: %s", str(exc),
+            "Keyring is skipped due to an exception: %s",
+            str(exc),
         )
-        keyring = None
+        keyring = None  # type: ignore[assignment]
     return None
 
 
 class MultiDomainBasicAuth(AuthBase):
-
-    def __init__(self, prompting=True, index_urls=None):
-        # type: (bool, Optional[List[str]]) -> None
+    def __init__(
+        self, prompting: bool = True, index_urls: Optional[List[str]] = None
+    ) -> None:
         self.prompting = prompting
         self.index_urls = index_urls
-        self.passwords = {}  # type: Dict[str, AuthInfo]
+        self.passwords: Dict[str, AuthInfo] = {}
         # When the user is prompted to enter credentials and keyring is
         # available, we will offer to save them. If the user accepts,
         # this value is set to the credentials they entered. After the
         # request authenticates, the caller should call
         # ``save_credentials`` to save these.
-        self._credentials_to_save = None  # type: Optional[Credentials]
+        self._credentials_to_save: Optional[Credentials] = None
 
-    def _get_index_url(self, url):
-        # type: (str) -> Optional[str]
+    def _get_index_url(self, url: str) -> Optional[str]:
         """Return the original index URL matching the requested URL.
 
         Cached or dynamically generated credentials may work against
@@ -111,9 +106,12 @@ def _get_index_url(self, url):
                 return u
         return None
 
-    def _get_new_credentials(self, original_url, allow_netrc=True,
-                             allow_keyring=True):
-        # type: (str, bool, bool) -> AuthInfo
+    def _get_new_credentials(
+        self,
+        original_url: str,
+        allow_netrc: bool = True,
+        allow_keyring: bool = False,
+    ) -> AuthInfo:
         """Find and return credentials for the specified URL."""
         # Split the credentials and netloc from the url.
         url, netloc, url_user_password = split_auth_netloc_from_url(
@@ -152,18 +150,21 @@ def _get_new_credentials(self, original_url, allow_netrc=True,
         # If we don't have a password and keyring is available, use it.
         if allow_keyring:
             # The index url is more specific than the netloc, so try it first
+            # fmt: off
             kr_auth = (
                 get_keyring_auth(index_url, username) or
                 get_keyring_auth(netloc, username)
             )
+            # fmt: on
             if kr_auth:
                 logger.debug("Found credentials in keyring for %s", netloc)
                 return kr_auth
 
         return username, password
 
-    def _get_url_and_credentials(self, original_url):
-        # type: (str) -> Tuple[str, Optional[str], Optional[str]]
+    def _get_url_and_credentials(
+        self, original_url: str
+    ) -> Tuple[str, Optional[str], Optional[str]]:
         """Return the credentials to use for the provided URL.
 
         If allowed, netrc and keyring may be used to obtain the
@@ -175,13 +176,19 @@ def _get_url_and_credentials(self, original_url):
         """
         url, netloc, _ = split_auth_netloc_from_url(original_url)
 
-        # Use any stored credentials that we have for this netloc
-        username, password = self.passwords.get(netloc, (None, None))
+        # Try to get credentials from original url
+        username, password = self._get_new_credentials(original_url)
 
-        if username is None and password is None:
-            # No stored credentials. Acquire new credentials without prompting
-            # the user. (e.g. from netrc, keyring, or the URL itself)
-            username, password = self._get_new_credentials(original_url)
+        # If credentials not found, use any stored credentials for this netloc.
+        # Do this if either the username or the password is missing.
+        # This accounts for the situation in which the user has specified
+        # the username in the index url, but the password comes from keyring.
+        if (username is None or password is None) and netloc in self.passwords:
+            un, pw = self.passwords[netloc]
+            # It is possible that the cached credentials are for a different username,
+            # in which case the cache should be ignored.
+            if username is None or username == un:
+                username, password = un, pw
 
         if username is not None or password is not None:
             # Convert the username and password if they're None, so that
@@ -196,15 +203,14 @@ def _get_url_and_credentials(self, original_url):
 
         assert (
             # Credentials were found
-            (username is not None and password is not None) or
+            (username is not None and password is not None)
             # Credentials were not found
-            (username is None and password is None)
+            or (username is None and password is None)
         ), f"Could not load credentials from url: {original_url}"
 
         return url, username, password
 
-    def __call__(self, req):
-        # type: (Request) -> Request
+    def __call__(self, req: Request) -> Request:
         # Get credentials for this request
         url, username, password = self._get_url_and_credentials(req.url)
 
@@ -221,8 +227,9 @@ def __call__(self, req):
         return req
 
     # Factored out to allow for easy patching in tests
-    def _prompt_for_password(self, netloc):
-        # type: (str) -> Tuple[Optional[str], Optional[str], bool]
+    def _prompt_for_password(
+        self, netloc: str
+    ) -> Tuple[Optional[str], Optional[str], bool]:
         username = ask_input(f"User for {netloc}: ")
         if not username:
             return None, None, False
@@ -233,14 +240,12 @@ def _prompt_for_password(self, netloc):
         return username, password, True
 
     # Factored out to allow for easy patching in tests
-    def _should_save_password_to_keyring(self):
-        # type: () -> bool
+    def _should_save_password_to_keyring(self) -> bool:
         if not keyring:
             return False
         return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y"
 
-    def handle_401(self, resp, **kwargs):
-        # type: (Response, **Any) -> Response
+    def handle_401(self, resp: Response, **kwargs: Any) -> Response:
         # We only care about 401 responses, anything else we want to just
         #   pass through the actual response
         if resp.status_code != 401:
@@ -252,8 +257,17 @@ def handle_401(self, resp, **kwargs):
 
         parsed = urllib.parse.urlparse(resp.url)
 
+        # Query the keyring for credentials:
+        username, password = self._get_new_credentials(
+            resp.url,
+            allow_netrc=False,
+            allow_keyring=True,
+        )
+
         # Prompt the user for a new username and password
-        username, password, save = self._prompt_for_password(parsed.netloc)
+        save = False
+        if not username and not password:
+            username, password, save = self._prompt_for_password(parsed.netloc)
 
         # Store the new username and password to use for future requests
         self._credentials_to_save = None
@@ -285,16 +299,15 @@ def handle_401(self, resp, **kwargs):
 
         return new_resp
 
-    def warn_on_401(self, resp, **kwargs):
-        # type: (Response, **Any) -> None
+    def warn_on_401(self, resp: Response, **kwargs: Any) -> None:
         """Response callback to warn about incorrect credentials."""
         if resp.status_code == 401:
             logger.warning(
-                '401 Error, Credentials not correct for %s', resp.request.url,
+                "401 Error, Credentials not correct for %s",
+                resp.request.url,
             )
 
-    def save_credentials(self, resp, **kwargs):
-        # type: (Response, **Any) -> None
+    def save_credentials(self, resp: Response, **kwargs: Any) -> None:
         """Response callback to save credentials on success."""
         assert keyring is not None, "should never reach here without keyring"
         if not keyring:
@@ -304,7 +317,7 @@ def save_credentials(self, resp, **kwargs):
         self._credentials_to_save = None
         if creds and resp.status_code < 400:
             try:
-                logger.info('Saving credentials to keyring')
+                logger.info("Saving credentials to keyring")
                 keyring.set_password(*creds)
             except Exception:
-                logger.exception('Failed to save credentials')
+                logger.exception("Failed to save credentials")
diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py
index 9253b204769..9dba7edf9cd 100644
--- a/src/pip/_internal/network/cache.py
+++ b/src/pip/_internal/network/cache.py
@@ -3,6 +3,7 @@
 
 import os
 from contextlib import contextmanager
+from typing import Iterator, Optional
 
 from pip._vendor.cachecontrol.cache import BaseCache
 from pip._vendor.cachecontrol.caches import FileCache
@@ -10,20 +11,14 @@
 
 from pip._internal.utils.filesystem import adjacent_tmp_file, replace
 from pip._internal.utils.misc import ensure_dir
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import Iterator, Optional
 
-
-def is_from_cache(response):
-    # type: (Response) -> bool
+def is_from_cache(response: Response) -> bool:
     return getattr(response, "from_cache", False)
 
 
 @contextmanager
-def suppressed_cache_errors():
-    # type: () -> Iterator[None]
+def suppressed_cache_errors() -> Iterator[None]:
     """If we can't access the cache then we can just skip caching and process
     requests as if caching wasn't enabled.
     """
@@ -39,14 +34,12 @@ class SafeFileCache(BaseCache):
     not be accessible or writable.
     """
 
-    def __init__(self, directory):
-        # type: (str) -> None
+    def __init__(self, directory: str) -> None:
         assert directory is not None, "Cache directory must not be None."
         super().__init__()
         self.directory = directory
 
-    def _get_cache_path(self, name):
-        # type: (str) -> str
+    def _get_cache_path(self, name: str) -> str:
         # From cachecontrol.caches.file_cache.FileCache._fn, brought into our
         # class for backwards-compatibility and to avoid using a non-public
         # method.
@@ -54,15 +47,13 @@ def _get_cache_path(self, name):
         parts = list(hashed[:5]) + [hashed]
         return os.path.join(self.directory, *parts)
 
-    def get(self, key):
-        # type: (str) -> Optional[bytes]
+    def get(self, key: str) -> Optional[bytes]:
         path = self._get_cache_path(key)
         with suppressed_cache_errors():
-            with open(path, 'rb') as f:
+            with open(path, "rb") as f:
                 return f.read()
 
-    def set(self, key, value):
-        # type: (str, bytes) -> None
+    def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
         path = self._get_cache_path(key)
         with suppressed_cache_errors():
             ensure_dir(os.path.dirname(path))
@@ -72,8 +63,7 @@ def set(self, key, value):
 
             replace(f.name, path)
 
-    def delete(self, key):
-        # type: (str) -> None
+    def delete(self, key: str) -> None:
         path = self._get_cache_path(key)
         with suppressed_cache_errors():
             os.remove(path)
diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py
index 32396573cae..35bc970e260 100644
--- a/src/pip/_internal/network/download.py
+++ b/src/pip/_internal/network/download.py
@@ -4,42 +4,34 @@
 import logging
 import mimetypes
 import os
+from typing import Iterable, Optional, Tuple
 
-from pip._vendor.requests.models import CONTENT_CHUNK_SIZE
+from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
 
-from pip._internal.cli.progress_bars import DownloadProgressProvider
+from pip._internal.cli.progress_bars import get_download_progress_renderer
 from pip._internal.exceptions import NetworkConnectionError
 from pip._internal.models.index import PyPI
+from pip._internal.models.link import Link
 from pip._internal.network.cache import is_from_cache
+from pip._internal.network.session import PipSession
 from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
 from pip._internal.utils.misc import format_size, redact_auth_from_url, splitext
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Iterable, Optional, Tuple
-
-    from pip._vendor.requests.models import Response
-
-    from pip._internal.models.link import Link
-    from pip._internal.network.session import PipSession
 
 logger = logging.getLogger(__name__)
 
 
-def _get_http_response_size(resp):
-    # type: (Response) -> Optional[int]
+def _get_http_response_size(resp: Response) -> Optional[int]:
     try:
-        return int(resp.headers['content-length'])
+        return int(resp.headers["content-length"])
     except (ValueError, KeyError, TypeError):
         return None
 
 
 def _prepare_download(
-    resp,  # type: Response
-    link,  # type: Link
-    progress_bar  # type: str
-):
-    # type: (...) -> Iterable[bytes]
+    resp: Response,
+    link: Link,
+    progress_bar: str,
+) -> Iterable[bytes]:
     total_length = _get_http_response_size(resp)
 
     if link.netloc == PyPI.file_storage_domain:
@@ -50,7 +42,7 @@ def _prepare_download(
     logged_url = redact_auth_from_url(url)
 
     if total_length:
-        logged_url = '{} ({})'.format(logged_url, format_size(total_length))
+        logged_url = "{} ({})".format(logged_url, format_size(total_length))
 
     if is_from_cache(resp):
         logger.info("Using cached %s", logged_url)
@@ -73,27 +65,24 @@ def _prepare_download(
     if not show_progress:
         return chunks
 
-    return DownloadProgressProvider(
-        progress_bar, max=total_length
-    )(chunks)
+    renderer = get_download_progress_renderer(bar_type=progress_bar, size=total_length)
+    return renderer(chunks)
 
 
-def sanitize_content_filename(filename):
-    # type: (str) -> str
+def sanitize_content_filename(filename: str) -> str:
     """
     Sanitize the "filename" value from a Content-Disposition header.
     """
     return os.path.basename(filename)
 
 
-def parse_content_disposition(content_disposition, default_filename):
-    # type: (str, str) -> str
+def parse_content_disposition(content_disposition: str, default_filename: str) -> str:
     """
     Parse the "filename" value from a Content-Disposition header, and
     return the default filename if the result is empty.
     """
     _type, params = cgi.parse_header(content_disposition)
-    filename = params.get('filename')
+    filename = params.get("filename")
     if filename:
         # We need to sanitize the filename to prevent directory traversal
         # in case the filename contains ".." path parts.
@@ -101,21 +90,18 @@ def parse_content_disposition(content_disposition, default_filename):
     return filename or default_filename
 
 
-def _get_http_response_filename(resp, link):
-    # type: (Response, Link) -> str
+def _get_http_response_filename(resp: Response, link: Link) -> str:
     """Get an ideal filename from the given HTTP response, falling back to
     the link filename if not provided.
     """
     filename = link.filename  # fallback
     # Have a look at the Content-Disposition header for a better guess
-    content_disposition = resp.headers.get('content-disposition')
+    content_disposition = resp.headers.get("content-disposition")
     if content_disposition:
         filename = parse_content_disposition(content_disposition, filename)
-    ext = splitext(filename)[1]  # type: Optional[str]
+    ext: Optional[str] = splitext(filename)[1]
     if not ext:
-        ext = mimetypes.guess_extension(
-            resp.headers.get('content-type', '')
-        )
+        ext = mimetypes.guess_extension(resp.headers.get("content-type", ""))
         if ext:
             filename += ext
     if not ext and link.url != resp.url:
@@ -125,9 +111,8 @@ def _get_http_response_filename(resp, link):
     return filename
 
 
-def _http_get_download(session, link):
-    # type: (PipSession, Link) -> Response
-    target_url = link.url.split('#', 1)[0]
+def _http_get_download(session: PipSession, link: Link) -> Response:
+    target_url = link.url.split("#", 1)[0]
     resp = session.get(target_url, headers=HEADERS, stream=True)
     raise_for_status(resp)
     return resp
@@ -136,15 +121,13 @@ def _http_get_download(session, link):
 class Downloader:
     def __init__(
         self,
-        session,  # type: PipSession
-        progress_bar,  # type: str
-    ):
-        # type: (...) -> None
+        session: PipSession,
+        progress_bar: str,
+    ) -> None:
         self._session = session
         self._progress_bar = progress_bar
 
-    def __call__(self, link, location):
-        # type: (Link, str) -> Tuple[str, str]
+    def __call__(self, link: Link, location: str) -> Tuple[str, str]:
         """Download the file given by link into location."""
         try:
             resp = _http_get_download(self._session, link)
@@ -159,26 +142,25 @@ def __call__(self, link, location):
         filepath = os.path.join(location, filename)
 
         chunks = _prepare_download(resp, link, self._progress_bar)
-        with open(filepath, 'wb') as content_file:
+        with open(filepath, "wb") as content_file:
             for chunk in chunks:
                 content_file.write(chunk)
-        content_type = resp.headers.get('Content-Type', '')
+        content_type = resp.headers.get("Content-Type", "")
         return filepath, content_type
 
 
 class BatchDownloader:
-
     def __init__(
         self,
-        session,  # type: PipSession
-        progress_bar,  # type: str
-    ):
-        # type: (...) -> None
+        session: PipSession,
+        progress_bar: str,
+    ) -> None:
         self._session = session
         self._progress_bar = progress_bar
 
-    def __call__(self, links, location):
-        # type: (Iterable[Link], str) -> Iterable[Tuple[str, Tuple[str, str]]]
+    def __call__(
+        self, links: Iterable[Link], location: str
+    ) -> Iterable[Tuple[Link, Tuple[str, str]]]:
         """Download the files given by links into location."""
         for link in links:
             try:
@@ -187,7 +169,8 @@ def __call__(self, links, location):
                 assert e.response is not None
                 logger.critical(
                     "HTTP error %s while getting %s",
-                    e.response.status_code, link,
+                    e.response.status_code,
+                    link,
                 )
                 raise
 
@@ -195,8 +178,8 @@ def __call__(self, links, location):
             filepath = os.path.join(location, filename)
 
             chunks = _prepare_download(resp, link, self._progress_bar)
-            with open(filepath, 'wb') as content_file:
+            with open(filepath, "wb") as content_file:
                 for chunk in chunks:
                     content_file.write(chunk)
-            content_type = resp.headers.get('Content-Type', '')
-            yield link.url, (filepath, content_type)
+            content_type = resp.headers.get("Content-Type", "")
+            yield link, (filepath, content_type)
diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py
index c5176a4bb2a..c9e44d5be58 100644
--- a/src/pip/_internal/network/lazy_wheel.py
+++ b/src/pip/_internal/network/lazy_wheel.py
@@ -1,47 +1,40 @@
 """Lazy ZIP over HTTP"""
 
-__all__ = ['HTTPRangeRequestUnsupported', 'dist_from_wheel_url']
+__all__ = ["HTTPRangeRequestUnsupported", "dist_from_wheel_url"]
 
 from bisect import bisect_left, bisect_right
 from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
+from typing import Any, Dict, Iterator, List, Optional, Tuple
 from zipfile import BadZipfile, ZipFile
 
-from pip._vendor.requests.models import CONTENT_CHUNK_SIZE
+from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
 
+from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution
+from pip._internal.network.session import PipSession
 from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Iterator, List, Optional, Tuple
-
-    from pip._vendor.pkg_resources import Distribution
-    from pip._vendor.requests.models import Response
-
-    from pip._internal.network.session import PipSession
 
 
 class HTTPRangeRequestUnsupported(Exception):
     pass
 
 
-def dist_from_wheel_url(name, url, session):
-    # type: (str, str, PipSession) -> Distribution
-    """Return a pkg_resources.Distribution from the given wheel URL.
+def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistribution:
+    """Return a distribution object from the given wheel URL.
 
     This uses HTTP range requests to only fetch the potion of the wheel
     containing metadata, just enough for the object to be constructed.
     If such requests are not supported, HTTPRangeRequestUnsupported
     is raised.
     """
-    with LazyZipOverHTTP(url, session) as wheel:
+    with LazyZipOverHTTP(url, session) as zf:
         # For read-only ZIP files, ZipFile only needs methods read,
         # seek, seekable and tell, not the whole IO protocol.
-        zip_file = ZipFile(wheel)  # type: ignore
+        wheel = MemoryWheel(zf.name, zf)  # type: ignore
         # After context manager exit, wheel.name
         # is an invalid file by intention.
-        return pkg_resources_distribution_for_wheel(zip_file, name, wheel.name)
+        return get_wheel_distribution(wheel, canonicalize_name(name))
 
 
 class LazyZipOverHTTP:
@@ -53,51 +46,46 @@ class LazyZipOverHTTP:
     during initialization.
     """
 
-    def __init__(self, url, session, chunk_size=CONTENT_CHUNK_SIZE):
-        # type: (str, PipSession, int) -> None
+    def __init__(
+        self, url: str, session: PipSession, chunk_size: int = CONTENT_CHUNK_SIZE
+    ) -> None:
         head = session.head(url, headers=HEADERS)
         raise_for_status(head)
         assert head.status_code == 200
         self._session, self._url, self._chunk_size = session, url, chunk_size
-        self._length = int(head.headers['Content-Length'])
+        self._length = int(head.headers["Content-Length"])
         self._file = NamedTemporaryFile()
         self.truncate(self._length)
-        self._left = []  # type: List[int]
-        self._right = []  # type: List[int]
-        if 'bytes' not in head.headers.get('Accept-Ranges', 'none'):
-            raise HTTPRangeRequestUnsupported('range request is not supported')
+        self._left: List[int] = []
+        self._right: List[int] = []
+        if "bytes" not in head.headers.get("Accept-Ranges", "none"):
+            raise HTTPRangeRequestUnsupported("range request is not supported")
         self._check_zip()
 
     @property
-    def mode(self):
-        # type: () -> str
+    def mode(self) -> str:
         """Opening mode, which is always rb."""
-        return 'rb'
+        return "rb"
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         """Path to the underlying file."""
         return self._file.name
 
-    def seekable(self):
-        # type: () -> bool
+    def seekable(self) -> bool:
         """Return whether random access is supported, which is True."""
         return True
 
-    def close(self):
-        # type: () -> None
+    def close(self) -> None:
         """Close the file."""
         self._file.close()
 
     @property
-    def closed(self):
-        # type: () -> bool
+    def closed(self) -> bool:
         """Whether the file is closed."""
         return self._file.closed
 
-    def read(self, size=-1):
-        # type: (int) -> bytes
+    def read(self, size: int = -1) -> bytes:
         """Read up to size bytes from the object and return them.
 
         As a convenience, if size is unspecified or -1,
@@ -106,18 +94,16 @@ def read(self, size=-1):
         """
         download_size = max(size, self._chunk_size)
         start, length = self.tell(), self._length
-        stop = length if size < 0 else min(start+download_size, length)
-        start = max(0, stop-download_size)
-        self._download(start, stop-1)
+        stop = length if size < 0 else min(start + download_size, length)
+        start = max(0, stop - download_size)
+        self._download(start, stop - 1)
         return self._file.read(size)
 
-    def readable(self):
-        # type: () -> bool
+    def readable(self) -> bool:
         """Return whether the file is readable, which is True."""
         return True
 
-    def seek(self, offset, whence=0):
-        # type: (int, int) -> int
+    def seek(self, offset: int, whence: int = 0) -> int:
         """Change stream position and return the new absolute position.
 
         Seek to offset relative position indicated by whence:
@@ -127,13 +113,11 @@ def seek(self, offset, whence=0):
         """
         return self._file.seek(offset, whence)
 
-    def tell(self):
-        # type: () -> int
-        """Return the current possition."""
+    def tell(self) -> int:
+        """Return the current position."""
         return self._file.tell()
 
-    def truncate(self, size=None):
-        # type: (Optional[int]) -> int
+    def truncate(self, size: Optional[int] = None) -> int:
         """Resize the stream to the given size in bytes.
 
         If size is unspecified resize to the current position.
@@ -143,23 +127,19 @@ def truncate(self, size=None):
         """
         return self._file.truncate(size)
 
-    def writable(self):
-        # type: () -> bool
+    def writable(self) -> bool:
         """Return False."""
         return False
 
-    def __enter__(self):
-        # type: () -> LazyZipOverHTTP
+    def __enter__(self) -> "LazyZipOverHTTP":
         self._file.__enter__()
         return self
 
-    def __exit__(self, *exc):
-        # type: (*Any) -> Optional[bool]
+    def __exit__(self, *exc: Any) -> Optional[bool]:
         return self._file.__exit__(*exc)
 
     @contextmanager
-    def _stay(self):
-        # type: ()-> Iterator[None]
+    def _stay(self) -> Iterator[None]:
         """Return a context manager keeping the position.
 
         At the end of the block, seek back to original position.
@@ -170,8 +150,7 @@ def _stay(self):
         finally:
             self.seek(pos)
 
-    def _check_zip(self):
-        # type: () -> None
+    def _check_zip(self) -> None:
         """Check and download until the file is a valid ZIP."""
         end = self._length - 1
         for start in reversed(range(0, end, self._chunk_size)):
@@ -186,17 +165,19 @@ def _check_zip(self):
                 else:
                     break
 
-    def _stream_response(self, start, end, base_headers=HEADERS):
-        # type: (int, int, Dict[str, str]) -> Response
+    def _stream_response(
+        self, start: int, end: int, base_headers: Dict[str, str] = HEADERS
+    ) -> Response:
         """Return HTTP response to a range request from start to end."""
         headers = base_headers.copy()
-        headers['Range'] = f'bytes={start}-{end}'
+        headers["Range"] = f"bytes={start}-{end}"
         # TODO: Get range requests to be correctly cached
-        headers['Cache-Control'] = 'no-cache'
+        headers["Cache-Control"] = "no-cache"
         return self._session.get(self._url, headers=headers, stream=True)
 
-    def _merge(self, start, end, left, right):
-        # type: (int, int, int, int) -> Iterator[Tuple[int, int]]
+    def _merge(
+        self, start: int, end: int, left: int, right: int
+    ) -> Iterator[Tuple[int, int]]:
         """Return an iterator of intervals to be fetched.
 
         Args:
@@ -206,18 +187,17 @@ def _merge(self, start, end, left, right):
             right (int): Index after last overlapping downloaded data
         """
         lslice, rslice = self._left[left:right], self._right[left:right]
-        i = start = min([start]+lslice[:1])
-        end = max([end]+rslice[-1:])
+        i = start = min([start] + lslice[:1])
+        end = max([end] + rslice[-1:])
         for j, k in zip(lslice, rslice):
             if j > i:
-                yield i, j-1
+                yield i, j - 1
             i = k + 1
         if i <= end:
             yield i, end
         self._left[left:right], self._right[left:right] = [start], [end]
 
-    def _download(self, start, end):
-        # type: (int, int) -> None
+    def _download(self, start: int, end: int) -> None:
         """Download bytes from start to end inclusively."""
         with self._stay():
             left = bisect_left(self._right, start)
diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py
index 5021b8eefaa..e06ac2d3ee1 100644
--- a/src/pip/_internal/network/session.py
+++ b/src/pip/_internal/network/session.py
@@ -2,58 +2,51 @@
 network request configuration and behavior.
 """
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import email.utils
+import io
 import ipaddress
 import json
 import logging
 import mimetypes
 import os
 import platform
+import shutil
+import subprocess
 import sys
 import urllib.parse
 import warnings
+from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple, Union
 
-from pip._vendor import requests, six, urllib3
+from pip._vendor import requests, urllib3
 from pip._vendor.cachecontrol import CacheControlAdapter
 from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
-from pip._vendor.requests.models import Response
+from pip._vendor.requests.models import PreparedRequest, Response
 from pip._vendor.requests.structures import CaseInsensitiveDict
+from pip._vendor.urllib3.connectionpool import ConnectionPool
 from pip._vendor.urllib3.exceptions import InsecureRequestWarning
 
 from pip import __version__
+from pip._internal.metadata import get_default_environment
+from pip._internal.models.link import Link
 from pip._internal.network.auth import MultiDomainBasicAuth
 from pip._internal.network.cache import SafeFileCache
 
 # Import ssl from compat so the initial import occurs in only one place.
 from pip._internal.utils.compat import has_tls
 from pip._internal.utils.glibc import libc_ver
-from pip._internal.utils.misc import (
-    build_url_from_netloc,
-    get_installed_version,
-    parse_netloc,
-)
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.misc import build_url_from_netloc, parse_netloc
 from pip._internal.utils.urls import url_to_path
 
-if MYPY_CHECK_RUNNING:
-    from typing import Iterator, List, Optional, Tuple, Union
-
-    from pip._internal.models.link import Link
-
-    SecureOrigin = Tuple[str, str, Optional[Union[int, str]]]
-
-
 logger = logging.getLogger(__name__)
 
+SecureOrigin = Tuple[str, str, Optional[Union[int, str]]]
+
 
 # Ignore warning raised when using --trusted-host.
 warnings.filterwarnings("ignore", category=InsecureRequestWarning)
 
 
-SECURE_ORIGINS = [
+SECURE_ORIGINS: List[SecureOrigin] = [
     # protocol, hostname, port
     # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
     ("https", "*", "*"),
@@ -63,7 +56,7 @@
     ("file", "*", None),
     # ssh is always secure.
     ("ssh", "*", "*"),
-]  # type: List[SecureOrigin]
+]
 
 
 # These are environment variables present when running under various
@@ -75,18 +68,17 @@
 # For more background, see: https://github.com/pypa/pip/issues/5499
 CI_ENVIRONMENT_VARIABLES = (
     # Azure Pipelines
-    'BUILD_BUILDID',
+    "BUILD_BUILDID",
     # Jenkins
-    'BUILD_ID',
+    "BUILD_ID",
     # AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI
-    'CI',
+    "CI",
     # Explicit environment variable.
-    'PIP_IS_CI',
+    "PIP_IS_CI",
 )
 
 
-def looks_like_ci():
-    # type: () -> bool
+def looks_like_ci() -> bool:
     """
     Return whether it looks like pip is running under CI.
     """
@@ -96,11 +88,11 @@ def looks_like_ci():
     return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
 
 
-def user_agent():
+def user_agent() -> str:
     """
     Return a string representing the user agent.
     """
-    data = {
+    data: Dict[str, Any] = {
         "installer": {"name": "pip", "version": __version__},
         "python": platform.python_version(),
         "implementation": {
@@ -108,33 +100,38 @@ def user_agent():
         },
     }
 
-    if data["implementation"]["name"] == 'CPython':
+    if data["implementation"]["name"] == "CPython":
         data["implementation"]["version"] = platform.python_version()
-    elif data["implementation"]["name"] == 'PyPy':
-        if sys.pypy_version_info.releaselevel == 'final':
-            pypy_version_info = sys.pypy_version_info[:3]
-        else:
-            pypy_version_info = sys.pypy_version_info
+    elif data["implementation"]["name"] == "PyPy":
+        pypy_version_info = sys.pypy_version_info  # type: ignore
+        if pypy_version_info.releaselevel == "final":
+            pypy_version_info = pypy_version_info[:3]
         data["implementation"]["version"] = ".".join(
             [str(x) for x in pypy_version_info]
         )
-    elif data["implementation"]["name"] == 'Jython':
+    elif data["implementation"]["name"] == "Jython":
         # Complete Guess
         data["implementation"]["version"] = platform.python_version()
-    elif data["implementation"]["name"] == 'IronPython':
+    elif data["implementation"]["name"] == "IronPython":
         # Complete Guess
         data["implementation"]["version"] = platform.python_version()
 
     if sys.platform.startswith("linux"):
         from pip._vendor import distro
-        distro_infos = dict(filter(
-            lambda x: x[1],
-            zip(["name", "version", "id"], distro.linux_distribution()),
-        ))
-        libc = dict(filter(
-            lambda x: x[1],
-            zip(["lib", "version"], libc_ver()),
-        ))
+
+        linux_distribution = distro.name(), distro.version(), distro.codename()
+        distro_infos: Dict[str, Any] = dict(
+            filter(
+                lambda x: x[1],
+                zip(["name", "version", "id"], linux_distribution),
+            )
+        )
+        libc = dict(
+            filter(
+                lambda x: x[1],
+                zip(["lib", "version"], libc_ver()),
+            )
+        )
         if libc:
             distro_infos["libc"] = libc
         if distro_infos:
@@ -154,11 +151,27 @@ def user_agent():
 
     if has_tls():
         import _ssl as ssl
+
         data["openssl_version"] = ssl.OPENSSL_VERSION
 
-    setuptools_version = get_installed_version("setuptools")
-    if setuptools_version is not None:
-        data["setuptools_version"] = setuptools_version
+    setuptools_dist = get_default_environment().get_distribution("setuptools")
+    if setuptools_dist is not None:
+        data["setuptools_version"] = str(setuptools_dist.version)
+
+    if shutil.which("rustc") is not None:
+        # If for any reason `rustc --version` fails, silently ignore it
+        try:
+            rustc_output = subprocess.check_output(
+                ["rustc", "--version"], stderr=subprocess.STDOUT, timeout=0.5
+            )
+        except Exception:
+            pass
+        else:
+            if rustc_output.startswith(b"rustc "):
+                # The format of `rustc --version` is:
+                # `b'rustc 1.52.1 (9bc8c42bb 2021-05-09)\n'`
+                # We extract just the middle (1.52.1) part
+                data["rustc_version"] = rustc_output.split(b" ")[1].decode()
 
     # Use None rather than False so as not to give the impression that
     # pip knows it is not being run under CI.  Rather, it is a null or
@@ -177,9 +190,15 @@ def user_agent():
 
 
 class LocalFSAdapter(BaseAdapter):
-
-    def send(self, request, stream=None, timeout=None, verify=None, cert=None,
-             proxies=None):
+    def send(
+        self,
+        request: PreparedRequest,
+        stream: bool = False,
+        timeout: Optional[Union[float, Tuple[float, float]]] = None,
+        verify: Union[bool, str] = True,
+        cert: Optional[Union[str, Tuple[str, str]]] = None,
+        proxies: Optional[Mapping[str, str]] = None,
+    ) -> Response:
         pathname = url_to_path(request.url)
 
         resp = Response()
@@ -189,57 +208,75 @@ def send(self, request, stream=None, timeout=None, verify=None, cert=None,
         try:
             stats = os.stat(pathname)
         except OSError as exc:
+            # format the exception raised as a io.BytesIO object,
+            # to return a better error message:
             resp.status_code = 404
-            resp.raw = exc
+            resp.reason = type(exc).__name__
+            resp.raw = io.BytesIO(f"{resp.reason}: {exc}".encode("utf8"))
         else:
             modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
             content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
-            resp.headers = CaseInsensitiveDict({
-                "Content-Type": content_type,
-                "Content-Length": stats.st_size,
-                "Last-Modified": modified,
-            })
+            resp.headers = CaseInsensitiveDict(
+                {
+                    "Content-Type": content_type,
+                    "Content-Length": stats.st_size,
+                    "Last-Modified": modified,
+                }
+            )
 
             resp.raw = open(pathname, "rb")
             resp.close = resp.raw.close
 
         return resp
 
-    def close(self):
+    def close(self) -> None:
         pass
 
 
 class InsecureHTTPAdapter(HTTPAdapter):
-
-    def cert_verify(self, conn, url, verify, cert):
+    def cert_verify(
+        self,
+        conn: ConnectionPool,
+        url: str,
+        verify: Union[bool, str],
+        cert: Optional[Union[str, Tuple[str, str]]],
+    ) -> None:
         super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
 
 
 class InsecureCacheControlAdapter(CacheControlAdapter):
-
-    def cert_verify(self, conn, url, verify, cert):
+    def cert_verify(
+        self,
+        conn: ConnectionPool,
+        url: str,
+        verify: Union[bool, str],
+        cert: Optional[Union[str, Tuple[str, str]]],
+    ) -> None:
         super().cert_verify(conn=conn, url=url, verify=False, cert=cert)
 
 
 class PipSession(requests.Session):
 
-    timeout = None  # type: Optional[int]
-
-    def __init__(self, *args, **kwargs):
+    timeout: Optional[int] = None
+
+    def __init__(
+        self,
+        *args: Any,
+        retries: int = 0,
+        cache: Optional[str] = None,
+        trusted_hosts: Sequence[str] = (),
+        index_urls: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
         """
         :param trusted_hosts: Domains not to emit warnings for when not using
             HTTPS.
         """
-        retries = kwargs.pop("retries", 0)
-        cache = kwargs.pop("cache", None)
-        trusted_hosts = kwargs.pop("trusted_hosts", [])  # type: List[str]
-        index_urls = kwargs.pop("index_urls", None)
-
         super().__init__(*args, **kwargs)
 
         # Namespace the attribute with "pip_" just in case to prevent
         # possible conflicts with the base class.
-        self.pip_trusted_origins = []  # type: List[Tuple[str, Optional[int]]]
+        self.pip_trusted_origins: List[Tuple[str, Optional[int]]] = []
 
         # Attach our User Agent to the request
         self.headers["User-Agent"] = user_agent()
@@ -253,7 +290,6 @@ def __init__(self, *args, **kwargs):
             # Set the total number of retries that a particular request can
             # have.
             total=retries,
-
             # A 503 error from PyPI typically means that the Fastly -> Origin
             # connection got interrupted in some way. A 503 error in general
             # is typically considered a transient error so we'll go ahead and
@@ -261,11 +297,10 @@ def __init__(self, *args, **kwargs):
             # A 500 may indicate transient error in Amazon S3
             # A 520 or 527 - may indicate transient error in CloudFlare
             status_forcelist=[500, 503, 520, 527],
-
             # Add a small amount of back off between failed requests in
             # order to prevent hammering the service.
             backoff_factor=0.25,
-        )
+        )  # type: ignore
 
         # Our Insecure HTTPAdapter disables HTTPS validation. It does not
         # support caching so we'll use it for all http:// URLs.
@@ -301,16 +336,16 @@ def __init__(self, *args, **kwargs):
         for host in trusted_hosts:
             self.add_trusted_host(host, suppress_logging=True)
 
-    def update_index_urls(self, new_index_urls):
-        # type: (List[str]) -> None
+    def update_index_urls(self, new_index_urls: List[str]) -> None:
         """
         :param new_index_urls: New index urls to update the authentication
             handler with.
         """
         self.auth.index_urls = new_index_urls
 
-    def add_trusted_host(self, host, source=None, suppress_logging=False):
-        # type: (str, Optional[str], bool) -> None
+    def add_trusted_host(
+        self, host: str, source: Optional[str] = None, suppress_logging: bool = False
+    ) -> None:
         """
         :param host: It is okay to provide a host that has previously been
             added.
@@ -318,9 +353,9 @@ def add_trusted_host(self, host, source=None, suppress_logging=False):
             string came from.
         """
         if not suppress_logging:
-            msg = f'adding trusted host: {host!r}'
+            msg = f"adding trusted host: {host!r}"
             if source is not None:
-                msg += f' (from {source})'
+                msg += f" (from {source})"
             logger.info(msg)
 
         host_port = parse_netloc(host)
@@ -328,35 +363,36 @@ def add_trusted_host(self, host, source=None, suppress_logging=False):
             self.pip_trusted_origins.append(host_port)
 
         self.mount(
-            build_url_from_netloc(host) + '/',
-            self._trusted_host_adapter
+            build_url_from_netloc(host, scheme="http") + "/", self._trusted_host_adapter
         )
+        self.mount(build_url_from_netloc(host) + "/", self._trusted_host_adapter)
         if not host_port[1]:
-            # Mount wildcard ports for the same host.
             self.mount(
-                build_url_from_netloc(host) + ':',
-                self._trusted_host_adapter
+                build_url_from_netloc(host, scheme="http") + ":",
+                self._trusted_host_adapter,
             )
+            # Mount wildcard ports for the same host.
+            self.mount(build_url_from_netloc(host) + ":", self._trusted_host_adapter)
 
-    def iter_secure_origins(self):
-        # type: () -> Iterator[SecureOrigin]
+    def iter_secure_origins(self) -> Iterator[SecureOrigin]:
         yield from SECURE_ORIGINS
         for host, port in self.pip_trusted_origins:
-            yield ('*', host, '*' if port is None else port)
+            yield ("*", host, "*" if port is None else port)
 
-    def is_secure_origin(self, location):
-        # type: (Link) -> bool
+    def is_secure_origin(self, location: Link) -> bool:
         # Determine if this url used a secure transport mechanism
         parsed = urllib.parse.urlparse(str(location))
         origin_protocol, origin_host, origin_port = (
-            parsed.scheme, parsed.hostname, parsed.port,
+            parsed.scheme,
+            parsed.hostname,
+            parsed.port,
         )
 
         # The protocol to use to see if the protocol matches.
         # Don't count the repository type as part of the protocol: in
         # cases such as "git+ssh", only use "ssh". (I.e., Only verify against
         # the last scheme.)
-        origin_protocol = origin_protocol.rsplit('+', 1)[-1]
+        origin_protocol = origin_protocol.rsplit("+", 1)[-1]
 
         # Determine if our origin is a secure origin by looking through our
         # hardcoded list of secure origins, as well as any additional ones
@@ -367,21 +403,15 @@ def is_secure_origin(self, location):
                 continue
 
             try:
-                addr = ipaddress.ip_address(
-                    None
-                    if origin_host is None
-                    else six.ensure_text(origin_host)
-                )
-                network = ipaddress.ip_network(
-                    six.ensure_text(secure_host)
-                )
+                addr = ipaddress.ip_address(origin_host)
+                network = ipaddress.ip_network(secure_host)
             except ValueError:
                 # We don't have both a valid address or a valid network, so
                 # we'll check this origin against hostnames.
                 if (
-                    origin_host and
-                    origin_host.lower() != secure_host.lower() and
-                    secure_host != "*"
+                    origin_host
+                    and origin_host.lower() != secure_host.lower()
+                    and secure_host != "*"
                 ):
                     continue
             else:
@@ -392,9 +422,9 @@ def is_secure_origin(self, location):
 
             # Check to see if the port matches.
             if (
-                origin_port != secure_port and
-                secure_port != "*" and
-                secure_port is not None
+                origin_port != secure_port
+                and secure_port != "*"
+                and secure_port is not None
             ):
                 continue
 
@@ -416,9 +446,11 @@ def is_secure_origin(self, location):
 
         return False
 
-    def request(self, method, url, *args, **kwargs):
+    def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Response:
         # Allow setting a default timeout on a session
         kwargs.setdefault("timeout", self.timeout)
+        # Allow setting a default proxies on a session
+        kwargs.setdefault("proxies", self.proxies)
 
         # Dispatch the actual request
         return super().request(method, url, *args, **kwargs)
diff --git a/src/pip/_internal/network/utils.py b/src/pip/_internal/network/utils.py
index f4ff95010fc..094cf1b4a97 100644
--- a/src/pip/_internal/network/utils.py
+++ b/src/pip/_internal/network/utils.py
@@ -1,10 +1,8 @@
+from typing import Dict, Iterator
+
 from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
 
 from pip._internal.exceptions import NetworkConnectionError
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Dict, Iterator
 
 # The following comments and HTTP headers were originally added by
 # Donald Stufft in git commit 22c562429a61bb77172039e480873fb239dd8c03.
@@ -25,40 +23,41 @@
 # you're not asking for a compressed file and will then decompress it
 # before sending because if that's the case I don't think it'll ever be
 # possible to make this work.
-HEADERS = {'Accept-Encoding': 'identity'}  # type: Dict[str, str]
+HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"}
 
 
-def raise_for_status(resp):
-    # type: (Response) -> None
-    http_error_msg = ''
+def raise_for_status(resp: Response) -> None:
+    http_error_msg = ""
     if isinstance(resp.reason, bytes):
         # We attempt to decode utf-8 first because some servers
         # choose to localize their reason strings. If the string
         # isn't utf-8, we fall back to iso-8859-1 for all other
         # encodings.
         try:
-            reason = resp.reason.decode('utf-8')
+            reason = resp.reason.decode("utf-8")
         except UnicodeDecodeError:
-            reason = resp.reason.decode('iso-8859-1')
+            reason = resp.reason.decode("iso-8859-1")
     else:
         reason = resp.reason
 
     if 400 <= resp.status_code < 500:
-        http_error_msg = '%s Client Error: %s for url: %s' % (
-            resp.status_code, reason, resp.url)
+        http_error_msg = (
+            f"{resp.status_code} Client Error: {reason} for url: {resp.url}"
+        )
 
     elif 500 <= resp.status_code < 600:
-        http_error_msg = '%s Server Error: %s for url: %s' % (
-            resp.status_code, reason, resp.url)
+        http_error_msg = (
+            f"{resp.status_code} Server Error: {reason} for url: {resp.url}"
+        )
 
     if http_error_msg:
         raise NetworkConnectionError(http_error_msg, response=resp)
 
 
-def response_chunks(response, chunk_size=CONTENT_CHUNK_SIZE):
-    # type: (Response, int) -> Iterator[bytes]
-    """Given a requests Response, provide the data chunks.
-    """
+def response_chunks(
+    response: Response, chunk_size: int = CONTENT_CHUNK_SIZE
+) -> Iterator[bytes]:
+    """Given a requests Response, provide the data chunks."""
     try:
         # Special case for urllib3.
         for chunk in response.raw.stream(
diff --git a/src/pip/_internal/network/xmlrpc.py b/src/pip/_internal/network/xmlrpc.py
index 87490453259..4a7d55d0e50 100644
--- a/src/pip/_internal/network/xmlrpc.py
+++ b/src/pip/_internal/network/xmlrpc.py
@@ -3,44 +3,50 @@
 
 import logging
 import urllib.parse
-
-# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
-#       why we ignore the type on this import
-from pip._vendor.six.moves import xmlrpc_client  # type: ignore
+import xmlrpc.client
+from typing import TYPE_CHECKING, Tuple
 
 from pip._internal.exceptions import NetworkConnectionError
+from pip._internal.network.session import PipSession
 from pip._internal.network.utils import raise_for_status
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Dict
-
-    from pip._internal.network.session import PipSession
 
+if TYPE_CHECKING:
+    from xmlrpc.client import _HostType, _Marshallable
 
 logger = logging.getLogger(__name__)
 
 
-class PipXmlrpcTransport(xmlrpc_client.Transport):
+class PipXmlrpcTransport(xmlrpc.client.Transport):
     """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
     object.
     """
 
-    def __init__(self, index_url, session, use_datetime=False):
-        # type: (str, PipSession, bool) -> None
+    def __init__(
+        self, index_url: str, session: PipSession, use_datetime: bool = False
+    ) -> None:
         super().__init__(use_datetime)
         index_parts = urllib.parse.urlparse(index_url)
         self._scheme = index_parts.scheme
         self._session = session
 
-    def request(self, host, handler, request_body, verbose=False):
-        # type: (str, str, Dict[str, str], bool) -> None
+    def request(
+        self,
+        host: "_HostType",
+        handler: str,
+        request_body: bytes,
+        verbose: bool = False,
+    ) -> Tuple["_Marshallable", ...]:
+        assert isinstance(host, str)
         parts = (self._scheme, host, handler, None, None, None)
         url = urllib.parse.urlunparse(parts)
         try:
-            headers = {'Content-Type': 'text/xml'}
-            response = self._session.post(url, data=request_body,
-                                          headers=headers, stream=True)
+            headers = {"Content-Type": "text/xml"}
+            response = self._session.post(
+                url,
+                data=request_body,
+                headers=headers,
+                stream=True,
+            )
             raise_for_status(response)
             self.verbose = verbose
             return self.parse_response(response.raw)
@@ -48,6 +54,7 @@ def request(self, host, handler, request_body, verbose=False):
             assert exc.response
             logger.critical(
                 "HTTP error %s while getting %s",
-                exc.response.status_code, url,
+                exc.response.status_code,
+                url,
             )
             raise
diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/operations/build/build_tracker.py
similarity index 55%
rename from src/pip/_internal/req/req_tracker.py
rename to src/pip/_internal/operations/build/build_tracker.py
index daa5b44ca25..d6574f2ca17 100644
--- a/src/pip/_internal/req/req_tracker.py
+++ b/src/pip/_internal/operations/build/build_tracker.py
@@ -2,30 +2,23 @@
 import hashlib
 import logging
 import os
+from types import TracebackType
+from typing import Dict, Iterator, Optional, Set, Type, Union
 
-from pip._vendor import contextlib2
-
+from pip._internal.models.link import Link
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from types import TracebackType
-    from typing import Dict, Iterator, Optional, Set, Type, Union
-
-    from pip._internal.models.link import Link
-    from pip._internal.req.req_install import InstallRequirement
 
 logger = logging.getLogger(__name__)
 
 
 @contextlib.contextmanager
-def update_env_context_manager(**changes):
-    # type: (str) -> Iterator[None]
+def update_env_context_manager(**changes: str) -> Iterator[None]:
     target = os.environ
 
     # Save values from the target and change them.
     non_existent_marker = object()
-    saved_values = {}  # type: Dict[str, Union[object, str]]
+    saved_values: Dict[str, Union[object, str]] = {}
     for name, new_value in changes.items():
         try:
             saved_values[name] = target[name]
@@ -46,52 +39,42 @@ def update_env_context_manager(**changes):
 
 
 @contextlib.contextmanager
-def get_requirement_tracker():
-    # type: () -> Iterator[RequirementTracker]
-    root = os.environ.get('PIP_REQ_TRACKER')
-    with contextlib2.ExitStack() as ctx:
+def get_build_tracker() -> Iterator["BuildTracker"]:
+    root = os.environ.get("PIP_BUILD_TRACKER")
+    with contextlib.ExitStack() as ctx:
         if root is None:
-            root = ctx.enter_context(
-                TempDirectory(kind='req-tracker')
-            ).path
-            ctx.enter_context(update_env_context_manager(PIP_REQ_TRACKER=root))
+            root = ctx.enter_context(TempDirectory(kind="build-tracker")).path
+            ctx.enter_context(update_env_context_manager(PIP_BUILD_TRACKER=root))
             logger.debug("Initialized build tracking at %s", root)
 
-        with RequirementTracker(root) as tracker:
+        with BuildTracker(root) as tracker:
             yield tracker
 
 
-class RequirementTracker:
-
-    def __init__(self, root):
-        # type: (str) -> None
+class BuildTracker:
+    def __init__(self, root: str) -> None:
         self._root = root
-        self._entries = set()  # type: Set[InstallRequirement]
+        self._entries: Set[InstallRequirement] = set()
         logger.debug("Created build tracker: %s", self._root)
 
-    def __enter__(self):
-        # type: () -> RequirementTracker
+    def __enter__(self) -> "BuildTracker":
         logger.debug("Entered build tracker: %s", self._root)
         return self
 
     def __exit__(
         self,
-        exc_type,  # type: Optional[Type[BaseException]]
-        exc_val,  # type: Optional[BaseException]
-        exc_tb  # type: Optional[TracebackType]
-    ):
-        # type: (...) -> None
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
         self.cleanup()
 
-    def _entry_path(self, link):
-        # type: (Link) -> str
+    def _entry_path(self, link: Link) -> str:
         hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest()
         return os.path.join(self._root, hashed)
 
-    def add(self, req):
-        # type: (InstallRequirement) -> None
-        """Add an InstallRequirement to build tracking.
-        """
+    def add(self, req: InstallRequirement) -> None:
+        """Add an InstallRequirement to build tracking."""
 
         assert req.link
         # Get the file to write information about this requirement.
@@ -105,42 +88,37 @@ def add(self, req):
         except FileNotFoundError:
             pass
         else:
-            message = '{} is already being built: {}'.format(
-                req.link, contents)
+            message = "{} is already being built: {}".format(req.link, contents)
             raise LookupError(message)
 
         # If we're here, req should really not be building already.
         assert req not in self._entries
 
         # Start tracking this requirement.
-        with open(entry_path, 'w') as fp:
+        with open(entry_path, "w", encoding="utf-8") as fp:
             fp.write(str(req))
         self._entries.add(req)
 
-        logger.debug('Added %s to build tracker %r', req, self._root)
+        logger.debug("Added %s to build tracker %r", req, self._root)
 
-    def remove(self, req):
-        # type: (InstallRequirement) -> None
-        """Remove an InstallRequirement from build tracking.
-        """
+    def remove(self, req: InstallRequirement) -> None:
+        """Remove an InstallRequirement from build tracking."""
 
         assert req.link
         # Delete the created file and the corresponding entries.
         os.unlink(self._entry_path(req.link))
         self._entries.remove(req)
 
-        logger.debug('Removed %s from build tracker %r', req, self._root)
+        logger.debug("Removed %s from build tracker %r", req, self._root)
 
-    def cleanup(self):
-        # type: () -> None
+    def cleanup(self) -> None:
         for req in set(self._entries):
             self.remove(req)
 
         logger.debug("Removed build tracker: %r", self._root)
 
     @contextlib.contextmanager
-    def track(self, req):
-        # type: (InstallRequirement) -> Iterator[None]
+    def track(self, req: InstallRequirement) -> Iterator[None]:
         self.add(req)
         yield
         self.remove(req)
diff --git a/src/pip/_internal/operations/build/metadata.py b/src/pip/_internal/operations/build/metadata.py
index 5709962b09e..e2b7b444543 100644
--- a/src/pip/_internal/operations/build/metadata.py
+++ b/src/pip/_internal/operations/build/metadata.py
@@ -3,25 +3,25 @@
 
 import os
 
+from pip._vendor.pep517.wrappers import Pep517HookCaller
+
+from pip._internal.build_env import BuildEnvironment
+from pip._internal.exceptions import (
+    InstallationSubprocessError,
+    MetadataGenerationFailed,
+)
 from pip._internal.utils.subprocess import runner_with_spinner_message
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from pip._vendor.pep517.wrappers import Pep517HookCaller
-
-    from pip._internal.build_env import BuildEnvironment
 
 
-def generate_metadata(build_env, backend):
-    # type: (BuildEnvironment, Pep517HookCaller) -> str
+def generate_metadata(
+    build_env: BuildEnvironment, backend: Pep517HookCaller, details: str
+) -> str:
     """Generate metadata using mechanisms described in PEP 517.
 
     Returns the generated metadata directory.
     """
-    metadata_tmpdir = TempDirectory(
-        kind="modern-metadata", globally_managed=True
-    )
+    metadata_tmpdir = TempDirectory(kind="modern-metadata", globally_managed=True)
 
     metadata_dir = metadata_tmpdir.path
 
@@ -29,10 +29,11 @@ def generate_metadata(build_env, backend):
         # Note that Pep517HookCaller implements a fallback for
         # prepare_metadata_for_build_wheel, so we don't have to
         # consider the possibility that this hook doesn't exist.
-        runner = runner_with_spinner_message("Preparing wheel metadata")
+        runner = runner_with_spinner_message("Preparing metadata (pyproject.toml)")
         with backend.subprocess_runner(runner):
-            distinfo_dir = backend.prepare_metadata_for_build_wheel(
-                metadata_dir
-            )
+            try:
+                distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
+            except InstallationSubprocessError as error:
+                raise MetadataGenerationFailed(package_details=details) from error
 
     return os.path.join(metadata_dir, distinfo_dir)
diff --git a/src/pip/_internal/operations/build/metadata_editable.py b/src/pip/_internal/operations/build/metadata_editable.py
new file mode 100644
index 00000000000..4c3f48b6cdf
--- /dev/null
+++ b/src/pip/_internal/operations/build/metadata_editable.py
@@ -0,0 +1,41 @@
+"""Metadata generation logic for source distributions.
+"""
+
+import os
+
+from pip._vendor.pep517.wrappers import Pep517HookCaller
+
+from pip._internal.build_env import BuildEnvironment
+from pip._internal.exceptions import (
+    InstallationSubprocessError,
+    MetadataGenerationFailed,
+)
+from pip._internal.utils.subprocess import runner_with_spinner_message
+from pip._internal.utils.temp_dir import TempDirectory
+
+
+def generate_editable_metadata(
+    build_env: BuildEnvironment, backend: Pep517HookCaller, details: str
+) -> str:
+    """Generate metadata using mechanisms described in PEP 660.
+
+    Returns the generated metadata directory.
+    """
+    metadata_tmpdir = TempDirectory(kind="modern-metadata", globally_managed=True)
+
+    metadata_dir = metadata_tmpdir.path
+
+    with build_env:
+        # Note that Pep517HookCaller implements a fallback for
+        # prepare_metadata_for_build_wheel/editable, so we don't have to
+        # consider the possibility that this hook doesn't exist.
+        runner = runner_with_spinner_message(
+            "Preparing editable metadata (pyproject.toml)"
+        )
+        with backend.subprocess_runner(runner):
+            try:
+                distinfo_dir = backend.prepare_metadata_for_build_editable(metadata_dir)
+            except InstallationSubprocessError as error:
+                raise MetadataGenerationFailed(package_details=details) from error
+
+    return os.path.join(metadata_dir, distinfo_dir)
diff --git a/src/pip/_internal/operations/build/metadata_legacy.py b/src/pip/_internal/operations/build/metadata_legacy.py
index d44589666f4..e60988d643e 100644
--- a/src/pip/_internal/operations/build/metadata_legacy.py
+++ b/src/pip/_internal/operations/build/metadata_legacy.py
@@ -4,61 +4,53 @@
 import logging
 import os
 
-from pip._internal.exceptions import InstallationError
+from pip._internal.build_env import BuildEnvironment
+from pip._internal.cli.spinners import open_spinner
+from pip._internal.exceptions import (
+    InstallationError,
+    InstallationSubprocessError,
+    MetadataGenerationFailed,
+)
 from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from pip._internal.build_env import BuildEnvironment
 
 logger = logging.getLogger(__name__)
 
 
-def _find_egg_info(directory):
-    # type: (str) -> str
-    """Find an .egg-info subdirectory in `directory`.
-    """
-    filenames = [
-        f for f in os.listdir(directory) if f.endswith(".egg-info")
-    ]
+def _find_egg_info(directory: str) -> str:
+    """Find an .egg-info subdirectory in `directory`."""
+    filenames = [f for f in os.listdir(directory) if f.endswith(".egg-info")]
 
     if not filenames:
-        raise InstallationError(
-            f"No .egg-info directory found in {directory}"
-        )
+        raise InstallationError(f"No .egg-info directory found in {directory}")
 
     if len(filenames) > 1:
         raise InstallationError(
-            "More than one .egg-info directory found in {}".format(
-                directory
-            )
+            "More than one .egg-info directory found in {}".format(directory)
         )
 
     return os.path.join(directory, filenames[0])
 
 
 def generate_metadata(
-    build_env,  # type: BuildEnvironment
-    setup_py_path,  # type: str
-    source_dir,  # type: str
-    isolated,  # type: bool
-    details,  # type: str
-):
-    # type: (...) -> str
+    build_env: BuildEnvironment,
+    setup_py_path: str,
+    source_dir: str,
+    isolated: bool,
+    details: str,
+) -> str:
     """Generate metadata using setup.py-based defacto mechanisms.
 
     Returns the generated metadata directory.
     """
     logger.debug(
-        'Running setup.py (path:%s) egg_info for package %s',
-        setup_py_path, details,
+        "Running setup.py (path:%s) egg_info for package %s",
+        setup_py_path,
+        details,
     )
 
-    egg_info_dir = TempDirectory(
-        kind="pip-egg-info", globally_managed=True
-    ).path
+    egg_info_dir = TempDirectory(kind="pip-egg-info", globally_managed=True).path
 
     args = make_setuptools_egg_info_args(
         setup_py_path,
@@ -67,11 +59,16 @@ def generate_metadata(
     )
 
     with build_env:
-        call_subprocess(
-            args,
-            cwd=source_dir,
-            command_desc='python setup.py egg_info',
-        )
+        with open_spinner("Preparing metadata (setup.py)") as spinner:
+            try:
+                call_subprocess(
+                    args,
+                    cwd=source_dir,
+                    command_desc="python setup.py egg_info",
+                    spinner=spinner,
+                )
+            except InstallationSubprocessError as error:
+                raise MetadataGenerationFailed(package_details=details) from error
 
     # Return the .egg-info directory.
     return _find_egg_info(egg_info_dir)
diff --git a/src/pip/_internal/operations/build/wheel.py b/src/pip/_internal/operations/build/wheel.py
index d25f9c42f62..b0d2fc9eadb 100644
--- a/src/pip/_internal/operations/build/wheel.py
+++ b/src/pip/_internal/operations/build/wheel.py
@@ -1,40 +1,30 @@
 import logging
 import os
+from typing import Optional
 
-from pip._internal.utils.subprocess import runner_with_spinner_message
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
+from pip._vendor.pep517.wrappers import Pep517HookCaller
 
-    from pip._vendor.pep517.wrappers import Pep517HookCaller
+from pip._internal.utils.subprocess import runner_with_spinner_message
 
 logger = logging.getLogger(__name__)
 
 
 def build_wheel_pep517(
-    name,  # type: str
-    backend,  # type: Pep517HookCaller
-    metadata_directory,  # type: str
-    build_options,  # type: List[str]
-    tempd,  # type: str
-):
-    # type: (...) -> Optional[str]
+    name: str,
+    backend: Pep517HookCaller,
+    metadata_directory: str,
+    tempd: str,
+) -> Optional[str]:
     """Build one InstallRequirement using the PEP 517 build process.
 
     Returns path to wheel if successfully built. Otherwise, returns None.
     """
     assert metadata_directory is not None
-    if build_options:
-        # PEP 517 does not support --build-options
-        logger.error('Cannot build wheel for %s using PEP 517 when '
-                     '--build-option is present', name)
-        return None
     try:
-        logger.debug('Destination directory: %s', tempd)
+        logger.debug("Destination directory: %s", tempd)
 
         runner = runner_with_spinner_message(
-            f'Building wheel for {name} (PEP 517)'
+            f"Building wheel for {name} (pyproject.toml)"
         )
         with backend.subprocess_runner(runner):
             wheel_name = backend.build_wheel(
@@ -42,6 +32,6 @@ def build_wheel_pep517(
                 metadata_directory=metadata_directory,
             )
     except Exception:
-        logger.error('Failed building wheel for %s', name)
+        logger.error("Failed building wheel for %s", name)
         return None
     return os.path.join(tempd, wheel_name)
diff --git a/src/pip/_internal/operations/build/wheel_editable.py b/src/pip/_internal/operations/build/wheel_editable.py
new file mode 100644
index 00000000000..cf7b01aed5a
--- /dev/null
+++ b/src/pip/_internal/operations/build/wheel_editable.py
@@ -0,0 +1,46 @@
+import logging
+import os
+from typing import Optional
+
+from pip._vendor.pep517.wrappers import HookMissing, Pep517HookCaller
+
+from pip._internal.utils.subprocess import runner_with_spinner_message
+
+logger = logging.getLogger(__name__)
+
+
+def build_wheel_editable(
+    name: str,
+    backend: Pep517HookCaller,
+    metadata_directory: str,
+    tempd: str,
+) -> Optional[str]:
+    """Build one InstallRequirement using the PEP 660 build process.
+
+    Returns path to wheel if successfully built. Otherwise, returns None.
+    """
+    assert metadata_directory is not None
+    try:
+        logger.debug("Destination directory: %s", tempd)
+
+        runner = runner_with_spinner_message(
+            f"Building editable for {name} (pyproject.toml)"
+        )
+        with backend.subprocess_runner(runner):
+            try:
+                wheel_name = backend.build_editable(
+                    tempd,
+                    metadata_directory=metadata_directory,
+                )
+            except HookMissing as e:
+                logger.error(
+                    "Cannot build editable %s because the build "
+                    "backend does not have the %s hook",
+                    name,
+                    e,
+                )
+                return None
+    except Exception:
+        logger.error("Failed building editable for %s", name)
+        return None
+    return os.path.join(tempd, wheel_name)
diff --git a/src/pip/_internal/operations/build/wheel_legacy.py b/src/pip/_internal/operations/build/wheel_legacy.py
index 82fa44406e6..c5f0492ccbe 100644
--- a/src/pip/_internal/operations/build/wheel_legacy.py
+++ b/src/pip/_internal/operations/build/wheel_legacy.py
@@ -1,65 +1,54 @@
 import logging
 import os.path
+from typing import List, Optional
 
 from pip._internal.cli.spinners import open_spinner
 from pip._internal.utils.setuptools_build import make_setuptools_bdist_wheel_args
-from pip._internal.utils.subprocess import (
-    LOG_DIVIDER,
-    call_subprocess,
-    format_command_args,
-)
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
+from pip._internal.utils.subprocess import call_subprocess, format_command_args
 
 logger = logging.getLogger(__name__)
 
 
 def format_command_result(
-    command_args,  # type: List[str]
-    command_output,  # type: str
-):
-    # type: (...) -> str
+    command_args: List[str],
+    command_output: str,
+) -> str:
     """Format command information for logging."""
     command_desc = format_command_args(command_args)
-    text = f'Command arguments: {command_desc}\n'
+    text = f"Command arguments: {command_desc}\n"
 
     if not command_output:
-        text += 'Command output: None'
+        text += "Command output: None"
     elif logger.getEffectiveLevel() > logging.DEBUG:
-        text += 'Command output: [use --verbose to show]'
+        text += "Command output: [use --verbose to show]"
     else:
-        if not command_output.endswith('\n'):
-            command_output += '\n'
-        text += f'Command output:\n{command_output}{LOG_DIVIDER}'
+        if not command_output.endswith("\n"):
+            command_output += "\n"
+        text += f"Command output:\n{command_output}"
 
     return text
 
 
 def get_legacy_build_wheel_path(
-    names,  # type: List[str]
-    temp_dir,  # type: str
-    name,  # type: str
-    command_args,  # type: List[str]
-    command_output,  # type: str
-):
-    # type: (...) -> Optional[str]
+    names: List[str],
+    temp_dir: str,
+    name: str,
+    command_args: List[str],
+    command_output: str,
+) -> Optional[str]:
     """Return the path to the wheel in the temporary build directory."""
     # Sort for determinism.
     names = sorted(names)
     if not names:
-        msg = (
-            'Legacy build of wheel for {!r} created no files.\n'
-        ).format(name)
+        msg = ("Legacy build of wheel for {!r} created no files.\n").format(name)
         msg += format_command_result(command_args, command_output)
         logger.warning(msg)
         return None
 
     if len(names) > 1:
         msg = (
-            'Legacy build of wheel for {!r} created more than one file.\n'
-            'Filenames (choosing first): {}\n'
+            "Legacy build of wheel for {!r} created more than one file.\n"
+            "Filenames (choosing first): {}\n"
         ).format(name, names)
         msg += format_command_result(command_args, command_output)
         logger.warning(msg)
@@ -68,14 +57,13 @@ def get_legacy_build_wheel_path(
 
 
 def build_wheel_legacy(
-    name,  # type: str
-    setup_py_path,  # type: str
-    source_dir,  # type: str
-    global_options,  # type: List[str]
-    build_options,  # type: List[str]
-    tempd,  # type: str
-):
-    # type: (...) -> Optional[str]
+    name: str,
+    setup_py_path: str,
+    source_dir: str,
+    global_options: List[str],
+    build_options: List[str],
+    tempd: str,
+) -> Optional[str]:
     """Build one unpacked package using the "legacy" build process.
 
     Returns path to wheel if successfully built. Otherwise, returns None.
@@ -87,19 +75,20 @@ def build_wheel_legacy(
         destination_dir=tempd,
     )
 
-    spin_message = f'Building wheel for {name} (setup.py)'
+    spin_message = f"Building wheel for {name} (setup.py)"
     with open_spinner(spin_message) as spinner:
-        logger.debug('Destination directory: %s', tempd)
+        logger.debug("Destination directory: %s", tempd)
 
         try:
             output = call_subprocess(
                 wheel_args,
+                command_desc="python setup.py bdist_wheel",
                 cwd=source_dir,
                 spinner=spinner,
             )
         except Exception:
             spinner.finish("error")
-            logger.error('Failed building wheel for %s', name)
+            logger.error("Failed building wheel for %s", name)
             return None
 
         names = os.listdir(tempd)
diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py
index 5dee6bcb400..fb3ac8b9c9e 100644
--- a/src/pip/_internal/operations/check.py
+++ b/src/pip/_internal/operations/check.py
@@ -2,58 +2,55 @@
 """
 
 import logging
-from collections import namedtuple
+from typing import Callable, Dict, List, NamedTuple, Optional, Set, Tuple
 
-from pip._vendor.packaging.utils import canonicalize_name
-from pip._vendor.pkg_resources import RequirementParseError
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 
 from pip._internal.distributions import make_distribution_for_install_requirement
-from pip._internal.utils.misc import get_installed_distributions
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.metadata import get_default_environment
+from pip._internal.metadata.base import DistributionVersion
+from pip._internal.req.req_install import InstallRequirement
 
 logger = logging.getLogger(__name__)
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 
-    from pip._internal.req.req_install import InstallRequirement
+class PackageDetails(NamedTuple):
+    version: DistributionVersion
+    dependencies: List[Requirement]
 
-    # Shorthands
-    PackageSet = Dict[str, 'PackageDetails']
-    Missing = Tuple[str, Any]
-    Conflicting = Tuple[str, str, Any]
 
-    MissingDict = Dict[str, List[Missing]]
-    ConflictingDict = Dict[str, List[Conflicting]]
-    CheckResult = Tuple[MissingDict, ConflictingDict]
-    ConflictDetails = Tuple[PackageSet, CheckResult]
+# Shorthands
+PackageSet = Dict[NormalizedName, PackageDetails]
+Missing = Tuple[NormalizedName, Requirement]
+Conflicting = Tuple[NormalizedName, DistributionVersion, Requirement]
 
-PackageDetails = namedtuple('PackageDetails', ['version', 'requires'])
+MissingDict = Dict[NormalizedName, List[Missing]]
+ConflictingDict = Dict[NormalizedName, List[Conflicting]]
+CheckResult = Tuple[MissingDict, ConflictingDict]
+ConflictDetails = Tuple[PackageSet, CheckResult]
 
 
-def create_package_set_from_installed(**kwargs):
-    # type: (**Any) -> Tuple[PackageSet, bool]
-    """Converts a list of distributions into a PackageSet.
-    """
-    # Default to using all packages installed on the system
-    if kwargs == {}:
-        kwargs = {"local_only": False, "skip": ()}
-
+def create_package_set_from_installed() -> Tuple[PackageSet, bool]:
+    """Converts a list of distributions into a PackageSet."""
     package_set = {}
     problems = False
-    for dist in get_installed_distributions(**kwargs):
-        name = canonicalize_name(dist.project_name)
+    env = get_default_environment()
+    for dist in env.iter_installed_distributions(local_only=False, skip=()):
+        name = dist.canonical_name
         try:
-            package_set[name] = PackageDetails(dist.version, dist.requires())
-        except (OSError, RequirementParseError) as e:
-            # Don't crash on unreadable or broken metadata
+            dependencies = list(dist.iter_dependencies())
+            package_set[name] = PackageDetails(dist.version, dependencies)
+        except (OSError, ValueError) as e:
+            # Don't crash on unreadable or broken metadata.
             logger.warning("Error parsing requirements for %s: %s", name, e)
             problems = True
     return package_set, problems
 
 
-def check_package_set(package_set, should_ignore=None):
-    # type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult
+def check_package_set(
+    package_set: PackageSet, should_ignore: Optional[Callable[[str], bool]] = None
+) -> CheckResult:
     """Check if a package set is consistent
 
     If should_ignore is passed, it should be a callable that takes a
@@ -63,16 +60,16 @@ def check_package_set(package_set, should_ignore=None):
     missing = {}
     conflicting = {}
 
-    for package_name in package_set:
+    for package_name, package_detail in package_set.items():
         # Info about dependencies of package_name
-        missing_deps = set()  # type: Set[Missing]
-        conflicting_deps = set()  # type: Set[Conflicting]
+        missing_deps: Set[Missing] = set()
+        conflicting_deps: Set[Conflicting] = set()
 
         if should_ignore and should_ignore(package_name):
             continue
 
-        for req in package_set[package_name].requires:
-            name = canonicalize_name(req.project_name)  # type: str
+        for req in package_detail.dependencies:
+            name = canonicalize_name(req.name)
 
             # Check if it's missing
             if name not in package_set:
@@ -84,7 +81,7 @@ def check_package_set(package_set, should_ignore=None):
                 continue
 
             # Check if there's a conflict
-            version = package_set[name].version  # type: str
+            version = package_set[name].version
             if not req.specifier.contains(version, prereleases=True):
                 conflicting_deps.add((name, version, req))
 
@@ -96,8 +93,7 @@ def check_package_set(package_set, should_ignore=None):
     return missing, conflicting
 
 
-def check_install_conflicts(to_install):
-    # type: (List[InstallRequirement]) -> ConflictDetails
+def check_install_conflicts(to_install: List[InstallRequirement]) -> ConflictDetails:
     """For checking if the dependency graph would be consistent after \
     installing given requirements
     """
@@ -113,41 +109,39 @@ def check_install_conflicts(to_install):
         package_set,
         check_package_set(
             package_set, should_ignore=lambda name: name not in whitelist
-        )
+        ),
     )
 
 
-def _simulate_installation_of(to_install, package_set):
-    # type: (List[InstallRequirement], PackageSet) -> Set[str]
-    """Computes the version of packages after installing to_install.
-    """
-
+def _simulate_installation_of(
+    to_install: List[InstallRequirement], package_set: PackageSet
+) -> Set[NormalizedName]:
+    """Computes the version of packages after installing to_install."""
     # Keep track of packages that were installed
     installed = set()
 
     # Modify it as installing requirement_set would (assuming no errors)
     for inst_req in to_install:
         abstract_dist = make_distribution_for_install_requirement(inst_req)
-        dist = abstract_dist.get_pkg_resources_distribution()
-
-        assert dist is not None
-        name = canonicalize_name(dist.key)
-        package_set[name] = PackageDetails(dist.version, dist.requires())
+        dist = abstract_dist.get_metadata_distribution()
+        name = dist.canonical_name
+        package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))
 
         installed.add(name)
 
     return installed
 
 
-def _create_whitelist(would_be_installed, package_set):
-    # type: (Set[str], PackageSet) -> Set[str]
+def _create_whitelist(
+    would_be_installed: Set[NormalizedName], package_set: PackageSet
+) -> Set[NormalizedName]:
     packages_affected = set(would_be_installed)
 
     for package_name in package_set:
         if package_name in packages_affected:
             continue
 
-        for req in package_set[package_name].requires:
+        for req in package_set[package_name].dependencies:
             if canonicalize_name(req.name) in packages_affected:
                 packages_affected.add(package_name)
                 break
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 5d63c12fa1a..456554085df 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -1,79 +1,46 @@
 import collections
 import logging
 import os
+from typing import Container, Dict, Iterable, Iterator, List, NamedTuple, Optional, Set
 
 from pip._vendor.packaging.utils import canonicalize_name
-from pip._vendor.pkg_resources import RequirementParseError
+from pip._vendor.packaging.version import Version
 
 from pip._internal.exceptions import BadCommand, InstallationError
+from pip._internal.metadata import BaseDistribution, get_environment
 from pip._internal.req.constructors import (
     install_req_from_editable,
     install_req_from_line,
 )
 from pip._internal.req.req_file import COMMENT_RE
-from pip._internal.utils.direct_url_helpers import (
-    direct_url_as_pep440_direct_reference,
-    dist_get_direct_url,
-)
-from pip._internal.utils.misc import dist_is_editable, get_installed_distributions
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import (
-        Container,
-        Dict,
-        Iterable,
-        Iterator,
-        List,
-        Optional,
-        Set,
-        Tuple,
-        Union,
-    )
-
-    from pip._vendor.pkg_resources import Distribution, Requirement
-
-    RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]]
-
+from pip._internal.utils.direct_url_helpers import direct_url_as_pep440_direct_reference
 
 logger = logging.getLogger(__name__)
 
 
-def freeze(
-    requirement=None,  # type: Optional[List[str]]
-    find_links=None,  # type: Optional[List[str]]
-    local_only=False,  # type: bool
-    user_only=False,  # type: bool
-    paths=None,  # type: Optional[List[str]]
-    isolated=False,  # type: bool
-    exclude_editable=False,  # type: bool
-    skip=()  # type: Container[str]
-):
-    # type: (...) -> Iterator[str]
-    find_links = find_links or []
+class _EditableInfo(NamedTuple):
+    requirement: str
+    comments: List[str]
 
-    for link in find_links:
-        yield f'-f {link}'
-    installations = {}  # type: Dict[str, FrozenRequirement]
 
-    for dist in get_installed_distributions(
-            local_only=local_only,
-            skip=(),
-            user_only=user_only,
-            paths=paths
-    ):
-        try:
-            req = FrozenRequirement.from_dist(dist)
-        except RequirementParseError as exc:
-            # We include dist rather than dist.project_name because the
-            # dist string includes more information, like the version and
-            # location. We also include the exception message to aid
-            # troubleshooting.
-            logger.warning(
-                'Could not generate requirement for distribution %r: %s',
-                dist, exc
-            )
-            continue
+def freeze(
+    requirement: Optional[List[str]] = None,
+    local_only: bool = False,
+    user_only: bool = False,
+    paths: Optional[List[str]] = None,
+    isolated: bool = False,
+    exclude_editable: bool = False,
+    skip: Container[str] = (),
+) -> Iterator[str]:
+    installations: Dict[str, FrozenRequirement] = {}
+
+    dists = get_environment(paths).iter_installed_distributions(
+        local_only=local_only,
+        skip=(),
+        user_only=user_only,
+    )
+    for dist in dists:
+        req = FrozenRequirement.from_dist(dist)
         if exclude_editable and req.editable:
             continue
         installations[req.canonical_name] = req
@@ -83,42 +50,50 @@ def freeze(
         # should only be emitted once, even if the same option is in multiple
         # requirements files, so we need to keep track of what has been emitted
         # so that we don't emit it again if it's seen again
-        emitted_options = set()  # type: Set[str]
+        emitted_options: Set[str] = set()
         # keep track of which files a requirement is in so that we can
         # give an accurate warning if a requirement appears multiple times.
-        req_files = collections.defaultdict(list)  # type: Dict[str, List[str]]
+        req_files: Dict[str, List[str]] = collections.defaultdict(list)
         for req_file_path in requirement:
             with open(req_file_path) as req_file:
                 for line in req_file:
-                    if (not line.strip() or
-                            line.strip().startswith('#') or
-                            line.startswith((
-                                '-r', '--requirement',
-                                '-f', '--find-links',
-                                '-i', '--index-url',
-                                '--pre',
-                                '--trusted-host',
-                                '--process-dependency-links',
-                                '--extra-index-url',
-                                '--use-feature'))):
+                    if (
+                        not line.strip()
+                        or line.strip().startswith("#")
+                        or line.startswith(
+                            (
+                                "-r",
+                                "--requirement",
+                                "-f",
+                                "--find-links",
+                                "-i",
+                                "--index-url",
+                                "--pre",
+                                "--trusted-host",
+                                "--process-dependency-links",
+                                "--extra-index-url",
+                                "--use-feature",
+                            )
+                        )
+                    ):
                         line = line.rstrip()
                         if line not in emitted_options:
                             emitted_options.add(line)
                             yield line
                         continue
 
-                    if line.startswith('-e') or line.startswith('--editable'):
-                        if line.startswith('-e'):
+                    if line.startswith("-e") or line.startswith("--editable"):
+                        if line.startswith("-e"):
                             line = line[2:].strip()
                         else:
-                            line = line[len('--editable'):].strip().lstrip('=')
+                            line = line[len("--editable") :].strip().lstrip("=")
                         line_req = install_req_from_editable(
                             line,
                             isolated=isolated,
                         )
                     else:
                         line_req = install_req_from_line(
-                            COMMENT_RE.sub('', line).strip(),
+                            COMMENT_RE.sub("", line).strip(),
                             isolated=isolated,
                         )
 
@@ -126,15 +101,15 @@ def freeze(
                         logger.info(
                             "Skipping line in requirement file [%s] because "
                             "it's not clear what it would install: %s",
-                            req_file_path, line.strip(),
+                            req_file_path,
+                            line.strip(),
                         )
                         logger.info(
                             "  (add #egg=PackageName to the URL to avoid"
                             " this warning)"
                         )
                     else:
-                        line_req_canonical_name = canonicalize_name(
-                            line_req.name)
+                        line_req_canonical_name = canonicalize_name(line_req.name)
                         if line_req_canonical_name not in installations:
                             # either it's not installed, or it is installed
                             # but has been processed already
@@ -143,14 +118,13 @@ def freeze(
                                     "Requirement file [%s] contains %s, but "
                                     "package %r is not installed",
                                     req_file_path,
-                                    COMMENT_RE.sub('', line).strip(),
-                                    line_req.name
+                                    COMMENT_RE.sub("", line).strip(),
+                                    line_req.name,
                                 )
                             else:
                                 req_files[line_req.name].append(req_file_path)
                         else:
-                            yield str(installations[
-                                line_req_canonical_name]).rstrip()
+                            yield str(installations[line_req_canonical_name]).rstrip()
                             del installations[line_req_canonical_name]
                             req_files[line_req.name].append(req_file_path)
 
@@ -158,83 +132,98 @@ def freeze(
         # single requirements file or in different requirements files).
         for name, files in req_files.items():
             if len(files) > 1:
-                logger.warning("Requirement %s included multiple times [%s]",
-                               name, ', '.join(sorted(set(files))))
+                logger.warning(
+                    "Requirement %s included multiple times [%s]",
+                    name,
+                    ", ".join(sorted(set(files))),
+                )
 
-        yield(
-            '## The following requirements were added by '
-            'pip freeze:'
-        )
-    for installation in sorted(
-            installations.values(), key=lambda x: x.name.lower()):
+        yield ("## The following requirements were added by pip freeze:")
+    for installation in sorted(installations.values(), key=lambda x: x.name.lower()):
         if installation.canonical_name not in skip:
             yield str(installation).rstrip()
 
 
-def get_requirement_info(dist):
-    # type: (Distribution) -> RequirementInfo
+def _format_as_name_version(dist: BaseDistribution) -> str:
+    if isinstance(dist.version, Version):
+        return f"{dist.raw_name}=={dist.version}"
+    return f"{dist.raw_name}==={dist.version}"
+
+
+def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
     """
-    Compute and return values (req, editable, comments) for use in
+    Compute and return values (req, comments) for use in
     FrozenRequirement.from_dist().
     """
-    if not dist_is_editable(dist):
-        return (None, False, [])
+    editable_project_location = dist.editable_project_location
+    assert editable_project_location
+    location = os.path.normcase(os.path.abspath(editable_project_location))
 
-    location = os.path.normcase(os.path.abspath(dist.location))
+    from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs
 
-    from pip._internal.vcs import RemoteNotFoundError, vcs
     vcs_backend = vcs.get_backend_for_dir(location)
 
     if vcs_backend is None:
-        req = dist.as_requirement()
+        display = _format_as_name_version(dist)
         logger.debug(
-            'No VCS found for editable requirement "%s" in: %r', req,
+            'No VCS found for editable requirement "%s" in: %r',
+            display,
             location,
         )
-        comments = [
-            f'# Editable install with no version control ({req})'
-        ]
-        return (location, True, comments)
+        return _EditableInfo(
+            requirement=location,
+            comments=[f"# Editable install with no version control ({display})"],
+        )
+
+    vcs_name = type(vcs_backend).__name__
 
     try:
-        req = vcs_backend.get_src_requirement(location, dist.project_name)
+        req = vcs_backend.get_src_requirement(location, dist.raw_name)
     except RemoteNotFoundError:
-        req = dist.as_requirement()
-        comments = [
-            '# Editable {} install with no remote ({})'.format(
-                type(vcs_backend).__name__, req,
-            )
-        ]
-        return (location, True, comments)
-
+        display = _format_as_name_version(dist)
+        return _EditableInfo(
+            requirement=location,
+            comments=[f"# Editable {vcs_name} install with no remote ({display})"],
+        )
+    except RemoteNotValidError as ex:
+        display = _format_as_name_version(dist)
+        return _EditableInfo(
+            requirement=location,
+            comments=[
+                f"# Editable {vcs_name} install ({display}) with either a deleted "
+                f"local remote or invalid URI:",
+                f"# '{ex.url}'",
+            ],
+        )
     except BadCommand:
         logger.warning(
-            'cannot determine version of editable source in %s '
-            '(%s command not found in path)',
+            "cannot determine version of editable source in %s "
+            "(%s command not found in path)",
             location,
             vcs_backend.name,
         )
-        return (None, True, [])
-
+        return _EditableInfo(requirement=location, comments=[])
     except InstallationError as exc:
-        logger.warning(
-            "Error when trying to get requirement for VCS system %s, "
-            "falling back to uneditable format", exc
-        )
+        logger.warning("Error when trying to get requirement for VCS system %s", exc)
     else:
-        return (req, True, [])
+        return _EditableInfo(requirement=req, comments=[])
 
-    logger.warning(
-        'Could not determine repository location of %s', location
-    )
-    comments = ['## !! Could not determine repository location']
+    logger.warning("Could not determine repository location of %s", location)
 
-    return (None, False, comments)
+    return _EditableInfo(
+        requirement=location,
+        comments=["## !! Could not determine repository location"],
+    )
 
 
 class FrozenRequirement:
-    def __init__(self, name, req, editable, comments=()):
-        # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None
+    def __init__(
+        self,
+        name: str,
+        req: str,
+        editable: bool,
+        comments: Iterable[str] = (),
+    ) -> None:
         self.name = name
         self.canonical_name = canonicalize_name(name)
         self.req = req
@@ -242,29 +231,24 @@ def __init__(self, name, req, editable, comments=()):
         self.comments = comments
 
     @classmethod
-    def from_dist(cls, dist):
-        # type: (Distribution) -> FrozenRequirement
-        # TODO `get_requirement_info` is taking care of editable requirements.
-        # TODO This should be refactored when we will add detection of
-        #      editable that provide .dist-info metadata.
-        req, editable, comments = get_requirement_info(dist)
-        if req is None and not editable:
-            # if PEP 610 metadata is present, attempt to use it
-            direct_url = dist_get_direct_url(dist)
+    def from_dist(cls, dist: BaseDistribution) -> "FrozenRequirement":
+        editable = dist.editable
+        if editable:
+            req, comments = _get_editable_info(dist)
+        else:
+            comments = []
+            direct_url = dist.direct_url
             if direct_url:
-                req = direct_url_as_pep440_direct_reference(
-                    direct_url, dist.project_name
-                )
-                comments = []
-        if req is None:
-            # name==version requirement
-            req = dist.as_requirement()
+                # if PEP 610 metadata is present, use it
+                req = direct_url_as_pep440_direct_reference(direct_url, dist.raw_name)
+            else:
+                # name==version requirement
+                req = _format_as_name_version(dist)
 
-        return cls(dist.project_name, req, editable, comments=comments)
+        return cls(dist.raw_name, req, editable, comments=comments)
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         req = self.req
         if self.editable:
-            req = f'-e {req}'
-        return '\n'.join(list(self.comments) + [str(req)]) + '\n'
+            req = f"-e {req}"
+        return "\n".join(list(self.comments) + [str(req)]) + "\n"
diff --git a/src/pip/_internal/operations/install/editable_legacy.py b/src/pip/_internal/operations/install/editable_legacy.py
index a668a61dc60..bb548cdca75 100644
--- a/src/pip/_internal/operations/install/editable_legacy.py
+++ b/src/pip/_internal/operations/install/editable_legacy.py
@@ -1,38 +1,32 @@
 """Legacy editable installation process, i.e. `setup.py develop`.
 """
 import logging
+from typing import List, Optional, Sequence
 
+from pip._internal.build_env import BuildEnvironment
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.setuptools_build import make_setuptools_develop_args
 from pip._internal.utils.subprocess import call_subprocess
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional, Sequence
-
-    from pip._internal.build_env import BuildEnvironment
-
 
 logger = logging.getLogger(__name__)
 
 
 def install_editable(
-    install_options,  # type: List[str]
-    global_options,  # type: Sequence[str]
-    prefix,  # type: Optional[str]
-    home,  # type: Optional[str]
-    use_user_site,  # type: bool
-    name,  # type: str
-    setup_py_path,  # type: str
-    isolated,  # type: bool
-    build_env,  # type: BuildEnvironment
-    unpacked_source_directory,  # type: str
-):
-    # type: (...) -> None
+    install_options: List[str],
+    global_options: Sequence[str],
+    prefix: Optional[str],
+    home: Optional[str],
+    use_user_site: bool,
+    name: str,
+    setup_py_path: str,
+    isolated: bool,
+    build_env: BuildEnvironment,
+    unpacked_source_directory: str,
+) -> None:
     """Install a package in editable mode. Most arguments are pass-through
     to setuptools.
     """
-    logger.info('Running setup.py develop for %s', name)
+    logger.info("Running setup.py develop for %s", name)
 
     args = make_setuptools_develop_args(
         setup_py_path,
@@ -48,5 +42,6 @@ def install_editable(
         with build_env:
             call_subprocess(
                 args,
+                command_desc="python setup.py develop",
                 cwd=unpacked_source_directory,
             )
diff --git a/src/pip/_internal/operations/install/legacy.py b/src/pip/_internal/operations/install/legacy.py
index 63a693a91ed..5b7ef901718 100644
--- a/src/pip/_internal/operations/install/legacy.py
+++ b/src/pip/_internal/operations/install/legacy.py
@@ -3,56 +3,79 @@
 
 import logging
 import os
-import sys
 from distutils.util import change_root
+from typing import List, Optional, Sequence
 
-from pip._internal.exceptions import InstallationError
-from pip._internal.utils.logging import indent_log
+from pip._internal.build_env import BuildEnvironment
+from pip._internal.exceptions import InstallationError, LegacyInstallFailure
+from pip._internal.models.scheme import Scheme
 from pip._internal.utils.misc import ensure_dir
 from pip._internal.utils.setuptools_build import make_setuptools_install_args
 from pip._internal.utils.subprocess import runner_with_spinner_message
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional, Sequence
-
-    from pip._internal.build_env import BuildEnvironment
-    from pip._internal.models.scheme import Scheme
+logger = logging.getLogger(__name__)
 
 
-logger = logging.getLogger(__name__)
+def write_installed_files_from_setuptools_record(
+    record_lines: List[str],
+    root: Optional[str],
+    req_description: str,
+) -> None:
+    def prepend_root(path: str) -> str:
+        if root is None or not os.path.isabs(path):
+            return path
+        else:
+            return change_root(root, path)
 
+    for line in record_lines:
+        directory = os.path.dirname(line)
+        if directory.endswith(".egg-info"):
+            egg_info_dir = prepend_root(directory)
+            break
+    else:
+        message = (
+            "{} did not indicate that it installed an "
+            ".egg-info directory. Only setup.py projects "
+            "generating .egg-info directories are supported."
+        ).format(req_description)
+        raise InstallationError(message)
 
-class LegacyInstallFailure(Exception):
-    def __init__(self):
-        # type: () -> None
-        self.parent = sys.exc_info()
+    new_lines = []
+    for line in record_lines:
+        filename = line.strip()
+        if os.path.isdir(filename):
+            filename += os.path.sep
+        new_lines.append(os.path.relpath(prepend_root(filename), egg_info_dir))
+    new_lines.sort()
+    ensure_dir(egg_info_dir)
+    inst_files_path = os.path.join(egg_info_dir, "installed-files.txt")
+    with open(inst_files_path, "w") as f:
+        f.write("\n".join(new_lines) + "\n")
 
 
 def install(
-    install_options,  # type: List[str]
-    global_options,  # type: Sequence[str]
-    root,  # type: Optional[str]
-    home,  # type: Optional[str]
-    prefix,  # type: Optional[str]
-    use_user_site,  # type: bool
-    pycompile,  # type: bool
-    scheme,  # type: Scheme
-    setup_py_path,  # type: str
-    isolated,  # type: bool
-    req_name,  # type: str
-    build_env,  # type: BuildEnvironment
-    unpacked_source_directory,  # type: str
-    req_description,  # type: str
-):
-    # type: (...) -> bool
+    install_options: List[str],
+    global_options: Sequence[str],
+    root: Optional[str],
+    home: Optional[str],
+    prefix: Optional[str],
+    use_user_site: bool,
+    pycompile: bool,
+    scheme: Scheme,
+    setup_py_path: str,
+    isolated: bool,
+    req_name: str,
+    build_env: BuildEnvironment,
+    unpacked_source_directory: str,
+    req_description: str,
+) -> bool:
 
     header_dir = scheme.headers
 
     with TempDirectory(kind="record") as temp_dir:
         try:
-            record_filename = os.path.join(temp_dir.path, 'install-record.txt')
+            record_filename = os.path.join(temp_dir.path, "install-record.txt")
             install_args = make_setuptools_install_args(
                 setup_py_path,
                 global_options=global_options,
@@ -70,20 +93,20 @@ def install(
             runner = runner_with_spinner_message(
                 f"Running setup.py install for {req_name}"
             )
-            with indent_log(), build_env:
+            with build_env:
                 runner(
                     cmd=install_args,
                     cwd=unpacked_source_directory,
                 )
 
             if not os.path.exists(record_filename):
-                logger.debug('Record file %s not found', record_filename)
+                logger.debug("Record file %s not found", record_filename)
                 # Signal to the caller that we didn't install the new package
                 return False
 
-        except Exception:
+        except Exception as e:
             # Signal to the caller that we didn't install the new package
-            raise LegacyInstallFailure
+            raise LegacyInstallFailure(package_details=req_name) from e
 
         # At this point, we have successfully installed the requirement.
 
@@ -93,38 +116,5 @@ def install(
         with open(record_filename) as f:
             record_lines = f.read().splitlines()
 
-    def prepend_root(path):
-        # type: (str) -> str
-        if root is None or not os.path.isabs(path):
-            return path
-        else:
-            return change_root(root, path)
-
-    for line in record_lines:
-        directory = os.path.dirname(line)
-        if directory.endswith('.egg-info'):
-            egg_info_dir = prepend_root(directory)
-            break
-    else:
-        message = (
-            "{} did not indicate that it installed an "
-            ".egg-info directory. Only setup.py projects "
-            "generating .egg-info directories are supported."
-        ).format(req_description)
-        raise InstallationError(message)
-
-    new_lines = []
-    for line in record_lines:
-        filename = line.strip()
-        if os.path.isdir(filename):
-            filename += os.path.sep
-        new_lines.append(
-            os.path.relpath(prepend_root(filename), egg_info_dir)
-        )
-    new_lines.sort()
-    ensure_dir(egg_info_dir)
-    inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
-    with open(inst_files_path, 'w') as f:
-        f.write('\n'.join(new_lines) + '\n')
-
+    write_installed_files_from_setuptools_record(record_lines, root, req_description)
     return True
diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py
index 37dcb618c65..e191b13431d 100644
--- a/src/pip/_internal/operations/install/wheel.py
+++ b/src/pip/_internal/operations/install/wheel.py
@@ -13,147 +13,119 @@
 import sys
 import warnings
 from base64 import urlsafe_b64encode
+from email.message import Message
 from itertools import chain, filterfalse, starmap
-from zipfile import ZipFile
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    BinaryIO,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    NewType,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
+from zipfile import ZipFile, ZipInfo
 
-from pip._vendor import pkg_resources
 from pip._vendor.distlib.scripts import ScriptMaker
 from pip._vendor.distlib.util import get_export_entry
-from pip._vendor.six import ensure_str, ensure_text, reraise
+from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.exceptions import InstallationError
 from pip._internal.locations import get_major_minor_version
+from pip._internal.metadata import (
+    BaseDistribution,
+    FilesystemWheel,
+    get_wheel_distribution,
+)
 from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl
-from pip._internal.models.scheme import SCHEME_KEYS
+from pip._internal.models.scheme import SCHEME_KEYS, Scheme
 from pip._internal.utils.filesystem import adjacent_tmp_file, replace
 from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file, partition
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.unpacking import (
     current_umask,
     is_within_directory,
     set_extracted_file_to_default_mode_plus_executable,
     zip_item_is_executable,
 )
-from pip._internal.utils.wheel import parse_wheel, pkg_resources_distribution_for_wheel
-
-# Use the custom cast function at runtime to make cast work,
-# and import typing.cast when performing pre-commit and type
-# checks
-if not MYPY_CHECK_RUNNING:
-    from pip._internal.utils.typing import cast
-else:
-    from email.message import Message
-    from typing import (
-        IO,
-        Any,
-        BinaryIO,
-        Callable,
-        Dict,
-        Iterable,
-        Iterator,
-        List,
-        NewType,
-        Optional,
-        Protocol,
-        Sequence,
-        Set,
-        Tuple,
-        Union,
-        cast,
-    )
-    from zipfile import ZipInfo
+from pip._internal.utils.wheel import parse_wheel
 
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.models.scheme import Scheme
-
-    RecordPath = NewType('RecordPath', str)
-    InstalledCSVRow = Tuple[RecordPath, str, Union[int, str]]
+if TYPE_CHECKING:
+    from typing import Protocol
 
     class File(Protocol):
-        src_record_path = None  # type: RecordPath
-        dest_path = None  # type: str
-        changed = None  # type: bool
+        src_record_path: "RecordPath"
+        dest_path: str
+        changed: bool
 
-        def save(self):
-            # type: () -> None
+        def save(self) -> None:
             pass
 
 
 logger = logging.getLogger(__name__)
 
+RecordPath = NewType("RecordPath", str)
+InstalledCSVRow = Tuple[RecordPath, str, Union[int, str]]
+
 
-def rehash(path, blocksize=1 << 20):
-    # type: (str, int) -> Tuple[str, str]
+def rehash(path: str, blocksize: int = 1 << 20) -> Tuple[str, str]:
     """Return (encoded_digest, length) for path using hashlib.sha256()"""
     h, length = hash_file(path, blocksize)
-    digest = 'sha256=' + urlsafe_b64encode(
-        h.digest()
-    ).decode('latin1').rstrip('=')
+    digest = "sha256=" + urlsafe_b64encode(h.digest()).decode("latin1").rstrip("=")
     return (digest, str(length))
 
 
-def csv_io_kwargs(mode):
-    # type: (str) -> Dict[str, Any]
+def csv_io_kwargs(mode: str) -> Dict[str, Any]:
     """Return keyword arguments to properly open a CSV file
     in the given mode.
     """
-    return {'mode': mode, 'newline': '', 'encoding': 'utf-8'}
+    return {"mode": mode, "newline": "", "encoding": "utf-8"}
 
 
-def fix_script(path):
-    # type: (str) -> bool
+def fix_script(path: str) -> bool:
     """Replace #!python with #!/path/to/python
     Return True if file was changed.
     """
     # XXX RECORD hashes will need to be updated
     assert os.path.isfile(path)
 
-    with open(path, 'rb') as script:
+    with open(path, "rb") as script:
         firstline = script.readline()
-        if not firstline.startswith(b'#!python'):
+        if not firstline.startswith(b"#!python"):
             return False
         exename = sys.executable.encode(sys.getfilesystemencoding())
-        firstline = b'#!' + exename + os.linesep.encode("ascii")
+        firstline = b"#!" + exename + os.linesep.encode("ascii")
         rest = script.read()
-    with open(path, 'wb') as script:
+    with open(path, "wb") as script:
         script.write(firstline)
         script.write(rest)
     return True
 
 
-def wheel_root_is_purelib(metadata):
-    # type: (Message) -> bool
+def wheel_root_is_purelib(metadata: Message) -> bool:
     return metadata.get("Root-Is-Purelib", "").lower() == "true"
 
 
-def get_entrypoints(distribution):
-    # type: (Distribution) -> Tuple[Dict[str, str], Dict[str, str]]
-    # get the entry points and then the script names
-    try:
-        console = distribution.get_entry_map('console_scripts')
-        gui = distribution.get_entry_map('gui_scripts')
-    except KeyError:
-        # Our dict-based Distribution raises KeyError if entry_points.txt
-        # doesn't exist.
-        return {}, {}
-
-    def _split_ep(s):
-        # type: (pkg_resources.EntryPoint) -> Tuple[str, str]
-        """get the string representation of EntryPoint,
-        remove space and split on '='
-        """
-        split_parts = str(s).replace(" ", "").split("=")
-        return split_parts[0], split_parts[1]
-
-    # convert the EntryPoint objects into strings with module:function
-    console = dict(_split_ep(v) for v in console.values())
-    gui = dict(_split_ep(v) for v in gui.values())
-    return console, gui
-
-
-def message_about_scripts_not_on_PATH(scripts):
-    # type: (Sequence[str]) -> Optional[str]
+def get_entrypoints(dist: BaseDistribution) -> Tuple[Dict[str, str], Dict[str, str]]:
+    console_scripts = {}
+    gui_scripts = {}
+    for entry_point in dist.iter_entry_points():
+        if entry_point.group == "console_scripts":
+            console_scripts[entry_point.name] = entry_point.value
+        elif entry_point.group == "gui_scripts":
+            gui_scripts[entry_point.name] = entry_point.value
+    return console_scripts, gui_scripts
+
+
+def message_about_scripts_not_on_PATH(scripts: Sequence[str]) -> Optional[str]:
     """Determine if any scripts are not on PATH and format a warning.
     Returns a warning message if one or more scripts are not on PATH,
     otherwise None.
@@ -162,7 +134,7 @@ def message_about_scripts_not_on_PATH(scripts):
         return None
 
     # Group scripts by the path they were installed in
-    grouped_by_dir = collections.defaultdict(set)  # type: Dict[str, Set[str]]
+    grouped_by_dir: Dict[str, Set[str]] = collections.defaultdict(set)
     for destfile in scripts:
         parent_dir = os.path.dirname(destfile)
         script_name = os.path.basename(destfile)
@@ -170,23 +142,24 @@ def message_about_scripts_not_on_PATH(scripts):
 
     # We don't want to warn for directories that are on PATH.
     not_warn_dirs = [
-        os.path.normcase(i).rstrip(os.sep) for i in
-        os.environ.get("PATH", "").split(os.pathsep)
+        os.path.normcase(i).rstrip(os.sep)
+        for i in os.environ.get("PATH", "").split(os.pathsep)
     ]
     # If an executable sits with sys.executable, we don't warn for it.
     #     This covers the case of venv invocations without activating the venv.
     not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable)))
-    warn_for = {
-        parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items()
+    warn_for: Dict[str, Set[str]] = {
+        parent_dir: scripts
+        for parent_dir, scripts in grouped_by_dir.items()
         if os.path.normcase(parent_dir) not in not_warn_dirs
-    }  # type: Dict[str, Set[str]]
+    }
     if not warn_for:
         return None
 
     # Format a message
     msg_lines = []
     for parent_dir, dir_scripts in warn_for.items():
-        sorted_scripts = sorted(dir_scripts)  # type: List[str]
+        sorted_scripts: List[str] = sorted(dir_scripts)
         if len(sorted_scripts) == 1:
             start_text = "script {} is".format(sorted_scripts[0])
         else:
@@ -195,8 +168,9 @@ def message_about_scripts_not_on_PATH(scripts):
             )
 
         msg_lines.append(
-            "The {} installed in '{}' which is not on PATH."
-            .format(start_text, parent_dir)
+            "The {} installed in '{}' which is not on PATH.".format(
+                start_text, parent_dir
+            )
         )
 
     last_line_fmt = (
@@ -223,8 +197,9 @@ def message_about_scripts_not_on_PATH(scripts):
     return "\n".join(msg_lines)
 
 
-def _normalized_outrows(outrows):
-    # type: (Iterable[InstalledCSVRow]) -> List[Tuple[str, str, str]]
+def _normalized_outrows(
+    outrows: Iterable[InstalledCSVRow],
+) -> List[Tuple[str, str, str]]:
     """Normalize the given rows of a RECORD file.
 
     Items in each row are converted into str. Rows are then sorted to make
@@ -244,69 +219,60 @@ def _normalized_outrows(outrows):
     # For additional background, see--
     # https://github.com/pypa/pip/issues/5868
     return sorted(
-        (ensure_str(record_path, encoding='utf-8'), hash_, str(size))
-        for record_path, hash_, size in outrows
+        (record_path, hash_, str(size)) for record_path, hash_, size in outrows
     )
 
 
-def _record_to_fs_path(record_path):
-    # type: (RecordPath) -> str
+def _record_to_fs_path(record_path: RecordPath) -> str:
     return record_path
 
 
-def _fs_to_record_path(path, relative_to=None):
-    # type: (str, Optional[str]) -> RecordPath
+def _fs_to_record_path(path: str, relative_to: Optional[str] = None) -> RecordPath:
     if relative_to is not None:
         # On Windows, do not handle relative paths if they belong to different
         # logical disks
-        if os.path.splitdrive(path)[0].lower() == \
-                os.path.splitdrive(relative_to)[0].lower():
+        if (
+            os.path.splitdrive(path)[0].lower()
+            == os.path.splitdrive(relative_to)[0].lower()
+        ):
             path = os.path.relpath(path, relative_to)
-    path = path.replace(os.path.sep, '/')
-    return cast('RecordPath', path)
-
-
-def _parse_record_path(record_column):
-    # type: (str) -> RecordPath
-    p = ensure_text(record_column, encoding='utf-8')
-    return cast('RecordPath', p)
+    path = path.replace(os.path.sep, "/")
+    return cast("RecordPath", path)
 
 
 def get_csv_rows_for_installed(
-    old_csv_rows,  # type: List[List[str]]
-    installed,  # type: Dict[RecordPath, RecordPath]
-    changed,  # type: Set[RecordPath]
-    generated,  # type: List[str]
-    lib_dir,  # type: str
-):
-    # type: (...) -> List[InstalledCSVRow]
+    old_csv_rows: List[List[str]],
+    installed: Dict[RecordPath, RecordPath],
+    changed: Set[RecordPath],
+    generated: List[str],
+    lib_dir: str,
+) -> List[InstalledCSVRow]:
     """
     :param installed: A map from archive RECORD path to installation RECORD
         path.
     """
-    installed_rows = []  # type: List[InstalledCSVRow]
+    installed_rows: List[InstalledCSVRow] = []
     for row in old_csv_rows:
         if len(row) > 3:
-            logger.warning('RECORD line has more than three elements: %s', row)
-        old_record_path = _parse_record_path(row[0])
+            logger.warning("RECORD line has more than three elements: %s", row)
+        old_record_path = cast("RecordPath", row[0])
         new_record_path = installed.pop(old_record_path, old_record_path)
         if new_record_path in changed:
             digest, length = rehash(_record_to_fs_path(new_record_path))
         else:
-            digest = row[1] if len(row) > 1 else ''
-            length = row[2] if len(row) > 2 else ''
+            digest = row[1] if len(row) > 1 else ""
+            length = row[2] if len(row) > 2 else ""
         installed_rows.append((new_record_path, digest, length))
     for f in generated:
         path = _fs_to_record_path(f, lib_dir)
         digest, length = rehash(f)
         installed_rows.append((path, digest, length))
     for installed_record_path in installed.values():
-        installed_rows.append((installed_record_path, '', ''))
+        installed_rows.append((installed_record_path, "", ""))
     return installed_rows
 
 
-def get_console_script_specs(console):
-    # type: (Dict[str, str]) -> List[str]
+def get_console_script_specs(console: Dict[str, str]) -> List[str]:
     """
     Given the mapping from entrypoint name to callable, return the relevant
     console script specs.
@@ -349,62 +315,57 @@ def get_console_script_specs(console):
     # DEFAULT
     #   - The default behavior is to install pip, pipX, pipX.Y, easy_install
     #     and easy_install-X.Y.
-    pip_script = console.pop('pip', None)
+    pip_script = console.pop("pip", None)
     if pip_script:
         if "ENSUREPIP_OPTIONS" not in os.environ:
-            scripts_to_generate.append('pip = ' + pip_script)
+            scripts_to_generate.append("pip = " + pip_script)
 
         if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
             scripts_to_generate.append(
-                'pip{} = {}'.format(sys.version_info[0], pip_script)
+                "pip{} = {}".format(sys.version_info[0], pip_script)
             )
 
-        scripts_to_generate.append(
-            f'pip{get_major_minor_version()} = {pip_script}'
-        )
+        scripts_to_generate.append(f"pip{get_major_minor_version()} = {pip_script}")
         # Delete any other versioned pip entry points
-        pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)]
+        pip_ep = [k for k in console if re.match(r"pip(\d(\.\d)?)?$", k)]
         for k in pip_ep:
             del console[k]
-    easy_install_script = console.pop('easy_install', None)
+    easy_install_script = console.pop("easy_install", None)
     if easy_install_script:
         if "ENSUREPIP_OPTIONS" not in os.environ:
-            scripts_to_generate.append(
-                'easy_install = ' + easy_install_script
-            )
+            scripts_to_generate.append("easy_install = " + easy_install_script)
 
         scripts_to_generate.append(
-            'easy_install-{} = {}'.format(
+            "easy_install-{} = {}".format(
                 get_major_minor_version(), easy_install_script
             )
         )
         # Delete any other versioned easy_install entry points
         easy_install_ep = [
-            k for k in console if re.match(r'easy_install(-\d\.\d)?$', k)
+            k for k in console if re.match(r"easy_install(-\d\.\d)?$", k)
         ]
         for k in easy_install_ep:
             del console[k]
 
     # Generate the console entry points specified in the wheel
-    scripts_to_generate.extend(starmap('{} = {}'.format, console.items()))
+    scripts_to_generate.extend(starmap("{} = {}".format, console.items()))
 
     return scripts_to_generate
 
 
 class ZipBackedFile:
-    def __init__(self, src_record_path, dest_path, zip_file):
-        # type: (RecordPath, str, ZipFile) -> None
+    def __init__(
+        self, src_record_path: RecordPath, dest_path: str, zip_file: ZipFile
+    ) -> None:
         self.src_record_path = src_record_path
         self.dest_path = dest_path
         self._zip_file = zip_file
         self.changed = False
 
-    def _getinfo(self):
-        # type: () -> ZipInfo
+    def _getinfo(self) -> ZipInfo:
         return self._zip_file.getinfo(self.src_record_path)
 
-    def save(self):
-        # type: () -> None
+    def save(self) -> None:
         # directory creation is lazy and after file filtering
         # to ensure we don't install empty dirs; empty dirs can't be
         # uninstalled.
@@ -433,22 +394,19 @@ def save(self):
 
 
 class ScriptFile:
-    def __init__(self, file):
-        # type: (File) -> None
+    def __init__(self, file: "File") -> None:
         self._file = file
         self.src_record_path = self._file.src_record_path
         self.dest_path = self._file.dest_path
         self.changed = False
 
-    def save(self):
-        # type: () -> None
+    def save(self) -> None:
         self._file.save()
         self.changed = fix_script(self.dest_path)
 
 
 class MissingCallableSuffix(InstallationError):
-    def __init__(self, entry_point):
-        # type: (str) -> None
+    def __init__(self, entry_point: str) -> None:
         super().__init__(
             "Invalid script entry point: {} - A callable "
             "suffix is required. Cf https://packaging.python.org/"
@@ -457,31 +415,28 @@ def __init__(self, entry_point):
         )
 
 
-def _raise_for_invalid_entrypoint(specification):
-    # type: (str) -> None
+def _raise_for_invalid_entrypoint(specification: str) -> None:
     entry = get_export_entry(specification)
     if entry is not None and entry.suffix is None:
         raise MissingCallableSuffix(str(entry))
 
 
 class PipScriptMaker(ScriptMaker):
-    def make(self, specification, options=None):
-        # type: (str, Dict[str, Any]) -> List[str]
+    def make(self, specification: str, options: Dict[str, Any] = None) -> List[str]:
         _raise_for_invalid_entrypoint(specification)
         return super().make(specification, options)
 
 
 def _install_wheel(
-    name,  # type: str
-    wheel_zip,  # type: ZipFile
-    wheel_path,  # type: str
-    scheme,  # type: Scheme
-    pycompile=True,  # type: bool
-    warn_script_location=True,  # type: bool
-    direct_url=None,  # type: Optional[DirectUrl]
-    requested=False,  # type: bool
-):
-    # type: (...) -> None
+    name: str,
+    wheel_zip: ZipFile,
+    wheel_path: str,
+    scheme: Scheme,
+    pycompile: bool = True,
+    warn_script_location: bool = True,
+    direct_url: Optional[DirectUrl] = None,
+    requested: bool = False,
+) -> None:
     """Install a wheel.
 
     :param name: Name of the project to install
@@ -508,33 +463,23 @@ def _install_wheel(
     #   installed = files copied from the wheel to the destination
     #   changed = files changed while installing (scripts #! line typically)
     #   generated = files newly generated during the install (script wrappers)
-    installed = {}  # type: Dict[RecordPath, RecordPath]
-    changed = set()  # type: Set[RecordPath]
-    generated = []  # type: List[str]
+    installed: Dict[RecordPath, RecordPath] = {}
+    changed: Set[RecordPath] = set()
+    generated: List[str] = []
 
-    def record_installed(srcfile, destfile, modified=False):
-        # type: (RecordPath, str, bool) -> None
+    def record_installed(
+        srcfile: RecordPath, destfile: str, modified: bool = False
+    ) -> None:
         """Map archive RECORD paths to installation RECORD paths."""
         newpath = _fs_to_record_path(destfile, lib_dir)
         installed[srcfile] = newpath
         if modified:
             changed.add(_fs_to_record_path(destfile))
 
-    def all_paths():
-        # type: () -> Iterable[RecordPath]
-        names = wheel_zip.namelist()
-        # If a flag is set, names may be unicode in Python 2. We convert to
-        # text explicitly so these are valid for lookup in RECORD.
-        decoded_names = map(ensure_text, names)
-        for name in decoded_names:
-            yield cast("RecordPath", name)
-
-    def is_dir_path(path):
-        # type: (RecordPath) -> bool
+    def is_dir_path(path: RecordPath) -> bool:
         return path.endswith("/")
 
-    def assert_no_path_traversal(dest_dir_path, target_path):
-        # type: (str, str) -> None
+    def assert_no_path_traversal(dest_dir_path: str, target_path: str) -> None:
         if not is_within_directory(dest_dir_path, target_path):
             message = (
                 "The wheel {!r} has a file {!r} trying to install"
@@ -544,10 +489,10 @@ def assert_no_path_traversal(dest_dir_path, target_path):
                 message.format(wheel_path, target_path, dest_dir_path)
             )
 
-    def root_scheme_file_maker(zip_file, dest):
-        # type: (ZipFile, str) -> Callable[[RecordPath], File]
-        def make_root_scheme_file(record_path):
-            # type: (RecordPath) -> File
+    def root_scheme_file_maker(
+        zip_file: ZipFile, dest: str
+    ) -> Callable[[RecordPath], "File"]:
+        def make_root_scheme_file(record_path: RecordPath) -> "File":
             normed_path = os.path.normpath(record_path)
             dest_path = os.path.join(dest, normed_path)
             assert_no_path_traversal(dest, dest_path)
@@ -555,17 +500,12 @@ def make_root_scheme_file(record_path):
 
         return make_root_scheme_file
 
-    def data_scheme_file_maker(zip_file, scheme):
-        # type: (ZipFile, Scheme) -> Callable[[RecordPath], File]
-        scheme_paths = {}
-        for key in SCHEME_KEYS:
-            encoded_key = ensure_text(key)
-            scheme_paths[encoded_key] = ensure_text(
-                getattr(scheme, key), encoding=sys.getfilesystemencoding()
-            )
+    def data_scheme_file_maker(
+        zip_file: ZipFile, scheme: Scheme
+    ) -> Callable[[RecordPath], "File"]:
+        scheme_paths = {key: getattr(scheme, key) for key in SCHEME_KEYS}
 
-        def make_data_scheme_file(record_path):
-            # type: (RecordPath) -> File
+        def make_data_scheme_file(record_path: RecordPath) -> "File":
             normed_path = os.path.normpath(record_path)
             try:
                 _, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2)
@@ -584,9 +524,7 @@ def make_data_scheme_file(record_path):
                     "Unknown scheme key used in {}: {} (for file {!r}). .data"
                     " directory contents should be in subdirectories named"
                     " with a valid scheme key ({})"
-                ).format(
-                    wheel_path, scheme_key, record_path, valid_scheme_keys
-                )
+                ).format(wheel_path, scheme_key, record_path, valid_scheme_keys)
                 raise InstallationError(message)
 
             dest_path = os.path.join(scheme_path, dest_subpath)
@@ -595,30 +533,19 @@ def make_data_scheme_file(record_path):
 
         return make_data_scheme_file
 
-    def is_data_scheme_path(path):
-        # type: (RecordPath) -> bool
+    def is_data_scheme_path(path: RecordPath) -> bool:
         return path.split("/", 1)[0].endswith(".data")
 
-    paths = all_paths()
+    paths = cast(List[RecordPath], wheel_zip.namelist())
     file_paths = filterfalse(is_dir_path, paths)
-    root_scheme_paths, data_scheme_paths = partition(
-        is_data_scheme_path, file_paths
-    )
+    root_scheme_paths, data_scheme_paths = partition(is_data_scheme_path, file_paths)
 
-    make_root_scheme_file = root_scheme_file_maker(
-        wheel_zip,
-        ensure_text(lib_dir, encoding=sys.getfilesystemencoding()),
-    )
-    files = map(make_root_scheme_file, root_scheme_paths)
+    make_root_scheme_file = root_scheme_file_maker(wheel_zip, lib_dir)
+    files: Iterator[File] = map(make_root_scheme_file, root_scheme_paths)
 
-    def is_script_scheme_path(path):
-        # type: (RecordPath) -> bool
+    def is_script_scheme_path(path: RecordPath) -> bool:
         parts = path.split("/", 2)
-        return (
-            len(parts) > 2 and
-            parts[0].endswith(".data") and
-            parts[1] == "scripts"
-        )
+        return len(parts) > 2 and parts[0].endswith(".data") and parts[1] == "scripts"
 
     other_scheme_paths, script_scheme_paths = partition(
         is_script_scheme_path, data_scheme_paths
@@ -629,32 +556,32 @@ def is_script_scheme_path(path):
     files = chain(files, other_scheme_files)
 
     # Get the defined entry points
-    distribution = pkg_resources_distribution_for_wheel(
-        wheel_zip, name, wheel_path
+    distribution = get_wheel_distribution(
+        FilesystemWheel(wheel_path),
+        canonicalize_name(name),
     )
     console, gui = get_entrypoints(distribution)
 
-    def is_entrypoint_wrapper(file):
-        # type: (File) -> bool
+    def is_entrypoint_wrapper(file: "File") -> bool:
         # EP, EP.exe and EP-script.py are scripts generated for
         # entry point EP by setuptools
         path = file.dest_path
         name = os.path.basename(path)
-        if name.lower().endswith('.exe'):
+        if name.lower().endswith(".exe"):
             matchname = name[:-4]
-        elif name.lower().endswith('-script.py'):
+        elif name.lower().endswith("-script.py"):
             matchname = name[:-10]
         elif name.lower().endswith(".pya"):
             matchname = name[:-4]
         else:
             matchname = name
         # Ignore setuptools-generated scripts
-        return (matchname in console or matchname in gui)
+        return matchname in console or matchname in gui
 
-    script_scheme_files = map(make_data_scheme_file, script_scheme_paths)
-    script_scheme_files = filterfalse(
-        is_entrypoint_wrapper, script_scheme_files
+    script_scheme_files: Iterator[File] = map(
+        make_data_scheme_file, script_scheme_paths
     )
+    script_scheme_files = filterfalse(is_entrypoint_wrapper, script_scheme_files)
     script_scheme_files = map(ScriptFile, script_scheme_files)
     files = chain(files, script_scheme_files)
 
@@ -662,8 +589,7 @@ def is_entrypoint_wrapper(file):
         file.save()
         record_installed(file.src_record_path, file.dest_path, file.changed)
 
-    def pyc_source_file_paths():
-        # type: () -> Iterator[str]
+    def pyc_source_file_paths() -> Iterator[str]:
         # We de-duplicate installation paths, since there can be overlap (e.g.
         # file in .data maps to same location as file in wheel root).
         # Sorting installation paths makes it easier to reproduce and debug
@@ -672,30 +598,21 @@ def pyc_source_file_paths():
             full_installed_path = os.path.join(lib_dir, installed_path)
             if not os.path.isfile(full_installed_path):
                 continue
-            if not full_installed_path.endswith('.py'):
+            if not full_installed_path.endswith(".py"):
                 continue
             yield full_installed_path
 
-    def pyc_output_path(path):
-        # type: (str) -> str
-        """Return the path the pyc file would have been written to.
-        """
+    def pyc_output_path(path: str) -> str:
+        """Return the path the pyc file would have been written to."""
         return importlib.util.cache_from_source(path)
 
     # Compile all of the pyc files for the installed files
     if pycompile:
         with captured_stdout() as stdout:
             with warnings.catch_warnings():
-                warnings.filterwarnings('ignore')
+                warnings.filterwarnings("ignore")
                 for path in pyc_source_file_paths():
-                    # Python 2's `compileall.compile_file` requires a str in
-                    # error cases, so we must convert to the native type.
-                    path_arg = ensure_str(
-                        path, encoding=sys.getfilesystemencoding()
-                    )
-                    success = compileall.compile_file(
-                        path_arg, force=True, quiet=True
-                    )
+                    success = compileall.compile_file(path, force=True, quiet=True)
                     if success:
                         pyc_path = pyc_output_path(path)
                         assert os.path.exists(pyc_path)
@@ -714,7 +631,7 @@ def pyc_output_path(path):
     # Ensure we don't generate any variants for scripts because this is almost
     # never what somebody wants.
     # See https://bitbucket.org/pypa/distlib/issue/35/
-    maker.variants = {''}
+    maker.variants = {""}
 
     # This is required because otherwise distlib creates scripts that are not
     # executable.
@@ -724,14 +641,12 @@ def pyc_output_path(path):
     # Generate the console and GUI entry points specified in the wheel
     scripts_to_generate = get_console_script_specs(console)
 
-    gui_scripts_to_generate = list(starmap('{} = {}'.format, gui.items()))
+    gui_scripts_to_generate = list(starmap("{} = {}".format, gui.items()))
 
     generated_console_scripts = maker.make_multiple(scripts_to_generate)
     generated.extend(generated_console_scripts)
 
-    generated.extend(
-        maker.make_multiple(gui_scripts_to_generate, {'gui': True})
-    )
+    generated.extend(maker.make_multiple(gui_scripts_to_generate, {"gui": True}))
 
     if warn_script_location:
         msg = message_about_scripts_not_on_PATH(generated_console_scripts)
@@ -741,8 +656,7 @@ def pyc_output_path(path):
     generated_file_mode = 0o666 & ~current_umask()
 
     @contextlib.contextmanager
-    def _generate_file(path, **kwargs):
-        # type: (str, **Any) -> Iterator[BinaryIO]
+    def _generate_file(path: str, **kwargs: Any) -> Iterator[BinaryIO]:
         with adjacent_tmp_file(path, **kwargs) as f:
             yield f
         os.chmod(f.name, generated_file_mode)
@@ -751,9 +665,9 @@ def _generate_file(path, **kwargs):
     dest_info_dir = os.path.join(lib_dir, info_dir)
 
     # Record pip as the installer
-    installer_path = os.path.join(dest_info_dir, 'INSTALLER')
+    installer_path = os.path.join(dest_info_dir, "INSTALLER")
     with _generate_file(installer_path) as installer_file:
-        installer_file.write(b'pip\n')
+        installer_file.write(b"pip\n")
     generated.append(installer_path)
 
     # Record the PEP 610 direct URL reference
@@ -765,12 +679,12 @@ def _generate_file(path, **kwargs):
 
     # Record the REQUESTED file
     if requested:
-        requested_path = os.path.join(dest_info_dir, 'REQUESTED')
-        with open(requested_path, "w"):
+        requested_path = os.path.join(dest_info_dir, "REQUESTED")
+        with open(requested_path, "wb"):
             pass
         generated.append(requested_path)
 
-    record_text = distribution.get_metadata('RECORD')
+    record_text = distribution.read_text("RECORD")
     record_rows = list(csv.reader(record_text.splitlines()))
 
     rows = get_csv_rows_for_installed(
@@ -778,42 +692,38 @@ def _generate_file(path, **kwargs):
         installed=installed,
         changed=changed,
         generated=generated,
-        lib_dir=lib_dir)
+        lib_dir=lib_dir,
+    )
 
     # Record details of all files installed
-    record_path = os.path.join(dest_info_dir, 'RECORD')
+    record_path = os.path.join(dest_info_dir, "RECORD")
 
-    with _generate_file(record_path, **csv_io_kwargs('w')) as record_file:
-        # The type mypy infers for record_file is different for Python 3
-        # (typing.IO[Any]) and Python 2 (typing.BinaryIO). We explicitly
-        # cast to typing.IO[str] as a workaround.
-        writer = csv.writer(cast('IO[str]', record_file))
+    with _generate_file(record_path, **csv_io_kwargs("w")) as record_file:
+        # Explicitly cast to typing.IO[str] as a workaround for the mypy error:
+        # "writer" has incompatible type "BinaryIO"; expected "_Writer"
+        writer = csv.writer(cast("IO[str]", record_file))
         writer.writerows(_normalized_outrows(rows))
 
 
 @contextlib.contextmanager
-def req_error_context(req_description):
-    # type: (str) -> Iterator[None]
+def req_error_context(req_description: str) -> Iterator[None]:
     try:
         yield
     except InstallationError as e:
         message = "For req: {}. {}".format(req_description, e.args[0])
-        reraise(
-            InstallationError, InstallationError(message), sys.exc_info()[2]
-        )
+        raise InstallationError(message) from e
 
 
 def install_wheel(
-    name,  # type: str
-    wheel_path,  # type: str
-    scheme,  # type: Scheme
-    req_description,  # type: str
-    pycompile=True,  # type: bool
-    warn_script_location=True,  # type: bool
-    direct_url=None,  # type: Optional[DirectUrl]
-    requested=False,  # type: bool
-):
-    # type: (...) -> None
+    name: str,
+    wheel_path: str,
+    scheme: Scheme,
+    req_description: str,
+    pycompile: bool = True,
+    warn_script_location: bool = True,
+    direct_url: Optional[DirectUrl] = None,
+    requested: bool = False,
+) -> None:
     with ZipFile(wheel_path, allowZip64=True) as z:
         with req_error_context(req_description):
             _install_wheel(
diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py
index 977d83a1b4e..46252816dcc 100644
--- a/src/pip/_internal/operations/prepare.py
+++ b/src/pip/_internal/operations/prepare.py
@@ -8,6 +8,7 @@
 import mimetypes
 import os
 import shutil
+from typing import Dict, Iterable, List, Optional
 
 from pip._vendor.packaging.utils import canonicalize_name
 
@@ -22,62 +23,50 @@
     PreviousBuildDirError,
     VcsHashUnsupported,
 )
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
+from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
 from pip._internal.network.download import BatchDownloader, Downloader
 from pip._internal.network.lazy_wheel import (
     HTTPRangeRequestUnsupported,
     dist_from_wheel_url,
 )
+from pip._internal.network.session import PipSession
+from pip._internal.operations.build.build_tracker import BuildTracker
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.filesystem import copy2_fixed
-from pip._internal.utils.hashes import MissingHashes
+from pip._internal.utils.hashes import Hashes, MissingHashes
 from pip._internal.utils.logging import indent_log
-from pip._internal.utils.misc import display_path, hide_url, path_to_display, rmtree
+from pip._internal.utils.misc import display_path, hide_url, is_installable_dir, rmtree
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.unpacking import unpack_file
 from pip._internal.vcs import vcs
 
-if MYPY_CHECK_RUNNING:
-    from typing import Dict, Iterable, List, Optional, Tuple
-
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.index.package_finder import PackageFinder
-    from pip._internal.models.link import Link
-    from pip._internal.network.session import PipSession
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._internal.req.req_tracker import RequirementTracker
-    from pip._internal.utils.hashes import Hashes
-
-
 logger = logging.getLogger(__name__)
 
 
 def _get_prepared_distribution(
-    req,  # type: InstallRequirement
-    req_tracker,  # type: RequirementTracker
-    finder,  # type: PackageFinder
-    build_isolation,  # type: bool
-):
-    # type: (...) -> Distribution
+    req: InstallRequirement,
+    build_tracker: BuildTracker,
+    finder: PackageFinder,
+    build_isolation: bool,
+) -> BaseDistribution:
     """Prepare a distribution for installation."""
     abstract_dist = make_distribution_for_install_requirement(req)
-    with req_tracker.track(req):
+    with build_tracker.track(req):
         abstract_dist.prepare_distribution_metadata(finder, build_isolation)
-    return abstract_dist.get_pkg_resources_distribution()
+    return abstract_dist.get_metadata_distribution()
 
 
-def unpack_vcs_link(link, location):
-    # type: (Link, str) -> None
+def unpack_vcs_link(link: Link, location: str, verbosity: int) -> None:
     vcs_backend = vcs.get_backend_for_scheme(link.scheme)
     assert vcs_backend is not None
-    vcs_backend.unpack(location, url=hide_url(link.url))
+    vcs_backend.unpack(location, url=hide_url(link.url), verbosity=verbosity)
 
 
 class File:
-
-    def __init__(self, path, content_type):
-        # type: (str, Optional[str]) -> None
+    def __init__(self, path: str, content_type: Optional[str]) -> None:
         self.path = path
         if content_type is None:
             self.content_type = mimetypes.guess_type(path)[0]
@@ -86,19 +75,16 @@ def __init__(self, path, content_type):
 
 
 def get_http_url(
-    link,  # type: Link
-    download,  # type: Downloader
-    download_dir=None,  # type: Optional[str]
-    hashes=None,  # type: Optional[Hashes]
-):
-    # type: (...) -> File
+    link: Link,
+    download: Downloader,
+    download_dir: Optional[str] = None,
+    hashes: Optional[Hashes] = None,
+) -> File:
     temp_dir = TempDirectory(kind="unpack", globally_managed=True)
     # If a download dir is specified, is the file already downloaded there?
     already_downloaded_path = None
     if download_dir:
-        already_downloaded_path = _check_download_dir(
-            link, download_dir, hashes
-        )
+        already_downloaded_path = _check_download_dir(link, download_dir, hashes)
 
     if already_downloaded_path:
         from_path = already_downloaded_path
@@ -112,8 +98,7 @@ def get_http_url(
     return File(from_path, content_type)
 
 
-def _copy2_ignoring_special_files(src, dest):
-    # type: (str, str) -> None
+def _copy2_ignoring_special_files(src: str, dest: str) -> None:
     """Copying special files is not supported, but as a convenience to users
     we skip errors copying them. This supports tools that may create e.g.
     socket files in the project source directory.
@@ -128,26 +113,24 @@ def _copy2_ignoring_special_files(src, dest):
         logger.warning(
             "Ignoring special file error '%s' encountered copying %s to %s.",
             str(e),
-            path_to_display(src),
-            path_to_display(dest),
+            src,
+            dest,
         )
 
 
-def _copy_source_tree(source, target):
-    # type: (str, str) -> None
+def _copy_source_tree(source: str, target: str) -> None:
     target_abspath = os.path.abspath(target)
     target_basename = os.path.basename(target_abspath)
     target_dirname = os.path.dirname(target_abspath)
 
-    def ignore(d, names):
-        # type: (str, List[str]) -> List[str]
-        skipped = []  # type: List[str]
+    def ignore(d: str, names: List[str]) -> List[str]:
+        skipped: List[str] = []
         if d == source:
             # Pulling in those directories can potentially be very slow,
             # exclude the following directories if they appear in the top
             # level dir (and only it).
             # See discussion at https://github.com/pypa/pip/pull/6770
-            skipped += ['.tox', '.nox']
+            skipped += [".tox", ".nox"]
         if os.path.abspath(d) == target_dirname:
             # Prevent an infinite recursion if the target is in source.
             # This can happen when TMPDIR is set to ${PWD}/...
@@ -165,19 +148,13 @@ def ignore(d, names):
 
 
 def get_file_url(
-    link,  # type: Link
-    download_dir=None,  # type: Optional[str]
-    hashes=None  # type: Optional[Hashes]
-):
-    # type: (...) -> File
-    """Get file and optionally check its hash.
-    """
+    link: Link, download_dir: Optional[str] = None, hashes: Optional[Hashes] = None
+) -> File:
+    """Get file and optionally check its hash."""
     # If a download dir is specified, is the file already there and valid?
     already_downloaded_path = None
     if download_dir:
-        already_downloaded_path = _check_download_dir(
-            link, download_dir, hashes
-        )
+        already_downloaded_path = _check_download_dir(link, download_dir, hashes)
 
     if already_downloaded_path:
         from_path = already_downloaded_path
@@ -195,13 +172,13 @@ def get_file_url(
 
 
 def unpack_url(
-    link,  # type: Link
-    location,  # type: str
-    download,  # type: Downloader
-    download_dir=None,  # type: Optional[str]
-    hashes=None,  # type: Optional[Hashes]
-):
-    # type: (...) -> Optional[File]
+    link: Link,
+    location: str,
+    download: Downloader,
+    verbosity: int,
+    download_dir: Optional[str] = None,
+    hashes: Optional[Hashes] = None,
+) -> Optional[File]:
     """Unpack link into location, downloading if required.
 
     :param hashes: A Hashes object, one of whose embedded hashes must match,
@@ -211,10 +188,17 @@ def unpack_url(
     """
     # non-editable vcs urls
     if link.is_vcs:
-        unpack_vcs_link(link, location)
+        unpack_vcs_link(link, location, verbosity=verbosity)
         return None
 
-    # If it's a url to a local directory
+    # Once out-of-tree-builds are no longer supported, could potentially
+    # replace the below condition with `assert not link.is_existing_dir`
+    # - unpack_url does not need to be called for in-tree-builds.
+    #
+    # As further cleanup, _copy_source_tree and accompanying tests can
+    # be removed.
+    #
+    # TODO when use-deprecated=out-of-tree-build is removed
     if link.is_existing_dir():
         if os.path.isdir(location):
             rmtree(location)
@@ -242,10 +226,11 @@ def unpack_url(
     return file
 
 
-def _check_download_dir(link, download_dir, hashes):
-    # type: (Link, str, Optional[Hashes]) -> Optional[str]
-    """ Check download_dir for previously downloaded file with correct hash
-        If a correct file is found return its path else None
+def _check_download_dir(
+    link: Link, download_dir: str, hashes: Optional[Hashes]
+) -> Optional[str]:
+    """Check download_dir for previously downloaded file with correct hash
+    If a correct file is found return its path else None
     """
     download_path = os.path.join(download_dir, link.filename)
 
@@ -253,15 +238,14 @@ def _check_download_dir(link, download_dir, hashes):
         return None
 
     # If already downloaded, does its hash match?
-    logger.info('File was already downloaded %s', download_path)
+    logger.info("File was already downloaded %s", download_path)
     if hashes:
         try:
             hashes.check_against_path(download_path)
         except HashMismatch:
             logger.warning(
-                'Previously-downloaded file %s has bad hash. '
-                'Re-downloading.',
-                download_path
+                "Previously-downloaded file %s has bad hash. Re-downloading.",
+                download_path,
             )
             os.unlink(download_path)
             return None
@@ -269,29 +253,29 @@ def _check_download_dir(link, download_dir, hashes):
 
 
 class RequirementPreparer:
-    """Prepares a Requirement
-    """
+    """Prepares a Requirement"""
 
     def __init__(
         self,
-        build_dir,  # type: str
-        download_dir,  # type: Optional[str]
-        src_dir,  # type: str
-        build_isolation,  # type: bool
-        req_tracker,  # type: RequirementTracker
-        session,  # type: PipSession
-        progress_bar,  # type: str
-        finder,  # type: PackageFinder
-        require_hashes,  # type: bool
-        use_user_site,  # type: bool
-        lazy_wheel,  # type: bool
-    ):
-        # type: (...) -> None
+        build_dir: str,
+        download_dir: Optional[str],
+        src_dir: str,
+        build_isolation: bool,
+        build_tracker: BuildTracker,
+        session: PipSession,
+        progress_bar: str,
+        finder: PackageFinder,
+        require_hashes: bool,
+        use_user_site: bool,
+        lazy_wheel: bool,
+        verbosity: int,
+        in_tree_build: bool,
+    ) -> None:
         super().__init__()
 
         self.src_dir = src_dir
         self.build_dir = build_dir
-        self.req_tracker = req_tracker
+        self.build_tracker = build_tracker
         self._session = session
         self._download = Downloader(session, progress_bar)
         self._batch_download = BatchDownloader(session, progress_bar)
@@ -313,14 +297,19 @@ def __init__(
         # Should wheels be downloaded lazily?
         self.use_lazy_wheel = lazy_wheel
 
-        # Memoized downloaded files, as mapping of url: (path, mime type)
-        self._downloaded = {}  # type: Dict[str, Tuple[str, str]]
+        # How verbose should underlying tooling be?
+        self.verbosity = verbosity
+
+        # Should in-tree builds be used for local paths?
+        self.in_tree_build = in_tree_build
+
+        # Memoized downloaded files, as mapping of url: path.
+        self._downloaded: Dict[str, str] = {}
 
         # Previous "header" printed for a link-based InstallRequirement
         self._previous_requirement_header = ("", "")
 
-    def _log_preparing_link(self, req):
-        # type: (InstallRequirement) -> None
+    def _log_preparing_link(self, req: InstallRequirement) -> None:
         """Provide context for the requirement being prepared."""
         if req.link.is_file and not req.original_link_is_in_wheel_cache:
             message = "Processing %s"
@@ -337,8 +326,9 @@ def _log_preparing_link(self, req):
             with indent_log():
                 logger.info("Using cached %s", req.link.filename)
 
-    def _ensure_link_req_src_dir(self, req, parallel_builds):
-        # type: (InstallRequirement, bool) -> None
+    def _ensure_link_req_src_dir(
+        self, req: InstallRequirement, parallel_builds: bool
+    ) -> None:
         """Ensure source_dir of a linked InstallRequirement."""
         # Since source_dir is only set for editable requirements.
         if req.link.is_wheel:
@@ -346,6 +336,11 @@ def _ensure_link_req_src_dir(self, req, parallel_builds):
             # directory.
             return
         assert req.source_dir is None
+        if req.link.is_existing_dir() and self.in_tree_build:
+            # build local directories in-tree
+            req.source_dir = req.link.file_path
+            return
+
         # We always delete unpacked sdists after pip runs.
         req.ensure_has_source_dir(
             self.build_dir,
@@ -358,7 +353,8 @@ def _ensure_link_req_src_dir(self, req, parallel_builds):
         # installation.
         # FIXME: this won't upgrade when there's an existing
         # package unpacked in `req.source_dir`
-        if os.path.exists(os.path.join(req.source_dir, 'setup.py')):
+        # TODO: this check is now probably dead code
+        if is_installable_dir(req.source_dir):
             raise PreviousBuildDirError(
                 "pip can't proceed with requirements '{}' due to a"
                 "pre-existing build directory ({}). This is likely "
@@ -367,8 +363,7 @@ def _ensure_link_req_src_dir(self, req, parallel_builds):
                 "Please delete it and try again.".format(req, req.source_dir)
             )
 
-    def _get_linked_req_hashes(self, req):
-        # type: (InstallRequirement) -> Hashes
+    def _get_linked_req_hashes(self, req: InstallRequirement) -> Hashes:
         # By the time this is called, the requirement's link should have
         # been checked so we can tell what kind of requirements req is
         # and raise some more informative errors than otherwise.
@@ -400,18 +395,19 @@ def _get_linked_req_hashes(self, req):
         # showing the user what the hash should be.
         return req.hashes(trust_internet=False) or MissingHashes()
 
-    def _fetch_metadata_using_lazy_wheel(self, link):
-        # type: (Link) -> Optional[Distribution]
+    def _fetch_metadata_using_lazy_wheel(
+        self,
+        link: Link,
+    ) -> Optional[BaseDistribution]:
         """Fetch metadata using lazy wheel, if possible."""
         if not self.use_lazy_wheel:
             return None
         if self.require_hashes:
-            logger.debug('Lazy wheel is not used as hash checking is required')
+            logger.debug("Lazy wheel is not used as hash checking is required")
             return None
         if link.is_file or not link.is_wheel:
             logger.debug(
-                'Lazy wheel is not used as '
-                '%r does not points to a remote wheel',
+                "Lazy wheel is not used as %r does not points to a remote wheel",
                 link,
             )
             return None
@@ -419,18 +415,52 @@ def _fetch_metadata_using_lazy_wheel(self, link):
         wheel = Wheel(link.filename)
         name = canonicalize_name(wheel.name)
         logger.info(
-            'Obtaining dependency information from %s %s',
-            name, wheel.version,
+            "Obtaining dependency information from %s %s",
+            name,
+            wheel.version,
         )
-        url = link.url.split('#', 1)[0]
+        url = link.url.split("#", 1)[0]
         try:
             return dist_from_wheel_url(name, url, self._session)
         except HTTPRangeRequestUnsupported:
-            logger.debug('%s does not support range requests', url)
+            logger.debug("%s does not support range requests", url)
             return None
 
-    def prepare_linked_requirement(self, req, parallel_builds=False):
-        # type: (InstallRequirement, bool) -> Distribution
+    def _complete_partial_requirements(
+        self,
+        partially_downloaded_reqs: Iterable[InstallRequirement],
+        parallel_builds: bool = False,
+    ) -> None:
+        """Download any requirements which were only fetched by metadata."""
+        # Download to a temporary directory. These will be copied over as
+        # needed for downstream 'download', 'wheel', and 'install' commands.
+        temp_dir = TempDirectory(kind="unpack", globally_managed=True).path
+
+        # Map each link to the requirement that owns it. This allows us to set
+        # `req.local_file_path` on the appropriate requirement after passing
+        # all the links at once into BatchDownloader.
+        links_to_fully_download: Dict[Link, InstallRequirement] = {}
+        for req in partially_downloaded_reqs:
+            assert req.link
+            links_to_fully_download[req.link] = req
+
+        batch_download = self._batch_download(
+            links_to_fully_download.keys(),
+            temp_dir,
+        )
+        for link, (filepath, _) in batch_download:
+            logger.debug("Downloading link %s to %s", link, filepath)
+            req = links_to_fully_download[link]
+            req.local_file_path = filepath
+
+        # This step is necessary to ensure all lazy wheels are processed
+        # successfully by the 'download', 'wheel', and 'install' commands.
+        for req in partially_downloaded_reqs:
+            self._prepare_linked_requirement(req, parallel_builds)
+
+    def prepare_linked_requirement(
+        self, req: InstallRequirement, parallel_builds: bool = False
+    ) -> BaseDistribution:
         """Prepare a requirement to be obtained from req.link."""
         assert req.link
         link = req.link
@@ -445,7 +475,7 @@ def prepare_linked_requirement(self, req, parallel_builds=False):
 
             if file_path is not None:
                 # The file is already available, so mark it as downloaded
-                self._downloaded[req.link.url] = file_path, None
+                self._downloaded[req.link.url] = file_path
             else:
                 # The file is not available, attempt to fetch only metadata
                 wheel_dist = self._fetch_metadata_using_lazy_wheel(link)
@@ -456,41 +486,67 @@ def prepare_linked_requirement(self, req, parallel_builds=False):
             # None of the optimizations worked, fully prepare the requirement
             return self._prepare_linked_requirement(req, parallel_builds)
 
-    def prepare_linked_requirements_more(self, reqs, parallel_builds=False):
-        # type: (Iterable[InstallRequirement], bool) -> None
-        """Prepare a linked requirement more, if needed."""
+    def prepare_linked_requirements_more(
+        self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
+    ) -> None:
+        """Prepare linked requirements more, if needed."""
         reqs = [req for req in reqs if req.needs_more_preparation]
-        links = [req.link for req in reqs]
+        for req in reqs:
+            # Determine if any of these requirements were already downloaded.
+            if self.download_dir is not None and req.link.is_wheel:
+                hashes = self._get_linked_req_hashes(req)
+                file_path = _check_download_dir(req.link, self.download_dir, hashes)
+                if file_path is not None:
+                    self._downloaded[req.link.url] = file_path
+                    req.needs_more_preparation = False
 
-        # Let's download to a temporary directory.
-        tmpdir = TempDirectory(kind="unpack", globally_managed=True).path
-        self._downloaded.update(self._batch_download(links, tmpdir))
+        # Prepare requirements we found were already downloaded for some
+        # reason. The other downloads will be completed separately.
+        partially_downloaded_reqs: List[InstallRequirement] = []
         for req in reqs:
-            self._prepare_linked_requirement(req, parallel_builds)
+            if req.needs_more_preparation:
+                partially_downloaded_reqs.append(req)
+            else:
+                self._prepare_linked_requirement(req, parallel_builds)
 
-    def _prepare_linked_requirement(self, req, parallel_builds):
-        # type: (InstallRequirement, bool) -> Distribution
+        # TODO: separate this part out from RequirementPreparer when the v1
+        # resolver can be removed!
+        self._complete_partial_requirements(
+            partially_downloaded_reqs,
+            parallel_builds=parallel_builds,
+        )
+
+    def _prepare_linked_requirement(
+        self, req: InstallRequirement, parallel_builds: bool
+    ) -> BaseDistribution:
         assert req.link
         link = req.link
 
         self._ensure_link_req_src_dir(req, parallel_builds)
         hashes = self._get_linked_req_hashes(req)
-        if link.url not in self._downloaded:
+
+        if link.is_existing_dir() and self.in_tree_build:
+            local_file = None
+        elif link.url not in self._downloaded:
             try:
                 local_file = unpack_url(
-                    link, req.source_dir, self._download,
-                    self.download_dir, hashes,
+                    link,
+                    req.source_dir,
+                    self._download,
+                    self.verbosity,
+                    self.download_dir,
+                    hashes,
                 )
             except NetworkConnectionError as exc:
                 raise InstallationError(
-                    'Could not install requirement {} because of HTTP '
-                    'error {} for URL {}'.format(req, exc, link)
+                    "Could not install requirement {} because of HTTP "
+                    "error {} for URL {}".format(req, exc, link)
                 )
         else:
-            file_path, content_type = self._downloaded[link.url]
+            file_path = self._downloaded[link.url]
             if hashes:
                 hashes.check_against_path(file_path)
-            local_file = File(file_path, content_type)
+            local_file = File(file_path, content_type=None)
 
         # For use in later processing,
         # preserve the file path on the requirement.
@@ -498,12 +554,14 @@ def _prepare_linked_requirement(self, req, parallel_builds):
             req.local_file_path = local_file.path
 
         dist = _get_prepared_distribution(
-            req, self.req_tracker, self.finder, self.build_isolation,
+            req,
+            self.build_tracker,
+            self.finder,
+            self.build_isolation,
         )
         return dist
 
-    def save_linked_requirement(self, req):
-        # type: (InstallRequirement) -> None
+    def save_linked_requirement(self, req: InstallRequirement) -> None:
         assert self.download_dir is not None
         assert req.link is not None
         link = req.link
@@ -514,8 +572,9 @@ def save_linked_requirement(self, req):
 
         if link.is_existing_dir():
             logger.debug(
-                'Not copying link to destination directory '
-                'since it is a directory: %s', link,
+                "Not copying link to destination directory "
+                "since it is a directory: %s",
+                link,
             )
             return
         if req.local_file_path is None:
@@ -526,31 +585,32 @@ def save_linked_requirement(self, req):
         if not os.path.exists(download_location):
             shutil.copy(req.local_file_path, download_location)
             download_path = display_path(download_location)
-            logger.info('Saved %s', download_path)
+            logger.info("Saved %s", download_path)
 
     def prepare_editable_requirement(
         self,
-        req,  # type: InstallRequirement
-    ):
-        # type: (...) -> Distribution
-        """Prepare an editable requirement
-        """
+        req: InstallRequirement,
+    ) -> BaseDistribution:
+        """Prepare an editable requirement."""
         assert req.editable, "cannot prepare a non-editable req as editable"
 
-        logger.info('Obtaining %s', req)
+        logger.info("Obtaining %s", req)
 
         with indent_log():
             if self.require_hashes:
                 raise InstallationError(
-                    'The editable requirement {} cannot be installed when '
-                    'requiring hashes, because there is no single file to '
-                    'hash.'.format(req)
+                    "The editable requirement {} cannot be installed when "
+                    "requiring hashes, because there is no single file to "
+                    "hash.".format(req)
                 )
             req.ensure_has_source_dir(self.src_dir)
-            req.update_editable(self.download_dir is None)
+            req.update_editable()
 
             dist = _get_prepared_distribution(
-                req, self.req_tracker, self.finder, self.build_isolation,
+                req,
+                self.build_tracker,
+                self.finder,
+                self.build_isolation,
             )
 
             req.check_if_exists(self.use_user_site)
@@ -559,27 +619,24 @@ def prepare_editable_requirement(
 
     def prepare_installed_requirement(
         self,
-        req,  # type: InstallRequirement
-        skip_reason  # type: str
-    ):
-        # type: (...) -> Distribution
-        """Prepare an already-installed requirement
-        """
+        req: InstallRequirement,
+        skip_reason: str,
+    ) -> BaseDistribution:
+        """Prepare an already-installed requirement."""
         assert req.satisfied_by, "req should have been satisfied but isn't"
         assert skip_reason is not None, (
             "did not get skip reason skipped but req.satisfied_by "
             "is set to {}".format(req.satisfied_by)
         )
         logger.info(
-            'Requirement %s: %s (%s)',
-            skip_reason, req, req.satisfied_by.version
+            "Requirement %s: %s (%s)", skip_reason, req, req.satisfied_by.version
         )
         with indent_log():
             if self.require_hashes:
                 logger.debug(
-                    'Since it is already installed, we are trusting this '
-                    'package without checking its hash. To ensure a '
-                    'completely repeatable environment, install into an '
-                    'empty virtualenv.'
+                    "Since it is already installed, we are trusting this "
+                    "package without checking its hash. To ensure a "
+                    "completely repeatable environment, install into an "
+                    "empty virtualenv."
                 )
-            return InstalledDistribution(req).get_pkg_resources_distribution()
+            return InstalledDistribution(req).get_metadata_distribution()
diff --git a/src/pip/_internal/pyproject.py b/src/pip/_internal/pyproject.py
index 68ca53bf0bf..e183eaf8658 100644
--- a/src/pip/_internal/pyproject.py
+++ b/src/pip/_internal/pyproject.py
@@ -1,41 +1,33 @@
 import os
 from collections import namedtuple
+from typing import Any, List, Optional
 
-from pip._vendor import toml
+from pip._vendor import tomli
 from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
 
-from pip._internal.exceptions import InstallationError
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.exceptions import (
+    InstallationError,
+    InvalidPyProjectBuildRequires,
+    MissingPyProjectBuildRequires,
+)
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, List, Optional
 
+def _is_list_of_str(obj: Any) -> bool:
+    return isinstance(obj, list) and all(isinstance(item, str) for item in obj)
 
-def _is_list_of_str(obj):
-    # type: (Any) -> bool
-    return (
-        isinstance(obj, list) and
-        all(isinstance(item, str) for item in obj)
-    )
 
+def make_pyproject_path(unpacked_source_directory: str) -> str:
+    return os.path.join(unpacked_source_directory, "pyproject.toml")
 
-def make_pyproject_path(unpacked_source_directory):
-    # type: (str) -> str
-    return os.path.join(unpacked_source_directory, 'pyproject.toml')
 
-
-BuildSystemDetails = namedtuple('BuildSystemDetails', [
-    'requires', 'backend', 'check', 'backend_path'
-])
+BuildSystemDetails = namedtuple(
+    "BuildSystemDetails", ["requires", "backend", "check", "backend_path"]
+)
 
 
 def load_pyproject_toml(
-    use_pep517,  # type: Optional[bool]
-    pyproject_toml,  # type: str
-    setup_py,  # type: str
-    req_name  # type: str
-):
-    # type: (...) -> Optional[BuildSystemDetails]
+    use_pep517: Optional[bool], pyproject_toml: str, setup_py: str, req_name: str
+) -> Optional[BuildSystemDetails]:
     """Load the pyproject.toml file.
 
     Parameters:
@@ -60,9 +52,15 @@ def load_pyproject_toml(
     has_pyproject = os.path.isfile(pyproject_toml)
     has_setup = os.path.isfile(setup_py)
 
+    if not has_pyproject and not has_setup:
+        raise InstallationError(
+            f"{req_name} does not appear to be a Python project: "
+            f"neither 'setup.py' nor 'pyproject.toml' found."
+        )
+
     if has_pyproject:
         with open(pyproject_toml, encoding="utf-8") as f:
-            pp_toml = toml.load(f)
+            pp_toml = tomli.loads(f.read())
         build_system = pp_toml.get("build-system")
     else:
         build_system = None
@@ -85,9 +83,7 @@ def load_pyproject_toml(
             raise InstallationError(
                 "Disabling PEP 517 processing is invalid: "
                 "project specifies a build backend of {} "
-                "in pyproject.toml".format(
-                    build_system["build-backend"]
-                )
+                "in pyproject.toml".format(build_system["build-backend"])
             )
         use_pep517 = True
 
@@ -127,46 +123,32 @@ def load_pyproject_toml(
 
     # Ensure that the build-system section in pyproject.toml conforms
     # to PEP 518.
-    error_template = (
-        "{package} has a pyproject.toml file that does not comply "
-        "with PEP 518: {reason}"
-    )
 
     # Specifying the build-system table but not the requires key is invalid
     if "requires" not in build_system:
-        raise InstallationError(
-            error_template.format(package=req_name, reason=(
-                "it has a 'build-system' table but not "
-                "'build-system.requires' which is mandatory in the table"
-            ))
-        )
+        raise MissingPyProjectBuildRequires(package=req_name)
 
     # Error out if requires is not a list of strings
     requires = build_system["requires"]
     if not _is_list_of_str(requires):
-        raise InstallationError(error_template.format(
+        raise InvalidPyProjectBuildRequires(
             package=req_name,
-            reason="'build-system.requires' is not a list of strings.",
-        ))
+            reason="It is not a list of strings.",
+        )
 
     # Each requirement must be valid as per PEP 508
     for requirement in requires:
         try:
             Requirement(requirement)
-        except InvalidRequirement:
-            raise InstallationError(
-                error_template.format(
-                    package=req_name,
-                    reason=(
-                        "'build-system.requires' contains an invalid "
-                        "requirement: {!r}".format(requirement)
-                    ),
-                )
-            )
+        except InvalidRequirement as error:
+            raise InvalidPyProjectBuildRequires(
+                package=req_name,
+                reason=f"It contains an invalid requirement: {requirement!r}",
+            ) from error
 
     backend = build_system.get("build-backend")
     backend_path = build_system.get("backend-path", [])
-    check = []  # type: List[str]
+    check: List[str] = []
     if backend is None:
         # If the user didn't specify a backend, we assume they want to use
         # the setuptools backend. But we can't be sure they have included
diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py
index 352d8923f17..70dea27a6a8 100644
--- a/src/pip/_internal/req/__init__.py
+++ b/src/pip/_internal/req/__init__.py
@@ -1,55 +1,50 @@
 import collections
 import logging
+from typing import Iterator, List, Optional, Sequence, Tuple
 
 from pip._internal.utils.logging import indent_log
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 from .req_file import parse_requirements
 from .req_install import InstallRequirement
 from .req_set import RequirementSet
 
-if MYPY_CHECK_RUNNING:
-    from typing import Iterator, List, Optional, Sequence, Tuple
-
 __all__ = [
-    "RequirementSet", "InstallRequirement",
-    "parse_requirements", "install_given_reqs",
+    "RequirementSet",
+    "InstallRequirement",
+    "parse_requirements",
+    "install_given_reqs",
 ]
 
 logger = logging.getLogger(__name__)
 
 
 class InstallationResult:
-    def __init__(self, name):
-        # type: (str) -> None
+    def __init__(self, name: str) -> None:
         self.name = name
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return f"InstallationResult(name={self.name!r})"
 
 
 def _validate_requirements(
-    requirements,  # type: List[InstallRequirement]
-):
-    # type: (...) -> Iterator[Tuple[str, InstallRequirement]]
+    requirements: List[InstallRequirement],
+) -> Iterator[Tuple[str, InstallRequirement]]:
     for req in requirements:
         assert req.name, f"invalid to-be-installed requirement: {req}"
         yield req.name, req
 
 
 def install_given_reqs(
-    requirements,  # type: List[InstallRequirement]
-    install_options,  # type: List[str]
-    global_options,  # type: Sequence[str]
-    root,  # type: Optional[str]
-    home,  # type: Optional[str]
-    prefix,  # type: Optional[str]
-    warn_script_location,  # type: bool
-    use_user_site,  # type: bool
-    pycompile,  # type: bool
-):
-    # type: (...) -> List[InstallationResult]
+    requirements: List[InstallRequirement],
+    install_options: List[str],
+    global_options: Sequence[str],
+    root: Optional[str],
+    home: Optional[str],
+    prefix: Optional[str],
+    warn_script_location: bool,
+    use_user_site: bool,
+    pycompile: bool,
+) -> List[InstallationResult]:
     """
     Install everything in the given list.
 
@@ -59,8 +54,8 @@ def install_given_reqs(
 
     if to_install:
         logger.info(
-            'Installing collected packages: %s',
-            ', '.join(to_install.keys()),
+            "Installing collected packages: %s",
+            ", ".join(to_install.keys()),
         )
 
     installed = []
@@ -68,11 +63,9 @@ def install_given_reqs(
     with indent_log():
         for req_name, requirement in to_install.items():
             if requirement.should_reinstall:
-                logger.info('Attempting uninstall: %s', req_name)
+                logger.info("Attempting uninstall: %s", req_name)
                 with indent_log():
-                    uninstalled_pathset = requirement.uninstall(
-                        auto_confirm=True
-                    )
+                    uninstalled_pathset = requirement.uninstall(auto_confirm=True)
             else:
                 uninstalled_pathset = None
 
diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py
index d02dc636b0e..25bfb391d88 100644
--- a/src/pip/_internal/req/constructors.py
+++ b/src/pip/_internal/req/constructors.py
@@ -11,43 +11,36 @@
 import logging
 import os
 import re
+from typing import Any, Dict, Optional, Set, Tuple, Union
 
 from pip._vendor.packaging.markers import Marker
 from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
 from pip._vendor.packaging.specifiers import Specifier
-from pip._vendor.pkg_resources import RequirementParseError, parse_requirements
 
 from pip._internal.exceptions import InstallationError
 from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
-from pip._internal.pyproject import make_pyproject_path
+from pip._internal.req.req_file import ParsedRequirement
 from pip._internal.req.req_install import InstallRequirement
-from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.filetypes import is_archive_file
 from pip._internal.utils.misc import is_installable_dir
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.packaging import get_requirement
 from pip._internal.utils.urls import path_to_url
 from pip._internal.vcs import is_url, vcs
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Optional, Set, Tuple, Union
-
-    from pip._internal.req.req_file import ParsedRequirement
-
-
 __all__ = [
-    "install_req_from_editable", "install_req_from_line",
-    "parse_editable"
+    "install_req_from_editable",
+    "install_req_from_line",
+    "parse_editable",
 ]
 
 logger = logging.getLogger(__name__)
 operators = Specifier._operators.keys()
 
 
-def _strip_extras(path):
-    # type: (str) -> Tuple[str, Optional[str]]
-    m = re.match(r'^(.+)(\[[^\]]+\])$', path)
+def _strip_extras(path: str) -> Tuple[str, Optional[str]]:
+    m = re.match(r"^(.+)(\[[^\]]+\])$", path)
     extras = None
     if m:
         path_no_extras = m.group(1)
@@ -58,15 +51,13 @@ def _strip_extras(path):
     return path_no_extras, extras
 
 
-def convert_extras(extras):
-    # type: (Optional[str]) -> Set[str]
+def convert_extras(extras: Optional[str]) -> Set[str]:
     if not extras:
         return set()
-    return Requirement("placeholder" + extras.lower()).extras
+    return get_requirement("placeholder" + extras.lower()).extras
 
 
-def parse_editable(editable_req):
-    # type: (str) -> Tuple[Optional[str], str, Set[str]]
+def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]:
     """Parses an editable requirement into:
         - a requirement name
         - an URL
@@ -83,55 +74,36 @@ def parse_editable(editable_req):
     url_no_extras, extras = _strip_extras(url)
 
     if os.path.isdir(url_no_extras):
-        if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
-            msg = (
-                'File "setup.py" not found. Directory cannot be installed '
-                'in editable mode: {}'.format(os.path.abspath(url_no_extras))
-            )
-            pyproject_path = make_pyproject_path(url_no_extras)
-            if os.path.isfile(pyproject_path):
-                msg += (
-                    '\n(A "pyproject.toml" file was found, but editable '
-                    'mode currently requires a setup.py based build.)'
-                )
-            raise InstallationError(msg)
-
         # Treating it as code that has already been checked out
         url_no_extras = path_to_url(url_no_extras)
 
-    if url_no_extras.lower().startswith('file:'):
+    if url_no_extras.lower().startswith("file:"):
         package_name = Link(url_no_extras).egg_fragment
         if extras:
             return (
                 package_name,
                 url_no_extras,
-                Requirement("placeholder" + extras.lower()).extras,
+                get_requirement("placeholder" + extras.lower()).extras,
             )
         else:
             return package_name, url_no_extras, set()
 
     for version_control in vcs:
-        if url.lower().startswith(f'{version_control}:'):
-            url = f'{version_control}+{url}'
+        if url.lower().startswith(f"{version_control}:"):
+            url = f"{version_control}+{url}"
             break
 
-    if '+' not in url:
+    link = Link(url)
+
+    if not link.is_vcs:
+        backends = ", ".join(vcs.all_schemes)
         raise InstallationError(
-            '{} is not a valid editable requirement. '
-            'It should either be a path to a local project or a VCS URL '
-            '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req)
+            f"{editable_req} is not a valid editable requirement. "
+            f"It should either be a path to a local project or a VCS URL "
+            f"(beginning with {backends})."
         )
 
-    vc_type = url.split('+', 1)[0].lower()
-
-    if not vcs.get_backend(vc_type):
-        backends = ", ".join([bends.name + '+URL' for bends in vcs.backends])
-        error_message = "For --editable={}, " \
-                        "only {} are currently supported".format(
-                            editable_req, backends)
-        raise InstallationError(error_message)
-
-    package_name = Link(url).egg_fragment
+    package_name = link.egg_fragment
     if not package_name:
         raise InstallationError(
             "Could not detect requirement name for '{}', please specify one "
@@ -140,44 +112,66 @@ def parse_editable(editable_req):
     return package_name, url, set()
 
 
-def deduce_helpful_msg(req):
-    # type: (str) -> str
+def check_first_requirement_in_file(filename: str) -> None:
+    """Check if file is parsable as a requirements file.
+
+    This is heavily based on ``pkg_resources.parse_requirements``, but
+    simplified to just check the first meaningful line.
+
+    :raises InvalidRequirement: If the first meaningful line cannot be parsed
+        as an requirement.
+    """
+    with open(filename, encoding="utf-8", errors="ignore") as f:
+        # Create a steppable iterator, so we can handle \-continuations.
+        lines = (
+            line
+            for line in (line.strip() for line in f)
+            if line and not line.startswith("#")  # Skip blank lines/comments.
+        )
+
+        for line in lines:
+            # Drop comments -- a hash without a space may be in a URL.
+            if " #" in line:
+                line = line[: line.find(" #")]
+            # If there is a line continuation, drop it, and append the next line.
+            if line.endswith("\\"):
+                line = line[:-2].strip() + next(lines, "")
+            Requirement(line)
+            return
+
+
+def deduce_helpful_msg(req: str) -> str:
     """Returns helpful msg in case requirements file does not exist,
     or cannot be parsed.
 
     :params req: Requirements file path
     """
-    msg = ""
-    if os.path.exists(req):
-        msg = " The path does exist. "
-        # Try to parse and check if it is a requirements file.
-        try:
-            with open(req, 'r') as fp:
-                # parse first line only
-                next(parse_requirements(fp.read()))
-                msg += (
-                    "The argument you provided "
-                    "({}) appears to be a"
-                    " requirements file. If that is the"
-                    " case, use the '-r' flag to install"
-                    " the packages specified within it."
-                ).format(req)
-        except RequirementParseError:
-            logger.debug(
-                "Cannot parse '%s' as requirements file", req, exc_info=True
-            )
+    if not os.path.exists(req):
+        return f" File '{req}' does not exist."
+    msg = " The path does exist. "
+    # Try to parse and check if it is a requirements file.
+    try:
+        check_first_requirement_in_file(req)
+    except InvalidRequirement:
+        logger.debug("Cannot parse '%s' as requirements file", req)
     else:
-        msg += f" File '{req}' does not exist."
+        msg += (
+            f"The argument you provided "
+            f"({req}) appears to be a"
+            f" requirements file. If that is the"
+            f" case, use the '-r' flag to install"
+            f" the packages specified within it."
+        )
     return msg
 
 
 class RequirementParts:
     def __init__(
-            self,
-            requirement,  # type: Optional[Requirement]
-            link,         # type: Optional[Link]
-            markers,      # type: Optional[Marker]
-            extras,       # type: Set[str]
+        self,
+        requirement: Optional[Requirement],
+        link: Optional[Link],
+        markers: Optional[Marker],
+        extras: Set[str],
     ):
         self.requirement = requirement
         self.link = link
@@ -185,13 +179,12 @@ def __init__(
         self.extras = extras
 
 
-def parse_req_from_editable(editable_req):
-    # type: (str) -> RequirementParts
+def parse_req_from_editable(editable_req: str) -> RequirementParts:
     name, url, extras_override = parse_editable(editable_req)
 
     if name is not None:
         try:
-            req = Requirement(name)
+            req: Optional[Requirement] = Requirement(name)
         except InvalidRequirement:
             raise InstallationError(f"Invalid requirement: '{name}'")
     else:
@@ -206,15 +199,15 @@ def parse_req_from_editable(editable_req):
 
 
 def install_req_from_editable(
-    editable_req,  # type: str
-    comes_from=None,  # type: Optional[Union[InstallRequirement, str]]
-    use_pep517=None,  # type: Optional[bool]
-    isolated=False,  # type: bool
-    options=None,  # type: Optional[Dict[str, Any]]
-    constraint=False,  # type: bool
-    user_supplied=False,  # type: bool
-):
-    # type: (...) -> InstallRequirement
+    editable_req: str,
+    comes_from: Optional[Union[InstallRequirement, str]] = None,
+    use_pep517: Optional[bool] = None,
+    isolated: bool = False,
+    options: Optional[Dict[str, Any]] = None,
+    constraint: bool = False,
+    user_supplied: bool = False,
+    permit_editable_wheels: bool = False,
+) -> InstallRequirement:
 
     parts = parse_req_from_editable(editable_req)
 
@@ -223,6 +216,7 @@ def install_req_from_editable(
         comes_from=comes_from,
         user_supplied=user_supplied,
         editable=True,
+        permit_editable_wheels=permit_editable_wheels,
         link=parts.link,
         constraint=constraint,
         use_pep517=use_pep517,
@@ -234,8 +228,7 @@ def install_req_from_editable(
     )
 
 
-def _looks_like_path(name):
-    # type: (str) -> bool
+def _looks_like_path(name: str) -> bool:
     """Checks whether the string "looks like" a path on the filesystem.
 
     This does not check whether the target actually exists, only judge from the
@@ -254,11 +247,10 @@ def _looks_like_path(name):
     return False
 
 
-def _get_url_from_path(path, name):
-    # type: (str, str) -> Optional[str]
+def _get_url_from_path(path: str, name: str) -> Optional[str]:
     """
-    First, it checks whether a provided path is an installable directory
-    (e.g. it has a setup.py). If it is, returns the path.
+    First, it checks whether a provided path is an installable directory. If it
+    is, returns the path.
 
     If false, check if the path is an archive file (such as a .whl).
     The function checks if the path is a file. If false, if the path has
@@ -267,33 +259,33 @@ def _get_url_from_path(path, name):
     if _looks_like_path(name) and os.path.isdir(path):
         if is_installable_dir(path):
             return path_to_url(path)
+        # TODO: The is_installable_dir test here might not be necessary
+        #       now that it is done in load_pyproject_toml too.
         raise InstallationError(
-            "Directory {name!r} is not installable. Neither 'setup.py' "
-            "nor 'pyproject.toml' found.".format(**locals())
+            f"Directory {name!r} is not installable. Neither 'setup.py' "
+            "nor 'pyproject.toml' found."
         )
     if not is_archive_file(path):
         return None
     if os.path.isfile(path):
         return path_to_url(path)
-    urlreq_parts = name.split('@', 1)
+    urlreq_parts = name.split("@", 1)
     if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]):
         # If the path contains '@' and the part before it does not look
         # like a path, try to treat it as a PEP 440 URL req instead.
         return None
     logger.warning(
-        'Requirement %r looks like a filename, but the '
-        'file does not exist',
-        name
+        "Requirement %r looks like a filename, but the file does not exist",
+        name,
     )
     return path_to_url(path)
 
 
-def parse_req_from_line(name, line_source):
-    # type: (str, Optional[str]) -> RequirementParts
+def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementParts:
     if is_url(name):
-        marker_sep = '; '
+        marker_sep = "; "
     else:
-        marker_sep = ';'
+        marker_sep = ";"
     if marker_sep in name:
         name, markers_as_string = name.split(marker_sep, 1)
         markers_as_string = markers_as_string.strip()
@@ -320,13 +312,12 @@ def parse_req_from_line(name, line_source):
     # it's a local file, dir, or url
     if link:
         # Handle relative file URLs
-        if link.scheme == 'file' and re.search(r'\.\./', link.url):
-            link = Link(
-                path_to_url(os.path.normpath(os.path.abspath(link.path))))
+        if link.scheme == "file" and re.search(r"\.\./", link.url):
+            link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path))))
         # wheel file
         if link.is_wheel:
             wheel = Wheel(link.filename)  # can raise InvalidWheelFilename
-            req_as_string = "{wheel.name}=={wheel.version}".format(**locals())
+            req_as_string = f"{wheel.name}=={wheel.version}"
         else:
             # set the req to the egg fragment.  when it's not there, this
             # will become an 'unnamed' requirement
@@ -338,29 +329,27 @@ def parse_req_from_line(name, line_source):
 
     extras = convert_extras(extras_as_string)
 
-    def with_source(text):
-        # type: (str) -> str
+    def with_source(text: str) -> str:
         if not line_source:
             return text
-        return f'{text} (from {line_source})'
+        return f"{text} (from {line_source})"
 
-    if req_as_string is not None:
+    def _parse_req_string(req_as_string: str) -> Requirement:
         try:
-            req = Requirement(req_as_string)
+            req = get_requirement(req_as_string)
         except InvalidRequirement:
             if os.path.sep in req_as_string:
                 add_msg = "It looks like a path."
                 add_msg += deduce_helpful_msg(req_as_string)
-            elif ('=' in req_as_string and
-                  not any(op in req_as_string for op in operators)):
+            elif "=" in req_as_string and not any(
+                op in req_as_string for op in operators
+            ):
                 add_msg = "= is not a valid operator. Did you mean == ?"
             else:
-                add_msg = ''
-            msg = with_source(
-                f'Invalid requirement: {req_as_string!r}'
-            )
+                add_msg = ""
+            msg = with_source(f"Invalid requirement: {req_as_string!r}")
             if add_msg:
-                msg += f'\nHint: {add_msg}'
+                msg += f"\nHint: {add_msg}"
             raise InstallationError(msg)
         else:
             # Deprecate extras after specifiers: "name>=1.0[extras]"
@@ -369,10 +358,13 @@ def with_source(text):
             # RequirementParts
             for spec in req.specifier:
                 spec_str = str(spec)
-                if spec_str.endswith(']'):
+                if spec_str.endswith("]"):
                     msg = f"Extras after version '{spec_str}'."
-                    replace = "moving the extras before version specifiers"
-                    deprecated(msg, replacement=replace, gone_in="21.0")
+                    raise InstallationError(msg)
+        return req
+
+    if req_as_string is not None:
+        req: Optional[Requirement] = _parse_req_string(req_as_string)
     else:
         req = None
 
@@ -380,16 +372,15 @@ def with_source(text):
 
 
 def install_req_from_line(
-    name,  # type: str
-    comes_from=None,  # type: Optional[Union[str, InstallRequirement]]
-    use_pep517=None,  # type: Optional[bool]
-    isolated=False,  # type: bool
-    options=None,  # type: Optional[Dict[str, Any]]
-    constraint=False,  # type: bool
-    line_source=None,  # type: Optional[str]
-    user_supplied=False,  # type: bool
-):
-    # type: (...) -> InstallRequirement
+    name: str,
+    comes_from: Optional[Union[str, InstallRequirement]] = None,
+    use_pep517: Optional[bool] = None,
+    isolated: bool = False,
+    options: Optional[Dict[str, Any]] = None,
+    constraint: bool = False,
+    line_source: Optional[str] = None,
+    user_supplied: bool = False,
+) -> InstallRequirement:
     """Creates an InstallRequirement from a name, which might be a
     requirement, directory containing 'setup.py', filename, or URL.
 
@@ -399,8 +390,12 @@ def install_req_from_line(
     parts = parse_req_from_line(name, line_source)
 
     return InstallRequirement(
-        parts.requirement, comes_from, link=parts.link, markers=parts.markers,
-        use_pep517=use_pep517, isolated=isolated,
+        parts.requirement,
+        comes_from,
+        link=parts.link,
+        markers=parts.markers,
+        use_pep517=use_pep517,
+        isolated=isolated,
         install_options=options.get("install_options", []) if options else [],
         global_options=options.get("global_options", []) if options else [],
         hash_options=options.get("hashes", {}) if options else {},
@@ -411,15 +406,14 @@ def install_req_from_line(
 
 
 def install_req_from_req_string(
-    req_string,  # type: str
-    comes_from=None,  # type: Optional[InstallRequirement]
-    isolated=False,  # type: bool
-    use_pep517=None,  # type: Optional[bool]
-    user_supplied=False,  # type: bool
-):
-    # type: (...) -> InstallRequirement
+    req_string: str,
+    comes_from: Optional[InstallRequirement] = None,
+    isolated: bool = False,
+    use_pep517: Optional[bool] = None,
+    user_supplied: bool = False,
+) -> InstallRequirement:
     try:
-        req = Requirement(req_string)
+        req = get_requirement(req_string)
     except InvalidRequirement:
         raise InstallationError(f"Invalid requirement: '{req_string}'")
 
@@ -427,8 +421,12 @@ def install_req_from_req_string(
         PyPI.file_storage_domain,
         TestPyPI.file_storage_domain,
     ]
-    if (req.url and comes_from and comes_from.link and
-            comes_from.link.netloc in domains_not_allowed):
+    if (
+        req.url
+        and comes_from
+        and comes_from.link
+        and comes_from.link.netloc in domains_not_allowed
+    ):
         # Explicitly disallow pypi packages that depend on external urls
         raise InstallationError(
             "Packages installed from PyPI cannot depend on packages "
@@ -446,12 +444,11 @@ def install_req_from_req_string(
 
 
 def install_req_from_parsed_requirement(
-    parsed_req,  # type: ParsedRequirement
-    isolated=False,  # type: bool
-    use_pep517=None,  # type: Optional[bool]
-    user_supplied=False,  # type: bool
-):
-    # type: (...) -> InstallRequirement
+    parsed_req: ParsedRequirement,
+    isolated: bool = False,
+    use_pep517: Optional[bool] = None,
+    user_supplied: bool = False,
+) -> InstallRequirement:
     if parsed_req.is_editable:
         req = install_req_from_editable(
             parsed_req.requirement,
@@ -474,3 +471,20 @@ def install_req_from_parsed_requirement(
             user_supplied=user_supplied,
         )
     return req
+
+
+def install_req_from_link_and_ireq(
+    link: Link, ireq: InstallRequirement
+) -> InstallRequirement:
+    return InstallRequirement(
+        req=ireq.req,
+        comes_from=ireq.comes_from,
+        editable=ireq.editable,
+        link=link,
+        markers=ireq.markers,
+        use_pep517=ireq.use_pep517,
+        isolated=ireq.isolated,
+        install_options=ireq.install_options,
+        global_options=ireq.global_options,
+        hash_options=ireq.hash_options,
+    )
diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py
index 716005dc560..03ae50492c5 100644
--- a/src/pip/_internal/req/req_file.py
+++ b/src/pip/_internal/req/req_file.py
@@ -7,49 +7,50 @@
 import re
 import shlex
 import urllib.parse
+from optparse import Values
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+)
 
 from pip._internal.cli import cmdoptions
 from pip._internal.exceptions import InstallationError, RequirementsFileParseError
 from pip._internal.models.search_scope import SearchScope
+from pip._internal.network.session import PipSession
 from pip._internal.network.utils import raise_for_status
 from pip._internal.utils.encoding import auto_decode
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-from pip._internal.utils.urls import get_url_scheme, url_to_path
-
-if MYPY_CHECK_RUNNING:
-    from optparse import Values
-    from typing import (
-        Any,
-        Callable,
-        Dict,
-        Iterator,
-        List,
-        NoReturn,
-        Optional,
-        Text,
-        Tuple,
-    )
+from pip._internal.utils.urls import get_url_scheme
 
-    from pip._internal.index.package_finder import PackageFinder
-    from pip._internal.network.session import PipSession
+if TYPE_CHECKING:
+    # NoReturn introduced in 3.6.2; imported only for type checking to maintain
+    # pip compatibility with older patch versions of Python 3.6
+    from typing import NoReturn
 
-    ReqFileLines = Iterator[Tuple[int, Text]]
+    from pip._internal.index.package_finder import PackageFinder
 
-    LineParser = Callable[[Text], Tuple[str, Values]]
+__all__ = ["parse_requirements"]
 
+ReqFileLines = Iterable[Tuple[int, str]]
 
-__all__ = ['parse_requirements']
+LineParser = Callable[[str], Tuple[str, Values]]
 
-SCHEME_RE = re.compile(r'^(http|https|file):', re.I)
-COMMENT_RE = re.compile(r'(^|\s+)#.*$')
+SCHEME_RE = re.compile(r"^(http|https|file):", re.I)
+COMMENT_RE = re.compile(r"(^|\s+)#.*$")
 
 # Matches environment variable-style values in '${MY_VARIABLE_1}' with the
 # variable name consisting of only uppercase letters, digits or the '_'
 # (underscore). This follows the POSIX standard defined in IEEE Std 1003.1,
 # 2013 Edition.
-ENV_VAR_RE = re.compile(r'(?P\$\{(?P[A-Z0-9_]+)\})')
+ENV_VAR_RE = re.compile(r"(?P\$\{(?P[A-Z0-9_]+)\})")
 
-SUPPORTED_OPTIONS = [
+SUPPORTED_OPTIONS: List[Callable[..., optparse.Option]] = [
     cmdoptions.index_url,
     cmdoptions.extra_index_url,
     cmdoptions.no_index,
@@ -64,14 +65,14 @@
     cmdoptions.pre,
     cmdoptions.trusted_host,
     cmdoptions.use_new_feature,
-]  # type: List[Callable[..., optparse.Option]]
+]
 
 # options to be passed to requirements
-SUPPORTED_OPTIONS_REQ = [
+SUPPORTED_OPTIONS_REQ: List[Callable[..., optparse.Option]] = [
     cmdoptions.install_options,
     cmdoptions.global_options,
     cmdoptions.hash,
-]  # type: List[Callable[..., optparse.Option]]
+]
 
 # the 'dest' string values
 SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ]
@@ -80,14 +81,13 @@
 class ParsedRequirement:
     def __init__(
         self,
-        requirement,  # type:str
-        is_editable,  # type: bool
-        comes_from,  # type: str
-        constraint,  # type: bool
-        options=None,  # type: Optional[Dict[str, Any]]
-        line_source=None,  # type: Optional[str]
-    ):
-        # type: (...) -> None
+        requirement: str,
+        is_editable: bool,
+        comes_from: str,
+        constraint: bool,
+        options: Optional[Dict[str, Any]] = None,
+        line_source: Optional[str] = None,
+    ) -> None:
         self.requirement = requirement
         self.is_editable = is_editable
         self.comes_from = comes_from
@@ -99,13 +99,12 @@ def __init__(
 class ParsedLine:
     def __init__(
         self,
-        filename,  # type: str
-        lineno,  # type: int
-        args,  # type: str
-        opts,  # type: Values
-        constraint,  # type: bool
-    ):
-        # type: (...) -> None
+        filename: str,
+        lineno: int,
+        args: str,
+        opts: Values,
+        constraint: bool,
+    ) -> None:
         self.filename = filename
         self.lineno = lineno
         self.opts = opts
@@ -125,13 +124,12 @@ def __init__(
 
 
 def parse_requirements(
-    filename,  # type: str
-    session,  # type: PipSession
-    finder=None,  # type: Optional[PackageFinder]
-    options=None,  # type: Optional[optparse.Values]
-    constraint=False,  # type: bool
-):
-    # type: (...) -> Iterator[ParsedRequirement]
+    filename: str,
+    session: PipSession,
+    finder: Optional["PackageFinder"] = None,
+    options: Optional[optparse.Values] = None,
+    constraint: bool = False,
+) -> Iterator[ParsedRequirement]:
     """Parse a requirements file and yield ParsedRequirement instances.
 
     :param filename:    Path or url of requirements file.
@@ -146,22 +144,18 @@ def parse_requirements(
 
     for parsed_line in parser.parse(filename, constraint):
         parsed_req = handle_line(
-            parsed_line,
-            options=options,
-            finder=finder,
-            session=session
+            parsed_line, options=options, finder=finder, session=session
         )
         if parsed_req is not None:
             yield parsed_req
 
 
-def preprocess(content):
-    # type: (str) -> ReqFileLines
+def preprocess(content: str) -> ReqFileLines:
     """Split, filter, and join lines, and return a line iterator
 
     :param content: the content of the requirements file
     """
-    lines_enum = enumerate(content.splitlines(), start=1)  # type: ReqFileLines
+    lines_enum: ReqFileLines = enumerate(content.splitlines(), start=1)
     lines_enum = join_lines(lines_enum)
     lines_enum = ignore_comments(lines_enum)
     lines_enum = expand_env_variables(lines_enum)
@@ -169,14 +163,15 @@ def preprocess(content):
 
 
 def handle_requirement_line(
-    line,  # type: ParsedLine
-    options=None,  # type: Optional[optparse.Values]
-):
-    # type: (...) -> ParsedRequirement
+    line: ParsedLine,
+    options: Optional[optparse.Values] = None,
+) -> ParsedRequirement:
 
     # preserve for the nested code path
-    line_comes_from = '{} {} (line {})'.format(
-        '-c' if line.constraint else '-r', line.filename, line.lineno,
+    line_comes_from = "{} {} (line {})".format(
+        "-c" if line.constraint else "-r",
+        line.filename,
+        line.lineno,
     )
 
     assert line.is_requirement
@@ -201,7 +196,7 @@ def handle_requirement_line(
             if dest in line.opts.__dict__ and line.opts.__dict__[dest]:
                 req_options[dest] = line.opts.__dict__[dest]
 
-        line_source = f'line {line.lineno} of {line.filename}'
+        line_source = f"line {line.lineno} of {line.filename}"
         return ParsedRequirement(
             requirement=line.requirement,
             is_editable=line.is_editable,
@@ -213,14 +208,13 @@ def handle_requirement_line(
 
 
 def handle_option_line(
-    opts,  # type: Values
-    filename,  # type: str
-    lineno,  # type: int
-    finder=None,  # type: Optional[PackageFinder]
-    options=None,  # type: Optional[optparse.Values]
-    session=None,  # type: Optional[PipSession]
-):
-    # type:  (...) -> None
+    opts: Values,
+    filename: str,
+    lineno: int,
+    finder: Optional["PackageFinder"] = None,
+    options: Optional[optparse.Values] = None,
+    session: Optional[PipSession] = None,
+) -> None:
 
     if options:
         # percolate options upward
@@ -228,8 +222,7 @@ def handle_option_line(
             options.require_hashes = opts.require_hashes
         if opts.features_enabled:
             options.features_enabled.extend(
-                f for f in opts.features_enabled
-                if f not in options.features_enabled
+                f for f in opts.features_enabled if f not in options.features_enabled
             )
 
     # set finder options
@@ -271,17 +264,16 @@ def handle_option_line(
 
         if session:
             for host in opts.trusted_hosts or []:
-                source = f'line {lineno} of {filename}'
+                source = f"line {lineno} of {filename}"
                 session.add_trusted_host(host, source=source)
 
 
 def handle_line(
-    line,  # type: ParsedLine
-    options=None,  # type: Optional[optparse.Values]
-    finder=None,  # type: Optional[PackageFinder]
-    session=None,  # type: Optional[PipSession]
-):
-    # type: (...) -> Optional[ParsedRequirement]
+    line: ParsedLine,
+    options: Optional[optparse.Values] = None,
+    finder: Optional["PackageFinder"] = None,
+    session: Optional[PipSession] = None,
+) -> Optional[ParsedRequirement]:
     """Handle a single parsed requirements line; This can result in
     creating/yielding requirements, or updating the finder.
 
@@ -323,25 +315,22 @@ def handle_line(
 class RequirementsFileParser:
     def __init__(
         self,
-        session,  # type: PipSession
-        line_parser,  # type: LineParser
-    ):
-        # type: (...) -> None
+        session: PipSession,
+        line_parser: LineParser,
+    ) -> None:
         self._session = session
         self._line_parser = line_parser
 
-    def parse(self, filename, constraint):
-        # type: (str, bool) -> Iterator[ParsedLine]
-        """Parse a given file, yielding parsed lines.
-        """
+    def parse(self, filename: str, constraint: bool) -> Iterator[ParsedLine]:
+        """Parse a given file, yielding parsed lines."""
         yield from self._parse_and_recurse(filename, constraint)
 
-    def _parse_and_recurse(self, filename, constraint):
-        # type: (str, bool) -> Iterator[ParsedLine]
+    def _parse_and_recurse(
+        self, filename: str, constraint: bool
+    ) -> Iterator[ParsedLine]:
         for line in self._parse_file(filename, constraint):
-            if (
-                not line.is_requirement and
-                (line.opts.requirements or line.opts.constraints)
+            if not line.is_requirement and (
+                line.opts.requirements or line.opts.constraints
             ):
                 # parse a nested requirements file
                 if line.opts.requirements:
@@ -359,15 +348,15 @@ def _parse_and_recurse(self, filename, constraint):
                 elif not SCHEME_RE.search(req_path):
                     # do a join so relative paths work
                     req_path = os.path.join(
-                        os.path.dirname(filename), req_path,
+                        os.path.dirname(filename),
+                        req_path,
                     )
 
                 yield from self._parse_and_recurse(req_path, nested_constraint)
             else:
                 yield line
 
-    def _parse_file(self, filename, constraint):
-        # type: (str, bool) -> Iterator[ParsedLine]
+    def _parse_file(self, filename: str, constraint: bool) -> Iterator[ParsedLine]:
         _, content = get_file_content(filename, self._session)
 
         lines_enum = preprocess(content)
@@ -377,7 +366,7 @@ def _parse_file(self, filename, constraint):
                 args_str, opts = self._line_parser(line)
             except OptionParsingError as e:
                 # add offending line
-                msg = f'Invalid requirement: {line}\n{e.msg}'
+                msg = f"Invalid requirement: {line}\n{e.msg}"
                 raise RequirementsFileParseError(msg)
 
             yield ParsedLine(
@@ -389,10 +378,8 @@ def _parse_file(self, filename, constraint):
             )
 
 
-def get_line_parser(finder):
-    # type: (Optional[PackageFinder]) -> LineParser
-    def parse_line(line):
-        # type: (str) -> Tuple[str, Values]
+def get_line_parser(finder: Optional["PackageFinder"]) -> LineParser:
+    def parse_line(line: str) -> Tuple[str, Values]:
         # Build new parser for each line since it accumulates appendable
         # options.
         parser = build_parser()
@@ -410,32 +397,29 @@ def parse_line(line):
     return parse_line
 
 
-def break_args_options(line):
-    # type: (str) -> Tuple[str, str]
+def break_args_options(line: str) -> Tuple[str, str]:
     """Break up the line into an args and options string.  We only want to shlex
     (and then optparse) the options, not the args.  args can contain markers
     which are corrupted by shlex.
     """
-    tokens = line.split(' ')
+    tokens = line.split(" ")
     args = []
     options = tokens[:]
     for token in tokens:
-        if token.startswith('-') or token.startswith('--'):
+        if token.startswith("-") or token.startswith("--"):
             break
         else:
             args.append(token)
             options.pop(0)
-    return ' '.join(args), ' '.join(options)
+    return " ".join(args), " ".join(options)
 
 
 class OptionParsingError(Exception):
-    def __init__(self, msg):
-        # type: (str) -> None
+    def __init__(self, msg: str) -> None:
         self.msg = msg
 
 
-def build_parser():
-    # type: () -> optparse.OptionParser
+def build_parser() -> optparse.OptionParser:
     """
     Return a parser for parsing requirement lines
     """
@@ -448,9 +432,9 @@ def build_parser():
 
     # By default optparse sys.exits on parsing errors. We want to wrap
     # that in our own exception.
-    def parser_exit(self, msg):
-        # type: (Any, str) -> NoReturn
+    def parser_exit(self: Any, msg: str) -> "NoReturn":
         raise OptionParsingError(msg)
+
     # NOTE: mypy disallows assigning to a method
     #       https://github.com/python/mypy/issues/2427
     parser.exit = parser_exit  # type: ignore
@@ -458,52 +442,49 @@ def parser_exit(self, msg):
     return parser
 
 
-def join_lines(lines_enum):
-    # type: (ReqFileLines) -> ReqFileLines
+def join_lines(lines_enum: ReqFileLines) -> ReqFileLines:
     """Joins a line ending in '\' with the previous line (except when following
     comments).  The joined line takes on the index of the first line.
     """
     primary_line_number = None
-    new_line = []  # type: List[str]
+    new_line: List[str] = []
     for line_number, line in lines_enum:
-        if not line.endswith('\\') or COMMENT_RE.match(line):
+        if not line.endswith("\\") or COMMENT_RE.match(line):
             if COMMENT_RE.match(line):
                 # this ensures comments are always matched later
-                line = ' ' + line
+                line = " " + line
             if new_line:
                 new_line.append(line)
                 assert primary_line_number is not None
-                yield primary_line_number, ''.join(new_line)
+                yield primary_line_number, "".join(new_line)
                 new_line = []
             else:
                 yield line_number, line
         else:
             if not new_line:
                 primary_line_number = line_number
-            new_line.append(line.strip('\\'))
+            new_line.append(line.strip("\\"))
 
     # last line contains \
     if new_line:
         assert primary_line_number is not None
-        yield primary_line_number, ''.join(new_line)
+        yield primary_line_number, "".join(new_line)
 
     # TODO: handle space after '\'.
 
 
-def ignore_comments(lines_enum):
-    # type: (ReqFileLines) -> ReqFileLines
+def ignore_comments(lines_enum: ReqFileLines) -> ReqFileLines:
     """
     Strips comments and filter empty lines.
     """
     for line_number, line in lines_enum:
-        line = COMMENT_RE.sub('', line)
+        line = COMMENT_RE.sub("", line)
         line = line.strip()
         if line:
             yield line_number, line
 
 
-def expand_env_variables(lines_enum):
-    # type: (ReqFileLines) -> ReqFileLines
+def expand_env_variables(lines_enum: ReqFileLines) -> ReqFileLines:
     """Replace all environment variables that can be retrieved via `os.getenv`.
 
     The only allowed format for environment variables defined in the
@@ -530,8 +511,7 @@ def expand_env_variables(lines_enum):
         yield line_number, line
 
 
-def get_file_content(url, session):
-    # type: (str, PipSession) -> Tuple[str, str]
+def get_file_content(url: str, session: PipSession) -> Tuple[str, str]:
     """Gets the content of a file; it may be a filename, file: URL, or
     http: URL.  Returns (location, content).  Content is unicode.
     Respects # -*- coding: declarations on the retrieved files.
@@ -541,20 +521,16 @@ def get_file_content(url, session):
     """
     scheme = get_url_scheme(url)
 
-    if scheme in ['http', 'https']:
-        # FIXME: catch some errors
+    # Pip has special support for file:// URLs (LocalFSAdapter).
+    if scheme in ["http", "https", "file"]:
         resp = session.get(url)
         raise_for_status(resp)
         return resp.url, resp.text
 
-    elif scheme == 'file':
-        url = url_to_path(url)
-
+    # Assume this is a bare path.
     try:
-        with open(url, 'rb') as f:
+        with open(url, "rb") as f:
             content = auto_decode(f.read())
     except OSError as exc:
-        raise InstallationError(
-            f'Could not open requirements file: {exc}'
-        )
+        raise InstallationError(f"Could not open requirements file: {exc}")
     return url, content
diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py
index 6ec3c9c6975..637b6ce10af 100644
--- a/src/pip/_internal/req/req_install.py
+++ b/src/pip/_internal/req/req_install.py
@@ -1,97 +1,66 @@
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
 
+import functools
 import logging
 import os
 import shutil
 import sys
 import uuid
 import zipfile
+from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, Union
 
-from pip._vendor import pkg_resources, six
+from pip._vendor.packaging.markers import Marker
 from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.specifiers import SpecifierSet
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.packaging.version import Version
 from pip._vendor.packaging.version import parse as parse_version
 from pip._vendor.pep517.wrappers import Pep517HookCaller
 
-from pip._internal.build_env import NoOpBuildEnvironment
-from pip._internal.exceptions import InstallationError
+from pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment
+from pip._internal.exceptions import InstallationError, LegacyInstallFailure
 from pip._internal.locations import get_scheme
+from pip._internal.metadata import (
+    BaseDistribution,
+    get_default_environment,
+    get_directory_distribution,
+)
 from pip._internal.models.link import Link
 from pip._internal.operations.build.metadata import generate_metadata
+from pip._internal.operations.build.metadata_editable import generate_editable_metadata
 from pip._internal.operations.build.metadata_legacy import (
     generate_metadata as generate_metadata_legacy,
 )
 from pip._internal.operations.install.editable_legacy import (
     install_editable as install_editable_legacy,
 )
-from pip._internal.operations.install.legacy import LegacyInstallFailure
 from pip._internal.operations.install.legacy import install as install_legacy
 from pip._internal.operations.install.wheel import install_wheel
 from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
 from pip._internal.req.req_uninstall import UninstallPathSet
 from pip._internal.utils.deprecation import deprecated
-from pip._internal.utils.direct_url_helpers import direct_url_from_link
+from pip._internal.utils.direct_url_helpers import (
+    direct_url_for_editable,
+    direct_url_from_link,
+)
 from pip._internal.utils.hashes import Hashes
-from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import (
     ask_path_exists,
     backup_dir,
     display_path,
-    dist_in_site_packages,
-    dist_in_usersite,
-    get_distribution,
-    get_installed_version,
     hide_url,
     redact_auth_from_url,
 )
-from pip._internal.utils.packaging import get_metadata
+from pip._internal.utils.packaging import is_pinned, safe_extra
+from pip._internal.utils.subprocess import runner_with_spinner_message
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.virtualenv import running_under_virtualenv
 from pip._internal.vcs import vcs
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
-
-    from pip._vendor.packaging.markers import Marker
-    from pip._vendor.packaging.specifiers import SpecifierSet
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.build_env import BuildEnvironment
-
-
 logger = logging.getLogger(__name__)
 
 
-def _get_dist(metadata_directory):
-    # type: (str) -> Distribution
-    """Return a pkg_resources.Distribution for the provided
-    metadata directory.
-    """
-    dist_dir = metadata_directory.rstrip(os.sep)
-
-    # Build a PathMetadata object, from path to metadata. :wink:
-    base_dir, dist_dir_name = os.path.split(dist_dir)
-    metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
-
-    # Determine the correct Distribution object type.
-    if dist_dir.endswith(".egg-info"):
-        dist_cls = pkg_resources.Distribution
-        dist_name = os.path.splitext(dist_dir_name)[0]
-    else:
-        assert dist_dir.endswith(".dist-info")
-        dist_cls = pkg_resources.DistInfoDistribution
-        dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
-
-    return dist_cls(
-        base_dir,
-        project_name=dist_name,
-        metadata=metadata,
-    )
-
-
 class InstallRequirement:
     """
     Represents something that may be installed later on, may have information
@@ -101,40 +70,39 @@ class InstallRequirement:
 
     def __init__(
         self,
-        req,  # type: Optional[Requirement]
-        comes_from,  # type: Optional[Union[str, InstallRequirement]]
-        editable=False,  # type: bool
-        link=None,  # type: Optional[Link]
-        markers=None,  # type: Optional[Marker]
-        use_pep517=None,  # type: Optional[bool]
-        isolated=False,  # type: bool
-        install_options=None,  # type: Optional[List[str]]
-        global_options=None,  # type: Optional[List[str]]
-        hash_options=None,  # type: Optional[Dict[str, List[str]]]
-        constraint=False,  # type: bool
-        extras=(),  # type: Iterable[str]
-        user_supplied=False,  # type: bool
-    ):
-        # type: (...) -> None
+        req: Optional[Requirement],
+        comes_from: Optional[Union[str, "InstallRequirement"]],
+        editable: bool = False,
+        link: Optional[Link] = None,
+        markers: Optional[Marker] = None,
+        use_pep517: Optional[bool] = None,
+        isolated: bool = False,
+        install_options: Optional[List[str]] = None,
+        global_options: Optional[List[str]] = None,
+        hash_options: Optional[Dict[str, List[str]]] = None,
+        constraint: bool = False,
+        extras: Collection[str] = (),
+        user_supplied: bool = False,
+        permit_editable_wheels: bool = False,
+    ) -> None:
         assert req is None or isinstance(req, Requirement), req
         self.req = req
         self.comes_from = comes_from
         self.constraint = constraint
         self.editable = editable
-        self.legacy_install_reason = None  # type: Optional[int]
+        self.permit_editable_wheels = permit_editable_wheels
+        self.legacy_install_reason: Optional[int] = None
 
         # source_dir is the local directory where the linked requirement is
         # located, or unpacked. In case unpacking is needed, creating and
         # populating source_dir is done by the RequirementPreparer. Note this
         # is not necessarily the directory where pyproject.toml or setup.py is
         # located - that one is obtained via unpacked_source_directory.
-        self.source_dir = None  # type: Optional[str]
+        self.source_dir: Optional[str] = None
         if self.editable:
             assert link
             if link.is_file:
-                self.source_dir = os.path.normpath(
-                    os.path.abspath(link.file_path)
-                )
+                self.source_dir = os.path.normpath(os.path.abspath(link.file_path))
 
         if link is None and req and req.url:
             # PEP 508 URL requirement
@@ -143,32 +111,29 @@ def __init__(
         self.original_link_is_in_wheel_cache = False
 
         # Path to any downloaded or already-existing package.
-        self.local_file_path = None  # type: Optional[str]
+        self.local_file_path: Optional[str] = None
         if self.link and self.link.is_file:
             self.local_file_path = self.link.file_path
 
         if extras:
             self.extras = extras
         elif req:
-            self.extras = {
-                pkg_resources.safe_extra(extra) for extra in req.extras
-            }
+            self.extras = {safe_extra(extra) for extra in req.extras}
         else:
             self.extras = set()
         if markers is None and req:
             markers = req.marker
         self.markers = markers
 
-        # This holds the pkg_resources.Distribution object if this requirement
-        # is already available:
-        self.satisfied_by = None  # type: Optional[Distribution]
+        # This holds the Distribution object if this requirement is already installed.
+        self.satisfied_by: Optional[BaseDistribution] = None
         # Whether the installation process should try to uninstall an existing
         # distribution before installing this requirement.
         self.should_reinstall = False
         # Temporary build location
-        self._temp_build_dir = None  # type: Optional[TempDirectory]
+        self._temp_build_dir: Optional[TempDirectory] = None
         # Set to True after successful installation
-        self.install_succeeded = None  # type: Optional[bool]
+        self.install_succeeded: Optional[bool] = None
         # Supplied options
         self.install_options = install_options if install_options else []
         self.global_options = global_options if global_options else []
@@ -181,22 +146,22 @@ def __init__(
         self.user_supplied = user_supplied
 
         self.isolated = isolated
-        self.build_env = NoOpBuildEnvironment()  # type: BuildEnvironment
+        self.build_env: BuildEnvironment = NoOpBuildEnvironment()
 
         # For PEP 517, the directory where we request the project metadata
         # gets stored. We need this to pass to build_wheel, so the backend
         # can ensure that the wheel matches the metadata (see the PEP for
         # details).
-        self.metadata_directory = None  # type: Optional[str]
+        self.metadata_directory: Optional[str] = None
 
         # The static build requirements (from pyproject.toml)
-        self.pyproject_requires = None  # type: Optional[List[str]]
+        self.pyproject_requires: Optional[List[str]] = None
 
         # Build requirements that we will check are available
-        self.requirements_to_check = []  # type: List[str]
+        self.requirements_to_check: List[str] = []
 
         # The PEP 517 backend we should use to build the project
-        self.pep517_backend = None  # type: Optional[Pep517HookCaller]
+        self.pep517_backend: Optional[Pep517HookCaller] = None
 
         # Are we using PEP 517 for this requirement?
         # After pyproject.toml has been loaded, the only valid values are True
@@ -208,92 +173,87 @@ def __init__(
         # This requirement needs more preparation before it can be built
         self.needs_more_preparation = False
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         if self.req:
             s = str(self.req)
             if self.link:
-                s += ' from {}'.format(redact_auth_from_url(self.link.url))
+                s += " from {}".format(redact_auth_from_url(self.link.url))
         elif self.link:
             s = redact_auth_from_url(self.link.url)
         else:
-            s = ''
+            s = ""
         if self.satisfied_by is not None:
-            s += ' in {}'.format(display_path(self.satisfied_by.location))
+            s += " in {}".format(display_path(self.satisfied_by.location))
         if self.comes_from:
             if isinstance(self.comes_from, str):
-                comes_from = self.comes_from  # type: Optional[str]
+                comes_from: Optional[str] = self.comes_from
             else:
                 comes_from = self.comes_from.from_path()
             if comes_from:
-                s += f' (from {comes_from})'
+                s += f" (from {comes_from})"
         return s
 
-    def __repr__(self):
-        # type: () -> str
-        return '<{} object: {} editable={!r}>'.format(
-            self.__class__.__name__, str(self), self.editable)
+    def __repr__(self) -> str:
+        return "<{} object: {} editable={!r}>".format(
+            self.__class__.__name__, str(self), self.editable
+        )
 
-    def format_debug(self):
-        # type: () -> str
-        """An un-tested helper for getting state, for debugging.
-        """
+    def format_debug(self) -> str:
+        """An un-tested helper for getting state, for debugging."""
         attributes = vars(self)
         names = sorted(attributes)
 
-        state = (
-            "{}={!r}".format(attr, attributes[attr]) for attr in sorted(names)
-        )
-        return '<{name} object: {{{state}}}>'.format(
+        state = ("{}={!r}".format(attr, attributes[attr]) for attr in sorted(names))
+        return "<{name} object: {{{state}}}>".format(
             name=self.__class__.__name__,
             state=", ".join(state),
         )
 
     # Things that are valid for all kinds of requirements?
     @property
-    def name(self):
-        # type: () -> Optional[str]
+    def name(self) -> Optional[str]:
         if self.req is None:
             return None
-        return six.ensure_str(pkg_resources.safe_name(self.req.name))
+        return self.req.name
+
+    @functools.lru_cache()  # use cached_property in python 3.8+
+    def supports_pyproject_editable(self) -> bool:
+        if not self.use_pep517:
+            return False
+        assert self.pep517_backend
+        with self.build_env:
+            runner = runner_with_spinner_message(
+                "Checking if build backend supports build_editable"
+            )
+            with self.pep517_backend.subprocess_runner(runner):
+                return "build_editable" in self.pep517_backend._supported_features()
 
     @property
-    def specifier(self):
-        # type: () -> SpecifierSet
+    def specifier(self) -> SpecifierSet:
         return self.req.specifier
 
     @property
-    def is_pinned(self):
-        # type: () -> bool
+    def is_pinned(self) -> bool:
         """Return whether I am pinned to an exact version.
 
         For example, some-package==1.2 is pinned; some-package>1.2 is not.
         """
-        specifiers = self.specifier
-        return (len(specifiers) == 1 and
-                next(iter(specifiers)).operator in {'==', '==='})
-
-    @property
-    def installed_version(self):
-        # type: () -> Optional[str]
-        return get_installed_version(self.name)
+        return is_pinned(self.specifier)
 
-    def match_markers(self, extras_requested=None):
-        # type: (Optional[Iterable[str]]) -> bool
+    def match_markers(self, extras_requested: Optional[Iterable[str]] = None) -> bool:
         if not extras_requested:
             # Provide an extra to safely evaluate the markers
             # without matching any extra
-            extras_requested = ('',)
+            extras_requested = ("",)
         if self.markers is not None:
             return any(
-                self.markers.evaluate({'extra': extra})
-                for extra in extras_requested)
+                self.markers.evaluate({"extra": extra}) for extra in extras_requested
+            )
         else:
             return True
 
     @property
-    def has_hash_options(self):
-        # type: () -> bool
+    def has_hash_options(self) -> bool:
         """Return whether any known-good hashes are specified as options.
 
         These activate --require-hashes mode; hashes specified as part of a
@@ -302,8 +262,7 @@ def has_hash_options(self):
         """
         return bool(self.hash_options)
 
-    def hashes(self, trust_internet=True):
-        # type: (bool) -> Hashes
+    def hashes(self, trust_internet: bool = True) -> Hashes:
         """Return a hash-comparer that considers my option- and URL-based
         hashes to be known-good.
 
@@ -324,10 +283,8 @@ def hashes(self, trust_internet=True):
             good_hashes.setdefault(link.hash_name, []).append(link.hash)
         return Hashes(good_hashes)
 
-    def from_path(self):
-        # type: () -> Optional[str]
-        """Format a nice indicator to show where this "comes from"
-        """
+    def from_path(self) -> Optional[str]:
+        """Format a nice indicator to show where this "comes from" """
         if self.req is None:
             return None
         s = str(self.req)
@@ -337,11 +294,12 @@ def from_path(self):
             else:
                 comes_from = self.comes_from.from_path()
             if comes_from:
-                s += '->' + comes_from
+                s += "->" + comes_from
         return s
 
-    def ensure_build_location(self, build_dir, autodelete, parallel_builds):
-        # type: (str, bool, bool) -> str
+    def ensure_build_location(
+        self, build_dir: str, autodelete: bool, parallel_builds: bool
+    ) -> str:
         assert build_dir is not None
         if self._temp_build_dir is not None:
             assert self._temp_build_dir.path
@@ -362,14 +320,14 @@ def ensure_build_location(self, build_dir, autodelete, parallel_builds):
 
         # When parallel builds are enabled, add a UUID to the build directory
         # name so multiple builds do not interfere with each other.
-        dir_name = canonicalize_name(self.name)
+        dir_name: str = canonicalize_name(self.name)
         if parallel_builds:
             dir_name = f"{dir_name}_{uuid.uuid4().hex}"
 
         # FIXME: Is there a better place to create the build_dir? (hg and bzr
         # need this)
         if not os.path.exists(build_dir):
-            logger.debug('Creating directory %s', build_dir)
+            logger.debug("Creating directory %s", build_dir)
             os.makedirs(build_dir)
         actual_build_dir = os.path.join(build_dir, dir_name)
         # `None` indicates that we respect the globally-configured deletion
@@ -382,10 +340,8 @@ def ensure_build_location(self, build_dir, autodelete, parallel_builds):
             globally_managed=True,
         ).path
 
-    def _set_requirement(self):
-        # type: () -> None
-        """Set requirement after generating metadata.
-        """
+    def _set_requirement(self) -> None:
+        """Set requirement after generating metadata."""
         assert self.req is None
         assert self.metadata is not None
         assert self.source_dir is not None
@@ -397,15 +353,16 @@ def _set_requirement(self):
             op = "==="
 
         self.req = Requirement(
-            "".join([
-                self.metadata["Name"],
-                op,
-                self.metadata["Version"],
-            ])
+            "".join(
+                [
+                    self.metadata["Name"],
+                    op,
+                    self.metadata["Version"],
+                ]
+            )
         )
 
-    def warn_on_mismatching_name(self):
-        # type: () -> None
+    def warn_on_mismatching_name(self) -> None:
         metadata_name = canonicalize_name(self.metadata["Name"])
         if canonicalize_name(self.req.name) == metadata_name:
             # Everything is fine.
@@ -413,37 +370,40 @@ def warn_on_mismatching_name(self):
 
         # If we're here, there's a mismatch. Log a warning about it.
         logger.warning(
-            'Generating metadata for package %s '
-            'produced metadata for project name %s. Fix your '
-            '#egg=%s fragments.',
-            self.name, metadata_name, self.name
+            "Generating metadata for package %s "
+            "produced metadata for project name %s. Fix your "
+            "#egg=%s fragments.",
+            self.name,
+            metadata_name,
+            self.name,
         )
         self.req = Requirement(metadata_name)
 
-    def check_if_exists(self, use_user_site):
-        # type: (bool) -> None
+    def check_if_exists(self, use_user_site: bool) -> None:
         """Find an installed distribution that satisfies or conflicts
         with this requirement, and set self.satisfied_by or
         self.should_reinstall appropriately.
         """
         if self.req is None:
             return
-        existing_dist = get_distribution(self.req.name)
+        existing_dist = get_default_environment().get_distribution(self.req.name)
         if not existing_dist:
             return
 
-        existing_version = existing_dist.parsed_version
-        if not self.req.specifier.contains(existing_version, prereleases=True):
+        version_compatible = self.req.specifier.contains(
+            existing_dist.version,
+            prereleases=True,
+        )
+        if not version_compatible:
             self.satisfied_by = None
             if use_user_site:
-                if dist_in_usersite(existing_dist):
+                if existing_dist.in_usersite:
                     self.should_reinstall = True
-                elif (running_under_virtualenv() and
-                        dist_in_site_packages(existing_dist)):
+                elif running_under_virtualenv() and existing_dist.in_site_packages:
                     raise InstallationError(
-                        "Will not install to the user site because it will "
-                        "lack sys.path precedence to {} in {}".format(
-                            existing_dist.project_name, existing_dist.location)
+                        f"Will not install to the user site because it will "
+                        f"lack sys.path precedence to {existing_dist.raw_name} "
+                        f"in {existing_dist.location}"
                     )
             else:
                 self.should_reinstall = True
@@ -458,36 +418,38 @@ def check_if_exists(self, use_user_site):
 
     # Things valid for wheels
     @property
-    def is_wheel(self):
-        # type: () -> bool
+    def is_wheel(self) -> bool:
         if not self.link:
             return False
         return self.link.is_wheel
 
     # Things valid for sdists
     @property
-    def unpacked_source_directory(self):
-        # type: () -> str
+    def unpacked_source_directory(self) -> str:
         return os.path.join(
-            self.source_dir,
-            self.link and self.link.subdirectory_fragment or '')
+            self.source_dir, self.link and self.link.subdirectory_fragment or ""
+        )
 
     @property
-    def setup_py_path(self):
-        # type: () -> str
+    def setup_py_path(self) -> str:
         assert self.source_dir, f"No source dir for {self}"
-        setup_py = os.path.join(self.unpacked_source_directory, 'setup.py')
+        setup_py = os.path.join(self.unpacked_source_directory, "setup.py")
 
         return setup_py
 
     @property
-    def pyproject_toml_path(self):
-        # type: () -> str
+    def setup_cfg_path(self) -> str:
+        assert self.source_dir, f"No source dir for {self}"
+        setup_cfg = os.path.join(self.unpacked_source_directory, "setup.cfg")
+
+        return setup_cfg
+
+    @property
+    def pyproject_toml_path(self) -> str:
         assert self.source_dir, f"No source dir for {self}"
         return make_pyproject_path(self.unpacked_source_directory)
 
-    def load_pyproject_toml(self):
-        # type: () -> None
+    def load_pyproject_toml(self) -> None:
         """Load the pyproject.toml file.
 
         After calling this routine, all of the attributes related to PEP 517
@@ -496,10 +458,7 @@ def load_pyproject_toml(self):
         follow the PEP 517 or legacy (setup.py) code path.
         """
         pyproject_toml_data = load_pyproject_toml(
-            self.use_pep517,
-            self.pyproject_toml_path,
-            self.setup_py_path,
-            str(self)
+            self.use_pep517, self.pyproject_toml_path, self.setup_py_path, str(self)
         )
 
         if pyproject_toml_data is None:
@@ -511,42 +470,67 @@ def load_pyproject_toml(self):
         self.requirements_to_check = check
         self.pyproject_requires = requires
         self.pep517_backend = Pep517HookCaller(
-            self.unpacked_source_directory, backend, backend_path=backend_path,
+            self.unpacked_source_directory,
+            backend,
+            backend_path=backend_path,
         )
 
-    def _generate_metadata(self):
-        # type: () -> str
-        """Invokes metadata generator functions, with the required arguments.
-        """
-        if not self.use_pep517:
-            assert self.unpacked_source_directory
+    def isolated_editable_sanity_check(self) -> None:
+        """Check that an editable requirement if valid for use with PEP 517/518.
 
-            return generate_metadata_legacy(
-                build_env=self.build_env,
-                setup_py_path=self.setup_py_path,
-                source_dir=self.unpacked_source_directory,
-                isolated=self.isolated,
-                details=self.name or f"from {self.link}"
+        This verifies that an editable that has a pyproject.toml either supports PEP 660
+        or as a setup.py or a setup.cfg
+        """
+        if (
+            self.editable
+            and self.use_pep517
+            and not self.supports_pyproject_editable()
+            and not os.path.isfile(self.setup_py_path)
+            and not os.path.isfile(self.setup_cfg_path)
+        ):
+            raise InstallationError(
+                f"Project {self} has a 'pyproject.toml' and its build "
+                f"backend is missing the 'build_editable' hook. Since it does not "
+                f"have a 'setup.py' nor a 'setup.cfg', "
+                f"it cannot be installed in editable mode. "
+                f"Consider using a build backend that supports PEP 660."
             )
 
-        assert self.pep517_backend is not None
-
-        return generate_metadata(
-            build_env=self.build_env,
-            backend=self.pep517_backend,
-        )
-
-    def prepare_metadata(self):
-        # type: () -> None
+    def prepare_metadata(self) -> None:
         """Ensure that project metadata is available.
 
-        Under PEP 517, call the backend hook to prepare the metadata.
+        Under PEP 517 and PEP 660, call the backend hook to prepare the metadata.
         Under legacy processing, call setup.py egg-info.
         """
         assert self.source_dir
-
-        with indent_log():
-            self.metadata_directory = self._generate_metadata()
+        details = self.name or f"from {self.link}"
+
+        if self.use_pep517:
+            assert self.pep517_backend is not None
+            if (
+                self.editable
+                and self.permit_editable_wheels
+                and self.supports_pyproject_editable()
+            ):
+                self.metadata_directory = generate_editable_metadata(
+                    build_env=self.build_env,
+                    backend=self.pep517_backend,
+                    details=details,
+                )
+            else:
+                self.metadata_directory = generate_metadata(
+                    build_env=self.build_env,
+                    backend=self.pep517_backend,
+                    details=details,
+                )
+        else:
+            self.metadata_directory = generate_metadata_legacy(
+                build_env=self.build_env,
+                setup_py_path=self.setup_py_path,
+                source_dir=self.unpacked_source_directory,
+                isolated=self.isolated,
+                details=details,
+            )
 
         # Act on the newly generated metadata, based on the name and version.
         if not self.name:
@@ -557,30 +541,27 @@ def prepare_metadata(self):
         self.assert_source_matches_version()
 
     @property
-    def metadata(self):
-        # type: () -> Any
-        if not hasattr(self, '_metadata'):
-            self._metadata = get_metadata(self.get_dist())
+    def metadata(self) -> Any:
+        if not hasattr(self, "_metadata"):
+            self._metadata = self.get_dist().metadata
 
         return self._metadata
 
-    def get_dist(self):
-        # type: () -> Distribution
-        return _get_dist(self.metadata_directory)
+    def get_dist(self) -> BaseDistribution:
+        return get_directory_distribution(self.metadata_directory)
 
-    def assert_source_matches_version(self):
-        # type: () -> None
+    def assert_source_matches_version(self) -> None:
         assert self.source_dir
-        version = self.metadata['version']
+        version = self.metadata["version"]
         if self.req.specifier and version not in self.req.specifier:
             logger.warning(
-                'Requested %s, but installing version %s',
+                "Requested %s, but installing version %s",
                 self,
                 version,
             )
         else:
             logger.debug(
-                'Source in %s has version %s, which satisfies requirement %s',
+                "Source in %s has version %s, which satisfies requirement %s",
                 display_path(self.source_dir),
                 version,
                 self,
@@ -589,11 +570,10 @@ def assert_source_matches_version(self):
     # For both source distributions and editables
     def ensure_has_source_dir(
         self,
-        parent_dir,
-        autodelete=False,
-        parallel_builds=False,
-    ):
-        # type: (str, bool, bool) -> None
+        parent_dir: str,
+        autodelete: bool = False,
+        parallel_builds: bool = False,
+    ) -> None:
         """Ensure that a source_dir is set.
 
         This will create a temporary build dir if the name of the requirement
@@ -611,52 +591,29 @@ def ensure_has_source_dir(
             )
 
     # For editable installations
-    def update_editable(self, obtain=True):
-        # type: (bool) -> None
+    def update_editable(self) -> None:
         if not self.link:
             logger.debug(
-                "Cannot update repository at %s; repository location is "
-                "unknown",
+                "Cannot update repository at %s; repository location is unknown",
                 self.source_dir,
             )
             return
         assert self.editable
         assert self.source_dir
-        if self.link.scheme == 'file':
+        if self.link.scheme == "file":
             # Static paths don't get updated
             return
-        assert '+' in self.link.url, \
-            "bad url: {self.link.url!r}".format(**locals())
-        vc_type, url = self.link.url.split('+', 1)
-        vcs_backend = vcs.get_backend(vc_type)
-        if vcs_backend:
-            if not self.link.is_vcs:
-                reason = (
-                    "This form of VCS requirement is being deprecated: {}."
-                ).format(
-                    self.link.url
-                )
-                replacement = None
-                if self.link.url.startswith("git+git@"):
-                    replacement = (
-                        "git+https://git@example.com/..., "
-                        "git+ssh://git@example.com/..., "
-                        "or the insecure git+git://git@example.com/..."
-                    )
-                deprecated(reason, replacement, gone_in="21.0", issue=7554)
-            hidden_url = hide_url(self.link.url)
-            if obtain:
-                vcs_backend.obtain(self.source_dir, url=hidden_url)
-            else:
-                vcs_backend.export(self.source_dir, url=hidden_url)
-        else:
-            assert 0, (
-                'Unexpected version control type (in {}): {}'.format(
-                    self.link, vc_type))
+        vcs_backend = vcs.get_backend_for_scheme(self.link.scheme)
+        # Editable requirements are validated in Requirement constructors.
+        # So here, if it's neither a path nor a valid VCS URL, it's a bug.
+        assert vcs_backend, f"Unsupported VCS URL {self.link.url}"
+        hidden_url = hide_url(self.link.url)
+        vcs_backend.obtain(self.source_dir, url=hidden_url, verbosity=0)
 
     # Top-level Actions
-    def uninstall(self, auto_confirm=False, verbose=False):
-        # type: (bool, bool) -> Optional[UninstallPathSet]
+    def uninstall(
+        self, auto_confirm: bool = False, verbose: bool = False
+    ) -> Optional[UninstallPathSet]:
         """
         Uninstall the distribution currently satisfying this requirement.
 
@@ -670,35 +627,30 @@ def uninstall(self, auto_confirm=False, verbose=False):
 
         """
         assert self.req
-        dist = get_distribution(self.req.name)
+        dist = get_default_environment().get_distribution(self.req.name)
         if not dist:
             logger.warning("Skipping %s as it is not installed.", self.name)
             return None
-        logger.info('Found existing installation: %s', dist)
+        logger.info("Found existing installation: %s", dist)
 
         uninstalled_pathset = UninstallPathSet.from_dist(dist)
         uninstalled_pathset.remove(auto_confirm, verbose)
         return uninstalled_pathset
 
-    def _get_archive_name(self, path, parentdir, rootdir):
-        # type: (str, str, str) -> str
-
-        def _clean_zip_name(name, prefix):
-            # type: (str, str) -> str
-            assert name.startswith(prefix + os.path.sep), (
-                "name {name!r} doesn't start with prefix {prefix!r}"
-                .format(**locals())
-            )
-            name = name[len(prefix) + 1:]
-            name = name.replace(os.path.sep, '/')
+    def _get_archive_name(self, path: str, parentdir: str, rootdir: str) -> str:
+        def _clean_zip_name(name: str, prefix: str) -> str:
+            assert name.startswith(
+                prefix + os.path.sep
+            ), f"name {name!r} doesn't start with prefix {prefix!r}"
+            name = name[len(prefix) + 1 :]
+            name = name.replace(os.path.sep, "/")
             return name
 
         path = os.path.join(parentdir, path)
         name = _clean_zip_name(path, rootdir)
-        return self.name + '/' + name
+        return self.name + "/" + name
 
-    def archive(self, build_dir):
-        # type: (Optional[str]) -> None
+    def archive(self, build_dir: Optional[str]) -> None:
         """Saves archive to provided build_dir.
 
         Used for saving downloaded VCS requirements as part of `pip download`.
@@ -708,70 +660,74 @@ def archive(self, build_dir):
             return
 
         create_archive = True
-        archive_name = '{}-{}.zip'.format(self.name, self.metadata["version"])
+        archive_name = "{}-{}.zip".format(self.name, self.metadata["version"])
         archive_path = os.path.join(build_dir, archive_name)
 
         if os.path.exists(archive_path):
             response = ask_path_exists(
-                'The file {} exists. (i)gnore, (w)ipe, '
-                '(b)ackup, (a)bort '.format(
-                    display_path(archive_path)),
-                ('i', 'w', 'b', 'a'))
-            if response == 'i':
+                "The file {} exists. (i)gnore, (w)ipe, "
+                "(b)ackup, (a)bort ".format(display_path(archive_path)),
+                ("i", "w", "b", "a"),
+            )
+            if response == "i":
                 create_archive = False
-            elif response == 'w':
-                logger.warning('Deleting %s', display_path(archive_path))
+            elif response == "w":
+                logger.warning("Deleting %s", display_path(archive_path))
                 os.remove(archive_path)
-            elif response == 'b':
+            elif response == "b":
                 dest_file = backup_dir(archive_path)
                 logger.warning(
-                    'Backing up %s to %s',
+                    "Backing up %s to %s",
                     display_path(archive_path),
                     display_path(dest_file),
                 )
                 shutil.move(archive_path, dest_file)
-            elif response == 'a':
+            elif response == "a":
                 sys.exit(-1)
 
         if not create_archive:
             return
 
         zip_output = zipfile.ZipFile(
-            archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True,
+            archive_path,
+            "w",
+            zipfile.ZIP_DEFLATED,
+            allowZip64=True,
         )
         with zip_output:
-            dir = os.path.normcase(
-                os.path.abspath(self.unpacked_source_directory)
-            )
+            dir = os.path.normcase(os.path.abspath(self.unpacked_source_directory))
             for dirpath, dirnames, filenames in os.walk(dir):
                 for dirname in dirnames:
                     dir_arcname = self._get_archive_name(
-                        dirname, parentdir=dirpath, rootdir=dir,
+                        dirname,
+                        parentdir=dirpath,
+                        rootdir=dir,
                     )
-                    zipdir = zipfile.ZipInfo(dir_arcname + '/')
+                    zipdir = zipfile.ZipInfo(dir_arcname + "/")
                     zipdir.external_attr = 0x1ED << 16  # 0o755
-                    zip_output.writestr(zipdir, '')
+                    zip_output.writestr(zipdir, "")
                 for filename in filenames:
                     file_arcname = self._get_archive_name(
-                        filename, parentdir=dirpath, rootdir=dir,
+                        filename,
+                        parentdir=dirpath,
+                        rootdir=dir,
                     )
                     filename = os.path.join(dirpath, filename)
                     zip_output.write(filename, file_arcname)
 
-        logger.info('Saved %s', display_path(archive_path))
+        logger.info("Saved %s", display_path(archive_path))
 
     def install(
         self,
-        install_options,  # type: List[str]
-        global_options=None,  # type: Optional[Sequence[str]]
-        root=None,  # type: Optional[str]
-        home=None,  # type: Optional[str]
-        prefix=None,  # type: Optional[str]
-        warn_script_location=True,  # type: bool
-        use_user_site=False,  # type: bool
-        pycompile=True  # type: bool
-    ):
-        # type: (...) -> None
+        install_options: List[str],
+        global_options: Optional[Sequence[str]] = None,
+        root: Optional[str] = None,
+        home: Optional[str] = None,
+        prefix: Optional[str] = None,
+        warn_script_location: bool = True,
+        use_user_site: bool = False,
+        pycompile: bool = True,
+    ) -> None:
         scheme = get_scheme(
             self.name,
             user=use_user_site,
@@ -782,7 +738,7 @@ def install(
         )
 
         global_options = global_options if global_options is not None else []
-        if self.editable:
+        if self.editable and not self.is_wheel:
             install_editable_legacy(
                 install_options,
                 global_options,
@@ -801,7 +757,9 @@ def install(
         if self.is_wheel:
             assert self.local_file_path
             direct_url = None
-            if self.original_link:
+            if self.editable:
+                direct_url = direct_url_for_editable(self.unpacked_source_directory)
+            elif self.original_link:
                 direct_url = direct_url_from_link(
                     self.original_link,
                     self.source_dir,
@@ -849,7 +807,7 @@ def install(
             )
         except LegacyInstallFailure as exc:
             self.install_succeeded = False
-            six.reraise(*exc.parent)
+            raise exc
         except Exception:
             self.install_succeeded = True
             raise
@@ -860,8 +818,9 @@ def install(
             deprecated(
                 reason=(
                     "{} was installed using the legacy 'setup.py install' "
-                    "method, because a wheel could not be built for it.".
-                    format(self.name)
+                    "method, because a wheel could not be built for it.".format(
+                        self.name
+                    )
                 ),
                 replacement="to fix the wheel build issue reported above",
                 gone_in=None,
@@ -869,15 +828,14 @@ def install(
             )
 
 
-def check_invalid_constraint_type(req):
-    # type: (InstallRequirement) -> str
+def check_invalid_constraint_type(req: InstallRequirement) -> str:
 
     # Check for unsupported forms
     problem = ""
     if not req.name:
         problem = "Unnamed requirements are not allowed as constraints"
-    elif req.link:
-        problem = "Links are not allowed as constraints"
+    elif req.editable:
+        problem = "Editable requirements are not allowed as constraints"
     elif req.extras:
         problem = "Constraints cannot have extras"
 
@@ -890,12 +848,10 @@ def check_invalid_constraint_type(req):
                 "undocumented. The new implementation of the resolver no "
                 "longer supports these forms."
             ),
-            replacement=(
-                "replacing the constraint with a requirement."
-            ),
+            replacement="replacing the constraint with a requirement",
             # No plan yet for when the new resolver becomes default
             gone_in=None,
-            issue=8210
+            issue=8210,
         )
 
     return problem
diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py
index fa58be66341..6626c37e2e1 100644
--- a/src/pip/_internal/req/req_set.py
+++ b/src/pip/_internal/req/req_set.py
@@ -1,63 +1,51 @@
 import logging
 from collections import OrderedDict
+from typing import Dict, Iterable, List, Optional, Tuple
 
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.exceptions import InstallationError
 from pip._internal.models.wheel import Wheel
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils import compatibility_tags
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Dict, Iterable, List, Optional, Tuple
-
-    from pip._internal.req.req_install import InstallRequirement
-
 
 logger = logging.getLogger(__name__)
 
 
 class RequirementSet:
+    def __init__(self, check_supported_wheels: bool = True) -> None:
+        """Create a RequirementSet."""
 
-    def __init__(self, check_supported_wheels=True):
-        # type: (bool) -> None
-        """Create a RequirementSet.
-        """
-
-        self.requirements = OrderedDict()  # type: Dict[str, InstallRequirement]
+        self.requirements: Dict[str, InstallRequirement] = OrderedDict()
         self.check_supported_wheels = check_supported_wheels
 
-        self.unnamed_requirements = []  # type: List[InstallRequirement]
+        self.unnamed_requirements: List[InstallRequirement] = []
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         requirements = sorted(
             (req for req in self.requirements.values() if not req.comes_from),
-            key=lambda req: canonicalize_name(req.name),
+            key=lambda req: canonicalize_name(req.name or ""),
         )
-        return ' '.join(str(req.req) for req in requirements)
+        return " ".join(str(req.req) for req in requirements)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         requirements = sorted(
             self.requirements.values(),
-            key=lambda req: canonicalize_name(req.name),
+            key=lambda req: canonicalize_name(req.name or ""),
         )
 
-        format_string = '<{classname} object; {count} requirement(s): {reqs}>'
+        format_string = "<{classname} object; {count} requirement(s): {reqs}>"
         return format_string.format(
             classname=self.__class__.__name__,
             count=len(requirements),
-            reqs=', '.join(str(req.req) for req in requirements),
+            reqs=", ".join(str(req.req) for req in requirements),
         )
 
-    def add_unnamed_requirement(self, install_req):
-        # type: (InstallRequirement) -> None
+    def add_unnamed_requirement(self, install_req: InstallRequirement) -> None:
         assert not install_req.name
         self.unnamed_requirements.append(install_req)
 
-    def add_named_requirement(self, install_req):
-        # type: (InstallRequirement) -> None
+    def add_named_requirement(self, install_req: InstallRequirement) -> None:
         assert install_req.name
 
         project_name = canonicalize_name(install_req.name)
@@ -65,11 +53,10 @@ def add_named_requirement(self, install_req):
 
     def add_requirement(
         self,
-        install_req,  # type: InstallRequirement
-        parent_req_name=None,  # type: Optional[str]
-        extras_requested=None  # type: Optional[Iterable[str]]
-    ):
-        # type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]]
+        install_req: InstallRequirement,
+        parent_req_name: Optional[str] = None,
+        extras_requested: Optional[Iterable[str]] = None,
+    ) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]]:
         """Add install_req as a requirement to install.
 
         :param parent_req_name: The name of the requirement that needed this
@@ -88,7 +75,8 @@ def add_requirement(
         if not install_req.match_markers(extras_requested):
             logger.info(
                 "Ignoring %s: markers '%s' don't match your environment",
-                install_req.name, install_req.markers,
+                install_req.name,
+                install_req.markers,
             )
             return [], None
 
@@ -99,16 +87,17 @@ def add_requirement(
         if install_req.link and install_req.link.is_wheel:
             wheel = Wheel(install_req.link.filename)
             tags = compatibility_tags.get_supported()
-            if (self.check_supported_wheels and not wheel.supported(tags)):
+            if self.check_supported_wheels and not wheel.supported(tags):
                 raise InstallationError(
                     "{} is not a supported wheel on this platform.".format(
-                        wheel.filename)
+                        wheel.filename
+                    )
                 )
 
         # This next bit is really a sanity check.
-        assert not install_req.user_supplied or parent_req_name is None, (
-            "a user supplied req shouldn't have a parent"
-        )
+        assert (
+            not install_req.user_supplied or parent_req_name is None
+        ), "a user supplied req shouldn't have a parent"
 
         # Unnamed requirements are scanned again and the requirement won't be
         # added as a dependency until after scanning.
@@ -117,22 +106,26 @@ def add_requirement(
             return [install_req], None
 
         try:
-            existing_req = self.get_requirement(
-                install_req.name)  # type: Optional[InstallRequirement]
+            existing_req: Optional[InstallRequirement] = self.get_requirement(
+                install_req.name
+            )
         except KeyError:
             existing_req = None
 
         has_conflicting_requirement = (
-            parent_req_name is None and
-            existing_req and
-            not existing_req.constraint and
-            existing_req.extras == install_req.extras and
-            existing_req.req.specifier != install_req.req.specifier
+            parent_req_name is None
+            and existing_req
+            and not existing_req.constraint
+            and existing_req.extras == install_req.extras
+            and existing_req.req
+            and install_req.req
+            and existing_req.req.specifier != install_req.req.specifier
         )
         if has_conflicting_requirement:
             raise InstallationError(
-                "Double requirement given: {} (already in {}, name={!r})"
-                .format(install_req, existing_req, install_req.name)
+                "Double requirement given: {} (already in {}, name={!r})".format(
+                    install_req, existing_req, install_req.name
+                )
             )
 
         # When no existing requirement exists, add the requirement as a
@@ -147,12 +140,8 @@ def add_requirement(
         if install_req.constraint or not existing_req.constraint:
             return [], existing_req
 
-        does_not_satisfy_constraint = (
-            install_req.link and
-            not (
-                existing_req.link and
-                install_req.link.path == existing_req.link.path
-            )
+        does_not_satisfy_constraint = install_req.link and not (
+            existing_req.link and install_req.link.path == existing_req.link.path
         )
         if does_not_satisfy_constraint:
             raise InstallationError(
@@ -167,36 +156,34 @@ def add_requirement(
         # mark the existing object as such.
         if install_req.user_supplied:
             existing_req.user_supplied = True
-        existing_req.extras = tuple(sorted(
-            set(existing_req.extras) | set(install_req.extras)
-        ))
+        existing_req.extras = tuple(
+            sorted(set(existing_req.extras) | set(install_req.extras))
+        )
         logger.debug(
             "Setting %s extras to: %s",
-            existing_req, existing_req.extras,
+            existing_req,
+            existing_req.extras,
         )
         # Return the existing requirement for addition to the parent and
         # scanning again.
         return [existing_req], existing_req
 
-    def has_requirement(self, name):
-        # type: (str) -> bool
+    def has_requirement(self, name: str) -> bool:
         project_name = canonicalize_name(name)
 
         return (
-            project_name in self.requirements and
-            not self.requirements[project_name].constraint
+            project_name in self.requirements
+            and not self.requirements[project_name].constraint
         )
 
-    def get_requirement(self, name):
-        # type: (str) -> InstallRequirement
+    def get_requirement(self, name: str) -> InstallRequirement:
         project_name = canonicalize_name(name)
 
         if project_name in self.requirements:
             return self.requirements[project_name]
 
-        raise KeyError("No project with the name {name!r}".format(**locals()))
+        raise KeyError(f"No project with the name {name!r}")
 
     @property
-    def all_requirements(self):
-        # type: () -> List[InstallRequirement]
+    def all_requirements(self) -> List[InstallRequirement]:
         return self.unnamed_requirements + list(self.requirements.values())
diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py
index ba4983f38fc..0490707ca87 100644
--- a/src/pip/_internal/req/req_uninstall.py
+++ b/src/pip/_internal/req/req_uninstall.py
@@ -1,86 +1,53 @@
-import csv
 import functools
-import logging
 import os
 import sys
 import sysconfig
 from importlib.util import cache_from_source
-
-from pip._vendor import pkg_resources
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
 
 from pip._internal.exceptions import UninstallationError
-from pip._internal.locations import bin_py, bin_user
+from pip._internal.locations import get_bin_prefix, get_bin_user
+from pip._internal.metadata import BaseDistribution
 from pip._internal.utils.compat import WINDOWS
-from pip._internal.utils.logging import indent_log
-from pip._internal.utils.misc import (
-    ask,
-    dist_in_usersite,
-    dist_is_local,
-    egg_link_path,
-    is_local,
-    normalize_path,
-    renames,
-    rmtree,
-)
+from pip._internal.utils.egg_link import egg_link_path_from_location
+from pip._internal.utils.logging import getLogger, indent_log
+from pip._internal.utils.misc import ask, is_local, normalize_path, renames, rmtree
 from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import (
-        Any,
-        Callable,
-        Dict,
-        Iterable,
-        Iterator,
-        List,
-        Optional,
-        Set,
-        Tuple,
-    )
-
-    from pip._vendor.pkg_resources import Distribution
 
-logger = logging.getLogger(__name__)
+logger = getLogger(__name__)
 
 
-def _script_names(dist, script_name, is_gui):
-    # type: (Distribution, str, bool) -> List[str]
+def _script_names(bin_dir: str, script_name: str, is_gui: bool) -> Iterator[str]:
     """Create the fully qualified name of the files created by
     {console,gui}_scripts for the given ``dist``.
     Returns the list of file names
     """
-    if dist_in_usersite(dist):
-        bin_dir = bin_user
-    else:
-        bin_dir = bin_py
     exe_name = os.path.join(bin_dir, script_name)
-    paths_to_remove = [exe_name]
-    if WINDOWS:
-        paths_to_remove.append(exe_name + '.exe')
-        paths_to_remove.append(exe_name + '.exe.manifest')
-        if is_gui:
-            paths_to_remove.append(exe_name + '-script.pyw')
-        else:
-            paths_to_remove.append(exe_name + '-script.py')
-    return paths_to_remove
+    yield exe_name
+    if not WINDOWS:
+        return
+    yield f"{exe_name}.exe"
+    yield f"{exe_name}.exe.manifest"
+    if is_gui:
+        yield f"{exe_name}-script.pyw"
+    else:
+        yield f"{exe_name}-script.py"
 
 
-def _unique(fn):
-    # type: (Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]
+def _unique(fn: Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]:
     @functools.wraps(fn)
-    def unique(*args, **kw):
-        # type: (Any, Any) -> Iterator[Any]
-        seen = set()  # type: Set[Any]
+    def unique(*args: Any, **kw: Any) -> Iterator[Any]:
+        seen: Set[Any] = set()
         for item in fn(*args, **kw):
             if item not in seen:
                 seen.add(item)
                 yield item
+
     return unique
 
 
 @_unique
-def uninstallation_paths(dist):
-    # type: (Distribution) -> Iterator[str]
+def uninstallation_paths(dist: BaseDistribution) -> Iterator[str]:
     """
     Yield all the uninstallation paths for dist based on RECORD-without-.py[co]
 
@@ -88,33 +55,53 @@ def uninstallation_paths(dist):
     the .pyc and .pyo in the same directory.
 
     UninstallPathSet.add() takes care of the __pycache__ .py[co].
+
+    If RECORD is not found, raises UninstallationError,
+    with possible information from the INSTALLER file.
+
+    https://packaging.python.org/specifications/recording-installed-packages/
     """
-    r = csv.reader(dist.get_metadata_lines('RECORD'))
-    for row in r:
-        path = os.path.join(dist.location, row[0])
+    location = dist.location
+    assert location is not None, "not installed"
+
+    entries = dist.iter_declared_entries()
+    if entries is None:
+        msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist)
+        installer = dist.installer
+        if not installer or installer == "pip":
+            dep = "{}=={}".format(dist.raw_name, dist.version)
+            msg += (
+                " You might be able to recover from this via: "
+                "'pip install --force-reinstall --no-deps {}'.".format(dep)
+            )
+        else:
+            msg += " Hint: The package was installed by {}.".format(installer)
+        raise UninstallationError(msg)
+
+    for entry in entries:
+        path = os.path.join(location, entry)
         yield path
-        if path.endswith('.py'):
+        if path.endswith(".py"):
             dn, fn = os.path.split(path)
             base = fn[:-3]
-            path = os.path.join(dn, base + '.pyc')
+            path = os.path.join(dn, base + ".pyc")
             yield path
-            path = os.path.join(dn, base + '.pyo')
+            path = os.path.join(dn, base + ".pyo")
             yield path
 
 
-def compact(paths):
-    # type: (Iterable[str]) -> Set[str]
+def compact(paths: Iterable[str]) -> Set[str]:
     """Compact a path set to contain the minimal number of paths
     necessary to contain all paths in the set. If /a/path/ and
     /a/path/to/a/file.txt are both in the set, leave only the
     shorter path."""
 
     sep = os.path.sep
-    short_paths = set()  # type: Set[str]
+    short_paths: Set[str] = set()
     for path in sorted(paths, key=len):
         should_skip = any(
-            path.startswith(shortpath.rstrip("*")) and
-            path[len(shortpath.rstrip("*").rstrip(sep))] == sep
+            path.startswith(shortpath.rstrip("*"))
+            and path[len(shortpath.rstrip("*").rstrip(sep))] == sep
             for shortpath in short_paths
         )
         if not should_skip:
@@ -122,8 +109,7 @@ def compact(paths):
     return short_paths
 
 
-def compress_for_rename(paths):
-    # type: (Iterable[str]) -> Set[str]
+def compress_for_rename(paths: Iterable[str]) -> Set[str]:
     """Returns a set containing the paths that need to be renamed.
 
     This set may include directories when the original sequence of paths
@@ -132,25 +118,21 @@ def compress_for_rename(paths):
     case_map = {os.path.normcase(p): p for p in paths}
     remaining = set(case_map)
     unchecked = sorted({os.path.split(p)[0] for p in case_map.values()}, key=len)
-    wildcards = set()  # type: Set[str]
+    wildcards: Set[str] = set()
 
-    def norm_join(*a):
-        # type: (str) -> str
+    def norm_join(*a: str) -> str:
         return os.path.normcase(os.path.join(*a))
 
     for root in unchecked:
-        if any(os.path.normcase(root).startswith(w)
-               for w in wildcards):
+        if any(os.path.normcase(root).startswith(w) for w in wildcards):
             # This directory has already been handled.
             continue
 
-        all_files = set()  # type: Set[str]
-        all_subdirs = set()  # type: Set[str]
+        all_files: Set[str] = set()
+        all_subdirs: Set[str] = set()
         for dirname, subdirs, files in os.walk(root):
-            all_subdirs.update(norm_join(root, dirname, d)
-                               for d in subdirs)
-            all_files.update(norm_join(root, dirname, f)
-                             for f in files)
+            all_subdirs.update(norm_join(root, dirname, d) for d in subdirs)
+            all_files.update(norm_join(root, dirname, f) for f in files)
         # If all the files we found are in our remaining set of files to
         # remove, then remove them from the latter set and add a wildcard
         # for the directory.
@@ -161,8 +143,7 @@ def norm_join(*a):
     return set(map(case_map.__getitem__, remaining)) | wildcards
 
 
-def compress_for_output_listing(paths):
-    # type: (Iterable[str]) -> Tuple[Set[str], Set[str]]
+def compress_for_output_listing(paths: Iterable[str]) -> Tuple[Set[str], Set[str]]:
     """Returns a tuple of 2 sets of which paths to display to user
 
     The first set contains paths that would be deleted. Files of a package
@@ -200,14 +181,14 @@ def compress_for_output_listing(paths):
                     continue
 
                 file_ = os.path.join(dirpath, fname)
-                if (os.path.isfile(file_) and
-                        os.path.normcase(file_) not in _normcased_files):
+                if (
+                    os.path.isfile(file_)
+                    and os.path.normcase(file_) not in _normcased_files
+                ):
                     # We are skipping this file. Add it to the set.
                     will_skip.add(file_)
 
-    will_remove = files | {
-        os.path.join(folder, "*") for folder in folders
-    }
+    will_remove = files | {os.path.join(folder, "*") for folder in folders}
 
     return will_remove, will_skip
 
@@ -215,32 +196,30 @@ def compress_for_output_listing(paths):
 class StashedUninstallPathSet:
     """A set of file rename operations to stash files while
     tentatively uninstalling them."""
-    def __init__(self):
-        # type: () -> None
+
+    def __init__(self) -> None:
         # Mapping from source file root to [Adjacent]TempDirectory
         # for files under that directory.
-        self._save_dirs = {}  # type: Dict[str, TempDirectory]
+        self._save_dirs: Dict[str, TempDirectory] = {}
         # (old path, new path) tuples for each move that may need
         # to be undone.
-        self._moves = []  # type: List[Tuple[str, str]]
+        self._moves: List[Tuple[str, str]] = []
 
-    def _get_directory_stash(self, path):
-        # type: (str) -> str
+    def _get_directory_stash(self, path: str) -> str:
         """Stashes a directory.
 
         Directories are stashed adjacent to their original location if
         possible, or else moved/copied into the user's temp dir."""
 
         try:
-            save_dir = AdjacentTempDirectory(path)  # type: TempDirectory
+            save_dir: TempDirectory = AdjacentTempDirectory(path)
         except OSError:
             save_dir = TempDirectory(kind="uninstall")
         self._save_dirs[os.path.normcase(path)] = save_dir
 
         return save_dir.path
 
-    def _get_file_stash(self, path):
-        # type: (str) -> str
+    def _get_file_stash(self, path: str) -> str:
         """Stashes a file.
 
         If no root has been provided, one will be created for the directory
@@ -259,7 +238,7 @@ def _get_file_stash(self, path):
         else:
             # Did not find any suitable root
             head = os.path.dirname(path)
-            save_dir = TempDirectory(kind='uninstall')
+            save_dir = TempDirectory(kind="uninstall")
             self._save_dirs[head] = save_dir
 
         relpath = os.path.relpath(path, head)
@@ -267,8 +246,7 @@ def _get_file_stash(self, path):
             return os.path.join(save_dir.path, relpath)
         return save_dir.path
 
-    def stash(self, path):
-        # type: (str) -> str
+    def stash(self, path: str) -> str:
         """Stashes the directory or file and returns its new location.
         Handle symlinks as files to avoid modifying the symlink targets.
         """
@@ -279,7 +257,7 @@ def stash(self, path):
             new_path = self._get_file_stash(path)
 
         self._moves.append((path, new_path))
-        if (path_is_dir and os.path.isdir(new_path)):
+        if path_is_dir and os.path.isdir(new_path):
             # If we're moving a directory, we need to
             # remove the destination first or else it will be
             # moved to inside the existing directory.
@@ -289,23 +267,21 @@ def stash(self, path):
         renames(path, new_path)
         return new_path
 
-    def commit(self):
-        # type: () -> None
+    def commit(self) -> None:
         """Commits the uninstall by removing stashed files."""
         for _, save_dir in self._save_dirs.items():
             save_dir.cleanup()
         self._moves = []
         self._save_dirs = {}
 
-    def rollback(self):
-        # type: () -> None
+    def rollback(self) -> None:
         """Undoes the uninstall by moving stashed files back."""
         for p in self._moves:
             logger.info("Moving to %s\n from %s", *p)
 
         for new_path, path in self._moves:
             try:
-                logger.debug('Replacing %s from %s', new_path, path)
+                logger.debug("Replacing %s from %s", new_path, path)
                 if os.path.isfile(new_path) or os.path.islink(new_path):
                     os.unlink(new_path)
                 elif os.path.isdir(new_path):
@@ -318,24 +294,22 @@ def rollback(self):
         self.commit()
 
     @property
-    def can_rollback(self):
-        # type: () -> bool
+    def can_rollback(self) -> bool:
         return bool(self._moves)
 
 
 class UninstallPathSet:
     """A set of file paths to be removed in the uninstallation of a
     requirement."""
-    def __init__(self, dist):
-        # type: (Distribution) -> None
-        self.paths = set()  # type: Set[str]
-        self._refuse = set()  # type: Set[str]
-        self.pth = {}  # type: Dict[str, UninstallPthEntries]
-        self.dist = dist
+
+    def __init__(self, dist: BaseDistribution) -> None:
+        self._paths: Set[str] = set()
+        self._refuse: Set[str] = set()
+        self._pth: Dict[str, UninstallPthEntries] = {}
+        self._dist = dist
         self._moved_paths = StashedUninstallPathSet()
 
-    def _permitted(self, path):
-        # type: (str) -> bool
+    def _permitted(self, path: str) -> bool:
         """
         Return True if the given path is one we are permitted to
         remove/modify, False otherwise.
@@ -343,8 +317,7 @@ def _permitted(self, path):
         """
         return is_local(path)
 
-    def add(self, path):
-        # type: (str) -> None
+    def add(self, path: str) -> None:
         head, tail = os.path.split(path)
 
         # we normalize the head to resolve parent directory symlinks, but not
@@ -354,64 +327,57 @@ def add(self, path):
         if not os.path.exists(path):
             return
         if self._permitted(path):
-            self.paths.add(path)
+            self._paths.add(path)
         else:
             self._refuse.add(path)
 
         # __pycache__ files can show up after 'installed-files.txt' is created,
         # due to imports
-        if os.path.splitext(path)[1] == '.py':
+        if os.path.splitext(path)[1] == ".py":
             self.add(cache_from_source(path))
 
-    def add_pth(self, pth_file, entry):
-        # type: (str, str) -> None
+    def add_pth(self, pth_file: str, entry: str) -> None:
         pth_file = normalize_path(pth_file)
         if self._permitted(pth_file):
-            if pth_file not in self.pth:
-                self.pth[pth_file] = UninstallPthEntries(pth_file)
-            self.pth[pth_file].add(entry)
+            if pth_file not in self._pth:
+                self._pth[pth_file] = UninstallPthEntries(pth_file)
+            self._pth[pth_file].add(entry)
         else:
             self._refuse.add(pth_file)
 
-    def remove(self, auto_confirm=False, verbose=False):
-        # type: (bool, bool) -> None
-        """Remove paths in ``self.paths`` with confirmation (unless
+    def remove(self, auto_confirm: bool = False, verbose: bool = False) -> None:
+        """Remove paths in ``self._paths`` with confirmation (unless
         ``auto_confirm`` is True)."""
 
-        if not self.paths:
+        if not self._paths:
             logger.info(
                 "Can't uninstall '%s'. No files were found to uninstall.",
-                self.dist.project_name,
+                self._dist.raw_name,
             )
             return
 
-        dist_name_version = (
-            self.dist.project_name + "-" + self.dist.version
-        )
-        logger.info('Uninstalling %s:', dist_name_version)
+        dist_name_version = f"{self._dist.raw_name}-{self._dist.version}"
+        logger.info("Uninstalling %s:", dist_name_version)
 
         with indent_log():
             if auto_confirm or self._allowed_to_proceed(verbose):
                 moved = self._moved_paths
 
-                for_rename = compress_for_rename(self.paths)
+                for_rename = compress_for_rename(self._paths)
 
                 for path in sorted(compact(for_rename)):
                     moved.stash(path)
-                    logger.debug('Removing file or directory %s', path)
+                    logger.verbose("Removing file or directory %s", path)
 
-                for pth in self.pth.values():
+                for pth in self._pth.values():
                     pth.remove()
 
-                logger.info('Successfully uninstalled %s', dist_name_version)
+                logger.info("Successfully uninstalled %s", dist_name_version)
 
-    def _allowed_to_proceed(self, verbose):
-        # type: (bool) -> bool
-        """Display which files would be deleted and prompt for confirmation
-        """
+    def _allowed_to_proceed(self, verbose: bool) -> bool:
+        """Display which files would be deleted and prompt for confirmation"""
 
-        def _display(msg, paths):
-            # type: (str, Iterable[str]) -> None
+        def _display(msg: str, paths: Iterable[str]) -> None:
             if not paths:
                 return
 
@@ -421,182 +387,201 @@ def _display(msg, paths):
                     logger.info(path)
 
         if not verbose:
-            will_remove, will_skip = compress_for_output_listing(self.paths)
+            will_remove, will_skip = compress_for_output_listing(self._paths)
         else:
             # In verbose mode, display all the files that are going to be
             # deleted.
-            will_remove = set(self.paths)
+            will_remove = set(self._paths)
             will_skip = set()
 
-        _display('Would remove:', will_remove)
-        _display('Would not remove (might be manually added):', will_skip)
-        _display('Would not remove (outside of prefix):', self._refuse)
+        _display("Would remove:", will_remove)
+        _display("Would not remove (might be manually added):", will_skip)
+        _display("Would not remove (outside of prefix):", self._refuse)
         if verbose:
-            _display('Will actually move:', compress_for_rename(self.paths))
+            _display("Will actually move:", compress_for_rename(self._paths))
 
-        return ask('Proceed (y/n)? ', ('y', 'n')) == 'y'
+        return ask("Proceed (Y/n)? ", ("y", "n", "")) != "n"
 
-    def rollback(self):
-        # type: () -> None
+    def rollback(self) -> None:
         """Rollback the changes previously made by remove()."""
         if not self._moved_paths.can_rollback:
             logger.error(
                 "Can't roll back %s; was not uninstalled",
-                self.dist.project_name,
+                self._dist.raw_name,
             )
             return
-        logger.info('Rolling back uninstall of %s', self.dist.project_name)
+        logger.info("Rolling back uninstall of %s", self._dist.raw_name)
         self._moved_paths.rollback()
-        for pth in self.pth.values():
+        for pth in self._pth.values():
             pth.rollback()
 
-    def commit(self):
-        # type: () -> None
+    def commit(self) -> None:
         """Remove temporary save dir: rollback will no longer be possible."""
         self._moved_paths.commit()
 
     @classmethod
-    def from_dist(cls, dist):
-        # type: (Distribution) -> UninstallPathSet
-        dist_path = normalize_path(dist.location)
-        if not dist_is_local(dist):
+    def from_dist(cls, dist: BaseDistribution) -> "UninstallPathSet":
+        dist_location = dist.location
+        info_location = dist.info_location
+        if dist_location is None:
+            logger.info(
+                "Not uninstalling %s since it is not installed",
+                dist.canonical_name,
+            )
+            return cls(dist)
+
+        normalized_dist_location = normalize_path(dist_location)
+        if not dist.local:
             logger.info(
                 "Not uninstalling %s at %s, outside environment %s",
-                dist.key,
-                dist_path,
+                dist.canonical_name,
+                normalized_dist_location,
                 sys.prefix,
             )
             return cls(dist)
 
-        if dist_path in {p for p in {sysconfig.get_path("stdlib"),
-                                     sysconfig.get_path("platstdlib")}
-                         if p}:
+        if normalized_dist_location in {
+            p
+            for p in {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}
+            if p
+        }:
             logger.info(
                 "Not uninstalling %s at %s, as it is in the standard library.",
-                dist.key,
-                dist_path,
+                dist.canonical_name,
+                normalized_dist_location,
             )
             return cls(dist)
 
         paths_to_remove = cls(dist)
-        develop_egg_link = egg_link_path(dist)
-        develop_egg_link_egg_info = '{}.egg-info'.format(
-            pkg_resources.to_filename(dist.project_name))
-        egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info)
-        # Special case for distutils installed package
-        distutils_egg_info = getattr(dist._provider, 'path', None)
+        develop_egg_link = egg_link_path_from_location(dist.raw_name)
+
+        # Distribution is installed with metadata in a "flat" .egg-info
+        # directory. This means it is not a modern .dist-info installation, an
+        # egg, or legacy editable.
+        setuptools_flat_installation = (
+            dist.installed_with_setuptools_egg_info
+            and info_location is not None
+            and os.path.exists(info_location)
+            # If dist is editable and the location points to a ``.egg-info``,
+            # we are in fact in the legacy editable case.
+            and not info_location.endswith(f"{dist.setuptools_filename}.egg-info")
+        )
 
         # Uninstall cases order do matter as in the case of 2 installs of the
         # same package, pip needs to uninstall the currently detected version
-        if (egg_info_exists and dist.egg_info.endswith('.egg-info') and
-                not dist.egg_info.endswith(develop_egg_link_egg_info)):
-            # if dist.egg_info.endswith(develop_egg_link_egg_info), we
-            # are in fact in the develop_egg_link case
-            paths_to_remove.add(dist.egg_info)
-            if dist.has_metadata('installed-files.txt'):
-                for installed_file in dist.get_metadata(
-                        'installed-files.txt').splitlines():
-                    path = os.path.normpath(
-                        os.path.join(dist.egg_info, installed_file)
-                    )
-                    paths_to_remove.add(path)
+        if setuptools_flat_installation:
+            if info_location is not None:
+                paths_to_remove.add(info_location)
+            installed_files = dist.iter_declared_entries()
+            if installed_files is not None:
+                for installed_file in installed_files:
+                    paths_to_remove.add(os.path.join(dist_location, installed_file))
             # FIXME: need a test for this elif block
             # occurs with --single-version-externally-managed/--record outside
             # of pip
-            elif dist.has_metadata('top_level.txt'):
-                if dist.has_metadata('namespace_packages.txt'):
-                    namespaces = dist.get_metadata('namespace_packages.txt')
-                else:
+            elif dist.is_file("top_level.txt"):
+                try:
+                    namespace_packages = dist.read_text("namespace_packages.txt")
+                except FileNotFoundError:
                     namespaces = []
+                else:
+                    namespaces = namespace_packages.splitlines(keepends=False)
                 for top_level_pkg in [
-                        p for p
-                        in dist.get_metadata('top_level.txt').splitlines()
-                        if p and p not in namespaces]:
-                    path = os.path.join(dist.location, top_level_pkg)
+                    p
+                    for p in dist.read_text("top_level.txt").splitlines()
+                    if p and p not in namespaces
+                ]:
+                    path = os.path.join(dist_location, top_level_pkg)
                     paths_to_remove.add(path)
-                    paths_to_remove.add(path + '.py')
-                    paths_to_remove.add(path + '.pyc')
-                    paths_to_remove.add(path + '.pyo')
+                    paths_to_remove.add(f"{path}.py")
+                    paths_to_remove.add(f"{path}.pyc")
+                    paths_to_remove.add(f"{path}.pyo")
 
-        elif distutils_egg_info:
+        elif dist.installed_by_distutils:
             raise UninstallationError(
                 "Cannot uninstall {!r}. It is a distutils installed project "
                 "and thus we cannot accurately determine which files belong "
                 "to it which would lead to only a partial uninstall.".format(
-                    dist.project_name,
+                    dist.raw_name,
                 )
             )
 
-        elif dist.location.endswith('.egg'):
+        elif dist.installed_as_egg:
             # package installed by easy_install
             # We cannot match on dist.egg_name because it can slightly vary
             # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg
-            paths_to_remove.add(dist.location)
-            easy_install_egg = os.path.split(dist.location)[1]
-            easy_install_pth = os.path.join(os.path.dirname(dist.location),
-                                            'easy-install.pth')
-            paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg)
+            paths_to_remove.add(dist_location)
+            easy_install_egg = os.path.split(dist_location)[1]
+            easy_install_pth = os.path.join(
+                os.path.dirname(dist_location),
+                "easy-install.pth",
+            )
+            paths_to_remove.add_pth(easy_install_pth, "./" + easy_install_egg)
 
-        elif egg_info_exists and dist.egg_info.endswith('.dist-info'):
+        elif dist.installed_with_dist_info:
             for path in uninstallation_paths(dist):
                 paths_to_remove.add(path)
 
         elif develop_egg_link:
-            # develop egg
-            with open(develop_egg_link, 'r') as fh:
+            # PEP 660 modern editable is handled in the ``.dist-info`` case
+            # above, so this only covers the setuptools-style editable.
+            with open(develop_egg_link) as fh:
                 link_pointer = normalize_path(os.path.normcase(fh.readline().strip()))
-            assert (link_pointer == dist_path), (
-                'Egg-link {} does not match installed location of {} '
-                '(at {})'.format(
-                    link_pointer, dist.project_name, dist.location)
+            assert os.path.samefile(link_pointer, dist_location), (
+                f"Egg-link {link_pointer} does not match installed location of "
+                f"{dist.raw_name} (at {dist_location})"
             )
             paths_to_remove.add(develop_egg_link)
-            easy_install_pth = os.path.join(os.path.dirname(develop_egg_link),
-                                            'easy-install.pth')
-            paths_to_remove.add_pth(easy_install_pth, dist.location)
+            easy_install_pth = os.path.join(
+                os.path.dirname(develop_egg_link), "easy-install.pth"
+            )
+            paths_to_remove.add_pth(easy_install_pth, dist_location)
 
         else:
             logger.debug(
-                'Not sure how to uninstall: %s - Check: %s',
-                dist, dist.location,
+                "Not sure how to uninstall: %s - Check: %s",
+                dist,
+                dist_location,
             )
 
+        if dist.in_usersite:
+            bin_dir = get_bin_user()
+        else:
+            bin_dir = get_bin_prefix()
+
         # find distutils scripts= scripts
-        if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
-            for script in dist.metadata_listdir('scripts'):
-                if dist_in_usersite(dist):
-                    bin_dir = bin_user
-                else:
-                    bin_dir = bin_py
-                paths_to_remove.add(os.path.join(bin_dir, script))
+        try:
+            for script in dist.iterdir("scripts"):
+                paths_to_remove.add(os.path.join(bin_dir, script.name))
                 if WINDOWS:
-                    paths_to_remove.add(os.path.join(bin_dir, script) + '.bat')
-
-        # find console_scripts
-        _scripts_to_remove = []
-        console_scripts = dist.get_entry_map(group='console_scripts')
-        for name in console_scripts.keys():
-            _scripts_to_remove.extend(_script_names(dist, name, False))
-        # find gui_scripts
-        gui_scripts = dist.get_entry_map(group='gui_scripts')
-        for name in gui_scripts.keys():
-            _scripts_to_remove.extend(_script_names(dist, name, True))
-
-        for s in _scripts_to_remove:
+                    paths_to_remove.add(os.path.join(bin_dir, f"{script.name}.bat"))
+        except (FileNotFoundError, NotADirectoryError):
+            pass
+
+        # find console_scripts and gui_scripts
+        def iter_scripts_to_remove(
+            dist: BaseDistribution,
+            bin_dir: str,
+        ) -> Iterator[str]:
+            for entry_point in dist.iter_entry_points():
+                if entry_point.group == "console_scripts":
+                    yield from _script_names(bin_dir, entry_point.name, False)
+                elif entry_point.group == "gui_scripts":
+                    yield from _script_names(bin_dir, entry_point.name, True)
+
+        for s in iter_scripts_to_remove(dist, bin_dir):
             paths_to_remove.add(s)
 
         return paths_to_remove
 
 
 class UninstallPthEntries:
-    def __init__(self, pth_file):
-        # type: (str) -> None
+    def __init__(self, pth_file: str) -> None:
         self.file = pth_file
-        self.entries = set()  # type: Set[str]
-        self._saved_lines = None  # type: Optional[List[bytes]]
+        self.entries: Set[str] = set()
+        self._saved_lines: Optional[List[bytes]] = None
 
-    def add(self, entry):
-        # type: (str) -> None
+    def add(self, entry: str) -> None:
         entry = os.path.normcase(entry)
         # On Windows, os.path.normcase converts the entry to use
         # backslashes.  This is correct for entries that describe absolute
@@ -608,47 +593,41 @@ def add(self, entry):
         # have more than "\\sever\share". Valid examples: "\\server\share\" or
         # "\\server\share\folder".
         if WINDOWS and not os.path.splitdrive(entry)[0]:
-            entry = entry.replace('\\', '/')
+            entry = entry.replace("\\", "/")
         self.entries.add(entry)
 
-    def remove(self):
-        # type: () -> None
-        logger.debug('Removing pth entries from %s:', self.file)
+    def remove(self) -> None:
+        logger.verbose("Removing pth entries from %s:", self.file)
 
         # If the file doesn't exist, log a warning and return
         if not os.path.isfile(self.file):
-            logger.warning(
-                "Cannot remove entries from nonexistent file %s", self.file
-            )
+            logger.warning("Cannot remove entries from nonexistent file %s", self.file)
             return
-        with open(self.file, 'rb') as fh:
+        with open(self.file, "rb") as fh:
             # windows uses '\r\n' with py3k, but uses '\n' with py2.x
             lines = fh.readlines()
             self._saved_lines = lines
-        if any(b'\r\n' in line for line in lines):
-            endline = '\r\n'
+        if any(b"\r\n" in line for line in lines):
+            endline = "\r\n"
         else:
-            endline = '\n'
+            endline = "\n"
         # handle missing trailing newline
         if lines and not lines[-1].endswith(endline.encode("utf-8")):
             lines[-1] = lines[-1] + endline.encode("utf-8")
         for entry in self.entries:
             try:
-                logger.debug('Removing entry: %s', entry)
+                logger.verbose("Removing entry: %s", entry)
                 lines.remove((entry + endline).encode("utf-8"))
             except ValueError:
                 pass
-        with open(self.file, 'wb') as fh:
+        with open(self.file, "wb") as fh:
             fh.writelines(lines)
 
-    def rollback(self):
-        # type: () -> bool
+    def rollback(self) -> bool:
         if self._saved_lines is None:
-            logger.error(
-                'Cannot roll back changes to %s, none were made', self.file
-            )
+            logger.error("Cannot roll back changes to %s, none were made", self.file)
             return False
-        logger.debug('Rolling %s back to previous state', self.file)
-        with open(self.file, 'wb') as fh:
+        logger.debug("Rolling %s back to previous state", self.file)
+        with open(self.file, "wb") as fh:
             fh.writelines(self._saved_lines)
         return True
diff --git a/src/pip/_internal/resolution/base.py b/src/pip/_internal/resolution/base.py
index f2816ab71c2..42dade18c1e 100644
--- a/src/pip/_internal/resolution/base.py
+++ b/src/pip/_internal/resolution/base.py
@@ -1,21 +1,20 @@
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from typing import Callable, List, Optional
 
-if MYPY_CHECK_RUNNING:
-    from typing import Callable, List
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.req.req_set import RequirementSet
 
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._internal.req.req_set import RequirementSet
-
-    InstallRequirementProvider = Callable[
-        [str, InstallRequirement], InstallRequirement
-    ]
+InstallRequirementProvider = Callable[
+    [str, Optional[InstallRequirement]], InstallRequirement
+]
 
 
 class BaseResolver:
-    def resolve(self, root_reqs, check_supported_wheels):
-        # type: (List[InstallRequirement], bool) -> RequirementSet
+    def resolve(
+        self, root_reqs: List[InstallRequirement], check_supported_wheels: bool
+    ) -> RequirementSet:
         raise NotImplementedError()
 
-    def get_installation_order(self, req_set):
-        # type: (RequirementSet) -> List[InstallRequirement]
+    def get_installation_order(
+        self, req_set: RequirementSet
+    ) -> List[InstallRequirement]:
         raise NotImplementedError()
diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py
index 665dba128a9..8c149d437d7 100644
--- a/src/pip/_internal/resolution/legacy/resolver.py
+++ b/src/pip/_internal/resolution/legacy/resolver.py
@@ -12,54 +12,50 @@
 
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
-# mypy: disallow-untyped-defs=False
 
 import logging
 import sys
 from collections import defaultdict
 from itertools import chain
+from typing import DefaultDict, Iterable, List, Optional, Set, Tuple
 
 from pip._vendor.packaging import specifiers
+from pip._vendor.packaging.requirements import Requirement
 
+from pip._internal.cache import WheelCache
 from pip._internal.exceptions import (
     BestVersionAlreadyInstalled,
     DistributionNotFound,
     HashError,
     HashErrors,
+    NoneMetadataError,
     UnsupportedPythonVersion,
 )
-from pip._internal.req.req_install import check_invalid_constraint_type
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
+from pip._internal.models.link import Link
+from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req.req_install import (
+    InstallRequirement,
+    check_invalid_constraint_type,
+)
 from pip._internal.req.req_set import RequirementSet
-from pip._internal.resolution.base import BaseResolver
+from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
 from pip._internal.utils.compatibility_tags import get_supported
 from pip._internal.utils.logging import indent_log
-from pip._internal.utils.misc import dist_in_usersite, normalize_version_info
-from pip._internal.utils.packaging import check_requires_python, get_requires_python
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import DefaultDict, List, Optional, Set, Tuple
-
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.cache import WheelCache
-    from pip._internal.index.package_finder import PackageFinder
-    from pip._internal.models.link import Link
-    from pip._internal.operations.prepare import RequirementPreparer
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._internal.resolution.base import InstallRequirementProvider
-
-    DiscoveredDependencies = DefaultDict[str, List[InstallRequirement]]
+from pip._internal.utils.misc import normalize_version_info
+from pip._internal.utils.packaging import check_requires_python
 
 logger = logging.getLogger(__name__)
 
+DiscoveredDependencies = DefaultDict[str, List[InstallRequirement]]
+
 
 def _check_dist_requires_python(
-    dist,  # type: Distribution
-    version_info,  # type: Tuple[int, int, int]
-    ignore_requires_python=False,  # type: bool
-):
-    # type: (...) -> None
+    dist: BaseDistribution,
+    version_info: Tuple[int, int, int],
+    ignore_requires_python: bool = False,
+) -> None:
     """
     Check whether the given Python version is compatible with a distribution's
     "Requires-Python" value.
@@ -72,34 +68,42 @@ def _check_dist_requires_python(
     :raises UnsupportedPythonVersion: When the given Python version isn't
         compatible.
     """
-    requires_python = get_requires_python(dist)
+    # This idiosyncratically converts the SpecifierSet to str and let
+    # check_requires_python then parse it again into SpecifierSet. But this
+    # is the legacy resolver so I'm just not going to bother refactoring.
+    try:
+        requires_python = str(dist.requires_python)
+    except FileNotFoundError as e:
+        raise NoneMetadataError(dist, str(e))
     try:
         is_compatible = check_requires_python(
-            requires_python, version_info=version_info,
+            requires_python,
+            version_info=version_info,
         )
     except specifiers.InvalidSpecifier as exc:
         logger.warning(
-            "Package %r has an invalid Requires-Python: %s",
-            dist.project_name, exc,
+            "Package %r has an invalid Requires-Python: %s", dist.raw_name, exc
         )
         return
 
     if is_compatible:
         return
 
-    version = '.'.join(map(str, version_info))
+    version = ".".join(map(str, version_info))
     if ignore_requires_python:
         logger.debug(
-            'Ignoring failed Requires-Python check for package %r: '
-            '%s not in %r',
-            dist.project_name, version, requires_python,
+            "Ignoring failed Requires-Python check for package %r: %s not in %r",
+            dist.raw_name,
+            version,
+            requires_python,
         )
         return
 
     raise UnsupportedPythonVersion(
-        'Package {!r} requires a different Python: {} not in {!r}'.format(
-            dist.project_name, version, requires_python,
-        ))
+        "Package {!r} requires a different Python: {} not in {!r}".format(
+            dist.raw_name, version, requires_python
+        )
+    )
 
 
 class Resolver(BaseResolver):
@@ -111,19 +115,18 @@ class Resolver(BaseResolver):
 
     def __init__(
         self,
-        preparer,  # type: RequirementPreparer
-        finder,  # type: PackageFinder
-        wheel_cache,  # type: Optional[WheelCache]
-        make_install_req,  # type: InstallRequirementProvider
-        use_user_site,  # type: bool
-        ignore_dependencies,  # type: bool
-        ignore_installed,  # type: bool
-        ignore_requires_python,  # type: bool
-        force_reinstall,  # type: bool
-        upgrade_strategy,  # type: str
-        py_version_info=None,  # type: Optional[Tuple[int, ...]]
-    ):
-        # type: (...) -> None
+        preparer: RequirementPreparer,
+        finder: PackageFinder,
+        wheel_cache: Optional[WheelCache],
+        make_install_req: InstallRequirementProvider,
+        use_user_site: bool,
+        ignore_dependencies: bool,
+        ignore_installed: bool,
+        ignore_requires_python: bool,
+        force_reinstall: bool,
+        upgrade_strategy: str,
+        py_version_info: Optional[Tuple[int, ...]] = None,
+    ) -> None:
         super().__init__()
         assert upgrade_strategy in self._allowed_strategies
 
@@ -146,11 +149,11 @@ def __init__(
         self.use_user_site = use_user_site
         self._make_install_req = make_install_req
 
-        self._discovered_dependencies = \
-            defaultdict(list)  # type: DiscoveredDependencies
+        self._discovered_dependencies: DiscoveredDependencies = defaultdict(list)
 
-    def resolve(self, root_reqs, check_supported_wheels):
-        # type: (List[InstallRequirement], bool) -> RequirementSet
+    def resolve(
+        self, root_reqs: List[InstallRequirement], check_supported_wheels: bool
+    ) -> RequirementSet:
         """Resolve what operations need to be done
 
         As a side-effect of this method, the packages (and their dependencies)
@@ -161,9 +164,7 @@ def resolve(self, root_reqs, check_supported_wheels):
         possible to move the preparation to become a step separated from
         dependency resolution.
         """
-        requirement_set = RequirementSet(
-            check_supported_wheels=check_supported_wheels
-        )
+        requirement_set = RequirementSet(check_supported_wheels=check_supported_wheels)
         for req in root_reqs:
             if req.constraint:
                 check_invalid_constraint_type(req)
@@ -173,7 +174,7 @@ def resolve(self, root_reqs, check_supported_wheels):
         # exceptions cannot be checked ahead of time, because
         # _populate_link() needs to be called before we can make decisions
         # based on link type.
-        discovered_reqs = []  # type: List[InstallRequirement]
+        discovered_reqs: List[InstallRequirement] = []
         hash_errors = HashErrors()
         for req in chain(requirement_set.all_requirements, discovered_reqs):
             try:
@@ -187,8 +188,7 @@ def resolve(self, root_reqs, check_supported_wheels):
 
         return requirement_set
 
-    def _is_upgrade_allowed(self, req):
-        # type: (InstallRequirement) -> bool
+    def _is_upgrade_allowed(self, req: InstallRequirement) -> bool:
         if self.upgrade_strategy == "to-satisfy-only":
             return False
         elif self.upgrade_strategy == "eager":
@@ -197,19 +197,19 @@ def _is_upgrade_allowed(self, req):
             assert self.upgrade_strategy == "only-if-needed"
             return req.user_supplied or req.constraint
 
-    def _set_req_to_reinstall(self, req):
-        # type: (InstallRequirement) -> None
+    def _set_req_to_reinstall(self, req: InstallRequirement) -> None:
         """
         Set a requirement to be installed.
         """
         # Don't uninstall the conflict if doing a user install and the
         # conflict is not a user install.
-        if not self.use_user_site or dist_in_usersite(req.satisfied_by):
+        if not self.use_user_site or req.satisfied_by.in_usersite:
             req.should_reinstall = True
         req.satisfied_by = None
 
-    def _check_skip_installed(self, req_to_install):
-        # type: (InstallRequirement) -> Optional[str]
+    def _check_skip_installed(
+        self, req_to_install: InstallRequirement
+    ) -> Optional[str]:
         """Check if req_to_install should be skipped.
 
         This will check if the req is installed, and whether we should upgrade
@@ -240,8 +240,8 @@ def _check_skip_installed(self, req_to_install):
 
         if not self._is_upgrade_allowed(req_to_install):
             if self.upgrade_strategy == "only-if-needed":
-                return 'already satisfied, skipping upgrade'
-            return 'already satisfied'
+                return "already satisfied, skipping upgrade"
+            return "already satisfied"
 
         # Check for the possibility of an upgrade.  For link-based
         # requirements we have to pull the tree down and inspect to assess
@@ -251,7 +251,7 @@ def _check_skip_installed(self, req_to_install):
                 self.finder.find_requirement(req_to_install, upgrade=True)
             except BestVersionAlreadyInstalled:
                 # Then the best version is installed.
-                return 'already up-to-date'
+                return "already up-to-date"
             except DistributionNotFound:
                 # No distribution found, so we squash the error.  It will
                 # be raised later when we re-try later to do the install.
@@ -261,8 +261,7 @@ def _check_skip_installed(self, req_to_install):
         self._set_req_to_reinstall(req_to_install)
         return None
 
-    def _find_requirement_link(self, req):
-        # type: (InstallRequirement) -> Optional[Link]
+    def _find_requirement_link(self, req: InstallRequirement) -> Optional[Link]:
         upgrade = self._is_upgrade_allowed(req)
         best_candidate = self.finder.find_requirement(req, upgrade)
         if not best_candidate:
@@ -271,21 +270,20 @@ def _find_requirement_link(self, req):
         # Log a warning per PEP 592 if necessary before returning.
         link = best_candidate.link
         if link.is_yanked:
-            reason = link.yanked_reason or ''
+            reason = link.yanked_reason or ""
             msg = (
                 # Mark this as a unicode string to prevent
                 # "UnicodeEncodeError: 'ascii' codec can't encode character"
                 # in Python 2 when the reason contains non-ascii characters.
-                'The candidate selected for download or install is a '
-                'yanked version: {candidate}\n'
-                'Reason for being yanked: {reason}'
+                "The candidate selected for download or install is a "
+                "yanked version: {candidate}\n"
+                "Reason for being yanked: {reason}"
             ).format(candidate=best_candidate, reason=reason)
             logger.warning(msg)
 
         return link
 
-    def _populate_link(self, req):
-        # type: (InstallRequirement) -> None
+    def _populate_link(self, req: InstallRequirement) -> None:
         """Ensure that if a link can be found for this, that it is found.
 
         Note that req.link may still be None - if the requirement is already
@@ -309,13 +307,12 @@ def _populate_link(self, req):
             supported_tags=get_supported(),
         )
         if cache_entry is not None:
-            logger.debug('Using cached wheel link: %s', cache_entry.link)
+            logger.debug("Using cached wheel link: %s", cache_entry.link)
             if req.link is req.original_link and cache_entry.persistent:
                 req.original_link_is_in_wheel_cache = True
             req.link = cache_entry.link
 
-    def _get_dist_for(self, req):
-        # type: (InstallRequirement) -> Distribution
+    def _get_dist_for(self, req: InstallRequirement) -> BaseDistribution:
         """Takes a InstallRequirement and returns a single AbstractDist \
         representing a prepared variant of the same.
         """
@@ -328,9 +325,7 @@ def _get_dist_for(self, req):
         skip_reason = self._check_skip_installed(req)
 
         if req.satisfied_by:
-            return self.preparer.prepare_installed_requirement(
-                req, skip_reason
-            )
+            return self.preparer.prepare_installed_requirement(req, skip_reason)
 
         # We eagerly populate the link, since that's our "legacy" behavior.
         self._populate_link(req)
@@ -349,26 +344,25 @@ def _get_dist_for(self, req):
 
         if req.satisfied_by:
             should_modify = (
-                self.upgrade_strategy != "to-satisfy-only" or
-                self.force_reinstall or
-                self.ignore_installed or
-                req.link.scheme == 'file'
+                self.upgrade_strategy != "to-satisfy-only"
+                or self.force_reinstall
+                or self.ignore_installed
+                or req.link.scheme == "file"
             )
             if should_modify:
                 self._set_req_to_reinstall(req)
             else:
                 logger.info(
-                    'Requirement already satisfied (use --upgrade to upgrade):'
-                    ' %s', req,
+                    "Requirement already satisfied (use --upgrade to upgrade): %s",
+                    req,
                 )
         return dist
 
     def _resolve_one(
         self,
-        requirement_set,  # type: RequirementSet
-        req_to_install,  # type: InstallRequirement
-    ):
-        # type: (...) -> List[InstallRequirement]
+        requirement_set: RequirementSet,
+        req_to_install: InstallRequirement,
+    ) -> List[InstallRequirement]:
         """Prepare a single requirements file.
 
         :return: A list of additional InstallRequirements to also install.
@@ -386,17 +380,18 @@ def _resolve_one(
         # This will raise UnsupportedPythonVersion if the given Python
         # version isn't compatible with the distribution's Requires-Python.
         _check_dist_requires_python(
-            dist, version_info=self._py_version_info,
+            dist,
+            version_info=self._py_version_info,
             ignore_requires_python=self.ignore_requires_python,
         )
 
-        more_reqs = []  # type: List[InstallRequirement]
+        more_reqs: List[InstallRequirement] = []
 
-        def add_req(subreq, extras_requested):
-            sub_install_req = self._make_install_req(
-                str(subreq),
-                req_to_install,
-            )
+        def add_req(subreq: Requirement, extras_requested: Iterable[str]) -> None:
+            # This idiosyncratically converts the Requirement to str and let
+            # make_install_req then parse it again into Requirement. But this is
+            # the legacy resolver so I'm just not going to bother refactoring.
+            sub_install_req = self._make_install_req(str(subreq), req_to_install)
             parent_req_name = req_to_install.name
             to_scan_again, add_to_parent = requirement_set.add_requirement(
                 sub_install_req,
@@ -404,9 +399,7 @@ def add_req(subreq, extras_requested):
                 extras_requested=extras_requested,
             )
             if parent_req_name and add_to_parent:
-                self._discovered_dependencies[parent_req_name].append(
-                    add_to_parent
-                )
+                self._discovered_dependencies[parent_req_name].append(add_to_parent)
             more_reqs.extend(to_scan_again)
 
         with indent_log():
@@ -417,35 +410,36 @@ def add_req(subreq, extras_requested):
                 # 'unnamed' requirements can only come from being directly
                 # provided by the user.
                 assert req_to_install.user_supplied
-                requirement_set.add_requirement(
-                    req_to_install, parent_req_name=None,
-                )
+                requirement_set.add_requirement(req_to_install, parent_req_name=None)
 
             if not self.ignore_dependencies:
                 if req_to_install.extras:
                     logger.debug(
                         "Installing extra requirements: %r",
-                        ','.join(req_to_install.extras),
+                        ",".join(req_to_install.extras),
                     )
                 missing_requested = sorted(
-                    set(req_to_install.extras) - set(dist.extras)
+                    set(req_to_install.extras) - set(dist.iter_provided_extras())
                 )
                 for missing in missing_requested:
                     logger.warning(
-                        "%s does not provide the extra '%s'",
-                        dist, missing
+                        "%s %s does not provide the extra '%s'",
+                        dist.raw_name,
+                        dist.version,
+                        missing,
                     )
 
                 available_requested = sorted(
-                    set(dist.extras) & set(req_to_install.extras)
+                    set(dist.iter_provided_extras()) & set(req_to_install.extras)
                 )
-                for subreq in dist.requires(available_requested):
+                for subreq in dist.iter_dependencies(available_requested):
                     add_req(subreq, extras_requested=available_requested)
 
         return more_reqs
 
-    def get_installation_order(self, req_set):
-        # type: (RequirementSet) -> List[InstallRequirement]
+    def get_installation_order(
+        self, req_set: RequirementSet
+    ) -> List[InstallRequirement]:
         """Create the installation order.
 
         The installation order is topological - requirements are installed
@@ -456,9 +450,9 @@ def get_installation_order(self, req_set):
         # installs the user specified things in the order given, except when
         # dependencies must come earlier to achieve topological order.
         order = []
-        ordered_reqs = set()  # type: Set[InstallRequirement]
+        ordered_reqs: Set[InstallRequirement] = set()
 
-        def schedule(req):
+        def schedule(req: InstallRequirement) -> None:
             if req.satisfied_by or req in ordered_reqs:
                 return
             if req.constraint:
diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py
index 82c5ec7c72d..b206692a0a9 100644
--- a/src/pip/_internal/resolution/resolvelib/base.py
+++ b/src/pip/_internal/resolution/resolvelib/base.py
@@ -1,25 +1,18 @@
+from typing import FrozenSet, Iterable, Optional, Tuple, Union
+
 from pip._vendor.packaging.specifiers import SpecifierSet
-from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
+from pip._vendor.packaging.version import LegacyVersion, Version
 
+from pip._internal.models.link import Link, links_equivalent
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.hashes import Hashes
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import FrozenSet, Iterable, Optional, Tuple
-
-    from pip._vendor.packaging.version import _BaseVersion
-
-    from pip._internal.models.link import Link
 
-    CandidateLookup = Tuple[
-        Optional["Candidate"],
-        Optional[InstallRequirement],
-    ]
+CandidateLookup = Tuple[Optional["Candidate"], Optional[InstallRequirement]]
+CandidateVersion = Union[LegacyVersion, Version]
 
 
-def format_name(project, extras):
-    # type: (str, FrozenSet[str]) -> str
+def format_name(project: str, extras: FrozenSet[str]) -> str:
     if not extras:
         return project
     canonical_extras = sorted(canonicalize_name(e) for e in extras)
@@ -27,39 +20,39 @@ def format_name(project, extras):
 
 
 class Constraint:
-    def __init__(self, specifier, hashes):
-        # type: (SpecifierSet, Hashes) -> None
+    def __init__(
+        self, specifier: SpecifierSet, hashes: Hashes, links: FrozenSet[Link]
+    ) -> None:
         self.specifier = specifier
         self.hashes = hashes
+        self.links = links
 
     @classmethod
-    def empty(cls):
-        # type: () -> Constraint
-        return Constraint(SpecifierSet(), Hashes())
+    def empty(cls) -> "Constraint":
+        return Constraint(SpecifierSet(), Hashes(), frozenset())
 
     @classmethod
-    def from_ireq(cls, ireq):
-        # type: (InstallRequirement) -> Constraint
-        return Constraint(ireq.specifier, ireq.hashes(trust_internet=False))
+    def from_ireq(cls, ireq: InstallRequirement) -> "Constraint":
+        links = frozenset([ireq.link]) if ireq.link else frozenset()
+        return Constraint(ireq.specifier, ireq.hashes(trust_internet=False), links)
 
-    def __nonzero__(self):
-        # type: () -> bool
-        return bool(self.specifier) or bool(self.hashes)
+    def __bool__(self) -> bool:
+        return bool(self.specifier) or bool(self.hashes) or bool(self.links)
 
-    def __bool__(self):
-        # type: () -> bool
-        return self.__nonzero__()
-
-    def __and__(self, other):
-        # type: (InstallRequirement) -> Constraint
+    def __and__(self, other: InstallRequirement) -> "Constraint":
         if not isinstance(other, InstallRequirement):
             return NotImplemented
         specifier = self.specifier & other.specifier
         hashes = self.hashes & other.hashes(trust_internet=False)
-        return Constraint(specifier, hashes)
-
-    def is_satisfied_by(self, candidate):
-        # type: (Candidate) -> bool
+        links = self.links
+        if other.link:
+            links = links.union([other.link])
+        return Constraint(specifier, hashes, links)
+
+    def is_satisfied_by(self, candidate: "Candidate") -> bool:
+        # Reject if there are any mismatched URL constraints on this package.
+        if self.links and not all(_match_link(link, candidate) for link in self.links):
+            return False
         # We can safely always allow prereleases here since PackageFinder
         # already implements the prerelease logic, and would have filtered out
         # prerelease candidates if the user does not expect them.
@@ -68,8 +61,7 @@ def is_satisfied_by(self, candidate):
 
 class Requirement:
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
         """The "project name" of a requirement.
 
         This is different from ``name`` if this requirement contains extras,
@@ -79,8 +71,7 @@ def project_name(self):
         raise NotImplementedError("Subclass should override")
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         """The name identifying this requirement in the resolver.
 
         This is different from ``project_name`` if this requirement contains
@@ -88,23 +79,25 @@ def name(self):
         """
         raise NotImplementedError("Subclass should override")
 
-    def is_satisfied_by(self, candidate):
-        # type: (Candidate) -> bool
+    def is_satisfied_by(self, candidate: "Candidate") -> bool:
         return False
 
-    def get_candidate_lookup(self):
-        # type: () -> CandidateLookup
+    def get_candidate_lookup(self) -> CandidateLookup:
         raise NotImplementedError("Subclass should override")
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         raise NotImplementedError("Subclass should override")
 
 
+def _match_link(link: Link, candidate: "Candidate") -> bool:
+    if candidate.source_link:
+        return links_equivalent(link, candidate.source_link)
+    return False
+
+
 class Candidate:
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
         """The "project name" of the candidate.
 
         This is different from ``name`` if this candidate contains extras,
@@ -114,8 +107,7 @@ def project_name(self):
         raise NotImplementedError("Override in subclass")
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         """The name identifying this candidate in the resolver.
 
         This is different from ``project_name`` if this candidate contains
@@ -124,33 +116,26 @@ def name(self):
         raise NotImplementedError("Override in subclass")
 
     @property
-    def version(self):
-        # type: () -> _BaseVersion
+    def version(self) -> CandidateVersion:
         raise NotImplementedError("Override in subclass")
 
     @property
-    def is_installed(self):
-        # type: () -> bool
+    def is_installed(self) -> bool:
         raise NotImplementedError("Override in subclass")
 
     @property
-    def is_editable(self):
-        # type: () -> bool
+    def is_editable(self) -> bool:
         raise NotImplementedError("Override in subclass")
 
     @property
-    def source_link(self):
-        # type: () -> Optional[Link]
+    def source_link(self) -> Optional[Link]:
         raise NotImplementedError("Override in subclass")
 
-    def iter_dependencies(self, with_requires):
-        # type: (bool) -> Iterable[Optional[Requirement]]
+    def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
         raise NotImplementedError("Override in subclass")
 
-    def get_install_requirement(self):
-        # type: () -> Optional[InstallRequirement]
+    def get_install_requirement(self) -> Optional[InstallRequirement]:
         raise NotImplementedError("Override in subclass")
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         raise NotImplementedError("Subclass should override")
diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py
index 6725684a515..9b8450e86b8 100644
--- a/src/pip/_internal/resolution/resolvelib/candidates.py
+++ b/src/pip/_internal/resolution/resolvelib/candidates.py
@@ -1,46 +1,57 @@
 import logging
 import sys
+from typing import TYPE_CHECKING, Any, FrozenSet, Iterable, Optional, Tuple, Union, cast
 
-from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
-from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.packaging.version import Version
 
-from pip._internal.exceptions import HashError, MetadataInconsistent
+from pip._internal.exceptions import (
+    HashError,
+    InstallationSubprocessError,
+    MetadataInconsistent,
+)
+from pip._internal.metadata import BaseDistribution
+from pip._internal.models.link import Link, links_equivalent
 from pip._internal.models.wheel import Wheel
 from pip._internal.req.constructors import (
     install_req_from_editable,
     install_req_from_line,
 )
 from pip._internal.req.req_install import InstallRequirement
-from pip._internal.utils.misc import dist_is_editable, normalize_version_info
-from pip._internal.utils.packaging import get_requires_python
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-from .base import Candidate, format_name
+from pip._internal.utils.misc import normalize_version_info
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, FrozenSet, Iterable, Optional, Tuple, Union
+from .base import Candidate, CandidateVersion, Requirement, format_name
 
-    from pip._vendor.packaging.version import _BaseVersion
-    from pip._vendor.pkg_resources import Distribution
+if TYPE_CHECKING:
+    from .factory import Factory
 
-    from pip._internal.models.link import Link
+logger = logging.getLogger(__name__)
 
-    from .base import Requirement
-    from .factory import Factory
+BaseCandidate = Union[
+    "AlreadyInstalledCandidate",
+    "EditableCandidate",
+    "LinkCandidate",
+]
 
-    BaseCandidate = Union[
-        "AlreadyInstalledCandidate",
-        "EditableCandidate",
-        "LinkCandidate",
-    ]
+# Avoid conflicting with the PyPI package "Python".
+REQUIRES_PYTHON_IDENTIFIER = cast(NormalizedName, "")
 
 
-logger = logging.getLogger(__name__)
+def as_base_candidate(candidate: Candidate) -> Optional[BaseCandidate]:
+    """The runtime version of BaseCandidate."""
+    base_candidate_classes = (
+        AlreadyInstalledCandidate,
+        EditableCandidate,
+        LinkCandidate,
+    )
+    if isinstance(candidate, base_candidate_classes):
+        return candidate
+    return None
 
 
-def make_install_req_from_link(link, template):
-    # type: (Link, InstallRequirement) -> InstallRequirement
+def make_install_req_from_link(
+    link: Link, template: InstallRequirement
+) -> InstallRequirement:
     assert not template.editable, "template is editable"
     if template.req:
         line = str(template.req)
@@ -56,7 +67,7 @@ def make_install_req_from_link(link, template):
         options=dict(
             install_options=template.install_options,
             global_options=template.global_options,
-            hashes=template.hash_options
+            hashes=template.hash_options,
         ),
     )
     ireq.original_link = template.original_link
@@ -64,8 +75,9 @@ def make_install_req_from_link(link, template):
     return ireq
 
 
-def make_install_req_from_editable(link, template):
-    # type: (Link, InstallRequirement) -> InstallRequirement
+def make_install_req_from_editable(
+    link: Link, template: InstallRequirement
+) -> InstallRequirement:
     assert template.editable, "template not editable"
     return install_req_from_editable(
         link.url,
@@ -74,23 +86,24 @@ def make_install_req_from_editable(link, template):
         use_pep517=template.use_pep517,
         isolated=template.isolated,
         constraint=template.constraint,
+        permit_editable_wheels=template.permit_editable_wheels,
         options=dict(
             install_options=template.install_options,
             global_options=template.global_options,
-            hashes=template.hash_options
+            hashes=template.hash_options,
         ),
     )
 
 
-def make_install_req_from_dist(dist, template):
-    # type: (Distribution, InstallRequirement) -> InstallRequirement
-    project_name = canonicalize_name(dist.project_name)
+def _make_install_req_from_dist(
+    dist: BaseDistribution, template: InstallRequirement
+) -> InstallRequirement:
     if template.req:
         line = str(template.req)
     elif template.link:
-        line = f"{project_name} @ {template.link.url}"
+        line = f"{dist.canonical_name} @ {template.link.url}"
     else:
-        line = f"{project_name}=={dist.parsed_version}"
+        line = f"{dist.canonical_name}=={dist.version}"
     ireq = install_req_from_line(
         line,
         user_supplied=template.user_supplied,
@@ -101,7 +114,7 @@ def make_install_req_from_dist(dist, template):
         options=dict(
             install_options=template.install_options,
             global_options=template.global_options,
-            hashes=template.hash_options
+            hashes=template.hash_options,
         ),
     )
     ireq.satisfied_by = dist
@@ -123,18 +136,19 @@ class exposes appropriate information to the resolver.
         ``link`` would point to the wheel cache, while this points to the
         found remote link (e.g. from pypi.org).
     """
+
+    dist: BaseDistribution
     is_installed = False
 
     def __init__(
         self,
-        link,          # type: Link
-        source_link,   # type: Link
-        ireq,          # type: InstallRequirement
-        factory,       # type: Factory
-        name=None,     # type: Optional[str]
-        version=None,  # type: Optional[_BaseVersion]
-    ):
-        # type: (...) -> None
+        link: Link,
+        source_link: Link,
+        ireq: InstallRequirement,
+        factory: "Factory",
+        name: Optional[NormalizedName] = None,
+        version: Optional[CandidateVersion] = None,
+    ) -> None:
         self._link = link
         self._source_link = source_link
         self._factory = factory
@@ -143,76 +157,72 @@ def __init__(
         self._version = version
         self.dist = self._prepare()
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return f"{self.name} {self.version}"
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}({link!r})".format(
             class_name=self.__class__.__name__,
             link=str(self._link),
         )
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash((self.__class__, self._link))
 
-    def __eq__(self, other):
-        # type: (Any) -> bool
+    def __eq__(self, other: Any) -> bool:
         if isinstance(other, self.__class__):
-            return self._link == other._link
+            return links_equivalent(self._link, other._link)
         return False
 
     @property
-    def source_link(self):
-        # type: () -> Optional[Link]
+    def source_link(self) -> Optional[Link]:
         return self._source_link
 
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
         """The normalised name of the project the candidate refers to"""
         if self._name is None:
-            self._name = canonicalize_name(self.dist.project_name)
+            self._name = self.dist.canonical_name
         return self._name
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         return self.project_name
 
     @property
-    def version(self):
-        # type: () -> _BaseVersion
+    def version(self) -> CandidateVersion:
         if self._version is None:
-            self._version = self.dist.parsed_version
+            self._version = self.dist.version
         return self._version
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return "{} {} (from {})".format(
             self.name,
             self.version,
-            self._link.file_path if self._link.is_file else self._link
+            self._link.file_path if self._link.is_file else self._link,
         )
 
-    def _prepare_distribution(self):
-        # type: () -> Distribution
+    def _prepare_distribution(self) -> BaseDistribution:
         raise NotImplementedError("Override in subclass")
 
-    def _check_metadata_consistency(self, dist):
-        # type: (Distribution) -> None
+    def _check_metadata_consistency(self, dist: BaseDistribution) -> None:
         """Check for consistency of project name and version of dist."""
-        name = canonicalize_name(dist.project_name)
-        if self._name is not None and self._name != name:
-            raise MetadataInconsistent(self._ireq, "name", dist.project_name)
-        version = dist.parsed_version
-        if self._version is not None and self._version != version:
-            raise MetadataInconsistent(self._ireq, "version", dist.version)
-
-    def _prepare(self):
-        # type: () -> Distribution
+        if self._name is not None and self._name != dist.canonical_name:
+            raise MetadataInconsistent(
+                self._ireq,
+                "name",
+                self._name,
+                dist.canonical_name,
+            )
+        if self._version is not None and self._version != dist.version:
+            raise MetadataInconsistent(
+                self._ireq,
+                "version",
+                str(self._version),
+                str(dist.version),
+            )
+
+    def _prepare(self) -> BaseDistribution:
         try:
             dist = self._prepare_distribution()
         except HashError as e:
@@ -221,31 +231,21 @@ def _prepare(self):
             # offending line to the user.
             e.req = self._ireq
             raise
+        except InstallationSubprocessError as exc:
+            # The output has been presented already, so don't duplicate it.
+            exc.context = "See above for output."
+            raise
+
         self._check_metadata_consistency(dist)
         return dist
 
-    def _get_requires_python_dependency(self):
-        # type: () -> Optional[Requirement]
-        requires_python = get_requires_python(self.dist)
-        if requires_python is None:
-            return None
-        try:
-            spec = SpecifierSet(requires_python)
-        except InvalidSpecifier as e:
-            message = "Package %r has an invalid Requires-Python: %s"
-            logger.warning(message, self.name, e)
-            return None
-        return self._factory.make_requires_python_requirement(spec)
-
-    def iter_dependencies(self, with_requires):
-        # type: (bool) -> Iterable[Optional[Requirement]]
-        requires = self.dist.requires() if with_requires else ()
+    def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
+        requires = self.dist.iter_dependencies() if with_requires else ()
         for r in requires:
             yield self._factory.make_requirement_from_spec(str(r), self._ireq)
-        yield self._get_requires_python_dependency()
+        yield self._factory.make_requires_python_requirement(self.dist.requires_python)
 
-    def get_install_requirement(self):
-        # type: () -> Optional[InstallRequirement]
+    def get_install_requirement(self) -> Optional[InstallRequirement]:
         return self._ireq
 
 
@@ -254,13 +254,12 @@ class LinkCandidate(_InstallRequirementBackedCandidate):
 
     def __init__(
         self,
-        link,          # type: Link
-        template,        # type: InstallRequirement
-        factory,       # type: Factory
-        name=None,     # type: Optional[str]
-        version=None,  # type: Optional[_BaseVersion]
-    ):
-        # type: (...) -> None
+        link: Link,
+        template: InstallRequirement,
+        factory: "Factory",
+        name: Optional[NormalizedName] = None,
+        version: Optional[CandidateVersion] = None,
+    ) -> None:
         source_link = link
         cache_entry = factory.get_wheel_cache_entry(link, name)
         if cache_entry is not None:
@@ -271,21 +270,19 @@ def __init__(
         if ireq.link.is_wheel and not ireq.link.is_file:
             wheel = Wheel(ireq.link.filename)
             wheel_name = canonicalize_name(wheel.name)
-            assert name == wheel_name, (
-                f"{name!r} != {wheel_name!r} for wheel"
-            )
+            assert name == wheel_name, f"{name!r} != {wheel_name!r} for wheel"
             # Version may not be present for PEP 508 direct URLs
             if version is not None:
                 wheel_version = Version(wheel.version)
-                assert version == wheel_version, (
-                    "{!r} != {!r} for wheel {}".format(
-                        version, wheel_version, name
-                    )
+                assert version == wheel_version, "{!r} != {!r} for wheel {}".format(
+                    version, wheel_version, name
                 )
 
-        if (cache_entry is not None and
-                cache_entry.persistent and
-                template.link is template.original_link):
+        if (
+            cache_entry is not None
+            and cache_entry.persistent
+            and template.link is template.original_link
+        ):
             ireq.original_link_is_in_wheel_cache = True
 
         super().__init__(
@@ -297,11 +294,9 @@ def __init__(
             version=version,
         )
 
-    def _prepare_distribution(self):
-        # type: () -> Distribution
-        return self._factory.preparer.prepare_linked_requirement(
-            self._ireq, parallel_builds=True,
-        )
+    def _prepare_distribution(self) -> BaseDistribution:
+        preparer = self._factory.preparer
+        return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
 
 
 class EditableCandidate(_InstallRequirementBackedCandidate):
@@ -309,13 +304,12 @@ class EditableCandidate(_InstallRequirementBackedCandidate):
 
     def __init__(
         self,
-        link,          # type: Link
-        template,        # type: InstallRequirement
-        factory,       # type: Factory
-        name=None,     # type: Optional[str]
-        version=None,  # type: Optional[_BaseVersion]
-    ):
-        # type: (...) -> None
+        link: Link,
+        template: InstallRequirement,
+        factory: "Factory",
+        name: Optional[NormalizedName] = None,
+        version: Optional[CandidateVersion] = None,
+    ) -> None:
         super().__init__(
             link=link,
             source_link=link,
@@ -325,8 +319,7 @@ def __init__(
             version=version,
         )
 
-    def _prepare_distribution(self):
-        # type: () -> Distribution
+    def _prepare_distribution(self) -> BaseDistribution:
         return self._factory.preparer.prepare_editable_requirement(self._ireq)
 
 
@@ -336,76 +329,64 @@ class AlreadyInstalledCandidate(Candidate):
 
     def __init__(
         self,
-        dist,  # type: Distribution
-        template,  # type: InstallRequirement
-        factory,  # type: Factory
-    ):
-        # type: (...) -> None
+        dist: BaseDistribution,
+        template: InstallRequirement,
+        factory: "Factory",
+    ) -> None:
         self.dist = dist
-        self._ireq = make_install_req_from_dist(dist, template)
+        self._ireq = _make_install_req_from_dist(dist, template)
         self._factory = factory
 
         # This is just logging some messages, so we can do it eagerly.
         # The returned dist would be exactly the same as self.dist because we
-        # set satisfied_by in make_install_req_from_dist.
+        # set satisfied_by in _make_install_req_from_dist.
         # TODO: Supply reason based on force_reinstall and upgrade_strategy.
         skip_reason = "already satisfied"
         factory.preparer.prepare_installed_requirement(self._ireq, skip_reason)
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return str(self.dist)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}({distribution!r})".format(
             class_name=self.__class__.__name__,
             distribution=self.dist,
         )
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash((self.__class__, self.name, self.version))
 
-    def __eq__(self, other):
-        # type: (Any) -> bool
+    def __eq__(self, other: Any) -> bool:
         if isinstance(other, self.__class__):
             return self.name == other.name and self.version == other.version
         return False
 
     @property
-    def project_name(self):
-        # type: () -> str
-        return canonicalize_name(self.dist.project_name)
+    def project_name(self) -> NormalizedName:
+        return self.dist.canonical_name
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         return self.project_name
 
     @property
-    def version(self):
-        # type: () -> _BaseVersion
-        return self.dist.parsed_version
+    def version(self) -> CandidateVersion:
+        return self.dist.version
 
     @property
-    def is_editable(self):
-        # type: () -> bool
-        return dist_is_editable(self.dist)
+    def is_editable(self) -> bool:
+        return self.dist.editable
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return f"{self.name} {self.version} (Installed)"
 
-    def iter_dependencies(self, with_requires):
-        # type: (bool) -> Iterable[Optional[Requirement]]
+    def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
         if not with_requires:
             return
-        for r in self.dist.requires():
+        for r in self.dist.iter_dependencies():
             yield self._factory.make_requirement_from_spec(str(r), self._ireq)
 
-    def get_install_requirement(self):
-        # type: () -> Optional[InstallRequirement]
+    def get_install_requirement(self) -> Optional[InstallRequirement]:
         return None
 
 
@@ -433,78 +414,65 @@ class ExtrasCandidate(Candidate):
     version 2.0. Having those candidates depend on foo=1.0 and foo=2.0
     respectively forces the resolver to recognise that this is a conflict.
     """
+
     def __init__(
         self,
-        base,  # type: BaseCandidate
-        extras,  # type: FrozenSet[str]
-    ):
-        # type: (...) -> None
+        base: BaseCandidate,
+        extras: FrozenSet[str],
+    ) -> None:
         self.base = base
         self.extras = extras
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         name, rest = str(self.base).split(" ", 1)
         return "{}[{}] {}".format(name, ",".join(self.extras), rest)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}(base={base!r}, extras={extras!r})".format(
             class_name=self.__class__.__name__,
             base=self.base,
             extras=self.extras,
         )
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash((self.base, self.extras))
 
-    def __eq__(self, other):
-        # type: (Any) -> bool
+    def __eq__(self, other: Any) -> bool:
         if isinstance(other, self.__class__):
             return self.base == other.base and self.extras == other.extras
         return False
 
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
         return self.base.project_name
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         """The normalised name of the project the candidate refers to"""
         return format_name(self.base.project_name, self.extras)
 
     @property
-    def version(self):
-        # type: () -> _BaseVersion
+    def version(self) -> CandidateVersion:
         return self.base.version
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return "{} [{}]".format(
-            self.base.format_for_error(),
-            ", ".join(sorted(self.extras))
+            self.base.format_for_error(), ", ".join(sorted(self.extras))
         )
 
     @property
-    def is_installed(self):
-        # type: () -> bool
+    def is_installed(self) -> bool:
         return self.base.is_installed
 
     @property
-    def is_editable(self):
-        # type: () -> bool
+    def is_editable(self) -> bool:
         return self.base.is_editable
 
     @property
-    def source_link(self):
-        # type: () -> Optional[Link]
+    def source_link(self) -> Optional[Link]:
         return self.base.source_link
 
-    def iter_dependencies(self, with_requires):
-        # type: (bool) -> Iterable[Optional[Requirement]]
+    def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
         factory = self.base._factory
 
         # Add a dependency on the exact base
@@ -515,25 +483,24 @@ def iter_dependencies(self, with_requires):
 
         # The user may have specified extras that the candidate doesn't
         # support. We ignore any unsupported extras here.
-        valid_extras = self.extras.intersection(self.base.dist.extras)
-        invalid_extras = self.extras.difference(self.base.dist.extras)
+        valid_extras = self.extras.intersection(self.base.dist.iter_provided_extras())
+        invalid_extras = self.extras.difference(self.base.dist.iter_provided_extras())
         for extra in sorted(invalid_extras):
             logger.warning(
                 "%s %s does not provide the extra '%s'",
                 self.base.name,
                 self.version,
-                extra
+                extra,
             )
 
-        for r in self.base.dist.requires(valid_extras):
+        for r in self.base.dist.iter_dependencies(valid_extras):
             requirement = factory.make_requirement_from_spec(
-                str(r), self.base._ireq, valid_extras,
+                str(r), self.base._ireq, valid_extras
             )
             if requirement:
                 yield requirement
 
-    def get_install_requirement(self):
-        # type: () -> Optional[InstallRequirement]
+    def get_install_requirement(self) -> Optional[InstallRequirement]:
         # We don't return anything here, because we always
         # depend on the base candidate, and we'll get the
         # install requirement from that.
@@ -544,8 +511,7 @@ class RequiresPythonCandidate(Candidate):
     is_installed = False
     source_link = None
 
-    def __init__(self, py_version_info):
-        # type: (Optional[Tuple[int, ...]]) -> None
+    def __init__(self, py_version_info: Optional[Tuple[int, ...]]) -> None:
         if py_version_info is not None:
             version_info = normalize_version_info(py_version_info)
         else:
@@ -556,34 +522,26 @@ def __init__(self, py_version_info):
     # only one RequiresPythonCandidate in a resolution, i.e. the host Python.
     # The built-in object.__eq__() and object.__ne__() do exactly what we want.
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return f"Python {self._version}"
 
     @property
-    def project_name(self):
-        # type: () -> str
-        # Avoid conflicting with the PyPI package "Python".
-        return ""
+    def project_name(self) -> NormalizedName:
+        return REQUIRES_PYTHON_IDENTIFIER
 
     @property
-    def name(self):
-        # type: () -> str
-        return self.project_name
+    def name(self) -> str:
+        return REQUIRES_PYTHON_IDENTIFIER
 
     @property
-    def version(self):
-        # type: () -> _BaseVersion
+    def version(self) -> CandidateVersion:
         return self._version
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return f"Python {self.version}"
 
-    def iter_dependencies(self, with_requires):
-        # type: (bool) -> Iterable[Optional[Requirement]]
+    def iter_dependencies(self, with_requires: bool) -> Iterable[Optional[Requirement]]:
         return ()
 
-    def get_install_requirement(self):
-        # type: () -> Optional[InstallRequirement]
+    def get_install_requirement(self) -> Optional[InstallRequirement]:
         return None
diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py
index bfaa0520ace..3cfcac865ff 100644
--- a/src/pip/_internal/resolution/resolvelib/factory.py
+++ b/src/pip/_internal/resolution/resolvelib/factory.py
@@ -1,7 +1,29 @@
+import contextlib
+import functools
 import logging
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    FrozenSet,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Set,
+    Tuple,
+    TypeVar,
+    cast,
+)
 
-from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.requirements import InvalidRequirement
+from pip._vendor.packaging.specifiers import SpecifierSet
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
+from pip._vendor.resolvelib import ResolutionImpossible
 
+from pip._internal.cache import CacheEntry, WheelCache
 from pip._internal.exceptions import (
     DistributionNotFound,
     InstallationError,
@@ -10,27 +32,33 @@
     UnsupportedPythonVersion,
     UnsupportedWheel,
 )
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution, get_default_environment
+from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
-from pip._internal.req.req_install import InstallRequirement
+from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req.constructors import install_req_from_link_and_ireq
+from pip._internal.req.req_install import (
+    InstallRequirement,
+    check_invalid_constraint_type,
+)
+from pip._internal.resolution.base import InstallRequirementProvider
 from pip._internal.utils.compatibility_tags import get_supported
 from pip._internal.utils.hashes import Hashes
-from pip._internal.utils.misc import (
-    dist_in_site_packages,
-    dist_in_usersite,
-    get_installed_distributions,
-)
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.packaging import get_requirement, is_pinned
 from pip._internal.utils.virtualenv import running_under_virtualenv
 
-from .base import Constraint
+from .base import Candidate, CandidateVersion, Constraint, Requirement
 from .candidates import (
     AlreadyInstalledCandidate,
+    BaseCandidate,
     EditableCandidate,
     ExtrasCandidate,
     LinkCandidate,
     RequiresPythonCandidate,
+    as_base_candidate,
 )
-from .found_candidates import FoundCandidates
+from .found_candidates import FoundCandidates, IndexCandidateInfo
 from .requirements import (
     ExplicitRequirement,
     RequiresPythonRequirement,
@@ -38,54 +66,40 @@
     UnsatisfiableRequirement,
 )
 
-if MYPY_CHECK_RUNNING:
-    from typing import (
-        Dict,
-        FrozenSet,
-        Iterable,
-        Iterator,
-        List,
-        Optional,
-        Sequence,
-        Set,
-        Tuple,
-        TypeVar,
-    )
-
-    from pip._vendor.packaging.specifiers import SpecifierSet
-    from pip._vendor.packaging.version import _BaseVersion
-    from pip._vendor.pkg_resources import Distribution
-    from pip._vendor.resolvelib import ResolutionImpossible
-
-    from pip._internal.cache import CacheEntry, WheelCache
-    from pip._internal.index.package_finder import PackageFinder
-    from pip._internal.models.link import Link
-    from pip._internal.operations.prepare import RequirementPreparer
-    from pip._internal.resolution.base import InstallRequirementProvider
-
-    from .base import Candidate, Requirement
-    from .candidates import BaseCandidate
-
-    C = TypeVar("C")
-    Cache = Dict[Link, C]
+if TYPE_CHECKING:
+    from typing import Protocol
+
+    class ConflictCause(Protocol):
+        requirement: RequiresPythonRequirement
+        parent: Candidate
+
 
 logger = logging.getLogger(__name__)
 
+C = TypeVar("C")
+Cache = Dict[Link, C]
+
+
+class CollectedRootRequirements(NamedTuple):
+    requirements: List[Requirement]
+    constraints: Dict[str, Constraint]
+    user_requested: Dict[str, int]
+
 
 class Factory:
     def __init__(
         self,
-        finder,  # type: PackageFinder
-        preparer,  # type: RequirementPreparer
-        make_install_req,  # type: InstallRequirementProvider
-        wheel_cache,  # type: Optional[WheelCache]
-        use_user_site,  # type: bool
-        force_reinstall,  # type: bool
-        ignore_installed,  # type: bool
-        ignore_requires_python,  # type: bool
-        py_version_info=None,  # type: Optional[Tuple[int, ...]]
-    ):
-        # type: (...) -> None
+        finder: PackageFinder,
+        preparer: RequirementPreparer,
+        make_install_req: InstallRequirementProvider,
+        wheel_cache: Optional[WheelCache],
+        use_user_site: bool,
+        force_reinstall: bool,
+        ignore_installed: bool,
+        ignore_requires_python: bool,
+        suppress_build_failures: bool,
+        py_version_info: Optional[Tuple[int, ...]] = None,
+    ) -> None:
         self._finder = finder
         self.preparer = preparer
         self._wheel_cache = wheel_cache
@@ -94,51 +108,72 @@ def __init__(
         self._use_user_site = use_user_site
         self._force_reinstall = force_reinstall
         self._ignore_requires_python = ignore_requires_python
+        self._suppress_build_failures = suppress_build_failures
 
-        self._build_failures = {}  # type: Cache[InstallationError]
-        self._link_candidate_cache = {}  # type: Cache[LinkCandidate]
-        self._editable_candidate_cache = {}  # type: Cache[EditableCandidate]
-        self._installed_candidate_cache = {
-        }  # type: Dict[str, AlreadyInstalledCandidate]
+        self._build_failures: Cache[InstallationError] = {}
+        self._link_candidate_cache: Cache[LinkCandidate] = {}
+        self._editable_candidate_cache: Cache[EditableCandidate] = {}
+        self._installed_candidate_cache: Dict[str, AlreadyInstalledCandidate] = {}
+        self._extras_candidate_cache: Dict[
+            Tuple[int, FrozenSet[str]], ExtrasCandidate
+        ] = {}
 
         if not ignore_installed:
+            env = get_default_environment()
             self._installed_dists = {
-                canonicalize_name(dist.project_name): dist
-                for dist in get_installed_distributions(local_only=False)
+                dist.canonical_name: dist
+                for dist in env.iter_installed_distributions(local_only=False)
             }
         else:
             self._installed_dists = {}
 
     @property
-    def force_reinstall(self):
-        # type: () -> bool
+    def force_reinstall(self) -> bool:
         return self._force_reinstall
 
+    def _fail_if_link_is_unsupported_wheel(self, link: Link) -> None:
+        if not link.is_wheel:
+            return
+        wheel = Wheel(link.filename)
+        if wheel.supported(self._finder.target_python.get_tags()):
+            return
+        msg = f"{link.filename} is not a supported wheel on this platform."
+        raise UnsupportedWheel(msg)
+
+    def _make_extras_candidate(
+        self, base: BaseCandidate, extras: FrozenSet[str]
+    ) -> ExtrasCandidate:
+        cache_key = (id(base), extras)
+        try:
+            candidate = self._extras_candidate_cache[cache_key]
+        except KeyError:
+            candidate = ExtrasCandidate(base, extras)
+            self._extras_candidate_cache[cache_key] = candidate
+        return candidate
+
     def _make_candidate_from_dist(
         self,
-        dist,  # type: Distribution
-        extras,  # type: FrozenSet[str]
-        template,  # type: InstallRequirement
-    ):
-        # type: (...) -> Candidate
+        dist: BaseDistribution,
+        extras: FrozenSet[str],
+        template: InstallRequirement,
+    ) -> Candidate:
         try:
-            base = self._installed_candidate_cache[dist.key]
+            base = self._installed_candidate_cache[dist.canonical_name]
         except KeyError:
             base = AlreadyInstalledCandidate(dist, template, factory=self)
-            self._installed_candidate_cache[dist.key] = base
-        if extras:
-            return ExtrasCandidate(base, extras)
-        return base
+            self._installed_candidate_cache[dist.canonical_name] = base
+        if not extras:
+            return base
+        return self._make_extras_candidate(base, extras)
 
     def _make_candidate_from_link(
         self,
-        link,  # type: Link
-        extras,  # type: FrozenSet[str]
-        template,  # type: InstallRequirement
-        name,  # type: Optional[str]
-        version,  # type: Optional[_BaseVersion]
-    ):
-        # type: (...) -> Optional[Candidate]
+        link: Link,
+        extras: FrozenSet[str],
+        template: InstallRequirement,
+        name: Optional[NormalizedName],
+        version: Optional[CandidateVersion],
+    ) -> Optional[Candidate]:
         # TODO: Check already installed candidate, and use it if the link and
         # editable flag match.
 
@@ -151,39 +186,68 @@ def _make_candidate_from_link(
             if link not in self._editable_candidate_cache:
                 try:
                     self._editable_candidate_cache[link] = EditableCandidate(
-                        link, template, factory=self,
-                        name=name, version=version,
+                        link,
+                        template,
+                        factory=self,
+                        name=name,
+                        version=version,
+                    )
+                except MetadataInconsistent as e:
+                    logger.info(
+                        "Discarding [blue underline]%s[/]: [yellow]%s[reset]",
+                        link,
+                        e,
+                        extra={"markup": True},
                     )
-                except (InstallationSubprocessError, MetadataInconsistent) as e:
-                    logger.warning("Discarding %s. %s", link, e)
                     self._build_failures[link] = e
                     return None
-            base = self._editable_candidate_cache[link]  # type: BaseCandidate
+                except InstallationSubprocessError as e:
+                    if not self._suppress_build_failures:
+                        raise
+                    logger.warning("Discarding %s due to build failure: %s", link, e)
+                    self._build_failures[link] = e
+                    return None
+
+            base: BaseCandidate = self._editable_candidate_cache[link]
         else:
             if link not in self._link_candidate_cache:
                 try:
                     self._link_candidate_cache[link] = LinkCandidate(
-                        link, template, factory=self,
-                        name=name, version=version,
+                        link,
+                        template,
+                        factory=self,
+                        name=name,
+                        version=version,
                     )
-                except (InstallationSubprocessError, MetadataInconsistent) as e:
-                    logger.warning("Discarding %s. %s", link, e)
+                except MetadataInconsistent as e:
+                    logger.info(
+                        "Discarding [blue underline]%s[/]: [yellow]%s[reset]",
+                        link,
+                        e,
+                        extra={"markup": True},
+                    )
+                    self._build_failures[link] = e
+                    return None
+                except InstallationSubprocessError as e:
+                    if not self._suppress_build_failures:
+                        raise
+                    logger.warning("Discarding %s due to build failure: %s", link, e)
                     self._build_failures[link] = e
                     return None
             base = self._link_candidate_cache[link]
 
-        if extras:
-            return ExtrasCandidate(base, extras)
-        return base
+        if not extras:
+            return base
+        return self._make_extras_candidate(base, extras)
 
     def _iter_found_candidates(
         self,
-        ireqs,  # type: Sequence[InstallRequirement]
-        specifier,  # type: SpecifierSet
-        hashes,  # type: Hashes
-        prefers_installed,  # type: bool
-    ):
-        # type: (...) -> Iterable[Candidate]
+        ireqs: Sequence[InstallRequirement],
+        specifier: SpecifierSet,
+        hashes: Hashes,
+        prefers_installed: bool,
+        incompatible_ids: Set[int],
+    ) -> Iterable[Candidate]:
         if not ireqs:
             return ()
 
@@ -192,29 +256,41 @@ def _iter_found_candidates(
         # all of them.
         # Hopefully the Project model can correct this mismatch in the future.
         template = ireqs[0]
+        assert template.req, "Candidates found on index must be PEP 508"
         name = canonicalize_name(template.req.name)
 
-        extras = frozenset()  # type: FrozenSet[str]
+        extras: FrozenSet[str] = frozenset()
         for ireq in ireqs:
+            assert ireq.req, "Candidates found on index must be PEP 508"
             specifier &= ireq.req.specifier
             hashes &= ireq.hashes(trust_internet=False)
             extras |= frozenset(ireq.extras)
 
-        # Get the installed version, if it matches, unless the user
-        # specified `--force-reinstall`, when we want the version from
-        # the index instead.
-        installed_candidate = None
-        if not self._force_reinstall and name in self._installed_dists:
-            installed_dist = self._installed_dists[name]
-            if specifier.contains(installed_dist.version, prereleases=True):
-                installed_candidate = self._make_candidate_from_dist(
-                    dist=installed_dist,
-                    extras=extras,
-                    template=template,
-                )
+        def _get_installed_candidate() -> Optional[Candidate]:
+            """Get the candidate for the currently-installed version."""
+            # If --force-reinstall is set, we want the version from the index
+            # instead, so we "pretend" there is nothing installed.
+            if self._force_reinstall:
+                return None
+            try:
+                installed_dist = self._installed_dists[name]
+            except KeyError:
+                return None
+            # Don't use the installed distribution if its version does not fit
+            # the current dependency graph.
+            if not specifier.contains(installed_dist.version, prereleases=True):
+                return None
+            candidate = self._make_candidate_from_dist(
+                dist=installed_dist,
+                extras=extras,
+                template=template,
+            )
+            # The candidate is a known incompatibility. Don't use it.
+            if id(candidate) in incompatible_ids:
+                return None
+            return candidate
 
-        def iter_index_candidates():
-            # type: () -> Iterator[Candidate]
+        def iter_index_candidate_infos() -> Iterator[IndexCandidateInfo]:
             result = self._finder.find_best_candidate(
                 project_name=name,
                 specifier=specifier,
@@ -222,52 +298,135 @@ def iter_index_candidates():
             )
             icans = list(result.iter_applicable())
 
-            # PEP 592: Yanked releases must be ignored unless only yanked
-            # releases can satisfy the version range. So if this is false,
-            # all yanked icans need to be skipped.
+            # PEP 592: Yanked releases are ignored unless the specifier
+            # explicitly pins a version (via '==' or '===') that can be
+            # solely satisfied by a yanked release.
             all_yanked = all(ican.link.is_yanked for ican in icans)
 
+            pinned = is_pinned(specifier)
+
+            if not template.is_pinned:
+                assert template.req, "Candidates found on index must be PEP 508"
+                template.req.specifier = specifier
+                template.hash_options = hashes.allowed
+
             # PackageFinder returns earlier versions first, so we reverse.
-            versions_found = set()  # type: Set[_BaseVersion]
             for ican in reversed(icans):
-                if not all_yanked and ican.link.is_yanked:
-                    continue
-                if ican.version in versions_found:
+                if not (all_yanked and pinned) and ican.link.is_yanked:
                     continue
-                candidate = self._make_candidate_from_link(
+                func = functools.partial(
+                    self._make_candidate_from_link,
                     link=ican.link,
                     extras=extras,
                     template=template,
                     name=name,
                     version=ican.version,
                 )
-                if candidate is None:
-                    continue
-                yield candidate
-                versions_found.add(ican.version)
+                yield ican.version, func
 
         return FoundCandidates(
-            iter_index_candidates,
-            installed_candidate,
+            iter_index_candidate_infos,
+            _get_installed_candidate(),
             prefers_installed,
+            incompatible_ids,
         )
 
+    def _iter_explicit_candidates_from_base(
+        self,
+        base_requirements: Iterable[Requirement],
+        extras: FrozenSet[str],
+    ) -> Iterator[Candidate]:
+        """Produce explicit candidates from the base given an extra-ed package.
+
+        :param base_requirements: Requirements known to the resolver. The
+            requirements are guaranteed to not have extras.
+        :param extras: The extras to inject into the explicit requirements'
+            candidates.
+        """
+        for req in base_requirements:
+            lookup_cand, _ = req.get_candidate_lookup()
+            if lookup_cand is None:  # Not explicit.
+                continue
+            # We've stripped extras from the identifier, and should always
+            # get a BaseCandidate here, unless there's a bug elsewhere.
+            base_cand = as_base_candidate(lookup_cand)
+            assert base_cand is not None, "no extras here"
+            yield self._make_extras_candidate(base_cand, extras)
+
+    def _iter_candidates_from_constraints(
+        self,
+        identifier: str,
+        constraint: Constraint,
+        template: InstallRequirement,
+    ) -> Iterator[Candidate]:
+        """Produce explicit candidates from constraints.
+
+        This creates "fake" InstallRequirement objects that are basically clones
+        of what "should" be the template, but with original_link set to link.
+        """
+        for link in constraint.links:
+            self._fail_if_link_is_unsupported_wheel(link)
+            candidate = self._make_candidate_from_link(
+                link,
+                extras=frozenset(),
+                template=install_req_from_link_and_ireq(link, template),
+                name=canonicalize_name(identifier),
+                version=None,
+            )
+            if candidate:
+                yield candidate
+
     def find_candidates(
         self,
-        requirements,  # type: Sequence[Requirement]
-        constraint,  # type: Constraint
-        prefers_installed,  # type: bool
-    ):
-        # type: (...) -> Iterable[Candidate]
-        explicit_candidates = set()  # type: Set[Candidate]
-        ireqs = []  # type: List[InstallRequirement]
-        for req in requirements:
+        identifier: str,
+        requirements: Mapping[str, Iterable[Requirement]],
+        incompatibilities: Mapping[str, Iterator[Candidate]],
+        constraint: Constraint,
+        prefers_installed: bool,
+    ) -> Iterable[Candidate]:
+        # Collect basic lookup information from the requirements.
+        explicit_candidates: Set[Candidate] = set()
+        ireqs: List[InstallRequirement] = []
+        for req in requirements[identifier]:
             cand, ireq = req.get_candidate_lookup()
             if cand is not None:
                 explicit_candidates.add(cand)
             if ireq is not None:
                 ireqs.append(ireq)
 
+        # If the current identifier contains extras, add explicit candidates
+        # from entries from extra-less identifier.
+        with contextlib.suppress(InvalidRequirement):
+            parsed_requirement = get_requirement(identifier)
+            explicit_candidates.update(
+                self._iter_explicit_candidates_from_base(
+                    requirements.get(parsed_requirement.name, ()),
+                    frozenset(parsed_requirement.extras),
+                ),
+            )
+
+        # Add explicit candidates from constraints. We only do this if there are
+        # known ireqs, which represent requirements not already explicit. If
+        # there are no ireqs, we're constraining already-explicit requirements,
+        # which is handled later when we return the explicit candidates.
+        if ireqs:
+            try:
+                explicit_candidates.update(
+                    self._iter_candidates_from_constraints(
+                        identifier,
+                        constraint,
+                        template=ireqs[0],
+                    ),
+                )
+            except UnsupportedWheel:
+                # If we're constrained to install a wheel incompatible with the
+                # target architecture, no candidates will ever be valid.
+                return ()
+
+        # Since we cache all the candidates, incompatibility identification
+        # can be made quicker by comparing only the id() values.
+        incompat_ids = {id(c) for c in incompatibilities.get(identifier, ())}
+
         # If none of the requirements want an explicit candidate, we can ask
         # the finder for candidates.
         if not explicit_candidates:
@@ -276,31 +435,30 @@ def find_candidates(
                 constraint.specifier,
                 constraint.hashes,
                 prefers_installed,
+                incompat_ids,
             )
 
         return (
-            c for c in explicit_candidates
-            if constraint.is_satisfied_by(c)
-            and all(req.is_satisfied_by(c) for req in requirements)
+            c
+            for c in explicit_candidates
+            if id(c) not in incompat_ids
+            and constraint.is_satisfied_by(c)
+            and all(req.is_satisfied_by(c) for req in requirements[identifier])
         )
 
-    def make_requirement_from_install_req(self, ireq, requested_extras):
-        # type: (InstallRequirement, Iterable[str]) -> Optional[Requirement]
+    def _make_requirement_from_install_req(
+        self, ireq: InstallRequirement, requested_extras: Iterable[str]
+    ) -> Optional[Requirement]:
         if not ireq.match_markers(requested_extras):
             logger.info(
                 "Ignoring %s: markers '%s' don't match your environment",
-                ireq.name, ireq.markers,
+                ireq.name,
+                ireq.markers,
             )
             return None
         if not ireq.link:
             return SpecifierRequirement(ireq)
-        if ireq.link.is_wheel:
-            wheel = Wheel(ireq.link.filename)
-            if not wheel.supported(self._finder.target_python.get_tags()):
-                msg = "{} is not a supported wheel on this platform.".format(
-                    wheel.filename,
-                )
-                raise UnsupportedWheel(msg)
+        self._fail_if_link_is_unsupported_wheel(ireq.link)
         cand = self._make_candidate_from_link(
             ireq.link,
             extras=frozenset(ireq.extras),
@@ -320,28 +478,64 @@ def make_requirement_from_install_req(self, ireq, requested_extras):
             return UnsatisfiableRequirement(canonicalize_name(ireq.name))
         return self.make_requirement_from_candidate(cand)
 
-    def make_requirement_from_candidate(self, candidate):
-        # type: (Candidate) -> ExplicitRequirement
+    def collect_root_requirements(
+        self, root_ireqs: List[InstallRequirement]
+    ) -> CollectedRootRequirements:
+        collected = CollectedRootRequirements([], {}, {})
+        for i, ireq in enumerate(root_ireqs):
+            if ireq.constraint:
+                # Ensure we only accept valid constraints
+                problem = check_invalid_constraint_type(ireq)
+                if problem:
+                    raise InstallationError(problem)
+                if not ireq.match_markers():
+                    continue
+                assert ireq.name, "Constraint must be named"
+                name = canonicalize_name(ireq.name)
+                if name in collected.constraints:
+                    collected.constraints[name] &= ireq
+                else:
+                    collected.constraints[name] = Constraint.from_ireq(ireq)
+            else:
+                req = self._make_requirement_from_install_req(
+                    ireq,
+                    requested_extras=(),
+                )
+                if req is None:
+                    continue
+                if ireq.user_supplied and req.name not in collected.user_requested:
+                    collected.user_requested[req.name] = i
+                collected.requirements.append(req)
+        return collected
+
+    def make_requirement_from_candidate(
+        self, candidate: Candidate
+    ) -> ExplicitRequirement:
         return ExplicitRequirement(candidate)
 
     def make_requirement_from_spec(
         self,
-        specifier,  # type: str
-        comes_from,  # type: InstallRequirement
-        requested_extras=(),  # type: Iterable[str]
-    ):
-        # type: (...) -> Optional[Requirement]
+        specifier: str,
+        comes_from: Optional[InstallRequirement],
+        requested_extras: Iterable[str] = (),
+    ) -> Optional[Requirement]:
         ireq = self._make_install_req_from_spec(specifier, comes_from)
-        return self.make_requirement_from_install_req(ireq, requested_extras)
+        return self._make_requirement_from_install_req(ireq, requested_extras)
 
-    def make_requires_python_requirement(self, specifier):
-        # type: (Optional[SpecifierSet]) -> Optional[Requirement]
-        if self._ignore_requires_python or specifier is None:
+    def make_requires_python_requirement(
+        self,
+        specifier: SpecifierSet,
+    ) -> Optional[Requirement]:
+        if self._ignore_requires_python:
+            return None
+        # Don't bother creating a dependency for an empty Requires-Python.
+        if not str(specifier):
             return None
         return RequiresPythonRequirement(specifier, self._python_candidate)
 
-    def get_wheel_cache_entry(self, link, name):
-        # type: (Link, Optional[str]) -> Optional[CacheEntry]
+    def get_wheel_cache_entry(
+        self, link: Link, name: Optional[str]
+    ) -> Optional[CacheEntry]:
         """Look up the link in the wheel cache.
 
         If ``preparer.require_hashes`` is True, don't use the wheel cache,
@@ -358,10 +552,9 @@ def get_wheel_cache_entry(self, link, name):
             supported_tags=get_supported(),
         )
 
-    def get_dist_to_uninstall(self, candidate):
-        # type: (Candidate) -> Optional[Distribution]
+    def get_dist_to_uninstall(self, candidate: Candidate) -> Optional[BaseDistribution]:
         # TODO: Are there more cases this needs to return True? Editable?
-        dist = self._installed_dists.get(candidate.name)
+        dist = self._installed_dists.get(candidate.project_name)
         if dist is None:  # Not installed, no uninstallation required.
             return None
 
@@ -372,52 +565,99 @@ def get_dist_to_uninstall(self, candidate):
             return dist
 
         # We're installing into user site. Remove the user site installation.
-        if dist_in_usersite(dist):
+        if dist.in_usersite:
             return dist
 
         # We're installing into user site, but the installed incompatible
         # package is in global site. We can't uninstall that, and would let
         # the new user installation to "shadow" it. But shadowing won't work
         # in virtual environments, so we error out.
-        if running_under_virtualenv() and dist_in_site_packages(dist):
-            raise InstallationError(
-                "Will not install to the user site because it will "
-                "lack sys.path precedence to {} in {}".format(
-                    dist.project_name, dist.location,
-                )
+        if running_under_virtualenv() and dist.in_site_packages:
+            message = (
+                f"Will not install to the user site because it will lack "
+                f"sys.path precedence to {dist.raw_name} in {dist.location}"
             )
+            raise InstallationError(message)
         return None
 
     def _report_requires_python_error(
-        self,
-        requirement,  # type: RequiresPythonRequirement
-        template,  # type: Candidate
-    ):
-        # type: (...) -> UnsupportedPythonVersion
-        message_format = (
-            "Package {package!r} requires a different Python: "
-            "{version} not in {specifier!r}"
-        )
-        message = message_format.format(
-            package=template.name,
-            version=self._python_candidate.version,
-            specifier=str(requirement.specifier),
-        )
+        self, causes: Sequence["ConflictCause"]
+    ) -> UnsupportedPythonVersion:
+        assert causes, "Requires-Python error reported with no cause"
+
+        version = self._python_candidate.version
+
+        if len(causes) == 1:
+            specifier = str(causes[0].requirement.specifier)
+            message = (
+                f"Package {causes[0].parent.name!r} requires a different "
+                f"Python: {version} not in {specifier!r}"
+            )
+            return UnsupportedPythonVersion(message)
+
+        message = f"Packages require a different Python. {version} not in:"
+        for cause in causes:
+            package = cause.parent.format_for_error()
+            specifier = str(cause.requirement.specifier)
+            message += f"\n{specifier!r} (required by {package})"
         return UnsupportedPythonVersion(message)
 
-    def get_installation_error(self, e):
-        # type: (ResolutionImpossible) -> InstallationError
+    def _report_single_requirement_conflict(
+        self, req: Requirement, parent: Optional[Candidate]
+    ) -> DistributionNotFound:
+        if parent is None:
+            req_disp = str(req)
+        else:
+            req_disp = f"{req} (from {parent.name})"
+
+        cands = self._finder.find_all_candidates(req.project_name)
+        skipped_by_requires_python = self._finder.requires_python_skipped_reasons()
+        versions = [str(v) for v in sorted({c.version for c in cands})]
+
+        if skipped_by_requires_python:
+            logger.critical(
+                "Ignored the following versions that require a different python "
+                "version: %s",
+                "; ".join(skipped_by_requires_python) or "none",
+            )
+        logger.critical(
+            "Could not find a version that satisfies the requirement %s "
+            "(from versions: %s)",
+            req_disp,
+            ", ".join(versions) or "none",
+        )
+        if str(req) == "requirements.txt":
+            logger.info(
+                "HINT: You are attempting to install a package literally "
+                'named "requirements.txt" (which cannot exist). Consider '
+                "using the '-r' flag to install the packages listed in "
+                "requirements.txt"
+            )
+
+        return DistributionNotFound(f"No matching distribution found for {req}")
+
+    def get_installation_error(
+        self,
+        e: "ResolutionImpossible[Requirement, Candidate]",
+        constraints: Dict[str, Constraint],
+    ) -> InstallationError:
 
         assert e.causes, "Installation error reported with no cause"
 
         # If one of the things we can't solve is "we need Python X.Y",
         # that is what we report.
-        for cause in e.causes:
-            if isinstance(cause.requirement, RequiresPythonRequirement):
-                return self._report_requires_python_error(
-                    cause.requirement,
-                    cause.parent,
-                )
+        requires_python_causes = [
+            cause
+            for cause in e.causes
+            if isinstance(cause.requirement, RequiresPythonRequirement)
+            and not cause.requirement.is_satisfied_by(self._python_candidate)
+        ]
+        if requires_python_causes:
+            # The comprehension above makes sure all Requirement instances are
+            # RequiresPythonRequirement, so let's cast for convenience.
+            return self._report_requires_python_error(
+                cast("Sequence[ConflictCause]", requires_python_causes),
+            )
 
         # Otherwise, we have a set of causes which can't all be satisfied
         # at once.
@@ -426,31 +666,20 @@ def get_installation_error(self, e):
         # satisfied. We just report that case.
         if len(e.causes) == 1:
             req, parent = e.causes[0]
-            if parent is None:
-                req_disp = str(req)
-            else:
-                req_disp = f'{req} (from {parent.name})'
-            logger.critical(
-                "Could not find a version that satisfies the requirement %s",
-                req_disp,
-            )
-            return DistributionNotFound(
-                f'No matching distribution found for {req}'
-            )
+            if req.name not in constraints:
+                return self._report_single_requirement_conflict(req, parent)
 
         # OK, we now have a list of requirements that can't all be
         # satisfied at once.
 
         # A couple of formatting helpers
-        def text_join(parts):
-            # type: (List[str]) -> str
+        def text_join(parts: List[str]) -> str:
             if len(parts) == 1:
                 return parts[0]
 
             return ", ".join(parts[:-1]) + " and " + parts[-1]
 
-        def describe_trigger(parent):
-            # type: (Candidate) -> str
+        def describe_trigger(parent: Candidate) -> str:
             ireq = parent.get_install_requirement()
             if not ireq or not ireq.comes_from:
                 return f"{parent.name}=={parent.version}"
@@ -472,31 +701,40 @@ def describe_trigger(parent):
         else:
             info = "the requested packages"
 
-        msg = "Cannot install {} because these package versions " \
+        msg = (
+            "Cannot install {} because these package versions "
             "have conflicting dependencies.".format(info)
+        )
         logger.critical(msg)
         msg = "\nThe conflict is caused by:"
+
+        relevant_constraints = set()
         for req, parent in e.causes:
+            if req.name in constraints:
+                relevant_constraints.add(req.name)
             msg = msg + "\n    "
             if parent:
-                msg = msg + "{} {} depends on ".format(
-                    parent.name,
-                    parent.version
-                )
+                msg = msg + f"{parent.name} {parent.version} depends on "
             else:
                 msg = msg + "The user requested "
             msg = msg + req.format_for_error()
-
-        msg = msg + "\n\n" + \
-            "To fix this you could try to:\n" + \
-            "1. loosen the range of package versions you've specified\n" + \
-            "2. remove package versions to allow pip attempt to solve " + \
-            "the dependency conflict\n"
+        for key in relevant_constraints:
+            spec = constraints[key].specifier
+            msg += f"\n    The user requested (constraint) {key}{spec}"
+
+        msg = (
+            msg
+            + "\n\n"
+            + "To fix this you could try to:\n"
+            + "1. loosen the range of package versions you've specified\n"
+            + "2. remove package versions to allow pip attempt to solve "
+            + "the dependency conflict\n"
+        )
 
         logger.info(msg)
 
         return DistributionNotFound(
             "ResolutionImpossible: for help visit "
-            "https://pip.pypa.io/en/latest/user_guide/"
-            "#fixing-conflicting-dependencies"
+            "https://pip.pypa.io/en/latest/topics/dependency-resolution/"
+            "#dealing-with-dependency-conflicts"
         )
diff --git a/src/pip/_internal/resolution/resolvelib/found_candidates.py b/src/pip/_internal/resolution/resolvelib/found_candidates.py
index c3f95c1d41d..8663097b447 100644
--- a/src/pip/_internal/resolution/resolvelib/found_candidates.py
+++ b/src/pip/_internal/resolution/resolvelib/found_candidates.py
@@ -9,20 +9,73 @@
 """
 
 import functools
-import itertools
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Set, Tuple
 
-from pip._vendor.six.moves import collections_abc  # type: ignore
+from pip._vendor.packaging.version import _BaseVersion
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from .base import Candidate
 
-if MYPY_CHECK_RUNNING:
-    from typing import Callable, Iterator, Optional
+IndexCandidateInfo = Tuple[_BaseVersion, Callable[[], Optional[Candidate]]]
 
-    from .base import Candidate
+if TYPE_CHECKING:
+    SequenceCandidate = Sequence[Candidate]
+else:
+    # For compatibility: Python before 3.9 does not support using [] on the
+    # Sequence class.
+    #
+    # >>> from collections.abc import Sequence
+    # >>> Sequence[str]
+    # Traceback (most recent call last):
+    #   File "", line 1, in 
+    # TypeError: 'ABCMeta' object is not subscriptable
+    #
+    # TODO: Remove this block after dropping Python 3.8 support.
+    SequenceCandidate = Sequence
 
 
-def _insert_installed(installed, others):
-    # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
+def _iter_built(infos: Iterator[IndexCandidateInfo]) -> Iterator[Candidate]:
+    """Iterator for ``FoundCandidates``.
+
+    This iterator is used when the package is not already installed. Candidates
+    from index come later in their normal ordering.
+    """
+    versions_found: Set[_BaseVersion] = set()
+    for version, func in infos:
+        if version in versions_found:
+            continue
+        candidate = func()
+        if candidate is None:
+            continue
+        yield candidate
+        versions_found.add(version)
+
+
+def _iter_built_with_prepended(
+    installed: Candidate, infos: Iterator[IndexCandidateInfo]
+) -> Iterator[Candidate]:
+    """Iterator for ``FoundCandidates``.
+
+    This iterator is used when the resolver prefers the already-installed
+    candidate and NOT to upgrade. The installed candidate is therefore
+    always yielded first, and candidates from index come later in their
+    normal ordering, except skipped when the version is already installed.
+    """
+    yield installed
+    versions_found: Set[_BaseVersion] = {installed.version}
+    for version, func in infos:
+        if version in versions_found:
+            continue
+        candidate = func()
+        if candidate is None:
+            continue
+        yield candidate
+        versions_found.add(version)
+
+
+def _iter_built_with_inserted(
+    installed: Candidate, infos: Iterator[IndexCandidateInfo]
+) -> Iterator[Candidate]:
     """Iterator for ``FoundCandidates``.
 
     This iterator is used when the resolver prefers to upgrade an
@@ -33,20 +86,26 @@ def _insert_installed(installed, others):
     the installed candidate exactly once before we start yielding older or
     equivalent candidates, or after all other candidates if they are all newer.
     """
-    installed_yielded = False
-    for candidate in others:
+    versions_found: Set[_BaseVersion] = set()
+    for version, func in infos:
+        if version in versions_found:
+            continue
         # If the installed candidate is better, yield it first.
-        if not installed_yielded and installed.version >= candidate.version:
+        if installed.version >= version:
             yield installed
-            installed_yielded = True
+            versions_found.add(installed.version)
+        candidate = func()
+        if candidate is None:
+            continue
         yield candidate
+        versions_found.add(version)
 
     # If the installed candidate is older than all other candidates.
-    if not installed_yielded:
+    if installed.version not in versions_found:
         yield installed
 
 
-class FoundCandidates(collections_abc.Sequence):
+class FoundCandidates(SequenceCandidate):
     """A lazy sequence to provide candidates to the resolver.
 
     The intended usage is to return this from `find_matches()` so the resolver
@@ -54,48 +113,43 @@ class FoundCandidates(collections_abc.Sequence):
     page when remote packages are actually needed. This improve performances
     when suitable candidates are already installed on disk.
     """
+
     def __init__(
         self,
-        get_others,  # type: Callable[[], Iterator[Candidate]]
-        installed,  # type: Optional[Candidate]
-        prefers_installed,  # type: bool
+        get_infos: Callable[[], Iterator[IndexCandidateInfo]],
+        installed: Optional[Candidate],
+        prefers_installed: bool,
+        incompatible_ids: Set[int],
     ):
-        self._get_others = get_others
+        self._get_infos = get_infos
         self._installed = installed
         self._prefers_installed = prefers_installed
+        self._incompatible_ids = incompatible_ids
 
-    def __getitem__(self, index):
-        # type: (int) -> Candidate
+    def __getitem__(self, index: Any) -> Any:
         # Implemented to satisfy the ABC check. This is not needed by the
         # resolver, and should not be used by the provider either (for
         # performance reasons).
         raise NotImplementedError("don't do this")
 
-    def __iter__(self):
-        # type: () -> Iterator[Candidate]
+    def __iter__(self) -> Iterator[Candidate]:
+        infos = self._get_infos()
         if not self._installed:
-            return self._get_others()
-        others = (
-            candidate
-            for candidate in self._get_others()
-            if candidate.version != self._installed.version
-        )
-        if self._prefers_installed:
-            return itertools.chain([self._installed], others)
-        return _insert_installed(self._installed, others)
-
-    def __len__(self):
-        # type: () -> int
+            iterator = _iter_built(infos)
+        elif self._prefers_installed:
+            iterator = _iter_built_with_prepended(self._installed, infos)
+        else:
+            iterator = _iter_built_with_inserted(self._installed, infos)
+        return (c for c in iterator if id(c) not in self._incompatible_ids)
+
+    def __len__(self) -> int:
         # Implemented to satisfy the ABC check. This is not needed by the
         # resolver, and should not be used by the provider either (for
         # performance reasons).
         raise NotImplementedError("don't do this")
 
     @functools.lru_cache(maxsize=1)
-    def __bool__(self):
-        # type: () -> bool
+    def __bool__(self) -> bool:
         if self._prefers_installed and self._installed:
             return True
         return any(self)
-
-    __nonzero__ = __bool__  # XXX: Python 2.
diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py
index 40a641a2a4d..e6ec9594f62 100644
--- a/src/pip/_internal/resolution/resolvelib/provider.py
+++ b/src/pip/_internal/resolution/resolvelib/provider.py
@@ -1,14 +1,31 @@
+import collections
+import math
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    Iterator,
+    Mapping,
+    Sequence,
+    TypeVar,
+    Union,
+)
+
 from pip._vendor.resolvelib.providers import AbstractProvider
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from .base import Candidate, Constraint, Requirement
+from .candidates import REQUIRES_PYTHON_IDENTIFIER
+from .factory import Factory
 
-from .base import Constraint
+if TYPE_CHECKING:
+    from pip._vendor.resolvelib.providers import Preference
+    from pip._vendor.resolvelib.resolvers import RequirementInformation
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Iterable, Optional, Sequence, Set, Tuple, Union
+    PreferenceInformation = RequirementInformation[Requirement, Candidate]
 
-    from .base import Candidate, Requirement
-    from .factory import Factory
+    _ProviderBase = AbstractProvider[Requirement, Candidate, str]
+else:
+    _ProviderBase = AbstractProvider
 
 # Notes on the relationship between the provider, the factory, and the
 # candidate and requirement classes.
@@ -29,7 +46,36 @@
 # services to those objects (access to pip's finder and preparer).
 
 
-class PipProvider(AbstractProvider):
+D = TypeVar("D")
+V = TypeVar("V")
+
+
+def _get_with_identifier(
+    mapping: Mapping[str, V],
+    identifier: str,
+    default: D,
+) -> Union[D, V]:
+    """Get item from a package name lookup mapping with a resolver identifier.
+
+    This extra logic is needed when the target mapping is keyed by package
+    name, which cannot be directly looked up with an identifier (which may
+    contain requested extras). Additional logic is added to also look up a value
+    by "cleaning up" the extras from the identifier.
+    """
+    if identifier in mapping:
+        return mapping[identifier]
+    # HACK: Theoretically we should check whether this identifier is a valid
+    # "NAME[EXTRAS]" format, and parse out the name part with packaging or
+    # some regular expression. But since pip's resolver only spits out three
+    # kinds of identifiers: normalized PEP 503 names, normalized names plus
+    # extras, and Requires-Python, we can cheat a bit here.
+    name, open_bracket, _ = identifier.partition("[")
+    if open_bracket and name in mapping:
+        return mapping[name]
+    return default
+
+
+class PipProvider(_ProviderBase):
     """Pip's provider implementation for resolvelib.
 
     :params constraints: A mapping of constraints specified by the user. Keys
@@ -42,30 +88,30 @@ class PipProvider(AbstractProvider):
 
     def __init__(
         self,
-        factory,  # type: Factory
-        constraints,  # type: Dict[str, Constraint]
-        ignore_dependencies,  # type: bool
-        upgrade_strategy,  # type: str
-        user_requested,  # type: Set[str]
-    ):
-        # type: (...) -> None
+        factory: Factory,
+        constraints: Dict[str, Constraint],
+        ignore_dependencies: bool,
+        upgrade_strategy: str,
+        user_requested: Dict[str, int],
+    ) -> None:
         self._factory = factory
         self._constraints = constraints
         self._ignore_dependencies = ignore_dependencies
         self._upgrade_strategy = upgrade_strategy
         self._user_requested = user_requested
+        self._known_depths: Dict[str, float] = collections.defaultdict(lambda: math.inf)
 
-    def identify(self, dependency):
-        # type: (Union[Requirement, Candidate]) -> str
-        return dependency.name
+    def identify(self, requirement_or_candidate: Union[Requirement, Candidate]) -> str:
+        return requirement_or_candidate.name
 
-    def get_preference(
+    def get_preference(  # type: ignore
         self,
-        resolution,  # type: Optional[Candidate]
-        candidates,  # type: Sequence[Candidate]
-        information  # type: Sequence[Tuple[Requirement, Candidate]]
-    ):
-        # type: (...) -> Any
+        identifier: str,
+        resolutions: Mapping[str, Candidate],
+        candidates: Mapping[str, Iterator[Candidate]],
+        information: Mapping[str, Iterable["PreferenceInformation"]],
+        backtrack_causes: Sequence["PreferenceInformation"],
+    ) -> "Preference":
         """Produce a sort key for given requirement based on preference.
 
         The lower the return value is, the more preferred this group of
@@ -73,50 +119,47 @@ def get_preference(
 
         Currently pip considers the followings in order:
 
-        * Prefer if any of the known requirements points to an explicit URL.
-        * If equal, prefer if any requirements contain ``===`` and ``==``.
-        * If equal, prefer if requirements include version constraints, e.g.
-          ``>=`` and ``<``.
-        * If equal, prefer user-specified (non-transitive) requirements.
+        * Prefer if any of the known requirements is "direct", e.g. points to an
+          explicit URL.
+        * If equal, prefer if any requirement is "pinned", i.e. contains
+          operator ``===`` or ``==``.
+        * If equal, calculate an approximate "depth" and resolve requirements
+          closer to the user-specified requirements first.
+        * Order user-specified requirements by the order they are specified.
+        * If equal, prefers "non-free" requirements, i.e. contains at least one
+          operator, such as ``>=`` or ``<``.
         * If equal, order alphabetically for consistency (helps debuggability).
         """
+        lookups = (r.get_candidate_lookup() for r, _ in information[identifier])
+        candidate, ireqs = zip(*lookups)
+        operators = [
+            specifier.operator
+            for specifier_set in (ireq.specifier for ireq in ireqs if ireq)
+            for specifier in specifier_set
+        ]
 
-        def _get_restrictive_rating(requirements):
-            # type: (Iterable[Requirement]) -> int
-            """Rate how restrictive a set of requirements are.
-
-            ``Requirement.get_candidate_lookup()`` returns a 2-tuple for
-            lookup. The first element is ``Optional[Candidate]`` and the
-            second ``Optional[InstallRequirement]``.
-
-            * If the requirement is an explicit one, the explicitly-required
-              candidate is returned as the first element.
-            * If the requirement is based on a PEP 508 specifier, the backing
-              ``InstallRequirement`` is returned as the second element.
-
-            We use the first element to check whether there is an explicit
-            requirement, and the second for equality operator.
-            """
-            lookups = (r.get_candidate_lookup() for r in requirements)
-            cands, ireqs = zip(*lookups)
-            if any(cand is not None for cand in cands):
-                return 0
-            spec_sets = (ireq.specifier for ireq in ireqs if ireq)
-            operators = [
-                specifier.operator
-                for spec_set in spec_sets
-                for specifier in spec_set
-            ]
-            if any(op in ("==", "===") for op in operators):
-                return 1
-            if operators:
-                return 2
-            # A "bare" requirement without any version requirements.
-            return 3
-
-        restrictive = _get_restrictive_rating(req for req, _ in information)
-        transitive = all(parent is not None for _, parent in information)
-        key = next(iter(candidates)).name if candidates else ""
+        direct = candidate is not None
+        pinned = any(op[:2] == "==" for op in operators)
+        unfree = bool(operators)
+
+        try:
+            requested_order: Union[int, float] = self._user_requested[identifier]
+        except KeyError:
+            requested_order = math.inf
+            parent_depths = (
+                self._known_depths[parent.name] if parent is not None else 0.0
+                for _, parent in information[identifier]
+            )
+            inferred_depth = min(d for d in parent_depths) + 1.0
+        else:
+            inferred_depth = 1.0
+        self._known_depths[identifier] = inferred_depth
+
+        requested_order = self._user_requested.get(identifier, math.inf)
+
+        # Requires-Python has only one candidate and the check is basically
+        # free, so we always do it first to avoid needless work if it fails.
+        requires_python = identifier == REQUIRES_PYTHON_IDENTIFIER
 
         # HACK: Setuptools have a very long and solid backward compatibility
         # track record, and extremely few projects would request a narrow,
@@ -124,20 +167,34 @@ def _get_restrictive_rating(requirements):
         # (Most projects specify it only to request for an installer feature,
         # which does not work, but that's another topic.) Intentionally
         # delaying Setuptools helps reduce branches the resolver has to check.
-        # This serves as a temporary fix for issues like "apache-airlfow[all]"
+        # This serves as a temporary fix for issues like "apache-airflow[all]"
         # while we work on "proper" branch pruning techniques.
-        delay_this = (key == "setuptools")
-
-        return (delay_this, restrictive, transitive, key)
-
-    def find_matches(self, requirements):
-        # type: (Sequence[Requirement]) -> Iterable[Candidate]
-        if not requirements:
-            return []
-        name = requirements[0].project_name
+        delay_this = identifier == "setuptools"
+
+        # Prefer the causes of backtracking on the assumption that the problem
+        # resolving the dependency tree is related to the failures that caused
+        # the backtracking
+        backtrack_cause = self.is_backtrack_cause(identifier, backtrack_causes)
+
+        return (
+            not requires_python,
+            delay_this,
+            not direct,
+            not pinned,
+            not backtrack_cause,
+            inferred_depth,
+            requested_order,
+            not unfree,
+            identifier,
+        )
 
-        def _eligible_for_upgrade(name):
-            # type: (str) -> bool
+    def find_matches(
+        self,
+        identifier: str,
+        requirements: Mapping[str, Iterator[Requirement]],
+        incompatibilities: Mapping[str, Iterator[Candidate]],
+    ) -> Iterable[Candidate]:
+        def _eligible_for_upgrade(identifier: str) -> bool:
             """Are upgrades allowed for this project?
 
             This checks the upgrade strategy, and whether the project was one
@@ -151,24 +208,41 @@ def _eligible_for_upgrade(name):
             if self._upgrade_strategy == "eager":
                 return True
             elif self._upgrade_strategy == "only-if-needed":
-                return (name in self._user_requested)
+                user_order = _get_with_identifier(
+                    self._user_requested,
+                    identifier,
+                    default=None,
+                )
+                return user_order is not None
             return False
 
+        constraint = _get_with_identifier(
+            self._constraints,
+            identifier,
+            default=Constraint.empty(),
+        )
         return self._factory.find_candidates(
-            requirements,
-            constraint=self._constraints.get(name, Constraint.empty()),
-            prefers_installed=(not _eligible_for_upgrade(name)),
+            identifier=identifier,
+            requirements=requirements,
+            constraint=constraint,
+            prefers_installed=(not _eligible_for_upgrade(identifier)),
+            incompatibilities=incompatibilities,
         )
 
-    def is_satisfied_by(self, requirement, candidate):
-        # type: (Requirement, Candidate) -> bool
+    def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> bool:
         return requirement.is_satisfied_by(candidate)
 
-    def get_dependencies(self, candidate):
-        # type: (Candidate) -> Sequence[Requirement]
+    def get_dependencies(self, candidate: Candidate) -> Sequence[Requirement]:
         with_requires = not self._ignore_dependencies
-        return [
-            r
-            for r in candidate.iter_dependencies(with_requires)
-            if r is not None
-        ]
+        return [r for r in candidate.iter_dependencies(with_requires) if r is not None]
+
+    @staticmethod
+    def is_backtrack_cause(
+        identifier: str, backtrack_causes: Sequence["PreferenceInformation"]
+    ) -> bool:
+        for backtrack_cause in backtrack_causes:
+            if identifier == backtrack_cause.requirement.name:
+                return True
+            if backtrack_cause.parent and identifier == backtrack_cause.parent.name:
+                return True
+        return False
diff --git a/src/pip/_internal/resolution/resolvelib/reporter.py b/src/pip/_internal/resolution/resolvelib/reporter.py
index d0ef3fadc67..6ced5329b81 100644
--- a/src/pip/_internal/resolution/resolvelib/reporter.py
+++ b/src/pip/_internal/resolution/resolvelib/reporter.py
@@ -1,24 +1,17 @@
 from collections import defaultdict
 from logging import getLogger
+from typing import Any, DefaultDict
 
 from pip._vendor.resolvelib.reporters import BaseReporter
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, DefaultDict
-
-    from .base import Candidate, Requirement
-
+from .base import Candidate, Requirement
 
 logger = getLogger(__name__)
 
 
 class PipReporter(BaseReporter):
-
-    def __init__(self):
-        # type: () -> None
-        self.backtracks_by_package = defaultdict(int)  # type: DefaultDict[str, int]
+    def __init__(self) -> None:
+        self.backtracks_by_package: DefaultDict[str, int] = defaultdict(int)
 
         self._messages_at_backtrack = {
             1: (
@@ -34,14 +27,12 @@ def __init__(self):
             13: (
                 "This is taking longer than usual. You might need to provide "
                 "the dependency resolver with stricter constraints to reduce "
-                "runtime. If you want to abort this run, you can press "
-                "Ctrl + C to do so. To improve how pip performs, tell us what "
-                "happened here: https://pip.pypa.io/surveys/backtracking"
-            )
+                "runtime. See https://pip.pypa.io/warnings/backtracking for "
+                "guidance. If you want to abort this run, press Ctrl + C."
+            ),
         }
 
-    def backtracking(self, candidate):
-        # type: (Candidate) -> None
+    def backtracking(self, candidate: Candidate) -> None:
         self.backtracks_by_package[candidate.name] += 1
 
         count = self.backtracks_by_package[candidate.name]
@@ -55,30 +46,23 @@ def backtracking(self, candidate):
 class PipDebuggingReporter(BaseReporter):
     """A reporter that does an info log for every event it sees."""
 
-    def starting(self):
-        # type: () -> None
+    def starting(self) -> None:
         logger.info("Reporter.starting()")
 
-    def starting_round(self, index):
-        # type: (int) -> None
+    def starting_round(self, index: int) -> None:
         logger.info("Reporter.starting_round(%r)", index)
 
-    def ending_round(self, index, state):
-        # type: (int, Any) -> None
+    def ending_round(self, index: int, state: Any) -> None:
         logger.info("Reporter.ending_round(%r, state)", index)
 
-    def ending(self, state):
-        # type: (Any) -> None
+    def ending(self, state: Any) -> None:
         logger.info("Reporter.ending(%r)", state)
 
-    def adding_requirement(self, requirement, parent):
-        # type: (Requirement, Candidate) -> None
+    def adding_requirement(self, requirement: Requirement, parent: Candidate) -> None:
         logger.info("Reporter.adding_requirement(%r, %r)", requirement, parent)
 
-    def backtracking(self, candidate):
-        # type: (Candidate) -> None
+    def backtracking(self, candidate: Candidate) -> None:
         logger.info("Reporter.backtracking(%r)", candidate)
 
-    def pinning(self, candidate):
-        # type: (Candidate) -> None
+    def pinning(self, candidate: Candidate) -> None:
         logger.info("Reporter.pinning(%r)", candidate)
diff --git a/src/pip/_internal/resolution/resolvelib/requirements.py b/src/pip/_internal/resolution/resolvelib/requirements.py
index 61c81e00eee..f561f1f1e27 100644
--- a/src/pip/_internal/resolution/resolvelib/requirements.py
+++ b/src/pip/_internal/resolution/resolvelib/requirements.py
@@ -1,88 +1,69 @@
-from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.packaging.specifiers import SpecifierSet
+from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.req.req_install import InstallRequirement
 
-from .base import Requirement, format_name
-
-if MYPY_CHECK_RUNNING:
-    from pip._vendor.packaging.specifiers import SpecifierSet
-
-    from pip._internal.req.req_install import InstallRequirement
-
-    from .base import Candidate, CandidateLookup
+from .base import Candidate, CandidateLookup, Requirement, format_name
 
 
 class ExplicitRequirement(Requirement):
-    def __init__(self, candidate):
-        # type: (Candidate) -> None
+    def __init__(self, candidate: Candidate) -> None:
         self.candidate = candidate
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return str(self.candidate)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}({candidate!r})".format(
             class_name=self.__class__.__name__,
             candidate=self.candidate,
         )
 
     @property
-    def project_name(self):
-        # type: () -> str
-        # No need to canonicalise - the candidate did this
+    def project_name(self) -> NormalizedName:
+        # No need to canonicalize - the candidate did this
         return self.candidate.project_name
 
     @property
-    def name(self):
-        # type: () -> str
-        # No need to canonicalise - the candidate did this
+    def name(self) -> str:
+        # No need to canonicalize - the candidate did this
         return self.candidate.name
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return self.candidate.format_for_error()
 
-    def get_candidate_lookup(self):
-        # type: () -> CandidateLookup
+    def get_candidate_lookup(self) -> CandidateLookup:
         return self.candidate, None
 
-    def is_satisfied_by(self, candidate):
-        # type: (Candidate) -> bool
+    def is_satisfied_by(self, candidate: Candidate) -> bool:
         return candidate == self.candidate
 
 
 class SpecifierRequirement(Requirement):
-    def __init__(self, ireq):
-        # type: (InstallRequirement) -> None
+    def __init__(self, ireq: InstallRequirement) -> None:
         assert ireq.link is None, "This is a link, not a specifier"
         self._ireq = ireq
         self._extras = frozenset(ireq.extras)
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return str(self._ireq.req)
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}({requirement!r})".format(
             class_name=self.__class__.__name__,
             requirement=str(self._ireq.req),
         )
 
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
+        assert self._ireq.req, "Specifier-backed ireq is always PEP 508"
         return canonicalize_name(self._ireq.req.name)
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         return format_name(self.project_name, self._extras)
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
 
         # Convert comma-separated specifiers into "A, B, ..., F and G"
         # This makes the specifier a bit more "human readable", without
@@ -96,63 +77,55 @@ def format_for_error(self):
 
         return ", ".join(parts[:-1]) + " and " + parts[-1]
 
-    def get_candidate_lookup(self):
-        # type: () -> CandidateLookup
+    def get_candidate_lookup(self) -> CandidateLookup:
         return None, self._ireq
 
-    def is_satisfied_by(self, candidate):
-        # type: (Candidate) -> bool
-        assert candidate.name == self.name, \
-            "Internal issue: Candidate is not for this requirement " \
-            " {} vs {}".format(candidate.name, self.name)
+    def is_satisfied_by(self, candidate: Candidate) -> bool:
+        assert candidate.name == self.name, (
+            f"Internal issue: Candidate is not for this requirement "
+            f"{candidate.name} vs {self.name}"
+        )
         # We can safely always allow prereleases here since PackageFinder
         # already implements the prerelease logic, and would have filtered out
         # prerelease candidates if the user does not expect them.
+        assert self._ireq.req, "Specifier-backed ireq is always PEP 508"
         spec = self._ireq.req.specifier
         return spec.contains(candidate.version, prereleases=True)
 
 
 class RequiresPythonRequirement(Requirement):
-    """A requirement representing Requires-Python metadata.
-    """
-    def __init__(self, specifier, match):
-        # type: (SpecifierSet, Candidate) -> None
+    """A requirement representing Requires-Python metadata."""
+
+    def __init__(self, specifier: SpecifierSet, match: Candidate) -> None:
         self.specifier = specifier
         self._candidate = match
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return f"Python {self.specifier}"
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}({specifier!r})".format(
             class_name=self.__class__.__name__,
             specifier=str(self.specifier),
         )
 
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
         return self._candidate.project_name
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         return self._candidate.name
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return str(self)
 
-    def get_candidate_lookup(self):
-        # type: () -> CandidateLookup
+    def get_candidate_lookup(self) -> CandidateLookup:
         if self.specifier.contains(self._candidate.version, prereleases=True):
             return self._candidate, None
         return None, None
 
-    def is_satisfied_by(self, candidate):
-        # type: (Candidate) -> bool
+    def is_satisfied_by(self, candidate: Candidate) -> bool:
         assert candidate.name == self._candidate.name, "Not Python candidate"
         # We can safely always allow prereleases here since PackageFinder
         # already implements the prerelease logic, and would have filtered out
@@ -161,41 +134,33 @@ def is_satisfied_by(self, candidate):
 
 
 class UnsatisfiableRequirement(Requirement):
-    """A requirement that cannot be satisfied.
-    """
-    def __init__(self, name):
-        # type: (str) -> None
+    """A requirement that cannot be satisfied."""
+
+    def __init__(self, name: NormalizedName) -> None:
         self._name = name
 
-    def __str__(self):
-        # type: () -> str
-        return "{} (unavailable)".format(self._name)
+    def __str__(self) -> str:
+        return f"{self._name} (unavailable)"
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return "{class_name}({name!r})".format(
             class_name=self.__class__.__name__,
             name=str(self._name),
         )
 
     @property
-    def project_name(self):
-        # type: () -> str
+    def project_name(self) -> NormalizedName:
         return self._name
 
     @property
-    def name(self):
-        # type: () -> str
+    def name(self) -> str:
         return self._name
 
-    def format_for_error(self):
-        # type: () -> str
+    def format_for_error(self) -> str:
         return str(self)
 
-    def get_candidate_lookup(self):
-        # type: () -> CandidateLookup
+    def get_candidate_lookup(self) -> CandidateLookup:
         return None, None
 
-    def is_satisfied_by(self, candidate):
-        # type: (Candidate) -> bool
+    def is_satisfied_by(self, candidate: Candidate) -> bool:
         return False
diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py
index d30d696fc46..32ef7899ba6 100644
--- a/src/pip/_internal/resolution/resolvelib/resolver.py
+++ b/src/pip/_internal/resolution/resolvelib/resolver.py
@@ -1,40 +1,32 @@
 import functools
 import logging
 import os
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, cast
 
-from pip._vendor import six
 from pip._vendor.packaging.utils import canonicalize_name
-from pip._vendor.resolvelib import ResolutionImpossible
+from pip._vendor.resolvelib import BaseReporter, ResolutionImpossible
 from pip._vendor.resolvelib import Resolver as RLResolver
+from pip._vendor.resolvelib.structs import DirectedGraph
 
-from pip._internal.exceptions import InstallationError
-from pip._internal.req.req_install import check_invalid_constraint_type
+from pip._internal.cache import WheelCache
+from pip._internal.index.package_finder import PackageFinder
+from pip._internal.operations.prepare import RequirementPreparer
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.req.req_set import RequirementSet
-from pip._internal.resolution.base import BaseResolver
+from pip._internal.resolution.base import BaseResolver, InstallRequirementProvider
 from pip._internal.resolution.resolvelib.provider import PipProvider
 from pip._internal.resolution.resolvelib.reporter import (
     PipDebuggingReporter,
     PipReporter,
 )
-from pip._internal.utils.deprecation import deprecated
-from pip._internal.utils.filetypes import is_archive_file
-from pip._internal.utils.misc import dist_is_editable
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-from .base import Constraint
+from .base import Candidate, Requirement
 from .factory import Factory
 
-if MYPY_CHECK_RUNNING:
-    from typing import Dict, List, Optional, Set, Tuple
+if TYPE_CHECKING:
+    from pip._vendor.resolvelib.resolvers import Result as RLResult
 
-    from pip._vendor.resolvelib.resolvers import Result
-    from pip._vendor.resolvelib.structs import Graph
-
-    from pip._internal.cache import WheelCache
-    from pip._internal.index.package_finder import PackageFinder
-    from pip._internal.operations.prepare import RequirementPreparer
-    from pip._internal.req.req_install import InstallRequirement
-    from pip._internal.resolution.base import InstallRequirementProvider
+    Result = RLResult[Requirement, Candidate, str]
 
 
 logger = logging.getLogger(__name__)
@@ -45,17 +37,18 @@ class Resolver(BaseResolver):
 
     def __init__(
         self,
-        preparer,  # type: RequirementPreparer
-        finder,  # type: PackageFinder
-        wheel_cache,  # type: Optional[WheelCache]
-        make_install_req,  # type: InstallRequirementProvider
-        use_user_site,  # type: bool
-        ignore_dependencies,  # type: bool
-        ignore_installed,  # type: bool
-        ignore_requires_python,  # type: bool
-        force_reinstall,  # type: bool
-        upgrade_strategy,  # type: str
-        py_version_info=None,  # type: Optional[Tuple[int, ...]]
+        preparer: RequirementPreparer,
+        finder: PackageFinder,
+        wheel_cache: Optional[WheelCache],
+        make_install_req: InstallRequirementProvider,
+        use_user_site: bool,
+        ignore_dependencies: bool,
+        ignore_installed: bool,
+        ignore_requires_python: bool,
+        force_reinstall: bool,
+        upgrade_strategy: str,
+        suppress_build_failures: bool,
+        py_version_info: Optional[Tuple[int, ...]] = None,
     ):
         super().__init__()
         assert upgrade_strategy in self._allowed_strategies
@@ -69,65 +62,48 @@ def __init__(
             force_reinstall=force_reinstall,
             ignore_installed=ignore_installed,
             ignore_requires_python=ignore_requires_python,
+            suppress_build_failures=suppress_build_failures,
             py_version_info=py_version_info,
         )
         self.ignore_dependencies = ignore_dependencies
         self.upgrade_strategy = upgrade_strategy
-        self._result = None  # type: Optional[Result]
-
-    def resolve(self, root_reqs, check_supported_wheels):
-        # type: (List[InstallRequirement], bool) -> RequirementSet
-
-        constraints = {}  # type: Dict[str, Constraint]
-        user_requested = set()  # type: Set[str]
-        requirements = []
-        for req in root_reqs:
-            if req.constraint:
-                # Ensure we only accept valid constraints
-                problem = check_invalid_constraint_type(req)
-                if problem:
-                    raise InstallationError(problem)
-                if not req.match_markers():
-                    continue
-                name = canonicalize_name(req.name)
-                if name in constraints:
-                    constraints[name] &= req
-                else:
-                    constraints[name] = Constraint.from_ireq(req)
-            else:
-                if req.user_supplied and req.name:
-                    user_requested.add(canonicalize_name(req.name))
-                r = self.factory.make_requirement_from_install_req(
-                    req, requested_extras=(),
-                )
-                if r is not None:
-                    requirements.append(r)
+        self._result: Optional[Result] = None
 
+    def resolve(
+        self, root_reqs: List[InstallRequirement], check_supported_wheels: bool
+    ) -> RequirementSet:
+        collected = self.factory.collect_root_requirements(root_reqs)
         provider = PipProvider(
             factory=self.factory,
-            constraints=constraints,
+            constraints=collected.constraints,
             ignore_dependencies=self.ignore_dependencies,
             upgrade_strategy=self.upgrade_strategy,
-            user_requested=user_requested,
+            user_requested=collected.user_requested,
         )
         if "PIP_RESOLVER_DEBUG" in os.environ:
-            reporter = PipDebuggingReporter()
+            reporter: BaseReporter = PipDebuggingReporter()
         else:
             reporter = PipReporter()
-        resolver = RLResolver(provider, reporter)
+        resolver: RLResolver[Requirement, Candidate, str] = RLResolver(
+            provider,
+            reporter,
+        )
 
         try:
             try_to_avoid_resolution_too_deep = 2000000
-            self._result = resolver.resolve(
-                requirements, max_rounds=try_to_avoid_resolution_too_deep,
+            result = self._result = resolver.resolve(
+                collected.requirements, max_rounds=try_to_avoid_resolution_too_deep
             )
 
         except ResolutionImpossible as e:
-            error = self.factory.get_installation_error(e)
-            six.raise_from(error, e)
+            error = self.factory.get_installation_error(
+                cast("ResolutionImpossible[Requirement, Candidate]", e),
+                collected.constraints,
+            )
+            raise error from e
 
         req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
-        for candidate in self._result.mapping.values():
+        for candidate in result.mapping.values():
             ireq = candidate.get_install_requirement()
             if ireq is None:
                 continue
@@ -141,14 +117,14 @@ def resolve(self, root_reqs, check_supported_wheels):
             elif self.factory.force_reinstall:
                 # The --force-reinstall flag is set -- reinstall.
                 ireq.should_reinstall = True
-            elif installed_dist.parsed_version != candidate.version:
+            elif installed_dist.version != candidate.version:
                 # The installation is different in version -- reinstall.
                 ireq.should_reinstall = True
-            elif candidate.is_editable or dist_is_editable(installed_dist):
+            elif candidate.is_editable or installed_dist.editable:
                 # The incoming distribution is editable, or different in
                 # editable-ness to installation -- reinstall.
                 ireq.should_reinstall = True
-            elif candidate.source_link.is_file:
+            elif candidate.source_link and candidate.source_link.is_file:
                 # The incoming distribution is under file://
                 if candidate.source_link.is_wheel:
                     # is a local wheel -- do nothing.
@@ -160,25 +136,6 @@ def resolve(self, root_reqs, check_supported_wheels):
                     )
                     continue
 
-                looks_like_sdist = (
-                    is_archive_file(candidate.source_link.file_path)
-                    and candidate.source_link.ext != ".zip"
-                )
-                if looks_like_sdist:
-                    # is a local sdist -- show a deprecation warning!
-                    reason = (
-                        "Source distribution is being reinstalled despite an "
-                        "installed package having the same name and version as "
-                        "the installed package."
-                    )
-                    replacement = "use --force-reinstall"
-                    deprecated(
-                        reason=reason,
-                        replacement=replacement,
-                        gone_in="21.1",
-                        issue=8711,
-                    )
-
                 # is a local sdist or path -- reinstall
                 ireq.should_reinstall = True
             else:
@@ -189,14 +146,14 @@ def resolve(self, root_reqs, check_supported_wheels):
                 # The reason can contain non-ASCII characters, Unicode
                 # is required for Python 2.
                 msg = (
-                    'The candidate selected for download or install is a '
-                    'yanked version: {name!r} candidate (version {version} '
-                    'at {link})\nReason for being yanked: {reason}'
+                    "The candidate selected for download or install is a "
+                    "yanked version: {name!r} candidate (version {version} "
+                    "at {link})\nReason for being yanked: {reason}"
                 ).format(
                     name=candidate.name,
                     version=candidate.version,
                     link=link,
-                    reason=link.yanked_reason or '',
+                    reason=link.yanked_reason or "",
                 )
                 logger.warning(msg)
 
@@ -206,8 +163,9 @@ def resolve(self, root_reqs, check_supported_wheels):
         self.factory.preparer.prepare_linked_requirements_more(reqs)
         return req_set
 
-    def get_installation_order(self, req_set):
-        # type: (RequirementSet) -> List[InstallRequirement]
+    def get_installation_order(
+        self, req_set: RequirementSet
+    ) -> List[InstallRequirement]:
         """Get order for installation of requirements in RequirementSet.
 
         The returned list contains a requirement before another that depends on
@@ -215,17 +173,19 @@ def get_installation_order(self, req_set):
         get installed one-by-one.
 
         The current implementation creates a topological ordering of the
-        dependency graph, while breaking any cycles in the graph at arbitrary
-        points. We make no guarantees about where the cycle would be broken,
-        other than they would be broken.
+        dependency graph, giving more weight to packages with less
+        or no dependencies, while breaking any cycles in the graph at
+        arbitrary points. We make no guarantees about where the cycle
+        would be broken, other than it *would* be broken.
         """
         assert self._result is not None, "must call resolve() first"
 
+        if not req_set.requirements:
+            # Nothing is left to install, so we do not need an order.
+            return []
+
         graph = self._result.graph
-        weights = get_topological_weights(
-            graph,
-            expected_node_count=len(self._result.mapping) + 1,
-        )
+        weights = get_topological_weights(graph, set(req_set.requirements.keys()))
 
         sorted_items = sorted(
             req_set.requirements.items(),
@@ -235,29 +195,38 @@ def get_installation_order(self, req_set):
         return [ireq for _, ireq in sorted_items]
 
 
-def get_topological_weights(graph, expected_node_count):
-    # type: (Graph, int) -> Dict[Optional[str], int]
+def get_topological_weights(
+    graph: "DirectedGraph[Optional[str]]", requirement_keys: Set[str]
+) -> Dict[Optional[str], int]:
     """Assign weights to each node based on how "deep" they are.
 
     This implementation may change at any point in the future without prior
     notice.
 
-    We take the length for the longest path to any node from root, ignoring any
-    paths that contain a single node twice (i.e. cycles). This is done through
-    a depth-first search through the graph, while keeping track of the path to
-    the node.
+    We first simplify the dependency graph by pruning any leaves and giving them
+    the highest weight: a package without any dependencies should be installed
+    first. This is done again and again in the same way, giving ever less weight
+    to the newly found leaves. The loop stops when no leaves are left: all
+    remaining packages have at least one dependency left in the graph.
+
+    Then we continue with the remaining graph, by taking the length for the
+    longest path to any node from root, ignoring any paths that contain a single
+    node twice (i.e. cycles). This is done through a depth-first search through
+    the graph, while keeping track of the path to the node.
 
     Cycles in the graph result would result in node being revisited while also
-    being it's own path. In this case, take no action. This helps ensure we
+    being on its own path. In this case, take no action. This helps ensure we
     don't get stuck in a cycle.
 
     When assigning weight, the longer path (i.e. larger length) is preferred.
+
+    We are only interested in the weights of packages that are in the
+    requirement_keys.
     """
-    path = set()  # type: Set[Optional[str]]
-    weights = {}  # type: Dict[Optional[str], int]
+    path: Set[Optional[str]] = set()
+    weights: Dict[Optional[str], int] = {}
 
-    def visit(node):
-        # type: (Optional[str]) -> None
+    def visit(node: Optional[str]) -> None:
         if node in path:
             # We hit a cycle, so we'll break it here.
             return
@@ -268,24 +237,57 @@ def visit(node):
             visit(child)
         path.remove(node)
 
+        if node not in requirement_keys:
+            return
+
         last_known_parent_count = weights.get(node, 0)
         weights[node] = max(last_known_parent_count, len(path))
 
+    # Simplify the graph, pruning leaves that have no dependencies.
+    # This is needed for large graphs (say over 200 packages) because the
+    # `visit` function is exponentially slower then, taking minutes.
+    # See https://github.com/pypa/pip/issues/10557
+    # We will loop until we explicitly break the loop.
+    while True:
+        leaves = set()
+        for key in graph:
+            if key is None:
+                continue
+            for _child in graph.iter_children(key):
+                # This means we have at least one child
+                break
+            else:
+                # No child.
+                leaves.add(key)
+        if not leaves:
+            # We are done simplifying.
+            break
+        # Calculate the weight for the leaves.
+        weight = len(graph) - 1
+        for leaf in leaves:
+            if leaf not in requirement_keys:
+                continue
+            weights[leaf] = weight
+        # Remove the leaves from the graph, making it simpler.
+        for leaf in leaves:
+            graph.remove(leaf)
+
+    # Visit the remaining graph.
     # `None` is guaranteed to be the root node by resolvelib.
     visit(None)
 
-    # Sanity checks
-    assert weights[None] == 0
-    assert len(weights) == expected_node_count
+    # Sanity check: all requirement keys should be in the weights,
+    # and no other keys should be in the weights.
+    difference = set(weights.keys()).difference(requirement_keys)
+    assert not difference, difference
 
     return weights
 
 
 def _req_set_item_sorter(
-    item,     # type: Tuple[str, InstallRequirement]
-    weights,  # type: Dict[Optional[str], int]
-):
-    # type: (...) -> Tuple[int, str]
+    item: Tuple[str, InstallRequirement],
+    weights: Dict[Optional[str], int],
+) -> Tuple[int, str]:
     """Key function used to sort install requirements for installation.
 
     Based on the "weight" mapping calculated in ``get_installation_order()``.
diff --git a/src/pip/_internal/self_outdated_check.py b/src/pip/_internal/self_outdated_check.py
index c22f06afe87..7300e0ea4c0 100644
--- a/src/pip/_internal/self_outdated_check.py
+++ b/src/pip/_internal/self_outdated_check.py
@@ -2,26 +2,20 @@
 import hashlib
 import json
 import logging
+import optparse
 import os.path
 import sys
+from typing import Any, Dict
 
-from pip._vendor.packaging import version as packaging_version
-from pip._vendor.six import ensure_binary
+from pip._vendor.packaging.version import parse as parse_version
 
 from pip._internal.index.collector import LinkCollector
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import get_default_environment
 from pip._internal.models.selection_prefs import SelectionPreferences
+from pip._internal.network.session import PipSession
 from pip._internal.utils.filesystem import adjacent_tmp_file, check_path_owner, replace
-from pip._internal.utils.misc import ensure_dir, get_distribution, get_installed_version
-from pip._internal.utils.packaging import get_installer
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    import optparse
-    from typing import Any, Dict
-
-    from pip._internal.network.session import PipSession
-
+from pip._internal.utils.misc import ensure_dir
 
 SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ"
 
@@ -29,17 +23,15 @@
 logger = logging.getLogger(__name__)
 
 
-def _get_statefile_name(key):
-    # type: (str) -> str
-    key_bytes = ensure_binary(key)
+def _get_statefile_name(key: str) -> str:
+    key_bytes = key.encode()
     name = hashlib.sha224(key_bytes).hexdigest()
     return name
 
 
 class SelfCheckState:
-    def __init__(self, cache_dir):
-        # type: (str) -> None
-        self.state = {}  # type: Dict[str, Any]
+    def __init__(self, cache_dir: str) -> None:
+        self.state: Dict[str, Any] = {}
         self.statefile_path = None
 
         # Try to load the existing state
@@ -48,7 +40,7 @@ def __init__(self, cache_dir):
                 cache_dir, "selfcheck", _get_statefile_name(self.key)
             )
             try:
-                with open(self.statefile_path) as statefile:
+                with open(self.statefile_path, encoding="utf-8") as statefile:
                     self.state = json.load(statefile)
             except (OSError, ValueError, KeyError):
                 # Explicitly suppressing exceptions, since we don't want to
@@ -56,12 +48,10 @@ def __init__(self, cache_dir):
                 pass
 
     @property
-    def key(self):
-        # type: () -> str
+    def key(self) -> str:
         return sys.prefix
 
-    def save(self, pypi_version, current_time):
-        # type: (str, datetime.datetime) -> None
+    def save(self, pypi_version: str, current_time: datetime.datetime) -> None:
         # If we do not have a path to cache in, don't bother saving.
         if not self.statefile_path:
             return
@@ -85,7 +75,7 @@ def save(self, pypi_version, current_time):
         text = json.dumps(state, sort_keys=True, separators=(",", ":"))
 
         with adjacent_tmp_file(self.statefile_path) as f:
-            f.write(ensure_binary(text))
+            f.write(text.encode())
 
         try:
             # Since we have a prefix-specific state file, we can just
@@ -96,32 +86,28 @@ def save(self, pypi_version, current_time):
             pass
 
 
-def was_installed_by_pip(pkg):
-    # type: (str) -> bool
+def was_installed_by_pip(pkg: str) -> bool:
     """Checks whether pkg was installed by pip
 
     This is used not to display the upgrade message when pip is in fact
     installed by system package manager, such as dnf on Fedora.
     """
-    dist = get_distribution(pkg)
-    if not dist:
-        return False
-    return "pip" == get_installer(dist)
+    dist = get_default_environment().get_distribution(pkg)
+    return dist is not None and "pip" == dist.installer
 
 
-def pip_self_version_check(session, options):
-    # type: (PipSession, optparse.Values) -> None
+def pip_self_version_check(session: PipSession, options: optparse.Values) -> None:
     """Check for an update for pip.
 
     Limit the frequency of checks to once per week. State is stored either in
     the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix
     of the pip script path.
     """
-    installed_version = get_installed_version("pip")
-    if not installed_version:
+    installed_dist = get_default_environment().get_distribution("pip")
+    if not installed_dist:
         return
 
-    pip_version = packaging_version.parse(installed_version)
+    pip_version = installed_dist.version
     pypi_version = None
 
     try:
@@ -131,8 +117,7 @@ def pip_self_version_check(session, options):
         # Determine if we need to refresh the state
         if "last_check" in state.state and "pypi_version" in state.state:
             last_check = datetime.datetime.strptime(
-                state.state["last_check"],
-                SELFCHECK_DATE_FMT
+                state.state["last_check"], SELFCHECK_DATE_FMT
             )
             if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60:
                 pypi_version = state.state["pypi_version"]
@@ -156,6 +141,9 @@ def pip_self_version_check(session, options):
             finder = PackageFinder.create(
                 link_collector=link_collector,
                 selection_prefs=selection_prefs,
+                use_deprecated_html5lib=(
+                    "html5lib" in options.deprecated_features_enabled
+                ),
             )
             best_candidate = finder.find_best_candidate("pip").best_candidate
             if best_candidate is None:
@@ -165,12 +153,12 @@ def pip_self_version_check(session, options):
             # save that we've performed a check
             state.save(pypi_version, current_time)
 
-        remote_version = packaging_version.parse(pypi_version)
+        remote_version = parse_version(pypi_version)
 
         local_version_is_older = (
-            pip_version < remote_version and
-            pip_version.base_version != remote_version.base_version and
-            was_installed_by_pip('pip')
+            pip_version < remote_version
+            and pip_version.base_version != remote_version.base_version
+            and was_installed_by_pip("pip")
         )
 
         # Determine if our pypi_version is older
@@ -180,13 +168,19 @@ def pip_self_version_check(session, options):
         # We cannot tell how the current pip is available in the current
         # command context, so be pragmatic here and suggest the command
         # that's always available. This does not accommodate spaces in
-        # `sys.executable`.
+        # `sys.executable` on purpose as it is not possible to do it
+        # correctly without knowing the user's shell. Thus,
+        # it won't be done until possible through the standard library.
+        # Do not be tempted to use the undocumented subprocess.list2cmdline.
+        # It is considered an internal implementation detail for a reason.
         pip_cmd = f"{sys.executable} -m pip"
         logger.warning(
             "You are using pip version %s; however, version %s is "
             "available.\nYou should consider upgrading via the "
             "'%s install --upgrade pip' command.",
-            pip_version, pypi_version, pip_cmd
+            pip_version,
+            pypi_version,
+            pip_cmd,
         )
     except Exception:
         logger.debug(
diff --git a/src/pip/_internal/utils/_log.py b/src/pip/_internal/utils/_log.py
new file mode 100644
index 00000000000..92c4c6a1938
--- /dev/null
+++ b/src/pip/_internal/utils/_log.py
@@ -0,0 +1,38 @@
+"""Customize logging
+
+Defines custom logger class for the `logger.verbose(...)` method.
+
+init_logging() must be called before any other modules that call logging.getLogger.
+"""
+
+import logging
+from typing import Any, cast
+
+# custom log level for `--verbose` output
+# between DEBUG and INFO
+VERBOSE = 15
+
+
+class VerboseLogger(logging.Logger):
+    """Custom Logger, defining a verbose log-level
+
+    VERBOSE is between INFO and DEBUG.
+    """
+
+    def verbose(self, msg: str, *args: Any, **kwargs: Any) -> None:
+        return self.log(VERBOSE, msg, *args, **kwargs)
+
+
+def getLogger(name: str) -> VerboseLogger:
+    """logging.getLogger, but ensures our VerboseLogger class is returned"""
+    return cast(VerboseLogger, logging.getLogger(name))
+
+
+def init_logging() -> None:
+    """Register our VerboseLogger and VERBOSE log level.
+
+    Should be called before any calls to getLogger(),
+    i.e. in pip._internal.__init__
+    """
+    logging.setLoggerClass(VerboseLogger)
+    logging.addLevelName(VERBOSE, "VERBOSE")
diff --git a/src/pip/_internal/utils/appdirs.py b/src/pip/_internal/utils/appdirs.py
index a0a37be8723..16933bf8afe 100644
--- a/src/pip/_internal/utils/appdirs.py
+++ b/src/pip/_internal/utils/appdirs.py
@@ -7,36 +7,46 @@
 """
 
 import os
+import sys
+from typing import List
 
-from pip._vendor import appdirs as _appdirs
+from pip._vendor import platformdirs as _appdirs
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import List
+def user_cache_dir(appname: str) -> str:
+    return _appdirs.user_cache_dir(appname, appauthor=False)
 
 
-def user_cache_dir(appname):
-    # type: (str) -> str
-    return _appdirs.user_cache_dir(appname, appauthor=False)
+def _macos_user_config_dir(appname: str, roaming: bool = True) -> str:
+    # Use ~/Application Support/pip, if the directory exists.
+    path = _appdirs.user_data_dir(appname, appauthor=False, roaming=roaming)
+    if os.path.isdir(path):
+        return path
+
+    # Use a Linux-like ~/.config/pip, by default.
+    linux_like_path = "~/.config/"
+    if appname:
+        linux_like_path = os.path.join(linux_like_path, appname)
+
+    return os.path.expanduser(linux_like_path)
 
 
-def user_config_dir(appname, roaming=True):
-    # type: (str, bool) -> str
-    path = _appdirs.user_config_dir(appname, appauthor=False, roaming=roaming)
-    if _appdirs.system == "darwin" and not os.path.isdir(path):
-        path = os.path.expanduser('~/.config/')
-        if appname:
-            path = os.path.join(path, appname)
-    return path
+def user_config_dir(appname: str, roaming: bool = True) -> str:
+    if sys.platform == "darwin":
+        return _macos_user_config_dir(appname, roaming)
+
+    return _appdirs.user_config_dir(appname, appauthor=False, roaming=roaming)
 
 
 # for the discussion regarding site_config_dir locations
 # see 
-def site_config_dirs(appname):
-    # type: (str) -> List[str]
+def site_config_dirs(appname: str) -> List[str]:
+    if sys.platform == "darwin":
+        return [_appdirs.site_data_dir(appname, appauthor=False, multipath=True)]
+
     dirval = _appdirs.site_config_dir(appname, appauthor=False, multipath=True)
-    if _appdirs.system not in ["win32", "darwin"]:
-        # always look in /etc directly as well
-        return dirval.split(os.pathsep) + ['/etc']
-    return [dirval]
+    if sys.platform == "win32":
+        return [dirval]
+
+    # Unix-y system. Look in /etc as well.
+    return dirval.split(os.pathsep) + ["/etc"]
diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py
index 0ae0483c813..3f4d300cef0 100644
--- a/src/pip/_internal/utils/compat.py
+++ b/src/pip/_internal/utils/compat.py
@@ -1,115 +1,30 @@
 """Stuff that differs in different Python versions and platform
 distributions."""
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
-import codecs
-import locale
 import logging
 import os
 import sys
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, Union
-
-
-__all__ = ["console_to_str", "get_path_uid", "stdlib_pkgs", "WINDOWS"]
+__all__ = ["get_path_uid", "stdlib_pkgs", "WINDOWS"]
 
 
 logger = logging.getLogger(__name__)
 
 
-def has_tls():
-    # type: () -> bool
+def has_tls() -> bool:
     try:
         import _ssl  # noqa: F401  # ignore unused
+
         return True
     except ImportError:
         pass
 
     from pip._vendor.urllib3.util import IS_PYOPENSSL
-    return IS_PYOPENSSL
-
-
-def str_to_display(data, desc=None):
-    # type: (Union[bytes, str], Optional[str]) -> str
-    """
-    For display or logging purposes, convert a bytes object (or text) to
-    text (e.g. unicode in Python 2) safe for output.
-
-    :param desc: An optional phrase describing the input data, for use in
-        the log message if a warning is logged. Defaults to "Bytes object".
-
-    This function should never error out and so can take a best effort
-    approach. It is okay to be lossy if needed since the return value is
-    just for display.
-
-    We assume the data is in the locale preferred encoding. If it won't
-    decode properly, we warn the user but decode as best we can.
 
-    We also ensure that the output can be safely written to standard output
-    without encoding errors.
-    """
-    if isinstance(data, str):
-        return data
-
-    # Otherwise, data is a bytes object (str in Python 2).
-    # First, get the encoding we assume. This is the preferred
-    # encoding for the locale, unless that is not found, or
-    # it is ASCII, in which case assume UTF-8
-    encoding = locale.getpreferredencoding()
-    if (not encoding) or codecs.lookup(encoding).name == "ascii":
-        encoding = "utf-8"
-
-    # Now try to decode the data - if we fail, warn the user and
-    # decode with replacement.
-    try:
-        decoded_data = data.decode(encoding)
-    except UnicodeDecodeError:
-        logger.warning(
-            '%s does not appear to be encoded as %s',
-            desc or 'Bytes object',
-            encoding,
-        )
-        decoded_data = data.decode(encoding, errors="backslashreplace")
-
-    # Make sure we can print the output, by encoding it to the output
-    # encoding with replacement of unencodable characters, and then
-    # decoding again.
-    # We use stderr's encoding because it's less likely to be
-    # redirected and if we don't find an encoding we skip this
-    # step (on the assumption that output is wrapped by something
-    # that won't fail).
-    # The double getattr is to deal with the possibility that we're
-    # being called in a situation where sys.__stderr__ doesn't exist,
-    # or doesn't have an encoding attribute. Neither of these cases
-    # should occur in normal pip use, but there's no harm in checking
-    # in case people use pip in (unsupported) unusual situations.
-    output_encoding = getattr(getattr(sys, "__stderr__", None),
-                              "encoding", None)
-
-    if output_encoding:
-        output_encoded = decoded_data.encode(
-            output_encoding,
-            errors="backslashreplace"
-        )
-        decoded_data = output_encoded.decode(output_encoding)
-
-    return decoded_data
-
-
-def console_to_str(data):
-    # type: (bytes) -> str
-    """Return a string, safe for output, of subprocess output.
-    """
-    return str_to_display(data, desc='Subprocess output')
+    return IS_PYOPENSSL
 
 
-def get_path_uid(path):
-    # type: (str) -> int
+def get_path_uid(path: str) -> int:
     """
     Return path's uid.
 
@@ -121,7 +36,7 @@ def get_path_uid(path):
 
     :raises OSError: When path is a symlink or can't be read.
     """
-    if hasattr(os, 'O_NOFOLLOW'):
+    if hasattr(os, "O_NOFOLLOW"):
         fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
         file_uid = os.fstat(fd).st_uid
         os.close(fd)
@@ -132,10 +47,7 @@ def get_path_uid(path):
             file_uid = os.stat(path).st_uid
         else:
             # raise OSError for parity with os.O_NOFOLLOW above
-            raise OSError(
-                "{} is a symlink; Will not return uid for symlinks".format(
-                    path)
-            )
+            raise OSError(f"{path} is a symlink; Will not return uid for symlinks")
     return file_uid
 
 
@@ -148,5 +60,4 @@ def get_path_uid(path):
 
 
 # windows detection, covers cpython and ironpython
-WINDOWS = (sys.platform.startswith("win") or
-           (sys.platform == 'cli' and os.name == 'nt'))
+WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
diff --git a/src/pip/_internal/utils/compatibility_tags.py b/src/pip/_internal/utils/compatibility_tags.py
index ac37c3a17ba..b6ed9a78e55 100644
--- a/src/pip/_internal/utils/compatibility_tags.py
+++ b/src/pip/_internal/utils/compatibility_tags.py
@@ -2,8 +2,10 @@
 """
 
 import re
+from typing import List, Optional, Tuple
 
 from pip._vendor.packaging.tags import (
+    PythonVersion,
     Tag,
     compatible_tags,
     cpython_tags,
@@ -13,24 +15,15 @@
     mac_platforms,
 )
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+_osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)")
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional, Tuple
 
-    from pip._vendor.packaging.tags import PythonVersion
-
-_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)')
-
-
-def version_info_to_nodot(version_info):
-    # type: (Tuple[int, ...]) -> str
+def version_info_to_nodot(version_info: Tuple[int, ...]) -> str:
     # Only use up to the first two numbers.
-    return ''.join(map(str, version_info[:2]))
+    return "".join(map(str, version_info[:2]))
 
 
-def _mac_platforms(arch):
-    # type: (str) -> List[str]
+def _mac_platforms(arch: str) -> List[str]:
     match = _osx_arch_pat.match(arch)
     if match:
         name, major, minor, actual_arch = match.groups()
@@ -41,7 +34,7 @@ def _mac_platforms(arch):
             # actual prefix provided by the user in case they provided
             # something like "macosxcustom_". It may be good to remove
             # this as undocumented or deprecate it in the future.
-            '{}_{}'.format(name, arch[len('macosx_'):])
+            "{}_{}".format(name, arch[len("macosx_") :])
             for arch in mac_platforms(mac_version, actual_arch)
         ]
     else:
@@ -50,42 +43,39 @@ def _mac_platforms(arch):
     return arches
 
 
-def _custom_manylinux_platforms(arch):
-    # type: (str) -> List[str]
+def _custom_manylinux_platforms(arch: str) -> List[str]:
     arches = [arch]
-    arch_prefix, arch_sep, arch_suffix = arch.partition('_')
-    if arch_prefix == 'manylinux2014':
+    arch_prefix, arch_sep, arch_suffix = arch.partition("_")
+    if arch_prefix == "manylinux2014":
         # manylinux1/manylinux2010 wheels run on most manylinux2014 systems
         # with the exception of wheels depending on ncurses. PEP 599 states
         # manylinux1/manylinux2010 wheels should be considered
         # manylinux2014 wheels:
         # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels
-        if arch_suffix in {'i686', 'x86_64'}:
-            arches.append('manylinux2010' + arch_sep + arch_suffix)
-            arches.append('manylinux1' + arch_sep + arch_suffix)
-    elif arch_prefix == 'manylinux2010':
+        if arch_suffix in {"i686", "x86_64"}:
+            arches.append("manylinux2010" + arch_sep + arch_suffix)
+            arches.append("manylinux1" + arch_sep + arch_suffix)
+    elif arch_prefix == "manylinux2010":
         # manylinux1 wheels run on most manylinux2010 systems with the
         # exception of wheels depending on ncurses. PEP 571 states
         # manylinux1 wheels should be considered manylinux2010 wheels:
         # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels
-        arches.append('manylinux1' + arch_sep + arch_suffix)
+        arches.append("manylinux1" + arch_sep + arch_suffix)
     return arches
 
 
-def _get_custom_platforms(arch):
-    # type: (str) -> List[str]
-    arch_prefix, arch_sep, arch_suffix = arch.partition('_')
-    if arch.startswith('macosx'):
+def _get_custom_platforms(arch: str) -> List[str]:
+    arch_prefix, arch_sep, arch_suffix = arch.partition("_")
+    if arch.startswith("macosx"):
         arches = _mac_platforms(arch)
-    elif arch_prefix in ['manylinux2014', 'manylinux2010']:
+    elif arch_prefix in ["manylinux2014", "manylinux2010"]:
         arches = _custom_manylinux_platforms(arch)
     else:
         arches = [arch]
     return arches
 
 
-def _expand_allowed_platforms(platforms):
-    # type: (Optional[List[str]]) -> Optional[List[str]]
+def _expand_allowed_platforms(platforms: Optional[List[str]]) -> Optional[List[str]]:
     if not platforms:
         return None
 
@@ -102,16 +92,16 @@ def _expand_allowed_platforms(platforms):
     return result
 
 
-def _get_python_version(version):
-    # type: (str) -> PythonVersion
+def _get_python_version(version: str) -> PythonVersion:
     if len(version) > 1:
         return int(version[0]), int(version[1:])
     else:
         return (int(version[0]),)
 
 
-def _get_custom_interpreter(implementation=None, version=None):
-    # type: (Optional[str], Optional[str]) -> str
+def _get_custom_interpreter(
+    implementation: Optional[str] = None, version: Optional[str] = None
+) -> str:
     if implementation is None:
         implementation = interpreter_name()
     if version is None:
@@ -120,12 +110,11 @@ def _get_custom_interpreter(implementation=None, version=None):
 
 
 def get_supported(
-    version=None,  # type: Optional[str]
-    platforms=None,  # type: Optional[List[str]]
-    impl=None,  # type: Optional[str]
-    abis=None  # type: Optional[List[str]]
-):
-    # type: (...) -> List[Tag]
+    version: Optional[str] = None,
+    platforms: Optional[List[str]] = None,
+    impl: Optional[str] = None,
+    abis: Optional[List[str]] = None,
+) -> List[Tag]:
     """Return a list of supported tags for each version specified in
     `versions`.
 
@@ -138,9 +127,9 @@ def get_supported(
     :param abis: specify a list of abis you want valid
         tags for, or None. If None, use the local interpreter abi.
     """
-    supported = []  # type: List[Tag]
+    supported: List[Tag] = []
 
-    python_version = None  # type: Optional[PythonVersion]
+    python_version: Optional[PythonVersion] = None
     if version is not None:
         python_version = _get_python_version(version)
 
diff --git a/src/pip/_internal/utils/datetime.py b/src/pip/_internal/utils/datetime.py
index b638646c8bb..8668b3b0ec1 100644
--- a/src/pip/_internal/utils/datetime.py
+++ b/src/pip/_internal/utils/datetime.py
@@ -4,8 +4,7 @@
 import datetime
 
 
-def today_is_later_than(year, month, day):
-    # type: (int, int, int) -> bool
+def today_is_later_than(year: int, month: int, day: int) -> bool:
     today = datetime.date.today()
     given = datetime.date(year, month, day)
 
diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py
index 534d3fde86c..72bd6f25a55 100644
--- a/src/pip/_internal/utils/deprecation.py
+++ b/src/pip/_internal/utils/deprecation.py
@@ -2,20 +2,13 @@
 A module that implements tooling to enable easy warnings about deprecations.
 """
 
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
 import warnings
+from typing import Any, Optional, TextIO, Type, Union
 
 from pip._vendor.packaging.version import parse
 
-from pip import __version__ as current_version
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Optional
-
+from pip import __version__ as current_version  # NOTE: tests patch this name.
 
 DEPRECATION_MSG_PREFIX = "DEPRECATION: "
 
@@ -24,29 +17,31 @@ class PipDeprecationWarning(Warning):
     pass
 
 
-_original_showwarning = None  # type: Any
+_original_showwarning: Any = None
 
 
 # Warnings <-> Logging Integration
-def _showwarning(message, category, filename, lineno, file=None, line=None):
+def _showwarning(
+    message: Union[Warning, str],
+    category: Type[Warning],
+    filename: str,
+    lineno: int,
+    file: Optional[TextIO] = None,
+    line: Optional[str] = None,
+) -> None:
     if file is not None:
         if _original_showwarning is not None:
-            _original_showwarning(
-                message, category, filename, lineno, file, line,
-            )
+            _original_showwarning(message, category, filename, lineno, file, line)
     elif issubclass(category, PipDeprecationWarning):
         # We use a specially named logger which will handle all of the
         # deprecation messages for pip.
         logger = logging.getLogger("pip._internal.deprecations")
         logger.warning(message)
     else:
-        _original_showwarning(
-            message, category, filename, lineno, file, line,
-        )
+        _original_showwarning(message, category, filename, lineno, file, line)
 
 
-def install_warning_logger():
-    # type: () -> None
+def install_warning_logger() -> None:
     # Enable our Deprecation Warnings
     warnings.simplefilter("default", PipDeprecationWarning, append=True)
 
@@ -57,46 +52,69 @@ def install_warning_logger():
         warnings.showwarning = _showwarning
 
 
-def deprecated(reason, replacement, gone_in, issue=None):
-    # type: (str, Optional[str], Optional[str], Optional[int]) -> None
+def deprecated(
+    *,
+    reason: str,
+    replacement: Optional[str],
+    gone_in: Optional[str],
+    feature_flag: Optional[str] = None,
+    issue: Optional[int] = None,
+) -> None:
     """Helper to deprecate existing functionality.
 
     reason:
         Textual reason shown to the user about why this functionality has
-        been deprecated.
+        been deprecated. Should be a complete sentence.
     replacement:
         Textual suggestion shown to the user about what alternative
         functionality they can use.
     gone_in:
         The version of pip does this functionality should get removed in.
-        Raises errors if pip's current version is greater than or equal to
+        Raises an error if pip's current version is greater than or equal to
         this.
+    feature_flag:
+        Command-line flag of the form --use-feature={feature_flag} for testing
+        upcoming functionality.
     issue:
         Issue number on the tracker that would serve as a useful place for
         users to find related discussion and provide feedback.
-
-    Always pass replacement, gone_in and issue as keyword arguments for clarity
-    at the call site.
     """
 
-    # Construct a nice message.
-    #   This is eagerly formatted as we want it to get logged as if someone
-    #   typed this entire message out.
-    sentences = [
-        (reason, DEPRECATION_MSG_PREFIX + "{}"),
-        (gone_in, "pip {} will remove support for this functionality."),
-        (replacement, "A possible replacement is {}."),
-        (issue, (
-            "You can find discussion regarding this at "
-            "https://github.com/pypa/pip/issues/{}."
-        )),
+    # Determine whether or not the feature is already gone in this version.
+    is_gone = gone_in is not None and parse(current_version) >= parse(gone_in)
+
+    message_parts = [
+        (reason, f"{DEPRECATION_MSG_PREFIX}{{}}"),
+        (
+            gone_in,
+            "pip {} will enforce this behaviour change."
+            if not is_gone
+            else "Since pip {}, this is no longer supported.",
+        ),
+        (
+            replacement,
+            "A possible replacement is {}.",
+        ),
+        (
+            feature_flag,
+            "You can use the flag --use-feature={} to test the upcoming behaviour."
+            if not is_gone
+            else None,
+        ),
+        (
+            issue,
+            "Discussion can be found at https://github.com/pypa/pip/issues/{}",
+        ),
     ]
+
     message = " ".join(
-        template.format(val) for val, template in sentences if val is not None
+        format_str.format(value)
+        for value, format_str in message_parts
+        if format_str is not None and value is not None
     )
 
-    # Raise as an error if it has to be removed.
-    if gone_in is not None and parse(current_version) >= parse(gone_in):
+    # Raise as an error if this behaviour is deprecated.
+    if is_gone:
         raise PipDeprecationWarning(message)
 
     warnings.warn(message, category=PipDeprecationWarning, stacklevel=2)
diff --git a/src/pip/_internal/utils/direct_url_helpers.py b/src/pip/_internal/utils/direct_url_helpers.py
index 9598137aa06..0e8e5e1608b 100644
--- a/src/pip/_internal/utils/direct_url_helpers.py
+++ b/src/pip/_internal/utils/direct_url_helpers.py
@@ -1,29 +1,12 @@
-import json
-import logging
+from typing import Optional
 
-from pip._internal.models.direct_url import (
-    DIRECT_URL_METADATA_NAME,
-    ArchiveInfo,
-    DirectUrl,
-    DirectUrlValidationError,
-    DirInfo,
-    VcsInfo,
-)
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.models.direct_url import ArchiveInfo, DirectUrl, DirInfo, VcsInfo
+from pip._internal.models.link import Link
+from pip._internal.utils.urls import path_to_url
 from pip._internal.vcs import vcs
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional
 
-    from pip._vendor.pkg_resources import Distribution
-
-    from pip._internal.models.link import Link
-
-logger = logging.getLogger(__name__)
-
-
-def direct_url_as_pep440_direct_reference(direct_url, name):
-    # type: (DirectUrl, str) -> str
+def direct_url_as_pep440_direct_reference(direct_url: DirectUrl, name: str) -> str:
     """Convert a DirectUrl to a pip requirement string."""
     direct_url.validate()  # if invalid, this is a pip bug
     requirement = name + " @ "
@@ -46,13 +29,21 @@ def direct_url_as_pep440_direct_reference(direct_url, name):
     return requirement
 
 
-def direct_url_from_link(link, source_dir=None, link_is_in_wheel_cache=False):
-    # type: (Link, Optional[str], bool) -> DirectUrl
+def direct_url_for_editable(source_dir: str) -> DirectUrl:
+    return DirectUrl(
+        url=path_to_url(source_dir),
+        info=DirInfo(editable=True),
+    )
+
+
+def direct_url_from_link(
+    link: Link, source_dir: Optional[str] = None, link_is_in_wheel_cache: bool = False
+) -> DirectUrl:
     if link.is_vcs:
         vcs_backend = vcs.get_backend_for_scheme(link.scheme)
         assert vcs_backend
-        url, requested_revision, _ = (
-            vcs_backend.get_url_rev_and_auth(link.url_without_fragment)
+        url, requested_revision, _ = vcs_backend.get_url_rev_and_auth(
+            link.url_without_fragment
         )
         # For VCS links, we need to find out and add commit_id.
         if link_is_in_wheel_cache:
@@ -94,28 +85,3 @@ def direct_url_from_link(link, source_dir=None, link_is_in_wheel_cache=False):
             info=ArchiveInfo(hash=hash),
             subdirectory=link.subdirectory_fragment,
         )
-
-
-def dist_get_direct_url(dist):
-    # type: (Distribution) -> Optional[DirectUrl]
-    """Obtain a DirectUrl from a pkg_resource.Distribution.
-
-    Returns None if the distribution has no `direct_url.json` metadata,
-    or if `direct_url.json` is invalid.
-    """
-    if not dist.has_metadata(DIRECT_URL_METADATA_NAME):
-        return None
-    try:
-        return DirectUrl.from_json(dist.get_metadata(DIRECT_URL_METADATA_NAME))
-    except (
-        DirectUrlValidationError,
-        json.JSONDecodeError,
-        UnicodeDecodeError
-    ) as e:
-        logger.warning(
-            "Error parsing %s for %s: %s",
-            DIRECT_URL_METADATA_NAME,
-            dist.project_name,
-            e,
-        )
-        return None
diff --git a/src/pip/_internal/utils/distutils_args.py b/src/pip/_internal/utils/distutils_args.py
index e38e402d733..e4aa5b827f6 100644
--- a/src/pip/_internal/utils/distutils_args.py
+++ b/src/pip/_internal/utils/distutils_args.py
@@ -1,11 +1,6 @@
 from distutils.errors import DistutilsArgError
 from distutils.fancy_getopt import FancyGetopt
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Dict, List
-
+from typing import Dict, List
 
 _options = [
     ("exec-prefix=", None, ""),
@@ -27,8 +22,7 @@
 _distutils_getopt = FancyGetopt(_options)  # type: ignore
 
 
-def parse_distutils_args(args):
-    # type: (List[str]) -> Dict[str, str]
+def parse_distutils_args(args: List[str]) -> Dict[str, str]:
     """Parse provided arguments, returning an object that has the
     matched arguments.
 
diff --git a/src/pip/_internal/utils/egg_link.py b/src/pip/_internal/utils/egg_link.py
new file mode 100644
index 00000000000..9e0da8d2d29
--- /dev/null
+++ b/src/pip/_internal/utils/egg_link.py
@@ -0,0 +1,75 @@
+# The following comment should be removed at some point in the future.
+# mypy: strict-optional=False
+
+import os
+import re
+import sys
+from typing import Optional
+
+from pip._internal.locations import site_packages, user_site
+from pip._internal.utils.virtualenv import (
+    running_under_virtualenv,
+    virtualenv_no_global,
+)
+
+__all__ = [
+    "egg_link_path_from_sys_path",
+    "egg_link_path_from_location",
+]
+
+
+def _egg_link_name(raw_name: str) -> str:
+    """
+    Convert a Name metadata value to a .egg-link name, by applying
+    the same substitution as pkg_resources's safe_name function.
+    Note: we cannot use canonicalize_name because it has a different logic.
+    """
+    return re.sub("[^A-Za-z0-9.]+", "-", raw_name) + ".egg-link"
+
+
+def egg_link_path_from_sys_path(raw_name: str) -> Optional[str]:
+    """
+    Look for a .egg-link file for project name, by walking sys.path.
+    """
+    egg_link_name = _egg_link_name(raw_name)
+    for path_item in sys.path:
+        egg_link = os.path.join(path_item, egg_link_name)
+        if os.path.isfile(egg_link):
+            return egg_link
+    return None
+
+
+def egg_link_path_from_location(raw_name: str) -> Optional[str]:
+    """
+    Return the path for the .egg-link file if it exists, otherwise, None.
+
+    There's 3 scenarios:
+    1) not in a virtualenv
+       try to find in site.USER_SITE, then site_packages
+    2) in a no-global virtualenv
+       try to find in site_packages
+    3) in a yes-global virtualenv
+       try to find in site_packages, then site.USER_SITE
+       (don't look in global location)
+
+    For #1 and #3, there could be odd cases, where there's an egg-link in 2
+    locations.
+
+    This method will just return the first one found.
+    """
+    sites = []
+    if running_under_virtualenv():
+        sites.append(site_packages)
+        if not virtualenv_no_global() and user_site:
+            sites.append(user_site)
+    else:
+        if user_site:
+            sites.append(user_site)
+        sites.append(site_packages)
+
+    egg_link_name = _egg_link_name(raw_name)
+    for site in sites:
+        egglink = os.path.join(site, egg_link_name)
+        if os.path.isfile(egglink):
+            return egglink
+    return None
diff --git a/src/pip/_internal/utils/encoding.py b/src/pip/_internal/utils/encoding.py
index 7df67987842..1c73f6c9a5d 100644
--- a/src/pip/_internal/utils/encoding.py
+++ b/src/pip/_internal/utils/encoding.py
@@ -2,39 +2,34 @@
 import locale
 import re
 import sys
+from typing import List, Tuple
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+BOMS: List[Tuple[bytes, str]] = [
+    (codecs.BOM_UTF8, "utf-8"),
+    (codecs.BOM_UTF16, "utf-16"),
+    (codecs.BOM_UTF16_BE, "utf-16-be"),
+    (codecs.BOM_UTF16_LE, "utf-16-le"),
+    (codecs.BOM_UTF32, "utf-32"),
+    (codecs.BOM_UTF32_BE, "utf-32-be"),
+    (codecs.BOM_UTF32_LE, "utf-32-le"),
+]
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Tuple
+ENCODING_RE = re.compile(br"coding[:=]\s*([-\w.]+)")
 
-BOMS = [
-    (codecs.BOM_UTF8, 'utf-8'),
-    (codecs.BOM_UTF16, 'utf-16'),
-    (codecs.BOM_UTF16_BE, 'utf-16-be'),
-    (codecs.BOM_UTF16_LE, 'utf-16-le'),
-    (codecs.BOM_UTF32, 'utf-32'),
-    (codecs.BOM_UTF32_BE, 'utf-32-be'),
-    (codecs.BOM_UTF32_LE, 'utf-32-le'),
-]  # type: List[Tuple[bytes, str]]
 
-ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)')
-
-
-def auto_decode(data):
-    # type: (bytes) -> str
+def auto_decode(data: bytes) -> str:
     """Check a bytes string for a BOM to correctly detect the encoding
 
     Fallback to locale.getpreferredencoding(False) like open() on Python3"""
     for bom, encoding in BOMS:
         if data.startswith(bom):
-            return data[len(bom):].decode(encoding)
+            return data[len(bom) :].decode(encoding)
     # Lets check the first two lines as in PEP263
-    for line in data.split(b'\n')[:2]:
-        if line[0:1] == b'#' and ENCODING_RE.search(line):
+    for line in data.split(b"\n")[:2]:
+        if line[0:1] == b"#" and ENCODING_RE.search(line):
             result = ENCODING_RE.search(line)
             assert result is not None
-            encoding = result.groups()[0].decode('ascii')
+            encoding = result.groups()[0].decode("ascii")
             return data.decode(encoding)
     return data.decode(
         locale.getpreferredencoding(False) or sys.getdefaultencoding(),
diff --git a/src/pip/_internal/utils/entrypoints.py b/src/pip/_internal/utils/entrypoints.py
index 64d1cb2bd0b..1504a12916b 100644
--- a/src/pip/_internal/utils/entrypoints.py
+++ b/src/pip/_internal/utils/entrypoints.py
@@ -1,14 +1,10 @@
 import sys
+from typing import List, Optional
 
 from pip._internal.cli.main import main
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
 
-
-def _wrapper(args=None):
-    # type: (Optional[List[str]]) -> int
+def _wrapper(args: Optional[List[str]] = None) -> int:
     """Central wrapper for all old entrypoints.
 
     Historically pip has had several entrypoints defined. Because of issues
diff --git a/src/pip/_internal/utils/filesystem.py b/src/pip/_internal/utils/filesystem.py
index 1af8c10eaaa..b7e6191abe6 100644
--- a/src/pip/_internal/utils/filesystem.py
+++ b/src/pip/_internal/utils/filesystem.py
@@ -7,21 +7,15 @@
 import sys
 from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
+from typing import Any, BinaryIO, Iterator, List, Union, cast
 
-# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
-#       why we ignore the type on this import.
-from pip._vendor.retrying import retry  # type: ignore
+from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
 
 from pip._internal.utils.compat import get_path_uid
 from pip._internal.utils.misc import format_size
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, BinaryIO, Iterator, List, Union
 
-
-def check_path_owner(path):
-    # type: (str) -> bool
+def check_path_owner(path: str) -> bool:
     # If we don't have a way to check the effective uid of this process, then
     # we'll just assume that we own the directory.
     if sys.platform == "win32" or not hasattr(os, "geteuid"):
@@ -48,8 +42,7 @@ def check_path_owner(path):
     return False  # assume we don't own the path
 
 
-def copy2_fixed(src, dest):
-    # type: (str, str) -> None
+def copy2_fixed(src: str, dest: str) -> None:
     """Wrap shutil.copy2() but map errors copying socket files to
     SpecialFileError as expected.
 
@@ -67,20 +60,17 @@ def copy2_fixed(src, dest):
                 pass
             else:
                 if is_socket_file:
-                    raise shutil.SpecialFileError(
-                        "`{f}` is a socket".format(**locals()))
+                    raise shutil.SpecialFileError(f"`{f}` is a socket")
 
         raise
 
 
-def is_socket(path):
-    # type: (str) -> bool
+def is_socket(path: str) -> bool:
     return stat.S_ISSOCK(os.lstat(path).st_mode)
 
 
 @contextmanager
-def adjacent_tmp_file(path, **kwargs):
-    # type: (str, **Any) -> Iterator[BinaryIO]
+def adjacent_tmp_file(path: str, **kwargs: Any) -> Iterator[BinaryIO]:
     """Return a file-like object pointing to a tmp file next to path.
 
     The file is created securely and is ensured to be written to disk
@@ -93,10 +83,10 @@ def adjacent_tmp_file(path, **kwargs):
         delete=False,
         dir=os.path.dirname(path),
         prefix=os.path.basename(path),
-        suffix='.tmp',
-        **kwargs
+        suffix=".tmp",
+        **kwargs,
     ) as f:
-        result = cast('BinaryIO', f)
+        result = cast(BinaryIO, f)
         try:
             yield result
         finally:
@@ -104,15 +94,15 @@ def adjacent_tmp_file(path, **kwargs):
             os.fsync(result.fileno())
 
 
-_replace_retry = retry(stop_max_delay=1000, wait_fixed=250)
+# Tenacity raises RetryError by default, explicitly raise the original exception
+_replace_retry = retry(reraise=True, stop=stop_after_delay(1), wait=wait_fixed(0.25))
 
 replace = _replace_retry(os.replace)
 
 
 # test_writable_dir and _test_writable_dir_win are copied from Flit,
 # with the author's agreement to also place them under pip's license.
-def test_writable_dir(path):
-    # type: (str) -> bool
+def test_writable_dir(path: str) -> bool:
     """Check if a directory is writable.
 
     Uses os.access() on POSIX, tries creating files on Windows.
@@ -124,20 +114,19 @@ def test_writable_dir(path):
             break  # Should never get here, but infinite loops are bad
         path = parent
 
-    if os.name == 'posix':
+    if os.name == "posix":
         return os.access(path, os.W_OK)
 
     return _test_writable_dir_win(path)
 
 
-def _test_writable_dir_win(path):
-    # type: (str) -> bool
+def _test_writable_dir_win(path: str) -> bool:
     # os.access doesn't work on Windows: http://bugs.python.org/issue2528
     # and we can't use tempfile: http://bugs.python.org/issue22107
-    basename = 'accesstest_deleteme_fishfingers_custard_'
-    alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789'
+    basename = "accesstest_deleteme_fishfingers_custard_"
+    alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
     for _ in range(10):
-        name = basename + ''.join(random.choice(alphabet) for _ in range(6))
+        name = basename + "".join(random.choice(alphabet) for _ in range(6))
         file = os.path.join(path, name)
         try:
             fd = os.open(file, os.O_RDWR | os.O_CREAT | os.O_EXCL)
@@ -156,37 +145,31 @@ def _test_writable_dir_win(path):
             return True
 
     # This should never be reached
-    raise OSError(
-        'Unexpected condition testing for writable directory'
-    )
+    raise OSError("Unexpected condition testing for writable directory")
 
 
-def find_files(path, pattern):
-    # type: (str, str) -> List[str]
+def find_files(path: str, pattern: str) -> List[str]:
     """Returns a list of absolute paths of files beneath path, recursively,
     with filenames which match the UNIX-style shell glob pattern."""
-    result = []  # type: List[str]
+    result: List[str] = []
     for root, _, files in os.walk(path):
         matches = fnmatch.filter(files, pattern)
         result.extend(os.path.join(root, f) for f in matches)
     return result
 
 
-def file_size(path):
-    # type: (str) -> Union[int, float]
+def file_size(path: str) -> Union[int, float]:
     # If it's a symlink, return 0.
     if os.path.islink(path):
         return 0
     return os.path.getsize(path)
 
 
-def format_file_size(path):
-    # type: (str) -> str
+def format_file_size(path: str) -> str:
     return format_size(file_size(path))
 
 
-def directory_size(path):
-    # type: (str) -> Union[int, float]
+def directory_size(path: str) -> Union[int, float]:
     size = 0.0
     for root, _dirs, files in os.walk(path):
         for filename in files:
@@ -195,6 +178,5 @@ def directory_size(path):
     return size
 
 
-def format_directory_size(path):
-    # type: (str) -> str
+def format_directory_size(path: str) -> str:
     return format_size(directory_size(path))
diff --git a/src/pip/_internal/utils/filetypes.py b/src/pip/_internal/utils/filetypes.py
index 201c6ebbed8..5948570178f 100644
--- a/src/pip/_internal/utils/filetypes.py
+++ b/src/pip/_internal/utils/filetypes.py
@@ -1,24 +1,25 @@
 """Filetype information.
 """
-from pip._internal.utils.misc import splitext
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import Tuple
+from typing import Tuple
+
+from pip._internal.utils.misc import splitext
 
-WHEEL_EXTENSION = '.whl'
-BZ2_EXTENSIONS = ('.tar.bz2', '.tbz')  # type: Tuple[str, ...]
-XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz',
-                 '.tar.lz', '.tar.lzma')  # type: Tuple[str, ...]
-ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION)  # type: Tuple[str, ...]
-TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar')  # type: Tuple[str, ...]
-ARCHIVE_EXTENSIONS = (
-    ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS
+WHEEL_EXTENSION = ".whl"
+BZ2_EXTENSIONS: Tuple[str, ...] = (".tar.bz2", ".tbz")
+XZ_EXTENSIONS: Tuple[str, ...] = (
+    ".tar.xz",
+    ".txz",
+    ".tlz",
+    ".tar.lz",
+    ".tar.lzma",
 )
+ZIP_EXTENSIONS: Tuple[str, ...] = (".zip", WHEEL_EXTENSION)
+TAR_EXTENSIONS: Tuple[str, ...] = (".tar.gz", ".tgz", ".tar")
+ARCHIVE_EXTENSIONS = ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS
 
 
-def is_archive_file(name):
-    # type: (str) -> bool
+def is_archive_file(name: str) -> bool:
     """Return True if `name` is a considered as an archive file."""
     ext = splitext(name)[1].lower()
     if ext in ARCHIVE_EXTENSIONS:
diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py
index 819979d8001..7bd3c20681d 100644
--- a/src/pip/_internal/utils/glibc.py
+++ b/src/pip/_internal/utils/glibc.py
@@ -3,21 +3,15 @@
 
 import os
 import sys
+from typing import Optional, Tuple
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, Tuple
-
-
-def glibc_version_string():
-    # type: () -> Optional[str]
+def glibc_version_string() -> Optional[str]:
     "Returns glibc version string, or None if not using glibc."
     return glibc_version_string_confstr() or glibc_version_string_ctypes()
 
 
-def glibc_version_string_confstr():
-    # type: () -> Optional[str]
+def glibc_version_string_confstr() -> Optional[str]:
     "Primary implementation of glibc_version_string using os.confstr."
     # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely
     # to be broken or missing. This strategy is used in the standard library
@@ -34,8 +28,7 @@ def glibc_version_string_confstr():
     return version
 
 
-def glibc_version_string_ctypes():
-    # type: () -> Optional[str]
+def glibc_version_string_ctypes() -> Optional[str]:
     "Fallback implementation of glibc_version_string using ctypes."
 
     try:
@@ -82,8 +75,7 @@ def glibc_version_string_ctypes():
 # versions that was generated by pip 8.1.2 and earlier is useless and
 # misleading. Solution: instead of using platform, use our code that actually
 # works.
-def libc_ver():
-    # type: () -> Tuple[str, str]
+def libc_ver() -> Tuple[str, str]:
     """Try to determine the glibc version
 
     Returns a tuple of strings (lib, version) which default to empty strings
diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py
index d5ff90063c5..2f89bbf22f5 100644
--- a/src/pip/_internal/utils/hashes.py
+++ b/src/pip/_internal/utils/hashes.py
@@ -1,22 +1,25 @@
 import hashlib
+from typing import TYPE_CHECKING, BinaryIO, Dict, Iterator, List
 
 from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError
 from pip._internal.utils.misc import read_chunks
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
-if MYPY_CHECK_RUNNING:
+if TYPE_CHECKING:
     from hashlib import _Hash
-    from typing import BinaryIO, Dict, Iterator, List, NoReturn
+
+    # NoReturn introduced in 3.6.2; imported only for type checking to maintain
+    # pip compatibility with older patch versions of Python 3.6
+    from typing import NoReturn
 
 
 # The recommended hash algo of the moment. Change this whenever the state of
 # the art changes; it won't hurt backward compatibility.
-FAVORITE_HASH = 'sha256'
+FAVORITE_HASH = "sha256"
 
 
 # Names of hashlib algorithms allowed by the --hash option and ``pip hash``
 # Currently, those are the ones at least as collision-resistant as sha256.
-STRONG_HASHES = ['sha256', 'sha384', 'sha512']
+STRONG_HASHES = ["sha256", "sha384", "sha512"]
 
 
 class Hashes:
@@ -24,8 +27,8 @@ class Hashes:
     known-good values
 
     """
-    def __init__(self, hashes=None):
-        # type: (Dict[str, List[str]]) -> None
+
+    def __init__(self, hashes: Dict[str, List[str]] = None) -> None:
         """
         :param hashes: A dict of algorithm names pointing to lists of allowed
             hex digests
@@ -37,8 +40,7 @@ def __init__(self, hashes=None):
                 allowed[alg] = sorted(keys)
         self._allowed = allowed
 
-    def __and__(self, other):
-        # type: (Hashes) -> Hashes
+    def __and__(self, other: "Hashes") -> "Hashes":
         if not isinstance(other, Hashes):
             return NotImplemented
 
@@ -58,21 +60,18 @@ def __and__(self, other):
         return Hashes(new)
 
     @property
-    def digest_count(self):
-        # type: () -> int
+    def digest_count(self) -> int:
         return sum(len(digests) for digests in self._allowed.values())
 
-    def is_hash_allowed(
-        self,
-        hash_name,   # type: str
-        hex_digest,  # type: str
-    ):
-        # type: (...) -> bool
+    @property
+    def allowed(self) -> Dict[str, List[str]]:
+        return self._allowed
+
+    def is_hash_allowed(self, hash_name: str, hex_digest: str) -> bool:
         """Return whether the given hex digest is allowed."""
         return hex_digest in self._allowed.get(hash_name, [])
 
-    def check_against_chunks(self, chunks):
-        # type: (Iterator[bytes]) -> None
+    def check_against_chunks(self, chunks: Iterator[bytes]) -> None:
         """Check good hashes against ones built from iterable of chunks of
         data.
 
@@ -84,9 +83,7 @@ def check_against_chunks(self, chunks):
             try:
                 gots[hash_name] = hashlib.new(hash_name)
             except (ValueError, TypeError):
-                raise InstallationError(
-                    f'Unknown hash name: {hash_name}'
-                )
+                raise InstallationError(f"Unknown hash name: {hash_name}")
 
         for chunk in chunks:
             for hash in gots.values():
@@ -97,12 +94,10 @@ def check_against_chunks(self, chunks):
                 return
         self._raise(gots)
 
-    def _raise(self, gots):
-        # type: (Dict[str, _Hash]) -> NoReturn
+    def _raise(self, gots: Dict[str, "_Hash"]) -> "NoReturn":
         raise HashMismatch(self._allowed, gots)
 
-    def check_against_file(self, file):
-        # type: (BinaryIO) -> None
+    def check_against_file(self, file: BinaryIO) -> None:
         """Check good hashes against a file-like object
 
         Raise HashMismatch if none match.
@@ -110,34 +105,28 @@ def check_against_file(self, file):
         """
         return self.check_against_chunks(read_chunks(file))
 
-    def check_against_path(self, path):
-        # type: (str) -> None
-        with open(path, 'rb') as file:
+    def check_against_path(self, path: str) -> None:
+        with open(path, "rb") as file:
             return self.check_against_file(file)
 
-    def __nonzero__(self):
-        # type: () -> bool
+    def __bool__(self) -> bool:
         """Return whether I know any known-good hashes."""
         return bool(self._allowed)
 
-    def __bool__(self):
-        # type: () -> bool
-        return self.__nonzero__()
-
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, Hashes):
             return NotImplemented
         return self._allowed == other._allowed
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(
-            ",".join(sorted(
-                ":".join((alg, digest))
-                for alg, digest_list in self._allowed.items()
-                for digest in digest_list
-            ))
+            ",".join(
+                sorted(
+                    ":".join((alg, digest))
+                    for alg, digest_list in self._allowed.items()
+                    for digest in digest_list
+                )
+            )
         )
 
 
@@ -148,13 +137,12 @@ class MissingHashes(Hashes):
     exception showing it to the user.
 
     """
-    def __init__(self):
-        # type: () -> None
+
+    def __init__(self) -> None:
         """Don't offer the ``hashes`` kwarg."""
         # Pass our favorite hash in to generate a "gotten hash". With the
         # empty list, it will never match, so an error will always raise.
         super().__init__(hashes={FAVORITE_HASH: []})
 
-    def _raise(self, gots):
-        # type: (Dict[str, _Hash]) -> NoReturn
+    def _raise(self, gots: Dict[str, "_Hash"]) -> "NoReturn":
         raise HashMissing(gots[FAVORITE_HASH].hexdigest())
diff --git a/src/pip/_internal/utils/inject_securetransport.py b/src/pip/_internal/utils/inject_securetransport.py
index 5b93b1d6730..276aa79bb81 100644
--- a/src/pip/_internal/utils/inject_securetransport.py
+++ b/src/pip/_internal/utils/inject_securetransport.py
@@ -10,8 +10,7 @@
 import sys
 
 
-def inject_securetransport():
-    # type: () -> None
+def inject_securetransport() -> None:
     # Only relevant on macOS
     if sys.platform != "darwin":
         return
@@ -22,7 +21,7 @@ def inject_securetransport():
         return
 
     # Checks for OpenSSL 1.0.1
-    if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100f:
+    if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100F:
         return
 
     try:
diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py
index 87b91d23d26..6e001c5d63c 100644
--- a/src/pip/_internal/utils/logging.py
+++ b/src/pip/_internal/utils/logging.py
@@ -1,68 +1,56 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import contextlib
 import errno
 import logging
 import logging.handlers
 import os
 import sys
-from logging import Filter, getLogger
-
+import threading
+from dataclasses import dataclass
+from logging import Filter
+from typing import IO, Any, ClassVar, Iterator, List, Optional, TextIO, Type
+
+from pip._vendor.rich.console import (
+    Console,
+    ConsoleOptions,
+    ConsoleRenderable,
+    RenderResult,
+)
+from pip._vendor.rich.highlighter import NullHighlighter
+from pip._vendor.rich.logging import RichHandler
+from pip._vendor.rich.segment import Segment
+from pip._vendor.rich.style import Style
+
+from pip._internal.exceptions import DiagnosticPipError
+from pip._internal.utils._log import VERBOSE, getLogger
 from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
 from pip._internal.utils.misc import ensure_dir
 
-try:
-    import threading
-except ImportError:
-    import dummy_threading as threading  # type: ignore
-
-
-try:
-    from pip._vendor import colorama
-# Lots of different errors can come from this, including SystemError and
-# ImportError.
-except Exception:
-    colorama = None
-
-
 _log_state = threading.local()
-subprocess_logger = getLogger('pip.subprocessor')
+subprocess_logger = getLogger("pip.subprocessor")
 
 
 class BrokenStdoutLoggingError(Exception):
     """
     Raised if BrokenPipeError occurs for the stdout stream while logging.
     """
-    pass
 
 
-# BrokenPipeError manifests differently in Windows and non-Windows.
-if WINDOWS:
-    # In Windows, a broken pipe can show up as EINVAL rather than EPIPE:
+def _is_broken_pipe_error(exc_class: Type[BaseException], exc: BaseException) -> bool:
+    if exc_class is BrokenPipeError:
+        return True
+
+    # On Windows, a broken pipe can show up as EINVAL rather than EPIPE:
     # https://bugs.python.org/issue19612
     # https://bugs.python.org/issue30418
-    def _is_broken_pipe_error(exc_class, exc):
-        """See the docstring for non-Windows below."""
-        return ((exc_class is BrokenPipeError) or
-                (exc_class is OSError and
-                 exc.errno in (errno.EINVAL, errno.EPIPE)))
-else:
-    # Then we are in the non-Windows case.
-    def _is_broken_pipe_error(exc_class, exc):
-        """
-        Return whether an exception is a broken pipe error.
+    if not WINDOWS:
+        return False
 
-        Args:
-          exc_class: an exception class.
-          exc: an exception instance.
-        """
-        return (exc_class is BrokenPipeError)
+    return isinstance(exc, OSError) and exc.errno in (errno.EINVAL, errno.EPIPE)
 
 
 @contextlib.contextmanager
-def indent_log(num=2):
+def indent_log(num: int = 2) -> Iterator[None]:
     """
     A context manager which will cause the log output to be indented for any
     log messages emitted inside it.
@@ -76,40 +64,45 @@ def indent_log(num=2):
         _log_state.indentation -= num
 
 
-def get_indentation():
-    return getattr(_log_state, 'indentation', 0)
+def get_indentation() -> int:
+    return getattr(_log_state, "indentation", 0)
 
 
 class IndentingFormatter(logging.Formatter):
     default_time_format = "%Y-%m-%dT%H:%M:%S"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(
+        self,
+        *args: Any,
+        add_timestamp: bool = False,
+        **kwargs: Any,
+    ) -> None:
         """
         A logging.Formatter that obeys the indent_log() context manager.
 
         :param add_timestamp: A bool indicating output lines should be prefixed
             with their record's timestamp.
         """
-        self.add_timestamp = kwargs.pop("add_timestamp", False)
+        self.add_timestamp = add_timestamp
         super().__init__(*args, **kwargs)
 
-    def get_message_start(self, formatted, levelno):
+    def get_message_start(self, formatted: str, levelno: int) -> str:
         """
         Return the start of the formatted log message (not counting the
         prefix to add to each line).
         """
         if levelno < logging.WARNING:
-            return ''
+            return ""
         if formatted.startswith(DEPRECATION_MSG_PREFIX):
             # Then the message already has a prefix.  We don't want it to
             # look like "WARNING: DEPRECATION: ...."
-            return ''
+            return ""
         if levelno < logging.ERROR:
-            return 'WARNING: '
+            return "WARNING: "
 
-        return 'ERROR: '
+        return "ERROR: "
 
-    def format(self, record):
+    def format(self, record: logging.LogRecord) -> str:
         """
         Calls the standard formatter, but will indent all of the log message
         lines by our current indentation level.
@@ -118,111 +111,98 @@ def format(self, record):
         message_start = self.get_message_start(formatted, record.levelno)
         formatted = message_start + formatted
 
-        prefix = ''
+        prefix = ""
         if self.add_timestamp:
             prefix = f"{self.formatTime(record)} "
         prefix += " " * get_indentation()
-        formatted = "".join([
-            prefix + line
-            for line in formatted.splitlines(True)
-        ])
+        formatted = "".join([prefix + line for line in formatted.splitlines(True)])
         return formatted
 
 
-def _color_wrap(*colors):
-    def wrapped(inp):
-        return "".join(list(colors) + [inp, colorama.Style.RESET_ALL])
-    return wrapped
-
-
-class ColorizedStreamHandler(logging.StreamHandler):
-
-    # Don't build up a list of colors if we don't have colorama
-    if colorama:
-        COLORS = [
-            # This needs to be in order from highest logging level to lowest.
-            (logging.ERROR, _color_wrap(colorama.Fore.RED)),
-            (logging.WARNING, _color_wrap(colorama.Fore.YELLOW)),
-        ]
-    else:
-        COLORS = []
-
-    def __init__(self, stream=None, no_color=None):
-        super().__init__(stream)
-        self._no_color = no_color
-
-        if WINDOWS and colorama:
-            self.stream = colorama.AnsiToWin32(self.stream)
-
-    def _using_stdout(self):
-        """
-        Return whether the handler is using sys.stdout.
-        """
-        if WINDOWS and colorama:
-            # Then self.stream is an AnsiToWin32 object.
-            return self.stream.wrapped is sys.stdout
-
-        return self.stream is sys.stdout
-
-    def should_color(self):
-        # Don't colorize things if we do not have colorama or if told not to
-        if not colorama or self._no_color:
-            return False
-
-        real_stream = (
-            self.stream if not isinstance(self.stream, colorama.AnsiToWin32)
-            else self.stream.wrapped
+@dataclass
+class IndentedRenderable:
+    renderable: ConsoleRenderable
+    indent: int
+
+    def __rich_console__(
+        self, console: Console, options: ConsoleOptions
+    ) -> RenderResult:
+        segments = console.render(self.renderable, options)
+        lines = Segment.split_lines(segments)
+        for line in lines:
+            yield Segment(" " * self.indent)
+            yield from line
+            yield Segment("\n")
+
+
+class RichPipStreamHandler(RichHandler):
+    KEYWORDS: ClassVar[Optional[List[str]]] = []
+
+    def __init__(self, stream: Optional[TextIO], no_color: bool) -> None:
+        super().__init__(
+            console=Console(file=stream, no_color=no_color, soft_wrap=True),
+            show_time=False,
+            show_level=False,
+            show_path=False,
+            highlighter=NullHighlighter(),
         )
 
-        # If the stream is a tty we should color it
-        if hasattr(real_stream, "isatty") and real_stream.isatty():
-            return True
-
-        # If we have an ANSI term we should color it
-        if os.environ.get("TERM") == "ANSI":
-            return True
-
-        # If anything else we should not color it
-        return False
-
-    def format(self, record):
-        msg = logging.StreamHandler.format(self, record)
-
-        if self.should_color():
-            for level, color in self.COLORS:
-                if record.levelno >= level:
-                    msg = color(msg)
-                    break
+    # Our custom override on Rich's logger, to make things work as we need them to.
+    def emit(self, record: logging.LogRecord) -> None:
+        style: Optional[Style] = None
+
+        # If we are given a diagnostic error to present, present it with indentation.
+        if record.msg == "[present-diagnostic] %s" and len(record.args) == 1:
+            diagnostic_error: DiagnosticPipError = record.args[0]  # type: ignore[index]
+            assert isinstance(diagnostic_error, DiagnosticPipError)
+
+            renderable: ConsoleRenderable = IndentedRenderable(
+                diagnostic_error, indent=get_indentation()
+            )
+        else:
+            message = self.format(record)
+            renderable = self.render_message(record, message)
+            if record.levelno is not None:
+                if record.levelno >= logging.ERROR:
+                    style = Style(color="red")
+                elif record.levelno >= logging.WARNING:
+                    style = Style(color="yellow")
+
+        try:
+            self.console.print(renderable, overflow="ignore", crop=False, style=style)
+        except Exception:
+            self.handleError(record)
+
+    def handleError(self, record: logging.LogRecord) -> None:
+        """Called when logging is unable to log some output."""
 
-        return msg
-
-    # The logging module says handleError() can be customized.
-    def handleError(self, record):
         exc_class, exc = sys.exc_info()[:2]
         # If a broken pipe occurred while calling write() or flush() on the
         # stdout stream in logging's Handler.emit(), then raise our special
         # exception so we can handle it in main() instead of logging the
         # broken pipe error and continuing.
-        if (exc_class and self._using_stdout() and
-                _is_broken_pipe_error(exc_class, exc)):
+        if (
+            exc_class
+            and exc
+            and self.console.file is sys.stdout
+            and _is_broken_pipe_error(exc_class, exc)
+        ):
             raise BrokenStdoutLoggingError()
 
         return super().handleError(record)
 
 
 class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler):
-
-    def _open(self):
+    def _open(self) -> IO[Any]:
         ensure_dir(os.path.dirname(self.baseFilename))
-        return logging.handlers.RotatingFileHandler._open(self)
+        return super()._open()
 
 
 class MaxLevelFilter(Filter):
-
-    def __init__(self, level):
+    def __init__(self, level: int) -> None:
         self.level = level
 
-    def filter(self, record):
+    def filter(self, record: logging.LogRecord) -> bool:
         return record.levelno < self.level
 
 
@@ -232,31 +212,33 @@ class ExcludeLoggerFilter(Filter):
     A logging Filter that excludes records from a logger (or its children).
     """
 
-    def filter(self, record):
+    def filter(self, record: logging.LogRecord) -> bool:
         # The base Filter class allows only records from a logger (or its
         # children).
         return not super().filter(record)
 
 
-def setup_logging(verbosity, no_color, user_log_file):
+def setup_logging(verbosity: int, no_color: bool, user_log_file: Optional[str]) -> int:
     """Configures and sets up all of the logging
 
     Returns the requested logging level, as its integer value.
     """
 
     # Determine the level to be logging at.
-    if verbosity >= 1:
-        level = "DEBUG"
+    if verbosity >= 2:
+        level_number = logging.DEBUG
+    elif verbosity == 1:
+        level_number = VERBOSE
     elif verbosity == -1:
-        level = "WARNING"
+        level_number = logging.WARNING
     elif verbosity == -2:
-        level = "ERROR"
+        level_number = logging.ERROR
     elif verbosity <= -3:
-        level = "CRITICAL"
+        level_number = logging.CRITICAL
     else:
-        level = "INFO"
+        level_number = logging.INFO
 
-    level_number = getattr(logging, level)
+    level = logging.getLevelName(level_number)
 
     # The "root" logger should match the "console" level *unless* we also need
     # to log to a user log file.
@@ -278,85 +260,84 @@ def setup_logging(verbosity, no_color, user_log_file):
         "stderr": "ext://sys.stderr",
     }
     handler_classes = {
-        "stream": "pip._internal.utils.logging.ColorizedStreamHandler",
+        "stream": "pip._internal.utils.logging.RichPipStreamHandler",
         "file": "pip._internal.utils.logging.BetterRotatingFileHandler",
     }
     handlers = ["console", "console_errors", "console_subprocess"] + (
         ["user_log"] if include_user_log else []
     )
 
-    logging.config.dictConfig({
-        "version": 1,
-        "disable_existing_loggers": False,
-        "filters": {
-            "exclude_warnings": {
-                "()": "pip._internal.utils.logging.MaxLevelFilter",
-                "level": logging.WARNING,
-            },
-            "restrict_to_subprocess": {
-                "()": "logging.Filter",
-                "name": subprocess_logger.name,
-            },
-            "exclude_subprocess": {
-                "()": "pip._internal.utils.logging.ExcludeLoggerFilter",
-                "name": subprocess_logger.name,
+    logging.config.dictConfig(
+        {
+            "version": 1,
+            "disable_existing_loggers": False,
+            "filters": {
+                "exclude_warnings": {
+                    "()": "pip._internal.utils.logging.MaxLevelFilter",
+                    "level": logging.WARNING,
+                },
+                "restrict_to_subprocess": {
+                    "()": "logging.Filter",
+                    "name": subprocess_logger.name,
+                },
+                "exclude_subprocess": {
+                    "()": "pip._internal.utils.logging.ExcludeLoggerFilter",
+                    "name": subprocess_logger.name,
+                },
             },
-        },
-        "formatters": {
-            "indent": {
-                "()": IndentingFormatter,
-                "format": "%(message)s",
+            "formatters": {
+                "indent": {
+                    "()": IndentingFormatter,
+                    "format": "%(message)s",
+                },
+                "indent_with_timestamp": {
+                    "()": IndentingFormatter,
+                    "format": "%(message)s",
+                    "add_timestamp": True,
+                },
             },
-            "indent_with_timestamp": {
-                "()": IndentingFormatter,
-                "format": "%(message)s",
-                "add_timestamp": True,
+            "handlers": {
+                "console": {
+                    "level": level,
+                    "class": handler_classes["stream"],
+                    "no_color": no_color,
+                    "stream": log_streams["stdout"],
+                    "filters": ["exclude_subprocess", "exclude_warnings"],
+                    "formatter": "indent",
+                },
+                "console_errors": {
+                    "level": "WARNING",
+                    "class": handler_classes["stream"],
+                    "no_color": no_color,
+                    "stream": log_streams["stderr"],
+                    "filters": ["exclude_subprocess"],
+                    "formatter": "indent",
+                },
+                # A handler responsible for logging to the console messages
+                # from the "subprocessor" logger.
+                "console_subprocess": {
+                    "level": level,
+                    "class": handler_classes["stream"],
+                    "stream": log_streams["stderr"],
+                    "no_color": no_color,
+                    "filters": ["restrict_to_subprocess"],
+                    "formatter": "indent",
+                },
+                "user_log": {
+                    "level": "DEBUG",
+                    "class": handler_classes["file"],
+                    "filename": additional_log_file,
+                    "encoding": "utf-8",
+                    "delay": True,
+                    "formatter": "indent_with_timestamp",
+                },
             },
-        },
-        "handlers": {
-            "console": {
-                "level": level,
-                "class": handler_classes["stream"],
-                "no_color": no_color,
-                "stream": log_streams["stdout"],
-                "filters": ["exclude_subprocess", "exclude_warnings"],
-                "formatter": "indent",
+            "root": {
+                "level": root_level,
+                "handlers": handlers,
             },
-            "console_errors": {
-                "level": "WARNING",
-                "class": handler_classes["stream"],
-                "no_color": no_color,
-                "stream": log_streams["stderr"],
-                "filters": ["exclude_subprocess"],
-                "formatter": "indent",
-            },
-            # A handler responsible for logging to the console messages
-            # from the "subprocessor" logger.
-            "console_subprocess": {
-                "level": level,
-                "class": handler_classes["stream"],
-                "no_color": no_color,
-                "stream": log_streams["stderr"],
-                "filters": ["restrict_to_subprocess"],
-                "formatter": "indent",
-            },
-            "user_log": {
-                "level": "DEBUG",
-                "class": handler_classes["file"],
-                "filename": additional_log_file,
-                "delay": True,
-                "formatter": "indent_with_timestamp",
-            },
-        },
-        "root": {
-            "level": root_level,
-            "handlers": handlers,
-        },
-        "loggers": {
-            "pip._vendor": {
-                "level": vendored_log_level
-            }
-        },
-    })
+            "loggers": {"pip._vendor": {"level": vendored_log_level}},
+        }
+    )
 
     return level_number
diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py
index a3bd49b9139..bfe0b9d3201 100644
--- a/src/pip/_internal/utils/misc.py
+++ b/src/pip/_internal/utils/misc.py
@@ -1,6 +1,5 @@
 # The following comment should be removed at some point in the future.
 # mypy: strict-optional=False
-# mypy: disallow-untyped-defs=False
 
 import contextlib
 import errno
@@ -16,70 +15,68 @@
 import urllib.parse
 from io import StringIO
 from itertools import filterfalse, tee, zip_longest
+from types import TracebackType
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    ContextManager,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    TextIO,
+    Tuple,
+    Type,
+    TypeVar,
+    cast,
+)
 
-from pip._vendor import pkg_resources
-from pip._vendor.packaging.utils import canonicalize_name
-
-# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
-#       why we ignore the type on this import.
-from pip._vendor.retrying import retry  # type: ignore
+from pip._vendor.tenacity import retry, stop_after_delay, wait_fixed
 
 from pip import __version__
 from pip._internal.exceptions import CommandError
-from pip._internal.locations import get_major_minor_version, site_packages, user_site
-from pip._internal.utils.compat import WINDOWS, stdlib_pkgs
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast
-from pip._internal.utils.virtualenv import (
-    running_under_virtualenv,
-    virtualenv_no_global,
-)
-
-if MYPY_CHECK_RUNNING:
-    from typing import (
-        Any,
-        AnyStr,
-        Callable,
-        Container,
-        Iterable,
-        Iterator,
-        List,
-        Optional,
-        Tuple,
-        TypeVar,
-    )
-
-    from pip._vendor.pkg_resources import Distribution
-
-    VersionInfo = Tuple[int, int, int]
-    T = TypeVar("T")
-
-
-__all__ = ['rmtree', 'display_path', 'backup_dir',
-           'ask', 'splitext',
-           'format_size', 'is_installable_dir',
-           'normalize_path',
-           'renames', 'get_prog',
-           'captured_stdout', 'ensure_dir',
-           'get_installed_version', 'remove_auth_from_url']
+from pip._internal.locations import get_major_minor_version
+from pip._internal.utils.compat import WINDOWS
+from pip._internal.utils.virtualenv import running_under_virtualenv
+
+__all__ = [
+    "rmtree",
+    "display_path",
+    "backup_dir",
+    "ask",
+    "splitext",
+    "format_size",
+    "is_installable_dir",
+    "normalize_path",
+    "renames",
+    "get_prog",
+    "captured_stdout",
+    "ensure_dir",
+    "remove_auth_from_url",
+]
 
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+ExcInfo = Tuple[Type[BaseException], BaseException, TracebackType]
+VersionInfo = Tuple[int, int, int]
+NetlocTuple = Tuple[str, Tuple[Optional[str], Optional[str]]]
 
-def get_pip_version():
-    # type: () -> str
+
+def get_pip_version() -> str:
     pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..")
     pip_pkg_dir = os.path.abspath(pip_pkg_dir)
 
-    return (
-        'pip {} from {} (python {})'.format(
-            __version__, pip_pkg_dir, get_major_minor_version(),
-        )
+    return "pip {} from {} (python {})".format(
+        __version__,
+        pip_pkg_dir,
+        get_major_minor_version(),
     )
 
 
-def normalize_version_info(py_version_info):
-    # type: (Tuple[int, ...]) -> Tuple[int, int, int]
+def normalize_version_info(py_version_info: Tuple[int, ...]) -> Tuple[int, int, int]:
     """
     Convert a tuple of ints representing a Python version to one of length
     three.
@@ -95,11 +92,10 @@ def normalize_version_info(py_version_info):
     elif len(py_version_info) > 3:
         py_version_info = py_version_info[:3]
 
-    return cast('VersionInfo', py_version_info)
+    return cast("VersionInfo", py_version_info)
 
 
-def ensure_dir(path):
-    # type: (AnyStr) -> None
+def ensure_dir(path: str) -> None:
     """os.path.makedirs without EEXIST."""
     try:
         os.makedirs(path)
@@ -109,28 +105,26 @@ def ensure_dir(path):
             raise
 
 
-def get_prog():
-    # type: () -> str
+def get_prog() -> str:
     try:
         prog = os.path.basename(sys.argv[0])
-        if prog in ('__main__.py', '-c'):
+        if prog in ("__main__.py", "-c"):
             return f"{sys.executable} -m pip"
         else:
             return prog
     except (AttributeError, TypeError, IndexError):
         pass
-    return 'pip'
+    return "pip"
 
 
 # Retry every half second for up to 3 seconds
-@retry(stop_max_delay=3000, wait_fixed=500)
-def rmtree(dir, ignore_errors=False):
-    # type: (AnyStr, bool) -> None
-    shutil.rmtree(dir, ignore_errors=ignore_errors,
-                  onerror=rmtree_errorhandler)
+# Tenacity raises RetryError by default, explicitly raise the original exception
+@retry(reraise=True, stop=stop_after_delay(3), wait=wait_fixed(0.5))
+def rmtree(dir: str, ignore_errors: bool = False) -> None:
+    shutil.rmtree(dir, ignore_errors=ignore_errors, onerror=rmtree_errorhandler)
 
 
-def rmtree_errorhandler(func, path, exc_info):
+def rmtree_errorhandler(func: Callable[..., Any], path: str, exc_info: ExcInfo) -> None:
     """On Windows, the files in .svn are read-only, so when rmtree() tries to
     remove them, an exception is thrown.  We catch that here, remove the
     read-only attribute, and hopefully continue without problems."""
@@ -150,42 +144,16 @@ def rmtree_errorhandler(func, path, exc_info):
         raise
 
 
-def path_to_display(path):
-    # type: (Optional[str]) -> Optional[str]
-    """
-    Convert a bytes (or text) path to text (unicode in Python 2) for display
-    and logging purposes.
-
-    This function should never error out. Also, this function is mainly needed
-    for Python 2 since in Python 3 str paths are already text.
-    """
-    if path is None:
-        return None
-    if isinstance(path, str):
-        return path
-    # Otherwise, path is a bytes object (str in Python 2).
-    try:
-        display_path = path.decode(sys.getfilesystemencoding(), 'strict')
-    except UnicodeDecodeError:
-        # Include the full bytes to make troubleshooting easier, even though
-        # it may not be very human readable.
-        display_path = ascii(path)
-
-    return display_path
-
-
-def display_path(path):
-    # type: (str) -> str
+def display_path(path: str) -> str:
     """Gives the display value for a given path, making it relative to cwd
     if possible."""
     path = os.path.normcase(os.path.abspath(path))
     if path.startswith(os.getcwd() + os.path.sep):
-        path = '.' + path[len(os.getcwd()):]
+        path = "." + path[len(os.getcwd()) :]
     return path
 
 
-def backup_dir(dir, ext='.bak'):
-    # type: (str, str) -> str
+def backup_dir(dir: str, ext: str = ".bak") -> str:
     """Figure out the name of a directory to back up the given dir to
     (adding .bak, .bak2, etc)"""
     n = 1
@@ -196,26 +164,22 @@ def backup_dir(dir, ext='.bak'):
     return dir + extension
 
 
-def ask_path_exists(message, options):
-    # type: (str, Iterable[str]) -> str
-    for action in os.environ.get('PIP_EXISTS_ACTION', '').split():
+def ask_path_exists(message: str, options: Iterable[str]) -> str:
+    for action in os.environ.get("PIP_EXISTS_ACTION", "").split():
         if action in options:
             return action
     return ask(message, options)
 
 
-def _check_no_input(message):
-    # type: (str) -> None
+def _check_no_input(message: str) -> None:
     """Raise an error if no input is allowed."""
-    if os.environ.get('PIP_NO_INPUT'):
+    if os.environ.get("PIP_NO_INPUT"):
         raise Exception(
-            'No input was expected ($PIP_NO_INPUT set); question: {}'.format(
-                message)
+            f"No input was expected ($PIP_NO_INPUT set); question: {message}"
         )
 
 
-def ask(message, options):
-    # type: (str, Iterable[str]) -> str
+def ask(message: str, options: Iterable[str]) -> str:
     """Ask the message interactively, with the given possible responses"""
     while 1:
         _check_no_input(message)
@@ -223,41 +187,53 @@ def ask(message, options):
         response = response.strip().lower()
         if response not in options:
             print(
-                'Your response ({!r}) was not one of the expected responses: '
-                '{}'.format(response, ', '.join(options))
+                "Your response ({!r}) was not one of the expected responses: "
+                "{}".format(response, ", ".join(options))
             )
         else:
             return response
 
 
-def ask_input(message):
-    # type: (str) -> str
+def ask_input(message: str) -> str:
     """Ask for input interactively."""
     _check_no_input(message)
     return input(message)
 
 
-def ask_password(message):
-    # type: (str) -> str
+def ask_password(message: str) -> str:
     """Ask for a password interactively."""
     _check_no_input(message)
     return getpass.getpass(message)
 
 
-def format_size(bytes):
-    # type: (float) -> str
+def strtobool(val: str) -> int:
+    """Convert a string representation of truth to true (1) or false (0).
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+    """
+    val = val.lower()
+    if val in ("y", "yes", "t", "true", "on", "1"):
+        return 1
+    elif val in ("n", "no", "f", "false", "off", "0"):
+        return 0
+    else:
+        raise ValueError(f"invalid truth value {val!r}")
+
+
+def format_size(bytes: float) -> str:
     if bytes > 1000 * 1000:
-        return '{:.1f} MB'.format(bytes / 1000.0 / 1000)
+        return "{:.1f} MB".format(bytes / 1000.0 / 1000)
     elif bytes > 10 * 1000:
-        return '{} kB'.format(int(bytes / 1000))
+        return "{} kB".format(int(bytes / 1000))
     elif bytes > 1000:
-        return '{:.1f} kB'.format(bytes / 1000.0)
+        return "{:.1f} kB".format(bytes / 1000.0)
     else:
-        return '{} bytes'.format(int(bytes))
+        return "{} bytes".format(int(bytes))
 
 
-def tabulate(rows):
-    # type: (Iterable[Iterable[Any]]) -> Tuple[List[str], List[int]]
+def tabulate(rows: Iterable[Iterable[Any]]) -> Tuple[List[str], List[int]]:
     """Return a list of formatted rows and a list of column sizes.
 
     For example::
@@ -266,27 +242,29 @@ def tabulate(rows):
     (['foobar     2000', '3735928559'], [10, 4])
     """
     rows = [tuple(map(str, row)) for row in rows]
-    sizes = [max(map(len, col)) for col in zip_longest(*rows, fillvalue='')]
+    sizes = [max(map(len, col)) for col in zip_longest(*rows, fillvalue="")]
     table = [" ".join(map(str.ljust, row, sizes)).rstrip() for row in rows]
     return table, sizes
 
 
-def is_installable_dir(path):
-    # type: (str) -> bool
-    """Is path is a directory containing setup.py or pyproject.toml?
+def is_installable_dir(path: str) -> bool:
+    """Is path is a directory containing pyproject.toml or setup.py?
+
+    If pyproject.toml exists, this is a PEP 517 project. Otherwise we look for
+    a legacy setuptools layout by identifying setup.py. We don't check for the
+    setup.cfg because using it without setup.py is only available for PEP 517
+    projects, which are already covered by the pyproject.toml check.
     """
     if not os.path.isdir(path):
         return False
-    setup_py = os.path.join(path, 'setup.py')
-    if os.path.isfile(setup_py):
+    if os.path.isfile(os.path.join(path, "pyproject.toml")):
         return True
-    pyproject_toml = os.path.join(path, 'pyproject.toml')
-    if os.path.isfile(pyproject_toml):
+    if os.path.isfile(os.path.join(path, "setup.py")):
         return True
     return False
 
 
-def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
+def read_chunks(file: BinaryIO, size: int = io.DEFAULT_BUFFER_SIZE) -> Iterator[bytes]:
     """Yield pieces of data from a file-like object until EOF."""
     while True:
         chunk = file.read(size)
@@ -295,8 +273,7 @@ def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
         yield chunk
 
 
-def normalize_path(path, resolve_symlinks=True):
-    # type: (str, bool) -> str
+def normalize_path(path: str, resolve_symlinks: bool = True) -> str:
     """
     Convert a path to its canonical, case-normalized, absolute version.
 
@@ -309,18 +286,16 @@ def normalize_path(path, resolve_symlinks=True):
     return os.path.normcase(path)
 
 
-def splitext(path):
-    # type: (str) -> Tuple[str, str]
+def splitext(path: str) -> Tuple[str, str]:
     """Like os.path.splitext, but take off .tar too"""
     base, ext = posixpath.splitext(path)
-    if base.lower().endswith('.tar'):
+    if base.lower().endswith(".tar"):
         ext = base[-4:] + ext
         base = base[:-4]
     return base, ext
 
 
-def renames(old, new):
-    # type: (str, str) -> None
+def renames(old: str, new: str) -> None:
     """Like os.renames(), but handles renaming across devices."""
     # Implementation borrowed from os.renames().
     head, tail = os.path.split(new)
@@ -337,8 +312,7 @@ def renames(old, new):
             pass
 
 
-def is_local(path):
-    # type: (str) -> bool
+def is_local(path: str) -> bool:
     """
     Return True if path is within sys.prefix, if we're running in a virtualenv.
 
@@ -352,238 +326,27 @@ def is_local(path):
     return path.startswith(normalize_path(sys.prefix))
 
 
-def dist_is_local(dist):
-    # type: (Distribution) -> bool
-    """
-    Return True if given Distribution object is installed locally
-    (i.e. within current virtualenv).
-
-    Always True if we're not in a virtualenv.
-
-    """
-    return is_local(dist_location(dist))
-
-
-def dist_in_usersite(dist):
-    # type: (Distribution) -> bool
-    """
-    Return True if given Distribution is installed in user site.
-    """
-    return dist_location(dist).startswith(normalize_path(user_site))
-
-
-def dist_in_site_packages(dist):
-    # type: (Distribution) -> bool
-    """
-    Return True if given Distribution is installed in
-    sysconfig.get_python_lib().
-    """
-    return dist_location(dist).startswith(normalize_path(site_packages))
-
-
-def dist_is_editable(dist):
-    # type: (Distribution) -> bool
-    """
-    Return True if given Distribution is an editable install.
-    """
-    for path_item in sys.path:
-        egg_link = os.path.join(path_item, dist.project_name + '.egg-link')
-        if os.path.isfile(egg_link):
-            return True
-    return False
-
-
-def get_installed_distributions(
-        local_only=True,  # type: bool
-        skip=stdlib_pkgs,  # type: Container[str]
-        include_editables=True,  # type: bool
-        editables_only=False,  # type: bool
-        user_only=False,  # type: bool
-        paths=None  # type: Optional[List[str]]
-):
-    # type: (...) -> List[Distribution]
-    """
-    Return a list of installed Distribution objects.
-
-    If ``local_only`` is True (default), only return installations
-    local to the current virtualenv, if in a virtualenv.
-
-    ``skip`` argument is an iterable of lower-case project names to
-    ignore; defaults to stdlib_pkgs
-
-    If ``include_editables`` is False, don't report editables.
-
-    If ``editables_only`` is True , only report editables.
-
-    If ``user_only`` is True , only report installations in the user
-    site directory.
-
-    If ``paths`` is set, only report the distributions present at the
-    specified list of locations.
-    """
-    if paths:
-        working_set = pkg_resources.WorkingSet(paths)
-    else:
-        working_set = pkg_resources.working_set
-
-    if local_only:
-        local_test = dist_is_local
-    else:
-        def local_test(d):
-            return True
-
-    if include_editables:
-        def editable_test(d):
-            return True
-    else:
-        def editable_test(d):
-            return not dist_is_editable(d)
-
-    if editables_only:
-        def editables_only_test(d):
-            return dist_is_editable(d)
-    else:
-        def editables_only_test(d):
-            return True
-
-    if user_only:
-        user_test = dist_in_usersite
-    else:
-        def user_test(d):
-            return True
-
-    return [d for d in working_set
-            if local_test(d) and
-            d.key not in skip and
-            editable_test(d) and
-            editables_only_test(d) and
-            user_test(d)
-            ]
-
-
-def _search_distribution(req_name):
-    # type: (str) -> Optional[Distribution]
-    """Find a distribution matching the ``req_name`` in the environment.
-
-    This searches from *all* distributions available in the environment, to
-    match the behavior of ``pkg_resources.get_distribution()``.
-    """
-    # Canonicalize the name before searching in the list of
-    # installed distributions and also while creating the package
-    # dictionary to get the Distribution object
-    req_name = canonicalize_name(req_name)
-    packages = get_installed_distributions(
-        local_only=False,
-        skip=(),
-        include_editables=True,
-        editables_only=False,
-        user_only=False,
-        paths=None,
-    )
-    pkg_dict = {canonicalize_name(p.key): p for p in packages}
-    return pkg_dict.get(req_name)
-
-
-def get_distribution(req_name):
-    # type: (str) -> Optional[Distribution]
-    """Given a requirement name, return the installed Distribution object.
-
-    This searches from *all* distributions available in the environment, to
-    match the behavior of ``pkg_resources.get_distribution()``.
-    """
-
-    # Search the distribution by looking through the working set
-    dist = _search_distribution(req_name)
-
-    # If distribution could not be found, call working_set.require
-    # to update the working set, and try to find the distribution
-    # again.
-    # This might happen for e.g. when you install a package
-    # twice, once using setup.py develop and again using setup.py install.
-    # Now when run pip uninstall twice, the package gets removed
-    # from the working set in the first uninstall, so we have to populate
-    # the working set again so that pip knows about it and the packages
-    # gets picked up and is successfully uninstalled the second time too.
-    if not dist:
-        try:
-            pkg_resources.working_set.require(req_name)
-        except pkg_resources.DistributionNotFound:
-            return None
-    return _search_distribution(req_name)
-
-
-def egg_link_path(dist):
-    # type: (Distribution) -> Optional[str]
-    """
-    Return the path for the .egg-link file if it exists, otherwise, None.
-
-    There's 3 scenarios:
-    1) not in a virtualenv
-       try to find in site.USER_SITE, then site_packages
-    2) in a no-global virtualenv
-       try to find in site_packages
-    3) in a yes-global virtualenv
-       try to find in site_packages, then site.USER_SITE
-       (don't look in global location)
-
-    For #1 and #3, there could be odd cases, where there's an egg-link in 2
-    locations.
-
-    This method will just return the first one found.
-    """
-    sites = []
-    if running_under_virtualenv():
-        sites.append(site_packages)
-        if not virtualenv_no_global() and user_site:
-            sites.append(user_site)
-    else:
-        if user_site:
-            sites.append(user_site)
-        sites.append(site_packages)
-
-    for site in sites:
-        egglink = os.path.join(site, dist.project_name) + '.egg-link'
-        if os.path.isfile(egglink):
-            return egglink
-    return None
-
-
-def dist_location(dist):
-    # type: (Distribution) -> str
-    """
-    Get the site-packages location of this distribution. Generally
-    this is dist.location, except in the case of develop-installed
-    packages, where dist.location is the source code location, and we
-    want to know where the egg-link file is.
-
-    The returned location is normalized (in particular, with symlinks removed).
-    """
-    egg_link = egg_link_path(dist)
-    if egg_link:
-        return normalize_path(egg_link)
-    return normalize_path(dist.location)
-
-
-def write_output(msg, *args):
-    # type: (Any, Any) -> None
+def write_output(msg: Any, *args: Any) -> None:
     logger.info(msg, *args)
 
 
 class StreamWrapper(StringIO):
+    orig_stream: TextIO = None
 
     @classmethod
-    def from_stream(cls, orig_stream):
+    def from_stream(cls, orig_stream: TextIO) -> "StreamWrapper":
         cls.orig_stream = orig_stream
         return cls()
 
     # compileall.compile_dir() needs stdout.encoding to print to stdout
+    # https://github.com/python/mypy/issues/4125
     @property
-    def encoding(self):
+    def encoding(self):  # type: ignore
         return self.orig_stream.encoding
 
 
 @contextlib.contextmanager
-def captured_output(stream_name):
+def captured_output(stream_name: str) -> Iterator[StreamWrapper]:
     """Return a context manager used by captured_stdout/stdin/stderr
     that temporarily replaces the sys stream *stream_name* with a StringIO.
 
@@ -597,7 +360,7 @@ def captured_output(stream_name):
         setattr(sys, stream_name, orig_stdout)
 
 
-def captured_stdout():
+def captured_stdout() -> ContextManager[StreamWrapper]:
     """Capture the output of sys.stdout:
 
        with captured_stdout() as stdout:
@@ -606,68 +369,47 @@ def captured_stdout():
 
     Taken from Lib/support/__init__.py in the CPython repo.
     """
-    return captured_output('stdout')
+    return captured_output("stdout")
 
 
-def captured_stderr():
+def captured_stderr() -> ContextManager[StreamWrapper]:
     """
     See captured_stdout().
     """
-    return captured_output('stderr')
-
-
-def get_installed_version(dist_name, working_set=None):
-    """Get the installed version of dist_name avoiding pkg_resources cache"""
-    # Create a requirement that we'll look for inside of setuptools.
-    req = pkg_resources.Requirement.parse(dist_name)
-
-    if working_set is None:
-        # We want to avoid having this cached, so we need to construct a new
-        # working set each time.
-        working_set = pkg_resources.WorkingSet()
-
-    # Get the installed distribution from our working set
-    dist = working_set.find(req)
-
-    # Check to see if we got an installed distribution or not, if we did
-    # we want to return it's version.
-    return dist.version if dist else None
+    return captured_output("stderr")
 
 
 # Simulates an enum
-def enum(*sequential, **named):
+def enum(*sequential: Any, **named: Any) -> Type[Any]:
     enums = dict(zip(sequential, range(len(sequential))), **named)
     reverse = {value: key for key, value in enums.items()}
-    enums['reverse_mapping'] = reverse
-    return type('Enum', (), enums)
+    enums["reverse_mapping"] = reverse
+    return type("Enum", (), enums)
 
 
-def build_netloc(host, port):
-    # type: (str, Optional[int]) -> str
+def build_netloc(host: str, port: Optional[int]) -> str:
     """
     Build a netloc from a host-port pair
     """
     if port is None:
         return host
-    if ':' in host:
+    if ":" in host:
         # Only wrap host with square brackets when it is IPv6
-        host = f'[{host}]'
-    return f'{host}:{port}'
+        host = f"[{host}]"
+    return f"{host}:{port}"
 
 
-def build_url_from_netloc(netloc, scheme='https'):
-    # type: (str, str) -> str
+def build_url_from_netloc(netloc: str, scheme: str = "https") -> str:
     """
     Build a full URL from a netloc.
     """
-    if netloc.count(':') >= 2 and '@' not in netloc and '[' not in netloc:
+    if netloc.count(":") >= 2 and "@" not in netloc and "[" not in netloc:
         # It must be a bare IPv6 address, so wrap it with brackets.
-        netloc = f'[{netloc}]'
-    return f'{scheme}://{netloc}'
+        netloc = f"[{netloc}]"
+    return f"{scheme}://{netloc}"
 
 
-def parse_netloc(netloc):
-    # type: (str) -> Tuple[str, Optional[int]]
+def parse_netloc(netloc: str) -> Tuple[str, Optional[int]]:
     """
     Return the host-port pair from a netloc.
     """
@@ -676,36 +418,36 @@ def parse_netloc(netloc):
     return parsed.hostname, parsed.port
 
 
-def split_auth_from_netloc(netloc):
+def split_auth_from_netloc(netloc: str) -> NetlocTuple:
     """
     Parse out and remove the auth information from a netloc.
 
     Returns: (netloc, (username, password)).
     """
-    if '@' not in netloc:
+    if "@" not in netloc:
         return netloc, (None, None)
 
     # Split from the right because that's how urllib.parse.urlsplit()
     # behaves if more than one @ is present (which can be checked using
     # the password attribute of urlsplit()'s return value).
-    auth, netloc = netloc.rsplit('@', 1)
-    if ':' in auth:
+    auth, netloc = netloc.rsplit("@", 1)
+    pw: Optional[str] = None
+    if ":" in auth:
         # Split from the left because that's how urllib.parse.urlsplit()
         # behaves if more than one : is present (which again can be checked
         # using the password attribute of the return value)
-        user_pass = auth.split(':', 1)
+        user, pw = auth.split(":", 1)
     else:
-        user_pass = auth, None
+        user, pw = auth, None
 
-    user_pass = tuple(
-        None if x is None else urllib.parse.unquote(x) for x in user_pass
-    )
+    user = urllib.parse.unquote(user)
+    if pw is not None:
+        pw = urllib.parse.unquote(pw)
 
-    return netloc, user_pass
+    return netloc, (user, pw)
 
 
-def redact_netloc(netloc):
-    # type: (str) -> str
+def redact_netloc(netloc: str) -> str:
     """
     Replace the sensitive data in a netloc with "****", if it exists.
 
@@ -717,17 +459,19 @@ def redact_netloc(netloc):
     if user is None:
         return netloc
     if password is None:
-        user = '****'
-        password = ''
+        user = "****"
+        password = ""
     else:
         user = urllib.parse.quote(user)
-        password = ':****'
-    return '{user}{password}@{netloc}'.format(user=user,
-                                              password=password,
-                                              netloc=netloc)
+        password = ":****"
+    return "{user}{password}@{netloc}".format(
+        user=user, password=password, netloc=netloc
+    )
 
 
-def _transform_url(url, transform_netloc):
+def _transform_url(
+    url: str, transform_netloc: Callable[[str], Tuple[Any, ...]]
+) -> Tuple[str, NetlocTuple]:
     """Transform and replace netloc in a url.
 
     transform_netloc is a function taking the netloc and returning a
@@ -740,23 +484,20 @@ def _transform_url(url, transform_netloc):
     purl = urllib.parse.urlsplit(url)
     netloc_tuple = transform_netloc(purl.netloc)
     # stripped url
-    url_pieces = (
-        purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment
-    )
+    url_pieces = (purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment)
     surl = urllib.parse.urlunsplit(url_pieces)
-    return surl, netloc_tuple
+    return surl, cast("NetlocTuple", netloc_tuple)
 
 
-def _get_netloc(netloc):
+def _get_netloc(netloc: str) -> NetlocTuple:
     return split_auth_from_netloc(netloc)
 
 
-def _redact_netloc(netloc):
+def _redact_netloc(netloc: str) -> Tuple[str]:
     return (redact_netloc(netloc),)
 
 
-def split_auth_netloc_from_url(url):
-    # type: (str) -> Tuple[str, str, Tuple[str, str]]
+def split_auth_netloc_from_url(url: str) -> Tuple[str, str, Tuple[str, str]]:
     """
     Parse a url into separate netloc, auth, and url with no auth.
 
@@ -766,112 +507,92 @@ def split_auth_netloc_from_url(url):
     return url_without_auth, netloc, auth
 
 
-def remove_auth_from_url(url):
-    # type: (str) -> str
+def remove_auth_from_url(url: str) -> str:
     """Return a copy of url with 'username:password@' removed."""
     # username/pass params are passed to subversion through flags
     # and are not recognized in the url.
     return _transform_url(url, _get_netloc)[0]
 
 
-def redact_auth_from_url(url):
-    # type: (str) -> str
+def redact_auth_from_url(url: str) -> str:
     """Replace the password in a given url with ****."""
     return _transform_url(url, _redact_netloc)[0]
 
 
 class HiddenText:
-    def __init__(
-        self,
-        secret,    # type: str
-        redacted,  # type: str
-    ):
-        # type: (...) -> None
+    def __init__(self, secret: str, redacted: str) -> None:
         self.secret = secret
         self.redacted = redacted
 
-    def __repr__(self):
-        # type: (...) -> str
-        return ''.format(str(self))
+    def __repr__(self) -> str:
+        return "".format(str(self))
 
-    def __str__(self):
-        # type: (...) -> str
+    def __str__(self) -> str:
         return self.redacted
 
     # This is useful for testing.
-    def __eq__(self, other):
-        # type: (Any) -> bool
+    def __eq__(self, other: Any) -> bool:
         if type(self) != type(other):
             return False
 
         # The string being used for redaction doesn't also have to match,
         # just the raw, original string.
-        return (self.secret == other.secret)
+        return self.secret == other.secret
 
 
-def hide_value(value):
-    # type: (str) -> HiddenText
-    return HiddenText(value, redacted='****')
+def hide_value(value: str) -> HiddenText:
+    return HiddenText(value, redacted="****")
 
 
-def hide_url(url):
-    # type: (str) -> HiddenText
+def hide_url(url: str) -> HiddenText:
     redacted = redact_auth_from_url(url)
     return HiddenText(url, redacted=redacted)
 
 
-def protect_pip_from_modification_on_windows(modifying_pip):
-    # type: (bool) -> None
+def protect_pip_from_modification_on_windows(modifying_pip: bool) -> None:
     """Protection of pip.exe from modification on Windows
 
     On Windows, any operation modifying pip should be run as:
         python -m pip ...
     """
     pip_names = [
-        "pip.exe",
-        "pip{}.exe".format(sys.version_info[0]),
-        "pip{}.{}.exe".format(*sys.version_info[:2])
+        "pip",
+        f"pip{sys.version_info.major}",
+        f"pip{sys.version_info.major}.{sys.version_info.minor}",
     ]
 
     # See https://github.com/pypa/pip/issues/1299 for more discussion
     should_show_use_python_msg = (
-        modifying_pip and
-        WINDOWS and
-        os.path.basename(sys.argv[0]) in pip_names
+        modifying_pip and WINDOWS and os.path.basename(sys.argv[0]) in pip_names
     )
 
     if should_show_use_python_msg:
-        new_command = [
-            sys.executable, "-m", "pip"
-        ] + sys.argv[1:]
+        new_command = [sys.executable, "-m", "pip"] + sys.argv[1:]
         raise CommandError(
-            'To modify pip, please run the following command:\n{}'
-            .format(" ".join(new_command))
+            "To modify pip, please run the following command:\n{}".format(
+                " ".join(new_command)
+            )
         )
 
 
-def is_console_interactive():
-    # type: () -> bool
-    """Is this console interactive?
-    """
+def is_console_interactive() -> bool:
+    """Is this console interactive?"""
     return sys.stdin is not None and sys.stdin.isatty()
 
 
-def hash_file(path, blocksize=1 << 20):
-    # type: (str, int) -> Tuple[Any, int]
-    """Return (hash, length) for path using hashlib.sha256()
-    """
+def hash_file(path: str, blocksize: int = 1 << 20) -> Tuple[Any, int]:
+    """Return (hash, length) for path using hashlib.sha256()"""
 
     h = hashlib.sha256()
     length = 0
-    with open(path, 'rb') as f:
+    with open(path, "rb") as f:
         for block in read_chunks(f, size=blocksize):
             length += len(block)
             h.update(block)
     return h, length
 
 
-def is_wheel_installed():
+def is_wheel_installed() -> bool:
     """
     Return whether the wheel package is installed.
     """
@@ -883,8 +604,7 @@ def is_wheel_installed():
     return True
 
 
-def pairwise(iterable):
-    # type: (Iterable[Any]) -> Iterator[Tuple[Any, Any]]
+def pairwise(iterable: Iterable[Any]) -> Iterator[Tuple[Any, Any]]:
     """
     Return paired elements.
 
@@ -896,10 +616,9 @@ def pairwise(iterable):
 
 
 def partition(
-    pred,  # type: Callable[[T], bool]
-    iterable,  # type: Iterable[T]
-):
-    # type: (...) -> Tuple[Iterable[T], Iterable[T]]
+    pred: Callable[[T], bool],
+    iterable: Iterable[T],
+) -> Tuple[Iterable[T], Iterable[T]]:
     """
     Use a predicate to partition entries into false entries and true entries,
     like
diff --git a/src/pip/_internal/utils/models.py b/src/pip/_internal/utils/models.py
index c14e9ff926e..b6bb21a8b26 100644
--- a/src/pip/_internal/utils/models.py
+++ b/src/pip/_internal/utils/models.py
@@ -1,40 +1,38 @@
 """Utilities for defining models
 """
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
 
 import operator
+from typing import Any, Callable, Type
 
 
 class KeyBasedCompareMixin:
-    """Provides comparison capabilities that is based on a key
-    """
+    """Provides comparison capabilities that is based on a key"""
 
-    __slots__ = ['_compare_key', '_defining_class']
+    __slots__ = ["_compare_key", "_defining_class"]
 
-    def __init__(self, key, defining_class):
+    def __init__(self, key: Any, defining_class: Type["KeyBasedCompareMixin"]) -> None:
         self._compare_key = key
         self._defining_class = defining_class
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self._compare_key)
 
-    def __lt__(self, other):
+    def __lt__(self, other: Any) -> bool:
         return self._compare(other, operator.__lt__)
 
-    def __le__(self, other):
+    def __le__(self, other: Any) -> bool:
         return self._compare(other, operator.__le__)
 
-    def __gt__(self, other):
+    def __gt__(self, other: Any) -> bool:
         return self._compare(other, operator.__gt__)
 
-    def __ge__(self, other):
+    def __ge__(self, other: Any) -> bool:
         return self._compare(other, operator.__ge__)
 
-    def __eq__(self, other):
+    def __eq__(self, other: Any) -> bool:
         return self._compare(other, operator.__eq__)
 
-    def _compare(self, other, method):
+    def _compare(self, other: Any, method: Callable[[Any, Any], bool]) -> bool:
         if not isinstance(other, self._defining_class):
             return NotImplemented
 
diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py
index fae06070c87..7c77371a205 100644
--- a/src/pip/_internal/utils/packaging.py
+++ b/src/pip/_internal/utils/packaging.py
@@ -1,25 +1,20 @@
+import functools
 import logging
-from email.parser import FeedParser
+import re
+from typing import NewType, Optional, Tuple, cast
 
-from pip._vendor import pkg_resources
 from pip._vendor.packaging import specifiers, version
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.specifiers import SpecifierSet
 
-from pip._internal.exceptions import NoneMetadataError
-from pip._internal.utils.misc import display_path
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from email.message import Message
-    from typing import Optional, Tuple
-
-    from pip._vendor.pkg_resources import Distribution
-
+NormalizedExtra = NewType("NormalizedExtra", str)
 
 logger = logging.getLogger(__name__)
 
 
-def check_requires_python(requires_python, version_info):
-    # type: (Optional[str], Tuple[int, ...]) -> bool
+def check_requires_python(
+    requires_python: Optional[str], version_info: Tuple[int, ...]
+) -> bool:
     """
     Check if the given Python version matches a "Requires-Python" specifier.
 
@@ -36,58 +31,40 @@ def check_requires_python(requires_python, version_info):
         return True
     requires_python_specifier = specifiers.SpecifierSet(requires_python)
 
-    python_version = version.parse('.'.join(map(str, version_info)))
+    python_version = version.parse(".".join(map(str, version_info)))
     return python_version in requires_python_specifier
 
 
-def get_metadata(dist):
-    # type: (Distribution) -> Message
-    """
-    :raises NoneMetadataError: if the distribution reports `has_metadata()`
-        True but `get_metadata()` returns None.
-    """
-    metadata_name = 'METADATA'
-    if (isinstance(dist, pkg_resources.DistInfoDistribution) and
-            dist.has_metadata(metadata_name)):
-        metadata = dist.get_metadata(metadata_name)
-    elif dist.has_metadata('PKG-INFO'):
-        metadata_name = 'PKG-INFO'
-        metadata = dist.get_metadata(metadata_name)
-    else:
-        logger.warning("No metadata found in %s", display_path(dist.location))
-        metadata = ''
-
-    if metadata is None:
-        raise NoneMetadataError(dist, metadata_name)
-
-    feed_parser = FeedParser()
-    # The following line errors out if with a "NoneType" TypeError if
-    # passed metadata=None.
-    feed_parser.feed(metadata)
-    return feed_parser.close()
-
-
-def get_requires_python(dist):
-    # type: (pkg_resources.Distribution) -> Optional[str]
-    """
-    Return the "Requires-Python" metadata for a distribution, or None
-    if not present.
-    """
-    pkg_info_dict = get_metadata(dist)
-    requires_python = pkg_info_dict.get('Requires-Python')
+@functools.lru_cache(maxsize=512)
+def get_requirement(req_string: str) -> Requirement:
+    """Construct a packaging.Requirement object with caching"""
+    # Parsing requirement strings is expensive, and is also expected to happen
+    # with a low diversity of different arguments (at least relative the number
+    # constructed). This method adds a cache to requirement object creation to
+    # minimize repeated parsing of the same string to construct equivalent
+    # Requirement objects.
+    return Requirement(req_string)
+
 
-    if requires_python is not None:
-        # Convert to a str to satisfy the type checker, since requires_python
-        # can be a Header object.
-        requires_python = str(requires_python)
+def safe_extra(extra: str) -> NormalizedExtra:
+    """Convert an arbitrary string to a standard 'extra' name
 
-    return requires_python
+    Any runs of non-alphanumeric characters are replaced with a single '_',
+    and the result is always lowercased.
 
+    This function is duplicated from ``pkg_resources``. Note that this is not
+    the same to either ``canonicalize_name`` or ``_egg_link_name``.
+    """
+    return cast(NormalizedExtra, re.sub("[^A-Za-z0-9.-]+", "_", extra).lower())
 
-def get_installer(dist):
-    # type: (Distribution) -> str
-    if dist.has_metadata('INSTALLER'):
-        for line in dist.get_metadata_lines('INSTALLER'):
-            if line.strip():
-                return line.strip()
-    return ''
+
+def is_pinned(specifier: SpecifierSet) -> bool:
+    for sp in specifier:
+        if sp.operator == "===":
+            return True
+        if sp.operator != "==":
+            continue
+        if sp.version.endswith(".*"):
+            continue
+        return True
+    return False
diff --git a/src/pip/_internal/utils/parallel.py b/src/pip/_internal/utils/parallel.py
deleted file mode 100644
index 57082367e18..00000000000
--- a/src/pip/_internal/utils/parallel.py
+++ /dev/null
@@ -1,105 +0,0 @@
-"""Convenient parallelization of higher order functions.
-
-This module provides two helper functions, with appropriate fallbacks on
-Python 2 and on systems lacking support for synchronization mechanisms:
-
-- map_multiprocess
-- map_multithread
-
-These helpers work like Python 3's map, with two differences:
-
-- They don't guarantee the order of processing of
-  the elements of the iterable.
-- The underlying process/thread pools chop the iterable into
-  a number of chunks, so that for very long iterables using
-  a large value for chunksize can make the job complete much faster
-  than using the default value of 1.
-"""
-
-__all__ = ['map_multiprocess', 'map_multithread']
-
-from contextlib import contextmanager
-from multiprocessing import Pool as ProcessPool
-from multiprocessing.dummy import Pool as ThreadPool
-
-from pip._vendor.requests.adapters import DEFAULT_POOLSIZE
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from multiprocessing import pool
-    from typing import Callable, Iterable, Iterator, TypeVar, Union
-
-    Pool = Union[pool.Pool, pool.ThreadPool]
-    S = TypeVar('S')
-    T = TypeVar('T')
-
-# On platforms without sem_open, multiprocessing[.dummy] Pool
-# cannot be created.
-try:
-    import multiprocessing.synchronize  # noqa
-except ImportError:
-    LACK_SEM_OPEN = True
-else:
-    LACK_SEM_OPEN = False
-
-# Incredibly large timeout to work around bpo-8296 on Python 2.
-TIMEOUT = 2000000
-
-
-@contextmanager
-def closing(pool):
-    # type: (Pool) -> Iterator[Pool]
-    """Return a context manager making sure the pool closes properly."""
-    try:
-        yield pool
-    finally:
-        # For Pool.imap*, close and join are needed
-        # for the returned iterator to begin yielding.
-        pool.close()
-        pool.join()
-        pool.terminate()
-
-
-def _map_fallback(func, iterable, chunksize=1):
-    # type: (Callable[[S], T], Iterable[S], int) -> Iterator[T]
-    """Make an iterator applying func to each element in iterable.
-
-    This function is the sequential fallback either on Python 2
-    where Pool.imap* doesn't react to KeyboardInterrupt
-    or when sem_open is unavailable.
-    """
-    return map(func, iterable)
-
-
-def _map_multiprocess(func, iterable, chunksize=1):
-    # type: (Callable[[S], T], Iterable[S], int) -> Iterator[T]
-    """Chop iterable into chunks and submit them to a process pool.
-
-    For very long iterables using a large value for chunksize can make
-    the job complete much faster than using the default value of 1.
-
-    Return an unordered iterator of the results.
-    """
-    with closing(ProcessPool()) as pool:
-        return pool.imap_unordered(func, iterable, chunksize)
-
-
-def _map_multithread(func, iterable, chunksize=1):
-    # type: (Callable[[S], T], Iterable[S], int) -> Iterator[T]
-    """Chop iterable into chunks and submit them to a thread pool.
-
-    For very long iterables using a large value for chunksize can make
-    the job complete much faster than using the default value of 1.
-
-    Return an unordered iterator of the results.
-    """
-    with closing(ThreadPool(DEFAULT_POOLSIZE)) as pool:
-        return pool.imap_unordered(func, iterable, chunksize)
-
-
-if LACK_SEM_OPEN:
-    map_multiprocess = map_multithread = _map_fallback
-else:
-    map_multiprocess = _map_multiprocess
-    map_multithread = _map_multithread
diff --git a/src/pip/_internal/utils/pkg_resources.py b/src/pip/_internal/utils/pkg_resources.py
deleted file mode 100644
index d5b26f53895..00000000000
--- a/src/pip/_internal/utils/pkg_resources.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from pip._vendor.pkg_resources import yield_lines
-from pip._vendor.six import ensure_str
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Dict, Iterable, List
-
-
-class DictMetadata:
-    """IMetadataProvider that reads metadata files from a dictionary.
-    """
-    def __init__(self, metadata):
-        # type: (Dict[str, bytes]) -> None
-        self._metadata = metadata
-
-    def has_metadata(self, name):
-        # type: (str) -> bool
-        return name in self._metadata
-
-    def get_metadata(self, name):
-        # type: (str) -> str
-        try:
-            return ensure_str(self._metadata[name])
-        except UnicodeDecodeError as e:
-            # Mirrors handling done in pkg_resources.NullProvider.
-            e.reason += f" in {name} file"
-            raise
-
-    def get_metadata_lines(self, name):
-        # type: (str) -> Iterable[str]
-        return yield_lines(self.get_metadata(name))
-
-    def metadata_isdir(self, name):
-        # type: (str) -> bool
-        return False
-
-    def metadata_listdir(self, name):
-        # type: (str) -> List[str]
-        return []
-
-    def run_script(self, script_name, namespace):
-        # type: (str, str) -> None
-        pass
diff --git a/src/pip/_internal/utils/setuptools_build.py b/src/pip/_internal/utils/setuptools_build.py
index 2a664b00703..f460c4003f3 100644
--- a/src/pip/_internal/utils/setuptools_build.py
+++ b/src/pip/_internal/utils/setuptools_build.py
@@ -1,32 +1,57 @@
 import sys
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional, Sequence
+import textwrap
+from typing import List, Optional, Sequence
 
 # Shim to wrap setup.py invocation with setuptools
-#
-# We set sys.argv[0] to the path to the underlying setup.py file so
-# setuptools / distutils don't take the path to the setup.py to be "-c" when
-# invoking via the shim.  This avoids e.g. the following manifest_maker
-# warning: "warning: manifest_maker: standard file '-c' not found".
-_SETUPTOOLS_SHIM = (
-    "import sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};"
-    "f=getattr(tokenize, 'open', open)(__file__);"
-    "code=f.read().replace('\\r\\n', '\\n');"
-    "f.close();"
-    "exec(compile(code, __file__, 'exec'))"
-)
+# Note that __file__ is handled via two {!r} *and* %r, to ensure that paths on
+# Windows are correctly handled (it should be "C:\\Users" not "C:\Users").
+_SETUPTOOLS_SHIM = textwrap.dedent(
+    """
+    exec(compile('''
+    # This is  -- a caller that pip uses to run setup.py
+    #
+    # - It imports setuptools before invoking setup.py, to enable projects that directly
+    #   import from `distutils.core` to work with newer packaging standards.
+    # - It provides a clear error message when setuptools is not installed.
+    # - It sets `sys.argv[0]` to the underlying `setup.py`, when invoking `setup.py` so
+    #   setuptools doesn't think the script is `-c`. This avoids the following warning:
+    #     manifest_maker: standard file '-c' not found".
+    # - It generates a shim setup.py, for handling setup.cfg-only projects.
+    import os, sys, tokenize
+
+    try:
+        import setuptools
+    except ImportError as error:
+        print(
+            "ERROR: Can not execute `setup.py` since setuptools is not available in "
+            "the build environment.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    __file__ = %r
+    sys.argv[0] = __file__
+
+    if os.path.exists(__file__):
+        filename = __file__
+        with tokenize.open(__file__) as f:
+            setup_py_code = f.read()
+    else:
+        filename = ""
+        setup_py_code = "from setuptools import setup; setup()"
+
+    exec(compile(setup_py_code, filename, "exec"))
+    ''' % ({!r},), "", "exec"))
+    """
+).rstrip()
 
 
 def make_setuptools_shim_args(
-    setup_py_path,  # type: str
-    global_options=None,  # type: Sequence[str]
-    no_user_config=False,  # type: bool
-    unbuffered_output=False  # type: bool
-):
-    # type: (...) -> List[str]
+    setup_py_path: str,
+    global_options: Sequence[str] = None,
+    no_user_config: bool = False,
+    unbuffered_output: bool = False,
+) -> List[str]:
     """
     Get setuptools command arguments with shim wrapped setup file invocation.
 
@@ -48,20 +73,17 @@ def make_setuptools_shim_args(
 
 
 def make_setuptools_bdist_wheel_args(
-    setup_py_path,  # type: str
-    global_options,  # type: Sequence[str]
-    build_options,  # type: Sequence[str]
-    destination_dir,  # type: str
-):
-    # type: (...) -> List[str]
+    setup_py_path: str,
+    global_options: Sequence[str],
+    build_options: Sequence[str],
+    destination_dir: str,
+) -> List[str]:
     # NOTE: Eventually, we'd want to also -S to the flags here, when we're
     # isolating. Currently, it breaks Python in virtualenvs, because it
     # relies on site.py to find parts of the standard library outside the
     # virtualenv.
     args = make_setuptools_shim_args(
-        setup_py_path,
-        global_options=global_options,
-        unbuffered_output=True
+        setup_py_path, global_options=global_options, unbuffered_output=True
     )
     args += ["bdist_wheel", "-d", destination_dir]
     args += build_options
@@ -69,29 +91,25 @@ def make_setuptools_bdist_wheel_args(
 
 
 def make_setuptools_clean_args(
-    setup_py_path,  # type: str
-    global_options,  # type: Sequence[str]
-):
-    # type: (...) -> List[str]
+    setup_py_path: str,
+    global_options: Sequence[str],
+) -> List[str]:
     args = make_setuptools_shim_args(
-        setup_py_path,
-        global_options=global_options,
-        unbuffered_output=True
+        setup_py_path, global_options=global_options, unbuffered_output=True
     )
     args += ["clean", "--all"]
     return args
 
 
 def make_setuptools_develop_args(
-    setup_py_path,  # type: str
-    global_options,  # type: Sequence[str]
-    install_options,  # type: Sequence[str]
-    no_user_config,  # type: bool
-    prefix,  # type: Optional[str]
-    home,  # type: Optional[str]
-    use_user_site,  # type: bool
-):
-    # type: (...) -> List[str]
+    setup_py_path: str,
+    global_options: Sequence[str],
+    install_options: Sequence[str],
+    no_user_config: bool,
+    prefix: Optional[str],
+    home: Optional[str],
+    use_user_site: bool,
+) -> List[str]:
     assert not (use_user_site and prefix)
 
     args = make_setuptools_shim_args(
@@ -107,7 +125,7 @@ def make_setuptools_develop_args(
     if prefix:
         args += ["--prefix", prefix]
     if home is not None:
-        args += ["--home", home]
+        args += ["--install-dir", home]
 
     if use_user_site:
         args += ["--user", "--prefix="]
@@ -116,14 +134,11 @@ def make_setuptools_develop_args(
 
 
 def make_setuptools_egg_info_args(
-    setup_py_path,  # type: str
-    egg_info_dir,  # type: Optional[str]
-    no_user_config,  # type: bool
-):
-    # type: (...) -> List[str]
-    args = make_setuptools_shim_args(
-        setup_py_path, no_user_config=no_user_config
-    )
+    setup_py_path: str,
+    egg_info_dir: Optional[str],
+    no_user_config: bool,
+) -> List[str]:
+    args = make_setuptools_shim_args(setup_py_path, no_user_config=no_user_config)
 
     args += ["egg_info"]
 
@@ -134,19 +149,18 @@ def make_setuptools_egg_info_args(
 
 
 def make_setuptools_install_args(
-    setup_py_path,  # type: str
-    global_options,  # type: Sequence[str]
-    install_options,  # type: Sequence[str]
-    record_filename,  # type: str
-    root,  # type: Optional[str]
-    prefix,  # type: Optional[str]
-    header_dir,  # type: Optional[str]
-    home,  # type: Optional[str]
-    use_user_site,  # type: bool
-    no_user_config,  # type: bool
-    pycompile  # type: bool
-):
-    # type: (...) -> List[str]
+    setup_py_path: str,
+    global_options: Sequence[str],
+    install_options: Sequence[str],
+    record_filename: str,
+    root: Optional[str],
+    prefix: Optional[str],
+    header_dir: Optional[str],
+    home: Optional[str],
+    use_user_site: bool,
+    no_user_config: bool,
+    pycompile: bool,
+) -> List[str]:
     assert not (use_user_site and prefix)
     assert not (use_user_site and root)
 
@@ -154,7 +168,7 @@ def make_setuptools_install_args(
         setup_py_path,
         global_options=global_options,
         no_user_config=no_user_config,
-        unbuffered_output=True
+        unbuffered_output=True,
     )
     args += ["install", "--record", record_filename]
     args += ["--single-version-externally-managed"]
diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py
index f685b03b34f..b5b762418f5 100644
--- a/src/pip/_internal/utils/subprocess.py
+++ b/src/pip/_internal/utils/subprocess.py
@@ -2,29 +2,38 @@
 import os
 import shlex
 import subprocess
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Union,
+)
+
+from pip._vendor.rich.markup import escape
 
 from pip._internal.cli.spinners import SpinnerInterface, open_spinner
 from pip._internal.exceptions import InstallationSubprocessError
-from pip._internal.utils.compat import console_to_str, str_to_display
-from pip._internal.utils.logging import subprocess_logger
-from pip._internal.utils.misc import HiddenText, path_to_display
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Callable, Iterable, List, Mapping, Optional, Union
-
-    CommandArgs = List[Union[str, HiddenText]]
+from pip._internal.utils.logging import VERBOSE, subprocess_logger
+from pip._internal.utils.misc import HiddenText
 
+if TYPE_CHECKING:
+    # Literal was introduced in Python 3.8.
+    #
+    # TODO: Remove `if TYPE_CHECKING` when dropping support for Python 3.7.
+    from typing import Literal
 
-LOG_DIVIDER = '----------------------------------------'
+CommandArgs = List[Union[str, HiddenText]]
 
 
-def make_command(*args):
-    # type: (Union[str, HiddenText, CommandArgs]) -> CommandArgs
+def make_command(*args: Union[str, HiddenText, CommandArgs]) -> CommandArgs:
     """
     Create a CommandArgs object.
     """
-    command_args = []  # type: CommandArgs
+    command_args: CommandArgs = []
     for arg in args:
         # Check for list instead of CommandArgs since CommandArgs is
         # only known during type-checking.
@@ -37,8 +46,7 @@ def make_command(*args):
     return command_args
 
 
-def format_command_args(args):
-    # type: (Union[List[str], CommandArgs]) -> str
+def format_command_args(args: Union[List[str], CommandArgs]) -> str:
     """
     Format command arguments for display.
     """
@@ -47,78 +55,33 @@ def format_command_args(args):
     # this can trigger a UnicodeDecodeError in Python 2 if the argument
     # has type unicode and includes a non-ascii character.  (The type
     # checker doesn't ensure the annotations are correct in all cases.)
-    return ' '.join(
-        shlex.quote(str(arg)) if isinstance(arg, HiddenText)
-        else shlex.quote(arg) for arg in args
+    return " ".join(
+        shlex.quote(str(arg)) if isinstance(arg, HiddenText) else shlex.quote(arg)
+        for arg in args
     )
 
 
-def reveal_command_args(args):
-    # type: (Union[List[str], CommandArgs]) -> List[str]
+def reveal_command_args(args: Union[List[str], CommandArgs]) -> List[str]:
     """
     Return the arguments in their raw, unredacted form.
     """
-    return [
-        arg.secret if isinstance(arg, HiddenText) else arg for arg in args
-    ]
-
-
-def make_subprocess_output_error(
-    cmd_args,     # type: Union[List[str], CommandArgs]
-    cwd,          # type: Optional[str]
-    lines,        # type: List[str]
-    exit_status,  # type: int
-):
-    # type: (...) -> str
-    """
-    Create and return the error message to use to log a subprocess error
-    with command output.
-
-    :param lines: A list of lines, each ending with a newline.
-    """
-    command = format_command_args(cmd_args)
-    # Convert `command` and `cwd` to text (unicode in Python 2) so we can use
-    # them as arguments in the unicode format string below. This avoids
-    # "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2
-    # if either contains a non-ascii character.
-    command_display = str_to_display(command, desc='command bytes')
-    cwd_display = path_to_display(cwd)
-
-    # We know the joined output value ends in a newline.
-    output = ''.join(lines)
-    msg = (
-        # Use a unicode string to avoid "UnicodeEncodeError: 'ascii'
-        # codec can't encode character ..." in Python 2 when a format
-        # argument (e.g. `output`) has a non-ascii character.
-        'Command errored out with exit status {exit_status}:\n'
-        ' command: {command_display}\n'
-        '     cwd: {cwd_display}\n'
-        'Complete output ({line_count} lines):\n{output}{divider}'
-    ).format(
-        exit_status=exit_status,
-        command_display=command_display,
-        cwd_display=cwd_display,
-        line_count=len(lines),
-        output=output,
-        divider=LOG_DIVIDER,
-    )
-    return msg
+    return [arg.secret if isinstance(arg, HiddenText) else arg for arg in args]
 
 
 def call_subprocess(
-    cmd,  # type: Union[List[str], CommandArgs]
-    show_stdout=False,  # type: bool
-    cwd=None,  # type: Optional[str]
-    on_returncode='raise',  # type: str
-    extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
-    command_desc=None,  # type: Optional[str]
-    extra_environ=None,  # type: Optional[Mapping[str, Any]]
-    unset_environ=None,  # type: Optional[Iterable[str]]
-    spinner=None,  # type: Optional[SpinnerInterface]
-    log_failed_cmd=True,  # type: Optional[bool]
-    stdout_only=False,  # type: Optional[bool]
-):
-    # type: (...) -> str
+    cmd: Union[List[str], CommandArgs],
+    show_stdout: bool = False,
+    cwd: Optional[str] = None,
+    on_returncode: 'Literal["raise", "warn", "ignore"]' = "raise",
+    extra_ok_returncodes: Optional[Iterable[int]] = None,
+    extra_environ: Optional[Mapping[str, Any]] = None,
+    unset_environ: Optional[Iterable[str]] = None,
+    spinner: Optional[SpinnerInterface] = None,
+    log_failed_cmd: Optional[bool] = True,
+    stdout_only: Optional[bool] = False,
+    *,
+    command_desc: str,
+) -> str:
     """
     Args:
       show_stdout: if true, use INFO to log the subprocess's stderr and
@@ -156,10 +119,10 @@ def call_subprocess(
         log_subprocess = subprocess_logger.info
         used_level = logging.INFO
     else:
-        # Then log the subprocess output using DEBUG.  This also ensures
+        # Then log the subprocess output using VERBOSE.  This also ensures
         # it will be logged to the log file (aka user_log), if enabled.
-        log_subprocess = subprocess_logger.debug
-        used_level = logging.DEBUG
+        log_subprocess = subprocess_logger.verbose
+        used_level = VERBOSE
 
     # Whether the subprocess will be visible in the console.
     showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level
@@ -168,9 +131,6 @@ def call_subprocess(
     # and we have a spinner.
     use_spinner = not showing_subprocess and spinner is not None
 
-    if command_desc is None:
-        command_desc = format_command_args(cmd)
-
     log_subprocess("Running command %s", command_desc)
     env = os.environ.copy()
     if extra_environ:
@@ -186,11 +146,14 @@ def call_subprocess(
             stderr=subprocess.STDOUT if not stdout_only else subprocess.PIPE,
             cwd=cwd,
             env=env,
+            errors="backslashreplace",
         )
     except Exception as exc:
         if log_failed_cmd:
             subprocess_logger.critical(
-                "Error %s while executing command %s", exc, command_desc,
+                "Error %s while executing command %s",
+                exc,
+                command_desc,
             )
         raise
     all_output = []
@@ -200,12 +163,11 @@ def call_subprocess(
         proc.stdin.close()
         # In this mode, stdout and stderr are in the same pipe.
         while True:
-            # The "line" value is a unicode string in Python 2.
-            line = console_to_str(proc.stdout.readline())
+            line: str = proc.stdout.readline()
             if not line:
                 break
             line = line.rstrip()
-            all_output.append(line + '\n')
+            all_output.append(line + "\n")
 
             # Show the line immediately.
             log_subprocess(line)
@@ -218,25 +180,21 @@ def call_subprocess(
         finally:
             if proc.stdout:
                 proc.stdout.close()
-        output = ''.join(all_output)
+        output = "".join(all_output)
     else:
         # In this mode, stdout and stderr are in different pipes.
         # We must use communicate() which is the only safe way to read both.
-        out_bytes, err_bytes = proc.communicate()
+        out, err = proc.communicate()
         # log line by line to preserve pip log indenting
-        out = console_to_str(out_bytes)
         for out_line in out.splitlines():
             log_subprocess(out_line)
         all_output.append(out)
-        err = console_to_str(err_bytes)
         for err_line in err.splitlines():
             log_subprocess(err_line)
         all_output.append(err)
         output = out
 
-    proc_had_error = (
-        proc.returncode and proc.returncode not in extra_ok_returncodes
-    )
+    proc_had_error = proc.returncode and proc.returncode not in extra_ok_returncodes
     if use_spinner:
         assert spinner
         if proc_had_error:
@@ -244,35 +202,41 @@ def call_subprocess(
         else:
             spinner.finish("done")
     if proc_had_error:
-        if on_returncode == 'raise':
-            if not showing_subprocess and log_failed_cmd:
-                # Then the subprocess streams haven't been logged to the
-                # console yet.
-                msg = make_subprocess_output_error(
-                    cmd_args=cmd,
-                    cwd=cwd,
-                    lines=all_output,
-                    exit_status=proc.returncode,
+        if on_returncode == "raise":
+            error = InstallationSubprocessError(
+                command_description=command_desc,
+                exit_code=proc.returncode,
+                output_lines=all_output if not showing_subprocess else None,
+            )
+            if log_failed_cmd:
+                subprocess_logger.error("[present-diagnostic] %s", error)
+                subprocess_logger.verbose(
+                    "[bold magenta]full command[/]: [blue]%s[/]",
+                    escape(format_command_args(cmd)),
+                    extra={"markup": True},
                 )
-                subprocess_logger.error(msg)
-            raise InstallationSubprocessError(proc.returncode, command_desc)
-        elif on_returncode == 'warn':
+                subprocess_logger.verbose(
+                    "[bold magenta]cwd[/]: %s",
+                    escape(cwd or "[inherit]"),
+                    extra={"markup": True},
+                )
+
+            raise error
+        elif on_returncode == "warn":
             subprocess_logger.warning(
                 'Command "%s" had error code %s in %s',
                 command_desc,
                 proc.returncode,
                 cwd,
             )
-        elif on_returncode == 'ignore':
+        elif on_returncode == "ignore":
             pass
         else:
-            raise ValueError('Invalid value: on_returncode={!r}'.format(
-                             on_returncode))
+            raise ValueError(f"Invalid value: on_returncode={on_returncode!r}")
     return output
 
 
-def runner_with_spinner_message(message):
-    # type: (str) -> Callable[..., None]
+def runner_with_spinner_message(message: str) -> Callable[..., None]:
     """Provide a subprocess_runner that shows a spinner message.
 
     Intended for use with for pep517's Pep517HookCaller. Thus, the runner has
@@ -280,14 +244,14 @@ def runner_with_spinner_message(message):
     """
 
     def runner(
-        cmd,  # type: List[str]
-        cwd=None,  # type: Optional[str]
-        extra_environ=None  # type: Optional[Mapping[str, Any]]
-    ):
-        # type: (...) -> None
+        cmd: List[str],
+        cwd: Optional[str] = None,
+        extra_environ: Optional[Mapping[str, Any]] = None,
+    ) -> None:
         with open_spinner(message) as spinner:
             call_subprocess(
                 cmd,
+                command_desc=message,
                 cwd=cwd,
                 extra_environ=extra_environ,
                 spinner=spinner,
diff --git a/src/pip/_internal/utils/temp_dir.py b/src/pip/_internal/utils/temp_dir.py
index 91b277df6ec..442679a758b 100644
--- a/src/pip/_internal/utils/temp_dir.py
+++ b/src/pip/_internal/utils/temp_dir.py
@@ -3,23 +3,15 @@
 import logging
 import os.path
 import tempfile
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
+from typing import Any, Dict, Iterator, Optional, TypeVar, Union
 
-from pip._vendor.contextlib2 import ExitStack
-from pip._vendor.six import ensure_text
-
-from pip._internal.utils.compat import WINDOWS
 from pip._internal.utils.misc import enum, rmtree
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Dict, Iterator, Optional, TypeVar, Union
-
-    _T = TypeVar('_T', bound='TempDirectory')
-
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T", bound="TempDirectory")
+
 
 # Kinds of temporary directories. Only needed for ones that are
 # globally-managed.
@@ -30,12 +22,11 @@
 )
 
 
-_tempdir_manager = None  # type: Optional[ExitStack]
+_tempdir_manager: Optional[ExitStack] = None
 
 
 @contextmanager
-def global_tempdir_manager():
-    # type: () -> Iterator[None]
+def global_tempdir_manager() -> Iterator[None]:
     global _tempdir_manager
     with ExitStack() as stack:
         old_tempdir_manager, _tempdir_manager = _tempdir_manager, stack
@@ -46,34 +37,29 @@ def global_tempdir_manager():
 
 
 class TempDirectoryTypeRegistry:
-    """Manages temp directory behavior
-    """
+    """Manages temp directory behavior"""
 
-    def __init__(self):
-        # type: () -> None
-        self._should_delete = {}  # type: Dict[str, bool]
+    def __init__(self) -> None:
+        self._should_delete: Dict[str, bool] = {}
 
-    def set_delete(self, kind, value):
-        # type: (str, bool) -> None
+    def set_delete(self, kind: str, value: bool) -> None:
         """Indicate whether a TempDirectory of the given kind should be
         auto-deleted.
         """
         self._should_delete[kind] = value
 
-    def get_delete(self, kind):
-        # type: (str) -> bool
+    def get_delete(self, kind: str) -> bool:
         """Get configured auto-delete flag for a given TempDirectory type,
         default True.
         """
         return self._should_delete.get(kind, True)
 
 
-_tempdir_registry = None  # type: Optional[TempDirectoryTypeRegistry]
+_tempdir_registry: Optional[TempDirectoryTypeRegistry] = None
 
 
 @contextmanager
-def tempdir_registry():
-    # type: () -> Iterator[TempDirectoryTypeRegistry]
+def tempdir_registry() -> Iterator[TempDirectoryTypeRegistry]:
     """Provides a scoped global tempdir registry that can be used to dictate
     whether directories should be deleted.
     """
@@ -116,10 +102,10 @@ class TempDirectory:
 
     def __init__(
         self,
-        path=None,    # type: Optional[str]
-        delete=_default,  # type: Union[bool, None, _Default]
-        kind="temp",  # type: str
-        globally_managed=False,  # type: bool
+        path: Optional[str] = None,
+        delete: Union[bool, None, _Default] = _default,
+        kind: str = "temp",
+        globally_managed: bool = False,
     ):
         super().__init__()
 
@@ -148,23 +134,17 @@ def __init__(
             _tempdir_manager.enter_context(self)
 
     @property
-    def path(self):
-        # type: () -> str
-        assert not self._deleted, (
-            f"Attempted to access deleted path: {self._path}"
-        )
+    def path(self) -> str:
+        assert not self._deleted, f"Attempted to access deleted path: {self._path}"
         return self._path
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         return f"<{self.__class__.__name__} {self.path!r}>"
 
-    def __enter__(self):
-        # type: (_T) -> _T
+    def __enter__(self: _T) -> _T:
         return self
 
-    def __exit__(self, exc, value, tb):
-        # type: (Any, Any, Any) -> None
+    def __exit__(self, exc: Any, value: Any, tb: Any) -> None:
         if self.delete is not None:
             delete = self.delete
         elif _tempdir_registry:
@@ -175,36 +155,22 @@ def __exit__(self, exc, value, tb):
         if delete:
             self.cleanup()
 
-    def _create(self, kind):
-        # type: (str) -> str
-        """Create a temporary directory and store its path in self.path
-        """
+    def _create(self, kind: str) -> str:
+        """Create a temporary directory and store its path in self.path"""
         # We realpath here because some systems have their default tmpdir
         # symlinked to another directory.  This tends to confuse build
         # scripts, so we canonicalize the path by traversing potential
         # symlinks here.
-        path = os.path.realpath(
-            tempfile.mkdtemp(prefix=f"pip-{kind}-")
-        )
+        path = os.path.realpath(tempfile.mkdtemp(prefix=f"pip-{kind}-"))
         logger.debug("Created temporary directory: %s", path)
         return path
 
-    def cleanup(self):
-        # type: () -> None
-        """Remove the temporary directory created and reset state
-        """
+    def cleanup(self) -> None:
+        """Remove the temporary directory created and reset state"""
         self._deleted = True
         if not os.path.exists(self._path):
             return
-        # Make sure to pass unicode on Python 2 to make the contents also
-        # use unicode, ensuring non-ASCII names and can be represented.
-        # This is only done on Windows because POSIX platforms use bytes
-        # natively for paths, and the bytes-text conversion omission avoids
-        # errors caused by the environment configuring encodings incorrectly.
-        if WINDOWS:
-            rmtree(ensure_text(self._path))
-        else:
-            rmtree(self._path)
+        rmtree(self._path)
 
 
 class AdjacentTempDirectory(TempDirectory):
@@ -221,6 +187,7 @@ class AdjacentTempDirectory(TempDirectory):
             (when used as a contextmanager)
 
     """
+
     # The characters that may be used to name the temp directory
     # We always prepend a ~ and then rotate through these until
     # a usable name is found.
@@ -228,14 +195,12 @@ class AdjacentTempDirectory(TempDirectory):
     # with leading '-' and invalid metadata
     LEADING_CHARS = "-~.=%0123456789"
 
-    def __init__(self, original, delete=None):
-        # type: (str, Optional[bool]) -> None
-        self.original = original.rstrip('/\\')
+    def __init__(self, original: str, delete: Optional[bool] = None) -> None:
+        self.original = original.rstrip("/\\")
         super().__init__(delete=delete)
 
     @classmethod
-    def _generate_names(cls, name):
-        # type: (str) -> Iterator[str]
+    def _generate_names(cls, name: str) -> Iterator[str]:
         """Generates a series of temporary names.
 
         The algorithm replaces the leading characters in the name
@@ -245,21 +210,22 @@ def _generate_names(cls, name):
         """
         for i in range(1, len(name)):
             for candidate in itertools.combinations_with_replacement(
-                    cls.LEADING_CHARS, i - 1):
-                new_name = '~' + ''.join(candidate) + name[i:]
+                cls.LEADING_CHARS, i - 1
+            ):
+                new_name = "~" + "".join(candidate) + name[i:]
                 if new_name != name:
                     yield new_name
 
         # If we make it this far, we will have to make a longer name
         for i in range(len(cls.LEADING_CHARS)):
             for candidate in itertools.combinations_with_replacement(
-                    cls.LEADING_CHARS, i):
-                new_name = '~' + ''.join(candidate) + name
+                cls.LEADING_CHARS, i
+            ):
+                new_name = "~" + "".join(candidate) + name
                 if new_name != name:
                     yield new_name
 
-    def _create(self, kind):
-        # type: (str) -> str
+    def _create(self, kind: str) -> str:
         root, name = os.path.split(self.original)
         for candidate in self._generate_names(name):
             path = os.path.join(root, candidate)
@@ -274,9 +240,7 @@ def _create(self, kind):
                 break
         else:
             # Final fallback on the default behavior.
-            path = os.path.realpath(
-                tempfile.mkdtemp(prefix=f"pip-{kind}-")
-            )
+            path = os.path.realpath(tempfile.mkdtemp(prefix=f"pip-{kind}-"))
 
         logger.debug("Created temporary directory: %s", path)
         return path
diff --git a/src/pip/_internal/utils/typing.py b/src/pip/_internal/utils/typing.py
deleted file mode 100644
index 8505a29b15d..00000000000
--- a/src/pip/_internal/utils/typing.py
+++ /dev/null
@@ -1,38 +0,0 @@
-"""For neatly implementing static typing in pip.
-
-`mypy` - the static type analysis tool we use - uses the `typing` module, which
-provides core functionality fundamental to mypy's functioning.
-
-Generally, `typing` would be imported at runtime and used in that fashion -
-it acts as a no-op at runtime and does not have any run-time overhead by
-design.
-
-As it turns out, `typing` is not vendorable - it uses separate sources for
-Python 2/Python 3. Thus, this codebase can not expect it to be present.
-To work around this, mypy allows the typing import to be behind a False-y
-optional to prevent it from running at runtime and type-comments can be used
-to remove the need for the types to be accessible directly during runtime.
-
-This module provides the False-y guard in a nicely named fashion so that a
-curious maintainer can reach here to read this.
-
-In pip, all static-typing related imports should be guarded as follows:
-
-    from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-    if MYPY_CHECK_RUNNING:
-        from typing import ...
-
-Ref: https://github.com/python/mypy/issues/3216
-"""
-
-MYPY_CHECK_RUNNING = False
-
-
-if MYPY_CHECK_RUNNING:
-    from typing import cast
-else:
-    # typing's cast() is needed at runtime, but we don't want to import typing.
-    # Thus, we use a dummy no-op version, which we tell mypy to ignore.
-    def cast(type_, value):  # type: ignore
-        return value
diff --git a/src/pip/_internal/utils/unpacking.py b/src/pip/_internal/utils/unpacking.py
index a24d7e55735..5f63f974b95 100644
--- a/src/pip/_internal/utils/unpacking.py
+++ b/src/pip/_internal/utils/unpacking.py
@@ -7,6 +7,8 @@
 import stat
 import tarfile
 import zipfile
+from typing import Iterable, List, Optional
+from zipfile import ZipInfo
 
 from pip._internal.exceptions import InstallationError
 from pip._internal.utils.filetypes import (
@@ -16,12 +18,6 @@
     ZIP_EXTENSIONS,
 )
 from pip._internal.utils.misc import ensure_dir
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import Iterable, List, Optional
-    from zipfile import ZipInfo
-
 
 logger = logging.getLogger(__name__)
 
@@ -30,44 +26,40 @@
 
 try:
     import bz2  # noqa
+
     SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS
 except ImportError:
-    logger.debug('bz2 module is not available')
+    logger.debug("bz2 module is not available")
 
 try:
     # Only for Python 3.3+
     import lzma  # noqa
+
     SUPPORTED_EXTENSIONS += XZ_EXTENSIONS
 except ImportError:
-    logger.debug('lzma module is not available')
+    logger.debug("lzma module is not available")
 
 
-def current_umask():
-    # type: () -> int
+def current_umask() -> int:
     """Get the current umask which involves having to set it temporarily."""
     mask = os.umask(0)
     os.umask(mask)
     return mask
 
 
-def split_leading_dir(path):
-    # type: (str) -> List[str]
-    path = path.lstrip('/').lstrip('\\')
-    if (
-        '/' in path and (
-            ('\\' in path and path.find('/') < path.find('\\')) or
-            '\\' not in path
-        )
+def split_leading_dir(path: str) -> List[str]:
+    path = path.lstrip("/").lstrip("\\")
+    if "/" in path and (
+        ("\\" in path and path.find("/") < path.find("\\")) or "\\" not in path
     ):
-        return path.split('/', 1)
-    elif '\\' in path:
-        return path.split('\\', 1)
+        return path.split("/", 1)
+    elif "\\" in path:
+        return path.split("\\", 1)
     else:
-        return [path, '']
+        return [path, ""]
 
 
-def has_leading_dir(paths):
-    # type: (Iterable[str]) -> bool
+def has_leading_dir(paths: Iterable[str]) -> bool:
     """Returns true if all the paths have the same leading path name
     (i.e., everything is in one subdirectory in an archive)"""
     common_prefix = None
@@ -82,8 +74,7 @@ def has_leading_dir(paths):
     return True
 
 
-def is_within_directory(directory, target):
-    # type: (str, str) -> bool
+def is_within_directory(directory: str, target: str) -> bool:
     """
     Return true if the absolute path of target is within the directory
     """
@@ -94,8 +85,7 @@ def is_within_directory(directory, target):
     return prefix == abs_directory
 
 
-def set_extracted_file_to_default_mode_plus_executable(path):
-    # type: (str) -> None
+def set_extracted_file_to_default_mode_plus_executable(path: str) -> None:
     """
     Make file present at path have execute for user/group/world
     (chmod +x) is no-op on windows per python docs
@@ -103,16 +93,14 @@ def set_extracted_file_to_default_mode_plus_executable(path):
     os.chmod(path, (0o777 & ~current_umask() | 0o111))
 
 
-def zip_item_is_executable(info):
-    # type: (ZipInfo) -> bool
+def zip_item_is_executable(info: ZipInfo) -> bool:
     mode = info.external_attr >> 16
     # if mode and regular file and any execute permissions for
     # user/group/world?
     return bool(mode and stat.S_ISREG(mode) and mode & 0o111)
 
 
-def unzip_file(filename, location, flatten=True):
-    # type: (str, str, bool) -> None
+def unzip_file(filename: str, location: str, flatten: bool = True) -> None:
     """
     Unzip the file (with path `filename`) to the destination `location`.  All
     files are written based on system defaults and umask (i.e. permissions are
@@ -122,7 +110,7 @@ def unzip_file(filename, location, flatten=True):
     no-ops per the python docs.
     """
     ensure_dir(location)
-    zipfp = open(filename, 'rb')
+    zipfp = open(filename, "rb")
     try:
         zip = zipfile.ZipFile(zipfp, allowZip64=True)
         leading = has_leading_dir(zip.namelist()) and flatten
@@ -135,11 +123,11 @@ def unzip_file(filename, location, flatten=True):
             dir = os.path.dirname(fn)
             if not is_within_directory(location, fn):
                 message = (
-                    'The zip file ({}) has a file ({}) trying to install '
-                    'outside target directory ({})'
+                    "The zip file ({}) has a file ({}) trying to install "
+                    "outside target directory ({})"
                 )
                 raise InstallationError(message.format(filename, fn, location))
-            if fn.endswith('/') or fn.endswith('\\'):
+            if fn.endswith("/") or fn.endswith("\\"):
                 # A directory
                 ensure_dir(fn)
             else:
@@ -148,7 +136,7 @@ def unzip_file(filename, location, flatten=True):
                 # chunk of memory for the file's content
                 fp = zip.open(name)
                 try:
-                    with open(fn, 'wb') as destfp:
+                    with open(fn, "wb") as destfp:
                         shutil.copyfileobj(fp, destfp)
                 finally:
                     fp.close()
@@ -158,8 +146,7 @@ def unzip_file(filename, location, flatten=True):
         zipfp.close()
 
 
-def untar_file(filename, location):
-    # type: (str, str) -> None
+def untar_file(filename: str, location: str) -> None:
     """
     Untar the file (with path `filename`) to the destination `location`.
     All files are written based on system defaults and umask (i.e. permissions
@@ -169,24 +156,23 @@ def untar_file(filename, location):
     no-ops per the python docs.
     """
     ensure_dir(location)
-    if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
-        mode = 'r:gz'
+    if filename.lower().endswith(".gz") or filename.lower().endswith(".tgz"):
+        mode = "r:gz"
     elif filename.lower().endswith(BZ2_EXTENSIONS):
-        mode = 'r:bz2'
+        mode = "r:bz2"
     elif filename.lower().endswith(XZ_EXTENSIONS):
-        mode = 'r:xz'
-    elif filename.lower().endswith('.tar'):
-        mode = 'r'
+        mode = "r:xz"
+    elif filename.lower().endswith(".tar"):
+        mode = "r"
     else:
         logger.warning(
-            'Cannot determine compression type for file %s', filename,
+            "Cannot determine compression type for file %s",
+            filename,
         )
-        mode = 'r:*'
-    tar = tarfile.open(filename, mode)
+        mode = "r:*"
+    tar = tarfile.open(filename, mode, encoding="utf-8")
     try:
-        leading = has_leading_dir([
-            member.name for member in tar.getmembers()
-        ])
+        leading = has_leading_dir([member.name for member in tar.getmembers()])
         for member in tar.getmembers():
             fn = member.name
             if leading:
@@ -194,12 +180,10 @@ def untar_file(filename, location):
             path = os.path.join(location, fn)
             if not is_within_directory(location, path):
                 message = (
-                    'The tar file ({}) has a file ({}) trying to install '
-                    'outside target directory ({})'
-                )
-                raise InstallationError(
-                    message.format(filename, path, location)
+                    "The tar file ({}) has a file ({}) trying to install "
+                    "outside target directory ({})"
                 )
+                raise InstallationError(message.format(filename, path, location))
             if member.isdir():
                 ensure_dir(path)
             elif member.issym():
@@ -210,8 +194,10 @@ def untar_file(filename, location):
                     # Some corrupt tar files seem to produce this
                     # (specifically bad symlinks)
                     logger.warning(
-                        'In the tar file %s the member %s is invalid: %s',
-                        filename, member.name, exc,
+                        "In the tar file %s the member %s is invalid: %s",
+                        filename,
+                        member.name,
+                        exc,
                     )
                     continue
             else:
@@ -221,13 +207,15 @@ def untar_file(filename, location):
                     # Some corrupt tar files seem to produce this
                     # (specifically bad symlinks)
                     logger.warning(
-                        'In the tar file %s the member %s is invalid: %s',
-                        filename, member.name, exc,
+                        "In the tar file %s the member %s is invalid: %s",
+                        filename,
+                        member.name,
+                        exc,
                     )
                     continue
                 ensure_dir(os.path.dirname(path))
                 assert fp is not None
-                with open(path, 'wb') as destfp:
+                with open(path, "wb") as destfp:
                     shutil.copyfileobj(fp, destfp)
                 fp.close()
                 # Update the timestamp (useful for cython compiled files)
@@ -240,38 +228,31 @@ def untar_file(filename, location):
 
 
 def unpack_file(
-        filename,  # type: str
-        location,  # type: str
-        content_type=None,  # type: Optional[str]
-):
-    # type: (...) -> None
+    filename: str,
+    location: str,
+    content_type: Optional[str] = None,
+) -> None:
     filename = os.path.realpath(filename)
     if (
-        content_type == 'application/zip' or
-        filename.lower().endswith(ZIP_EXTENSIONS) or
-        zipfile.is_zipfile(filename)
+        content_type == "application/zip"
+        or filename.lower().endswith(ZIP_EXTENSIONS)
+        or zipfile.is_zipfile(filename)
     ):
-        unzip_file(
-            filename,
-            location,
-            flatten=not filename.endswith('.whl')
-        )
+        unzip_file(filename, location, flatten=not filename.endswith(".whl"))
     elif (
-        content_type == 'application/x-gzip' or
-        tarfile.is_tarfile(filename) or
-        filename.lower().endswith(
-            TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS
-        )
+        content_type == "application/x-gzip"
+        or tarfile.is_tarfile(filename)
+        or filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)
     ):
         untar_file(filename, location)
     else:
         # FIXME: handle?
         # FIXME: magic signatures?
         logger.critical(
-            'Cannot unpack file %s (downloaded from %s, content-type: %s); '
-            'cannot detect archive format',
-            filename, location, content_type,
-        )
-        raise InstallationError(
-            f'Cannot determine archive format of {location}'
+            "Cannot unpack file %s (downloaded from %s, content-type: %s); "
+            "cannot detect archive format",
+            filename,
+            location,
+            content_type,
         )
+        raise InstallationError(f"Cannot determine archive format of {location}")
diff --git a/src/pip/_internal/utils/urls.py b/src/pip/_internal/utils/urls.py
index 0ef063c2198..6ba2e04f350 100644
--- a/src/pip/_internal/utils/urls.py
+++ b/src/pip/_internal/utils/urls.py
@@ -1,54 +1,62 @@
 import os
-import sys
+import string
 import urllib.parse
 import urllib.request
+from typing import Optional
 
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from .compat import WINDOWS
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional
 
-
-def get_url_scheme(url):
-    # type: (str) -> Optional[str]
-    if ':' not in url:
+def get_url_scheme(url: str) -> Optional[str]:
+    if ":" not in url:
         return None
-    return url.split(':', 1)[0].lower()
+    return url.split(":", 1)[0].lower()
 
 
-def path_to_url(path):
-    # type: (str) -> str
+def path_to_url(path: str) -> str:
     """
     Convert a path to a file: URL.  The path will be made absolute and have
     quoted path parts.
     """
     path = os.path.normpath(os.path.abspath(path))
-    url = urllib.parse.urljoin('file:', urllib.request.pathname2url(path))
+    url = urllib.parse.urljoin("file:", urllib.request.pathname2url(path))
     return url
 
 
-def url_to_path(url):
-    # type: (str) -> str
+def url_to_path(url: str) -> str:
     """
     Convert a file: URL to a path.
     """
-    assert url.startswith('file:'), (
-        "You can only turn file: urls into filenames (not {url!r})"
-        .format(**locals()))
+    assert url.startswith(
+        "file:"
+    ), f"You can only turn file: urls into filenames (not {url!r})"
 
     _, netloc, path, _, _ = urllib.parse.urlsplit(url)
 
-    if not netloc or netloc == 'localhost':
+    if not netloc or netloc == "localhost":
         # According to RFC 8089, same as empty authority.
-        netloc = ''
-    elif sys.platform == 'win32':
+        netloc = ""
+    elif WINDOWS:
         # If we have a UNC path, prepend UNC share notation.
-        netloc = '\\\\' + netloc
+        netloc = "\\\\" + netloc
     else:
         raise ValueError(
-            'non-local file URIs are not supported on this platform: {url!r}'
-            .format(**locals())
+            f"non-local file URIs are not supported on this platform: {url!r}"
         )
 
     path = urllib.request.url2pathname(netloc + path)
+
+    # On Windows, urlsplit parses the path as something like "/C:/Users/foo".
+    # This creates issues for path-related functions like io.open(), so we try
+    # to detect and strip the leading slash.
+    if (
+        WINDOWS
+        and not netloc  # Not UNC.
+        and len(path) >= 3
+        and path[0] == "/"  # Leading slash to strip.
+        and path[1] in string.ascii_letters  # Drive letter.
+        and path[2:4] in (":", ":/")  # Colon + end of string, or colon + absolute path.
+    ):
+        path = path[1:]
+
     return path
diff --git a/src/pip/_internal/utils/virtualenv.py b/src/pip/_internal/utils/virtualenv.py
index 3086bf2fc8d..c926db4c332 100644
--- a/src/pip/_internal/utils/virtualenv.py
+++ b/src/pip/_internal/utils/virtualenv.py
@@ -3,11 +3,7 @@
 import re
 import site
 import sys
-
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from typing import List, Optional
+from typing import List, Optional
 
 logger = logging.getLogger(__name__)
 _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile(
@@ -15,8 +11,7 @@
 )
 
 
-def _running_under_venv():
-    # type: () -> bool
+def _running_under_venv() -> bool:
     """Checks if sys.base_prefix and sys.prefix match.
 
     This handles PEP 405 compliant virtual environments.
@@ -24,41 +19,36 @@ def _running_under_venv():
     return sys.prefix != getattr(sys, "base_prefix", sys.prefix)
 
 
-def _running_under_regular_virtualenv():
-    # type: () -> bool
+def _running_under_regular_virtualenv() -> bool:
     """Checks if sys.real_prefix is set.
 
     This handles virtual environments created with pypa's virtualenv.
     """
     # pypa/virtualenv case
-    return hasattr(sys, 'real_prefix')
+    return hasattr(sys, "real_prefix")
 
 
-def running_under_virtualenv():
-    # type: () -> bool
-    """Return True if we're running inside a virtualenv, False otherwise.
-    """
+def running_under_virtualenv() -> bool:
+    """Return True if we're running inside a virtualenv, False otherwise."""
     return _running_under_venv() or _running_under_regular_virtualenv()
 
 
-def _get_pyvenv_cfg_lines():
-    # type: () -> Optional[List[str]]
+def _get_pyvenv_cfg_lines() -> Optional[List[str]]:
     """Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines
 
     Returns None, if it could not read/access the file.
     """
-    pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg')
+    pyvenv_cfg_file = os.path.join(sys.prefix, "pyvenv.cfg")
     try:
         # Although PEP 405 does not specify, the built-in venv module always
         # writes with UTF-8. (pypa/pip#8717)
-        with open(pyvenv_cfg_file, encoding='utf-8') as f:
+        with open(pyvenv_cfg_file, encoding="utf-8") as f:
             return f.read().splitlines()  # avoids trailing newlines
     except OSError:
         return None
 
 
-def _no_global_under_venv():
-    # type: () -> bool
+def _no_global_under_venv() -> bool:
     """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion
 
     PEP 405 specifies that when system site-packages are not supposed to be
@@ -82,13 +72,12 @@ def _no_global_under_venv():
 
     for line in cfg_lines:
         match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line)
-        if match is not None and match.group('value') == 'false':
+        if match is not None and match.group("value") == "false":
             return True
     return False
 
 
-def _no_global_under_regular_virtualenv():
-    # type: () -> bool
+def _no_global_under_regular_virtualenv() -> bool:
     """Check if "no-global-site-packages.txt" exists beside site.py
 
     This mirrors logic in pypa/virtualenv for determining whether system
@@ -96,15 +85,14 @@ def _no_global_under_regular_virtualenv():
     """
     site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
     no_global_site_packages_file = os.path.join(
-        site_mod_dir, 'no-global-site-packages.txt',
+        site_mod_dir,
+        "no-global-site-packages.txt",
     )
     return os.path.exists(no_global_site_packages_file)
 
 
-def virtualenv_no_global():
-    # type: () -> bool
-    """Returns a boolean, whether running in venv with no system site-packages.
-    """
+def virtualenv_no_global() -> bool:
+    """Returns a boolean, whether running in venv with no system site-packages."""
     # PEP 405 compliance needs to be checked first since virtualenv >=20 would
     # return True for both checks, but is only able to use the PEP 405 config.
     if _running_under_venv():
diff --git a/src/pip/_internal/utils/wheel.py b/src/pip/_internal/utils/wheel.py
index e8d9c4bc154..e5e3f34ed81 100644
--- a/src/pip/_internal/utils/wheel.py
+++ b/src/pip/_internal/utils/wheel.py
@@ -2,23 +2,14 @@
 """
 
 import logging
+from email.message import Message
 from email.parser import Parser
+from typing import Tuple
 from zipfile import BadZipFile, ZipFile
 
 from pip._vendor.packaging.utils import canonicalize_name
-from pip._vendor.pkg_resources import DistInfoDistribution
-from pip._vendor.six import ensure_str
 
 from pip._internal.exceptions import UnsupportedWheel
-from pip._internal.utils.pkg_resources import DictMetadata
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-
-if MYPY_CHECK_RUNNING:
-    from email.message import Message
-    from typing import Dict, Tuple
-
-    from pip._vendor.pkg_resources import Distribution
-
 
 VERSION_COMPATIBLE = (1, 0)
 
@@ -26,67 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
-class WheelMetadata(DictMetadata):
-    """Metadata provider that maps metadata decoding exceptions to our
-    internal exception type.
-    """
-    def __init__(self, metadata, wheel_name):
-        # type: (Dict[str, bytes], str) -> None
-        super().__init__(metadata)
-        self._wheel_name = wheel_name
-
-    def get_metadata(self, name):
-        # type: (str) -> str
-        try:
-            return super().get_metadata(name)
-        except UnicodeDecodeError as e:
-            # Augment the default error with the origin of the file.
-            raise UnsupportedWheel(
-                "Error decoding metadata for {}: {}".format(
-                    self._wheel_name, e
-                )
-            )
-
-
-def pkg_resources_distribution_for_wheel(wheel_zip, name, location):
-    # type: (ZipFile, str, str) -> Distribution
-    """Get a pkg_resources distribution given a wheel.
-
-    :raises UnsupportedWheel: on any errors
-    """
-    info_dir, _ = parse_wheel(wheel_zip, name)
-
-    metadata_files = [
-        p for p in wheel_zip.namelist() if p.startswith(f"{info_dir}/")
-    ]
-
-    metadata_text = {}  # type: Dict[str, bytes]
-    for path in metadata_files:
-        # If a flag is set, namelist entries may be unicode in Python 2.
-        # We coerce them to native str type to match the types used in the rest
-        # of the code. This cannot fail because unicode can always be encoded
-        # with UTF-8.
-        full_path = ensure_str(path)
-        _, metadata_name = full_path.split("/", 1)
-
-        try:
-            metadata_text[metadata_name] = read_wheel_metadata_file(
-                wheel_zip, full_path
-            )
-        except UnsupportedWheel as e:
-            raise UnsupportedWheel(
-                "{} has an invalid wheel, {}".format(name, str(e))
-            )
-
-    metadata = WheelMetadata(metadata_text, location)
-
-    return DistInfoDistribution(
-        location=location, metadata=metadata, project_name=name
-    )
-
-
-def parse_wheel(wheel_zip, name):
-    # type: (ZipFile, str) -> Tuple[str, Message]
+def parse_wheel(wheel_zip: ZipFile, name: str) -> Tuple[str, Message]:
     """Extract information from the provided wheel, ensuring it meets basic
     standards.
 
@@ -97,17 +28,14 @@ def parse_wheel(wheel_zip, name):
         metadata = wheel_metadata(wheel_zip, info_dir)
         version = wheel_version(metadata)
     except UnsupportedWheel as e:
-        raise UnsupportedWheel(
-            "{} has an invalid wheel, {}".format(name, str(e))
-        )
+        raise UnsupportedWheel("{} has an invalid wheel, {}".format(name, str(e)))
 
     check_compatibility(version, name)
 
     return info_dir, metadata
 
 
-def wheel_dist_info_dir(source, name):
-    # type: (ZipFile, str) -> str
+def wheel_dist_info_dir(source: ZipFile, name: str) -> str:
     """Returns the name of the contained .dist-info directory.
 
     Raises AssertionError or UnsupportedWheel if not found, >1 found, or
@@ -116,16 +44,14 @@ def wheel_dist_info_dir(source, name):
     # Zip file path separators must be /
     subdirs = {p.split("/", 1)[0] for p in source.namelist()}
 
-    info_dirs = [s for s in subdirs if s.endswith('.dist-info')]
+    info_dirs = [s for s in subdirs if s.endswith(".dist-info")]
 
     if not info_dirs:
         raise UnsupportedWheel(".dist-info directory not found")
 
     if len(info_dirs) > 1:
         raise UnsupportedWheel(
-            "multiple .dist-info directories found: {}".format(
-                ", ".join(info_dirs)
-            )
+            "multiple .dist-info directories found: {}".format(", ".join(info_dirs))
         )
 
     info_dir = info_dirs[0]
@@ -139,25 +65,19 @@ def wheel_dist_info_dir(source, name):
             )
         )
 
-    # Zip file paths can be unicode or str depending on the zip entry flags,
-    # so normalize it.
-    return ensure_str(info_dir)
+    return info_dir
 
 
-def read_wheel_metadata_file(source, path):
-    # type: (ZipFile, str) -> bytes
+def read_wheel_metadata_file(source: ZipFile, path: str) -> bytes:
     try:
         return source.read(path)
         # BadZipFile for general corruption, KeyError for missing entry,
         # and RuntimeError for password-protected files
     except (BadZipFile, KeyError, RuntimeError) as e:
-        raise UnsupportedWheel(
-            f"could not read {path!r} file: {e!r}"
-        )
+        raise UnsupportedWheel(f"could not read {path!r} file: {e!r}")
 
 
-def wheel_metadata(source, dist_info_dir):
-    # type: (ZipFile, str) -> Message
+def wheel_metadata(source: ZipFile, dist_info_dir: str) -> Message:
     """Return the WHEEL metadata of an extracted wheel, if possible.
     Otherwise, raise UnsupportedWheel.
     """
@@ -166,7 +86,7 @@ def wheel_metadata(source, dist_info_dir):
     wheel_contents = read_wheel_metadata_file(source, path)
 
     try:
-        wheel_text = ensure_str(wheel_contents)
+        wheel_text = wheel_contents.decode()
     except UnicodeDecodeError as e:
         raise UnsupportedWheel(f"error decoding {path!r}: {e!r}")
 
@@ -176,8 +96,7 @@ def wheel_metadata(source, dist_info_dir):
     return Parser().parsestr(wheel_text)
 
 
-def wheel_version(wheel_data):
-    # type: (Message) -> Tuple[int, ...]
+def wheel_version(wheel_data: Message) -> Tuple[int, ...]:
     """Given WHEEL metadata, return the parsed Wheel-Version.
     Otherwise, raise UnsupportedWheel.
     """
@@ -188,13 +107,12 @@ def wheel_version(wheel_data):
     version = version_text.strip()
 
     try:
-        return tuple(map(int, version.split('.')))
+        return tuple(map(int, version.split(".")))
     except ValueError:
         raise UnsupportedWheel(f"invalid Wheel-Version: {version!r}")
 
 
-def check_compatibility(version, name):
-    # type: (Tuple[int, ...], str) -> None
+def check_compatibility(version: Tuple[int, ...], name: str) -> None:
     """Raises errors or warns if called with an incompatible Wheel-Version.
 
     pip should refuse to install a Wheel-Version that's a major series
@@ -209,10 +127,10 @@ def check_compatibility(version, name):
     if version[0] > VERSION_COMPATIBLE[0]:
         raise UnsupportedWheel(
             "{}'s Wheel-Version ({}) is not compatible with this version "
-            "of pip".format(name, '.'.join(map(str, version)))
+            "of pip".format(name, ".".join(map(str, version)))
         )
     elif version > VERSION_COMPATIBLE:
         logger.warning(
-            'Installing from a newer Wheel-Version (%s)',
-            '.'.join(map(str, version)),
+            "Installing from a newer Wheel-Version (%s)",
+            ".".join(map(str, version)),
         )
diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py
index 2a4eb137576..b6beddbe6d2 100644
--- a/src/pip/_internal/vcs/__init__.py
+++ b/src/pip/_internal/vcs/__init__.py
@@ -1,7 +1,6 @@
 # Expose a limited set of classes and functions so callers outside of
 # the vcs package don't need to import deeper than `pip._internal.vcs`.
-# (The test directory and imports protected by MYPY_CHECK_RUNNING may
-# still need to import from a vcs sub-package.)
+# (The test directory may still need to import from a vcs sub-package.)
 # Import all vcs modules to register each VCS in the VcsSupport object.
 import pip._internal.vcs.bazaar
 import pip._internal.vcs.git
@@ -9,6 +8,7 @@
 import pip._internal.vcs.subversion  # noqa: F401
 from pip._internal.vcs.versioncontrol import (  # noqa: F401
     RemoteNotFoundError,
+    RemoteNotValidError,
     is_url,
     make_vcs_requirement_url,
     vcs,
diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py
index 3c9c8a47930..a7b16e2e052 100644
--- a/src/pip/_internal/vcs/bazaar.py
+++ b/src/pip/_internal/vcs/bazaar.py
@@ -1,95 +1,80 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
-import os
+from typing import List, Optional, Tuple
 
-from pip._internal.utils.misc import display_path, rmtree
+from pip._internal.utils.misc import HiddenText, display_path
 from pip._internal.utils.subprocess import make_command
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import path_to_url
-from pip._internal.vcs.versioncontrol import RemoteNotFoundError, VersionControl, vcs
-
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, Tuple
-
-    from pip._internal.utils.misc import HiddenText
-    from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions
-
+from pip._internal.vcs.versioncontrol import (
+    AuthInfo,
+    RemoteNotFoundError,
+    RevOptions,
+    VersionControl,
+    vcs,
+)
 
 logger = logging.getLogger(__name__)
 
 
 class Bazaar(VersionControl):
-    name = 'bzr'
-    dirname = '.bzr'
-    repo_name = 'branch'
+    name = "bzr"
+    dirname = ".bzr"
+    repo_name = "branch"
     schemes = (
-        'bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp',
-        'bzr+lp',
+        "bzr+http",
+        "bzr+https",
+        "bzr+ssh",
+        "bzr+sftp",
+        "bzr+ftp",
+        "bzr+lp",
+        "bzr+file",
     )
 
     @staticmethod
-    def get_base_rev_args(rev):
-        return ['-r', rev]
-
-    def export(self, location, url):
-        # type: (str, HiddenText) -> None
-        """
-        Export the Bazaar repository at the url to the destination location
-        """
-        # Remove the location to make sure Bazaar can export it correctly
-        if os.path.exists(location):
-            rmtree(location)
+    def get_base_rev_args(rev: str) -> List[str]:
+        return ["-r", rev]
 
-        url, rev_options = self.get_url_rev_options(url)
-        self.run_command(
-            make_command('export', location, url, rev_options.to_args()),
-            show_stdout=False,
-        )
-
-    def fetch_new(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def fetch_new(
+        self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
+    ) -> None:
         rev_display = rev_options.to_display()
         logger.info(
-            'Checking out %s%s to %s',
+            "Checking out %s%s to %s",
             url,
             rev_display,
             display_path(dest),
         )
-        cmd_args = (
-            make_command('branch', '-q', rev_options.to_args(), url, dest)
-        )
+        if verbosity <= 0:
+            flag = "--quiet"
+        elif verbosity == 1:
+            flag = ""
+        else:
+            flag = f"-{'v'*verbosity}"
+        cmd_args = make_command("branch", flag, rev_options.to_args(), url, dest)
         self.run_command(cmd_args)
 
-    def switch(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
-        self.run_command(make_command('switch', url), cwd=dest)
+    def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
+        self.run_command(make_command("switch", url), cwd=dest)
 
-    def update(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
-        cmd_args = make_command('pull', '-q', rev_options.to_args())
+    def update(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
+        cmd_args = make_command("pull", "-q", rev_options.to_args())
         self.run_command(cmd_args, cwd=dest)
 
     @classmethod
-    def get_url_rev_and_auth(cls, url):
-        # type: (str) -> Tuple[str, Optional[str], AuthInfo]
+    def get_url_rev_and_auth(cls, url: str) -> Tuple[str, Optional[str], AuthInfo]:
         # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it
         url, rev, user_pass = super().get_url_rev_and_auth(url)
-        if url.startswith('ssh://'):
-            url = 'bzr+' + url
+        if url.startswith("ssh://"):
+            url = "bzr+" + url
         return url, rev, user_pass
 
     @classmethod
-    def get_remote_url(cls, location):
-        # type: (str) -> str
+    def get_remote_url(cls, location: str) -> str:
         urls = cls.run_command(
-            ['info'], show_stdout=False, stdout_only=True, cwd=location
+            ["info"], show_stdout=False, stdout_only=True, cwd=location
         )
         for line in urls.splitlines():
             line = line.strip()
-            for x in ('checkout of branch: ',
-                      'parent branch: '):
+            for x in ("checkout of branch: ", "parent branch: "):
                 if line.startswith(x):
                     repo = line.split(x)[1]
                     if cls._is_local_repository(repo):
@@ -98,15 +83,17 @@ def get_remote_url(cls, location):
         raise RemoteNotFoundError
 
     @classmethod
-    def get_revision(cls, location):
-        # type: (str) -> str
+    def get_revision(cls, location: str) -> str:
         revision = cls.run_command(
-            ['revno'], show_stdout=False, stdout_only=True, cwd=location,
+            ["revno"],
+            show_stdout=False,
+            stdout_only=True,
+            cwd=location,
         )
         return revision.splitlines()[-1]
 
     @classmethod
-    def is_commit_id_equal(cls, dest, name):
+    def is_commit_id_equal(cls, dest: str, name: Optional[str]) -> bool:
         """Always assume the versions don't match"""
         return False
 
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index a2a1b5f0394..8d1d4993767 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -1,33 +1,24 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
 import os.path
+import pathlib
 import re
 import urllib.parse
 import urllib.request
-
-from pip._vendor.packaging.version import parse as parse_version
+from typing import List, Optional, Tuple
 
 from pip._internal.exceptions import BadCommand, InstallationError
-from pip._internal.utils.misc import display_path, hide_url
+from pip._internal.utils.misc import HiddenText, display_path, hide_url
 from pip._internal.utils.subprocess import make_command
-from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.vcs.versioncontrol import (
+    AuthInfo,
     RemoteNotFoundError,
+    RemoteNotValidError,
+    RevOptions,
     VersionControl,
-    find_path_to_setup_from_repo_root,
+    find_path_to_project_root_from_repo_root,
     vcs,
 )
 
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, Tuple
-
-    from pip._internal.utils.misc import HiddenText
-    from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions
-
-
 urlsplit = urllib.parse.urlsplit
 urlunsplit = urllib.parse.urlunsplit
 
@@ -35,31 +26,57 @@
 logger = logging.getLogger(__name__)
 
 
-HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$')
+GIT_VERSION_REGEX = re.compile(
+    r"^git version "  # Prefix.
+    r"(\d+)"  # Major.
+    r"\.(\d+)"  # Dot, minor.
+    r"(?:\.(\d+))?"  # Optional dot, patch.
+    r".*$"  # Suffix, including any pre- and post-release segments we don't care about.
+)
+
+HASH_REGEX = re.compile("^[a-fA-F0-9]{40}$")
+
+# SCP (Secure copy protocol) shorthand. e.g. 'git@example.com:foo/bar.git'
+SCP_REGEX = re.compile(
+    r"""^
+    # Optional user, e.g. 'git@'
+    (\w+@)?
+    # Server, e.g. 'github.com'.
+    ([^/:]+):
+    # The server-side path. e.g. 'user/project.git'. Must start with an
+    # alphanumeric character so as not to be confusable with a Windows paths
+    # like 'C:/foo/bar' or 'C:\foo\bar'.
+    (\w[^:]*)
+    $""",
+    re.VERBOSE,
+)
 
 
-def looks_like_hash(sha):
+def looks_like_hash(sha: str) -> bool:
     return bool(HASH_REGEX.match(sha))
 
 
 class Git(VersionControl):
-    name = 'git'
-    dirname = '.git'
-    repo_name = 'clone'
+    name = "git"
+    dirname = ".git"
+    repo_name = "clone"
     schemes = (
-        'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file',
+        "git+http",
+        "git+https",
+        "git+ssh",
+        "git+git",
+        "git+file",
     )
     # Prevent the user's environment variables from interfering with pip:
     # https://github.com/pypa/pip/issues/1130
-    unset_environ = ('GIT_DIR', 'GIT_WORK_TREE')
-    default_arg_rev = 'HEAD'
+    unset_environ = ("GIT_DIR", "GIT_WORK_TREE")
+    default_arg_rev = "HEAD"
 
     @staticmethod
-    def get_base_rev_args(rev):
+    def get_base_rev_args(rev: str) -> List[str]:
         return [rev]
 
-    def is_immutable_rev_checkout(self, url, dest):
-        # type: (str, str) -> bool
+    def is_immutable_rev_checkout(self, url: str, dest: str) -> bool:
         _, rev_options = self.get_url_rev_options(hide_url(url))
         if not rev_options.rev:
             return False
@@ -70,28 +87,24 @@ def is_immutable_rev_checkout(self, url, dest):
         # return False in the rare case rev is both a commit hash
         # and a tag or a branch; we don't want to cache in that case
         # because that branch/tag could point to something else in the future
-        is_tag_or_branch = bool(
-            self.get_revision_sha(dest, rev_options.rev)[0]
-        )
+        is_tag_or_branch = bool(self.get_revision_sha(dest, rev_options.rev)[0])
         return not is_tag_or_branch
 
-    def get_git_version(self):
-        VERSION_PFX = 'git version '
+    def get_git_version(self) -> Tuple[int, ...]:
         version = self.run_command(
-            ['version'], show_stdout=False, stdout_only=True
+            ["version"],
+            command_desc="git version",
+            show_stdout=False,
+            stdout_only=True,
         )
-        if version.startswith(VERSION_PFX):
-            version = version[len(VERSION_PFX):].split()[0]
-        else:
-            version = ''
-        # get first 3 positions of the git version because
-        # on windows it is x.y.z.windows.t, and this parses as
-        # LegacyVersion which always smaller than a Version.
-        version = '.'.join(version.split('.')[:3])
-        return parse_version(version)
+        match = GIT_VERSION_REGEX.match(version)
+        if not match:
+            logger.warning("Can't parse git version: %s", version)
+            return ()
+        return tuple(int(c) for c in match.groups())
 
     @classmethod
-    def get_current_branch(cls, location):
+    def get_current_branch(cls, location: str) -> Optional[str]:
         """
         Return the current branch, or None if HEAD isn't at a branch
         (e.g. detached HEAD).
@@ -100,36 +113,23 @@ def get_current_branch(cls, location):
         # HEAD rather than a symbolic ref.  In addition, the -q causes the
         # command to exit with status code 1 instead of 128 in this case
         # and to suppress the message to stderr.
-        args = ['symbolic-ref', '-q', 'HEAD']
+        args = ["symbolic-ref", "-q", "HEAD"]
         output = cls.run_command(
             args,
-            extra_ok_returncodes=(1, ),
+            extra_ok_returncodes=(1,),
             show_stdout=False,
             stdout_only=True,
             cwd=location,
         )
         ref = output.strip()
 
-        if ref.startswith('refs/heads/'):
-            return ref[len('refs/heads/'):]
+        if ref.startswith("refs/heads/"):
+            return ref[len("refs/heads/") :]
 
         return None
 
-    def export(self, location, url):
-        # type: (str, HiddenText) -> None
-        """Export the Git repository at the url to the destination location"""
-        if not location.endswith('/'):
-            location = location + '/'
-
-        with TempDirectory(kind="export") as temp_dir:
-            self.unpack(temp_dir.path, url=url)
-            self.run_command(
-                ['checkout-index', '-a', '-f', '--prefix', location],
-                show_stdout=False, cwd=temp_dir.path
-            )
-
     @classmethod
-    def get_revision_sha(cls, dest, rev):
+    def get_revision_sha(cls, dest: str, rev: str) -> Tuple[Optional[str], bool]:
         """
         Return (sha_or_none, is_branch), where sha_or_none is a commit hash
         if the revision names a remote branch or tag, otherwise None.
@@ -140,25 +140,31 @@ def get_revision_sha(cls, dest, rev):
         """
         # Pass rev to pre-filter the list.
         output = cls.run_command(
-            ['show-ref', rev],
+            ["show-ref", rev],
             cwd=dest,
             show_stdout=False,
             stdout_only=True,
-            on_returncode='ignore',
+            on_returncode="ignore",
         )
         refs = {}
-        for line in output.strip().splitlines():
+        # NOTE: We do not use splitlines here since that would split on other
+        #       unicode separators, which can be maliciously used to install a
+        #       different revision.
+        for line in output.strip().split("\n"):
+            line = line.rstrip("\r")
+            if not line:
+                continue
             try:
-                sha, ref = line.split()
+                ref_sha, ref_name = line.split(" ", maxsplit=2)
             except ValueError:
                 # Include the offending line to simplify troubleshooting if
                 # this error ever occurs.
-                raise ValueError(f'unexpected show-ref line: {line!r}')
+                raise ValueError(f"unexpected show-ref line: {line!r}")
 
-            refs[ref] = sha
+            refs[ref_name] = ref_sha
 
-        branch_ref = f'refs/remotes/origin/{rev}'
-        tag_ref = f'refs/tags/{rev}'
+        branch_ref = f"refs/remotes/origin/{rev}"
+        tag_ref = f"refs/tags/{rev}"
 
         sha = refs.get(branch_ref)
         if sha is not None:
@@ -169,7 +175,7 @@ def get_revision_sha(cls, dest, rev):
         return (sha, False)
 
     @classmethod
-    def _should_fetch(cls, dest, rev):
+    def _should_fetch(cls, dest: str, rev: str) -> bool:
         """
         Return true if rev is a ref or is a commit that we don't have locally.
 
@@ -192,8 +198,9 @@ def _should_fetch(cls, dest, rev):
         return True
 
     @classmethod
-    def resolve_revision(cls, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> RevOptions
+    def resolve_revision(
+        cls, dest: str, url: HiddenText, rev_options: RevOptions
+    ) -> RevOptions:
         """
         Resolve a revision to a new RevOptions object with the SHA1 of the
         branch, tag, or ref if found.
@@ -227,17 +234,17 @@ def resolve_revision(cls, dest, url, rev_options):
 
         # fetch the requested revision
         cls.run_command(
-            make_command('fetch', '-q', url, rev_options.to_args()),
+            make_command("fetch", "-q", url, rev_options.to_args()),
             cwd=dest,
         )
         # Change the revision to the SHA of the ref we fetched
-        sha = cls.get_revision(dest, rev='FETCH_HEAD')
+        sha = cls.get_revision(dest, rev="FETCH_HEAD")
         rev_options = rev_options.make_new(sha)
 
         return rev_options
 
     @classmethod
-    def is_commit_id_equal(cls, dest, name):
+    def is_commit_id_equal(cls, dest: str, name: Optional[str]) -> bool:
         """
         Return whether the current commit hash equals the given name.
 
@@ -251,65 +258,95 @@ def is_commit_id_equal(cls, dest, name):
 
         return cls.get_revision(dest) == name
 
-    def fetch_new(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def fetch_new(
+        self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
+    ) -> None:
         rev_display = rev_options.to_display()
-        logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest))
-        self.run_command(make_command('clone', '-q', url, dest))
+        logger.info("Cloning %s%s to %s", url, rev_display, display_path(dest))
+        if verbosity <= 0:
+            flags: Tuple[str, ...] = ("--quiet",)
+        elif verbosity == 1:
+            flags = ()
+        else:
+            flags = ("--verbose", "--progress")
+        if self.get_git_version() >= (2, 17):
+            # Git added support for partial clone in 2.17
+            # https://git-scm.com/docs/partial-clone
+            # Speeds up cloning by functioning without a complete copy of repository
+            self.run_command(
+                make_command(
+                    "clone",
+                    "--filter=blob:none",
+                    *flags,
+                    url,
+                    dest,
+                )
+            )
+        else:
+            self.run_command(make_command("clone", *flags, url, dest))
 
         if rev_options.rev:
             # Then a specific revision was requested.
             rev_options = self.resolve_revision(dest, url, rev_options)
-            branch_name = getattr(rev_options, 'branch_name', None)
+            branch_name = getattr(rev_options, "branch_name", None)
+            logger.debug("Rev options %s, branch_name %s", rev_options, branch_name)
             if branch_name is None:
                 # Only do a checkout if the current commit id doesn't match
                 # the requested revision.
                 if not self.is_commit_id_equal(dest, rev_options.rev):
                     cmd_args = make_command(
-                        'checkout', '-q', rev_options.to_args(),
+                        "checkout",
+                        "-q",
+                        rev_options.to_args(),
                     )
                     self.run_command(cmd_args, cwd=dest)
             elif self.get_current_branch(dest) != branch_name:
                 # Then a specific branch was requested, and that branch
                 # is not yet checked out.
-                track_branch = f'origin/{branch_name}'
+                track_branch = f"origin/{branch_name}"
                 cmd_args = [
-                    'checkout', '-b', branch_name, '--track', track_branch,
+                    "checkout",
+                    "-b",
+                    branch_name,
+                    "--track",
+                    track_branch,
                 ]
                 self.run_command(cmd_args, cwd=dest)
+        else:
+            sha = self.get_revision(dest)
+            rev_options = rev_options.make_new(sha)
+
+        logger.info("Resolved %s to commit %s", url, rev_options.rev)
 
         #: repo may contain submodules
         self.update_submodules(dest)
 
-    def switch(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
         self.run_command(
-            make_command('config', 'remote.origin.url', url),
+            make_command("config", "remote.origin.url", url),
             cwd=dest,
         )
-        cmd_args = make_command('checkout', '-q', rev_options.to_args())
+        cmd_args = make_command("checkout", "-q", rev_options.to_args())
         self.run_command(cmd_args, cwd=dest)
 
         self.update_submodules(dest)
 
-    def update(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def update(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
         # First fetch changes from the default remote
-        if self.get_git_version() >= parse_version('1.9.0'):
+        if self.get_git_version() >= (1, 9):
             # fetch tags in addition to everything else
-            self.run_command(['fetch', '-q', '--tags'], cwd=dest)
+            self.run_command(["fetch", "-q", "--tags"], cwd=dest)
         else:
-            self.run_command(['fetch', '-q'], cwd=dest)
+            self.run_command(["fetch", "-q"], cwd=dest)
         # Then reset to wanted revision (maybe even origin/master)
         rev_options = self.resolve_revision(dest, url, rev_options)
-        cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args())
+        cmd_args = make_command("reset", "--hard", "-q", rev_options.to_args())
         self.run_command(cmd_args, cwd=dest)
         #: update submodules
         self.update_submodules(dest)
 
     @classmethod
-    def get_remote_url(cls, location):
-        # type: (str) -> str
+    def get_remote_url(cls, location: str) -> str:
         """
         Return URL of the first remote encountered.
 
@@ -319,8 +356,8 @@ def get_remote_url(cls, location):
         # We need to pass 1 for extra_ok_returncodes since the command
         # exits with return code 1 if there are no matching lines.
         stdout = cls.run_command(
-            ['config', '--get-regexp', r'remote\..*\.url'],
-            extra_ok_returncodes=(1, ),
+            ["config", "--get-regexp", r"remote\..*\.url"],
+            extra_ok_returncodes=(1,),
             show_stdout=False,
             stdout_only=True,
             cwd=location,
@@ -332,20 +369,51 @@ def get_remote_url(cls, location):
             raise RemoteNotFoundError
 
         for remote in remotes:
-            if remote.startswith('remote.origin.url '):
+            if remote.startswith("remote.origin.url "):
                 found_remote = remote
                 break
-        url = found_remote.split(' ')[1]
-        return url.strip()
+        url = found_remote.split(" ")[1]
+        return cls._git_remote_to_pip_url(url.strip())
+
+    @staticmethod
+    def _git_remote_to_pip_url(url: str) -> str:
+        """
+        Convert a remote url from what git uses to what pip accepts.
+
+        There are 3 legal forms **url** may take:
+
+            1. A fully qualified url: ssh://git@example.com/foo/bar.git
+            2. A local project.git folder: /path/to/bare/repository.git
+            3. SCP shorthand for form 1: git@example.com:foo/bar.git
+
+        Form 1 is output as-is. Form 2 must be converted to URI and form 3 must
+        be converted to form 1.
+
+        See the corresponding test test_git_remote_url_to_pip() for examples of
+        sample inputs/outputs.
+        """
+        if re.match(r"\w+://", url):
+            # This is already valid. Pass it though as-is.
+            return url
+        if os.path.exists(url):
+            # A local bare remote (git clone --mirror).
+            # Needs a file:// prefix.
+            return pathlib.PurePath(url).as_uri()
+        scp_match = SCP_REGEX.match(url)
+        if scp_match:
+            # Add an ssh:// prefix and replace the ':' with a '/'.
+            return scp_match.expand(r"ssh://\1\2/\3")
+        # Otherwise, bail out.
+        raise RemoteNotValidError(url)
 
     @classmethod
-    def has_commit(cls, location, rev):
+    def has_commit(cls, location: str, rev: str) -> bool:
         """
         Check if rev is a commit that is available in the local repository.
         """
         try:
             cls.run_command(
-                ['rev-parse', '-q', '--verify', "sha^" + rev],
+                ["rev-parse", "-q", "--verify", "sha^" + rev],
                 cwd=location,
                 log_failed_cmd=False,
             )
@@ -355,12 +423,11 @@ def has_commit(cls, location, rev):
             return True
 
     @classmethod
-    def get_revision(cls, location, rev=None):
-        # type: (str, Optional[str]) -> str
+    def get_revision(cls, location: str, rev: Optional[str] = None) -> str:
         if rev is None:
-            rev = 'HEAD'
+            rev = "HEAD"
         current_rev = cls.run_command(
-            ['rev-parse', rev],
+            ["rev-parse", rev],
             show_stdout=False,
             stdout_only=True,
             cwd=location,
@@ -368,26 +435,25 @@ def get_revision(cls, location, rev=None):
         return current_rev.strip()
 
     @classmethod
-    def get_subdirectory(cls, location):
+    def get_subdirectory(cls, location: str) -> Optional[str]:
         """
-        Return the path to setup.py, relative to the repo root.
-        Return None if setup.py is in the repo root.
+        Return the path to Python project root, relative to the repo root.
+        Return None if the project root is in the repo root.
         """
         # find the repo root
         git_dir = cls.run_command(
-            ['rev-parse', '--git-dir'],
+            ["rev-parse", "--git-dir"],
             show_stdout=False,
             stdout_only=True,
             cwd=location,
         ).strip()
         if not os.path.isabs(git_dir):
             git_dir = os.path.join(location, git_dir)
-        repo_root = os.path.abspath(os.path.join(git_dir, '..'))
-        return find_path_to_setup_from_repo_root(location, repo_root)
+        repo_root = os.path.abspath(os.path.join(git_dir, ".."))
+        return find_path_to_project_root_from_repo_root(location, repo_root)
 
     @classmethod
-    def get_url_rev_and_auth(cls, url):
-        # type: (str) -> Tuple[str, Optional[str], AuthInfo]
+    def get_url_rev_and_auth(cls, url: str) -> Tuple[str, Optional[str], AuthInfo]:
         """
         Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
         That's required because although they use SSH they sometimes don't
@@ -397,58 +463,64 @@ def get_url_rev_and_auth(cls, url):
         # Works around an apparent Git bug
         # (see https://article.gmane.org/gmane.comp.version-control.git/146500)
         scheme, netloc, path, query, fragment = urlsplit(url)
-        if scheme.endswith('file'):
-            initial_slashes = path[:-len(path.lstrip('/'))]
-            newpath = (
-                initial_slashes +
-                urllib.request.url2pathname(path)
-                .replace('\\', '/').lstrip('/')
-            )
-            after_plus = scheme.find('+') + 1
+        if scheme.endswith("file"):
+            initial_slashes = path[: -len(path.lstrip("/"))]
+            newpath = initial_slashes + urllib.request.url2pathname(path).replace(
+                "\\", "/"
+            ).lstrip("/")
+            after_plus = scheme.find("+") + 1
             url = scheme[:after_plus] + urlunsplit(
                 (scheme[after_plus:], netloc, newpath, query, fragment),
             )
 
-        if '://' not in url:
-            assert 'file:' not in url
-            url = url.replace('git+', 'git+ssh://')
+        if "://" not in url:
+            assert "file:" not in url
+            url = url.replace("git+", "git+ssh://")
             url, rev, user_pass = super().get_url_rev_and_auth(url)
-            url = url.replace('ssh://', '')
+            url = url.replace("ssh://", "")
         else:
             url, rev, user_pass = super().get_url_rev_and_auth(url)
 
         return url, rev, user_pass
 
     @classmethod
-    def update_submodules(cls, location):
-        if not os.path.exists(os.path.join(location, '.gitmodules')):
+    def update_submodules(cls, location: str) -> None:
+        if not os.path.exists(os.path.join(location, ".gitmodules")):
             return
         cls.run_command(
-            ['submodule', 'update', '--init', '--recursive', '-q'],
+            ["submodule", "update", "--init", "--recursive", "-q"],
             cwd=location,
         )
 
     @classmethod
-    def get_repository_root(cls, location):
+    def get_repository_root(cls, location: str) -> Optional[str]:
         loc = super().get_repository_root(location)
         if loc:
             return loc
         try:
             r = cls.run_command(
-                ['rev-parse', '--show-toplevel'],
+                ["rev-parse", "--show-toplevel"],
                 cwd=location,
                 show_stdout=False,
                 stdout_only=True,
-                on_returncode='raise',
+                on_returncode="raise",
                 log_failed_cmd=False,
             )
         except BadCommand:
-            logger.debug("could not determine if %s is under git control "
-                         "because git is not available", location)
+            logger.debug(
+                "could not determine if %s is under git control "
+                "because git is not available",
+                location,
+            )
             return None
         except InstallationError:
             return None
-        return os.path.normpath(r.rstrip('\r\n'))
+        return os.path.normpath(r.rstrip("\r\n"))
+
+    @staticmethod
+    def should_add_vcs_url_prefix(repo_url: str) -> bool:
+        """In either https or ssh form, requirements must be prefixed with git+."""
+        return True
 
 
 vcs.register(Git)
diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py
index 640697550b1..2a005e0aff2 100644
--- a/src/pip/_internal/vcs/mercurial.py
+++ b/src/pip/_internal/vcs/mercurial.py
@@ -1,95 +1,85 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import configparser
 import logging
 import os
+from typing import List, Optional, Tuple
 
 from pip._internal.exceptions import BadCommand, InstallationError
-from pip._internal.utils.misc import display_path
+from pip._internal.utils.misc import HiddenText, display_path
 from pip._internal.utils.subprocess import make_command
-from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import path_to_url
 from pip._internal.vcs.versioncontrol import (
+    RevOptions,
     VersionControl,
-    find_path_to_setup_from_repo_root,
+    find_path_to_project_root_from_repo_root,
     vcs,
 )
 
-if MYPY_CHECK_RUNNING:
-    from pip._internal.utils.misc import HiddenText
-    from pip._internal.vcs.versioncontrol import RevOptions
-
-
 logger = logging.getLogger(__name__)
 
 
 class Mercurial(VersionControl):
-    name = 'hg'
-    dirname = '.hg'
-    repo_name = 'clone'
+    name = "hg"
+    dirname = ".hg"
+    repo_name = "clone"
     schemes = (
-        'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http',
+        "hg+file",
+        "hg+http",
+        "hg+https",
+        "hg+ssh",
+        "hg+static-http",
     )
 
     @staticmethod
-    def get_base_rev_args(rev):
+    def get_base_rev_args(rev: str) -> List[str]:
         return [rev]
 
-    def export(self, location, url):
-        # type: (str, HiddenText) -> None
-        """Export the Hg repository at the url to the destination location"""
-        with TempDirectory(kind="export") as temp_dir:
-            self.unpack(temp_dir.path, url=url)
-
-            self.run_command(
-                ['archive', location], show_stdout=False, cwd=temp_dir.path
-            )
-
-    def fetch_new(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def fetch_new(
+        self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
+    ) -> None:
         rev_display = rev_options.to_display()
         logger.info(
-            'Cloning hg %s%s to %s',
+            "Cloning hg %s%s to %s",
             url,
             rev_display,
             display_path(dest),
         )
-        self.run_command(make_command('clone', '--noupdate', '-q', url, dest))
+        if verbosity <= 0:
+            flags: Tuple[str, ...] = ("--quiet",)
+        elif verbosity == 1:
+            flags = ()
+        elif verbosity == 2:
+            flags = ("--verbose",)
+        else:
+            flags = ("--verbose", "--debug")
+        self.run_command(make_command("clone", "--noupdate", *flags, url, dest))
         self.run_command(
-            make_command('update', '-q', rev_options.to_args()),
+            make_command("update", *flags, rev_options.to_args()),
             cwd=dest,
         )
 
-    def switch(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
-        repo_config = os.path.join(dest, self.dirname, 'hgrc')
+    def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
+        repo_config = os.path.join(dest, self.dirname, "hgrc")
         config = configparser.RawConfigParser()
         try:
             config.read(repo_config)
-            config.set('paths', 'default', url.secret)
-            with open(repo_config, 'w') as config_file:
+            config.set("paths", "default", url.secret)
+            with open(repo_config, "w") as config_file:
                 config.write(config_file)
         except (OSError, configparser.NoSectionError) as exc:
-            logger.warning(
-                'Could not switch Mercurial repository to %s: %s', url, exc,
-            )
+            logger.warning("Could not switch Mercurial repository to %s: %s", url, exc)
         else:
-            cmd_args = make_command('update', '-q', rev_options.to_args())
+            cmd_args = make_command("update", "-q", rev_options.to_args())
             self.run_command(cmd_args, cwd=dest)
 
-    def update(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
-        self.run_command(['pull', '-q'], cwd=dest)
-        cmd_args = make_command('update', '-q', rev_options.to_args())
+    def update(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
+        self.run_command(["pull", "-q"], cwd=dest)
+        cmd_args = make_command("update", "-q", rev_options.to_args())
         self.run_command(cmd_args, cwd=dest)
 
     @classmethod
-    def get_remote_url(cls, location):
-        # type: (str) -> str
+    def get_remote_url(cls, location: str) -> str:
         url = cls.run_command(
-            ['showconfig', 'paths.default'],
+            ["showconfig", "paths.default"],
             show_stdout=False,
             stdout_only=True,
             cwd=location,
@@ -99,13 +89,12 @@ def get_remote_url(cls, location):
         return url.strip()
 
     @classmethod
-    def get_revision(cls, location):
-        # type: (str) -> str
+    def get_revision(cls, location: str) -> str:
         """
         Return the repository-local changeset revision number, as an integer.
         """
         current_revision = cls.run_command(
-            ['parents', '--template={rev}'],
+            ["parents", "--template={rev}"],
             show_stdout=False,
             stdout_only=True,
             cwd=location,
@@ -113,13 +102,13 @@ def get_revision(cls, location):
         return current_revision
 
     @classmethod
-    def get_requirement_revision(cls, location):
+    def get_requirement_revision(cls, location: str) -> str:
         """
         Return the changeset identification hash, as a 40-character
         hexadecimal string
         """
         current_rev_hash = cls.run_command(
-            ['parents', '--template={node}'],
+            ["parents", "--template={node}"],
             show_stdout=False,
             stdout_only=True,
             cwd=location,
@@ -127,45 +116,48 @@ def get_requirement_revision(cls, location):
         return current_rev_hash
 
     @classmethod
-    def is_commit_id_equal(cls, dest, name):
+    def is_commit_id_equal(cls, dest: str, name: Optional[str]) -> bool:
         """Always assume the versions don't match"""
         return False
 
     @classmethod
-    def get_subdirectory(cls, location):
+    def get_subdirectory(cls, location: str) -> Optional[str]:
         """
-        Return the path to setup.py, relative to the repo root.
-        Return None if setup.py is in the repo root.
+        Return the path to Python project root, relative to the repo root.
+        Return None if the project root is in the repo root.
         """
         # find the repo root
         repo_root = cls.run_command(
-            ['root'], show_stdout=False, stdout_only=True, cwd=location
+            ["root"], show_stdout=False, stdout_only=True, cwd=location
         ).strip()
         if not os.path.isabs(repo_root):
             repo_root = os.path.abspath(os.path.join(location, repo_root))
-        return find_path_to_setup_from_repo_root(location, repo_root)
+        return find_path_to_project_root_from_repo_root(location, repo_root)
 
     @classmethod
-    def get_repository_root(cls, location):
+    def get_repository_root(cls, location: str) -> Optional[str]:
         loc = super().get_repository_root(location)
         if loc:
             return loc
         try:
             r = cls.run_command(
-                ['root'],
+                ["root"],
                 cwd=location,
                 show_stdout=False,
                 stdout_only=True,
-                on_returncode='raise',
+                on_returncode="raise",
                 log_failed_cmd=False,
             )
         except BadCommand:
-            logger.debug("could not determine if %s is under hg control "
-                         "because hg is not available", location)
+            logger.debug(
+                "could not determine if %s is under hg control "
+                "because hg is not available",
+                location,
+            )
             return None
         except InstallationError:
             return None
-        return os.path.normpath(r.rstrip('\r\n'))
+        return os.path.normpath(r.rstrip("\r\n"))
 
 
 vcs.register(Mercurial)
diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py
index f397c427e67..89c8754ce09 100644
--- a/src/pip/_internal/vcs/subversion.py
+++ b/src/pip/_internal/vcs/subversion.py
@@ -1,55 +1,48 @@
-# The following comment should be removed at some point in the future.
-# mypy: disallow-untyped-defs=False
-
 import logging
 import os
 import re
+from typing import List, Optional, Tuple
 
-from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import (
+    HiddenText,
     display_path,
     is_console_interactive,
-    rmtree,
+    is_installable_dir,
     split_auth_from_netloc,
 )
-from pip._internal.utils.subprocess import make_command
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
-from pip._internal.vcs.versioncontrol import RemoteNotFoundError, VersionControl, vcs
+from pip._internal.utils.subprocess import CommandArgs, make_command
+from pip._internal.vcs.versioncontrol import (
+    AuthInfo,
+    RemoteNotFoundError,
+    RevOptions,
+    VersionControl,
+    vcs,
+)
+
+logger = logging.getLogger(__name__)
 
 _svn_xml_url_re = re.compile('url="([^"]+)"')
 _svn_rev_re = re.compile(r'committed-rev="(\d+)"')
 _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"')
-_svn_info_xml_url_re = re.compile(r'(.*)')
-
-
-if MYPY_CHECK_RUNNING:
-    from typing import Optional, Tuple
-
-    from pip._internal.utils.misc import HiddenText
-    from pip._internal.utils.subprocess import CommandArgs
-    from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions
-
-
-logger = logging.getLogger(__name__)
+_svn_info_xml_url_re = re.compile(r"(.*)")
 
 
 class Subversion(VersionControl):
-    name = 'svn'
-    dirname = '.svn'
-    repo_name = 'checkout'
-    schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https', 'svn+svn')
+    name = "svn"
+    dirname = ".svn"
+    repo_name = "checkout"
+    schemes = ("svn+ssh", "svn+http", "svn+https", "svn+svn", "svn+file")
 
     @classmethod
-    def should_add_vcs_url_prefix(cls, remote_url):
+    def should_add_vcs_url_prefix(cls, remote_url: str) -> bool:
         return True
 
     @staticmethod
-    def get_base_rev_args(rev):
-        return ['-r', rev]
+    def get_base_rev_args(rev: str) -> List[str]:
+        return ["-r", rev]
 
     @classmethod
-    def get_revision(cls, location):
-        # type: (str) -> str
+    def get_revision(cls, location: str) -> str:
         """
         Return the maximum revision for all files under a given location
         """
@@ -59,9 +52,9 @@ def get_revision(cls, location):
         for base, dirs, _ in os.walk(location):
             if cls.dirname not in dirs:
                 dirs[:] = []
-                continue    # no sense walking uncontrolled subdirs
+                continue  # no sense walking uncontrolled subdirs
             dirs.remove(cls.dirname)
-            entries_fn = os.path.join(base, cls.dirname, 'entries')
+            entries_fn = os.path.join(base, cls.dirname, "entries")
             if not os.path.exists(entries_fn):
                 # FIXME: should we warn?
                 continue
@@ -69,20 +62,23 @@ def get_revision(cls, location):
             dirurl, localrev = cls._get_svn_url_rev(base)
 
             if base == location:
-                base = dirurl + '/'   # save the root url
+                assert dirurl is not None
+                base = dirurl + "/"  # save the root url
             elif not dirurl or not dirurl.startswith(base):
                 dirs[:] = []
-                continue    # not part of the same svn tree, skip it
+                continue  # not part of the same svn tree, skip it
             revision = max(revision, localrev)
         return str(revision)
 
     @classmethod
-    def get_netloc_and_auth(cls, netloc, scheme):
+    def get_netloc_and_auth(
+        cls, netloc: str, scheme: str
+    ) -> Tuple[str, Tuple[Optional[str], Optional[str]]]:
         """
         This override allows the auth information to be passed to svn via the
         --username and --password options instead of via the URL.
         """
-        if scheme == 'ssh':
+        if scheme == "ssh":
             # The --username and --password options can't be used for
             # svn+ssh URLs, so keep the auth information in the URL.
             return super().get_netloc_and_auth(netloc, scheme)
@@ -90,40 +86,38 @@ def get_netloc_and_auth(cls, netloc, scheme):
         return split_auth_from_netloc(netloc)
 
     @classmethod
-    def get_url_rev_and_auth(cls, url):
-        # type: (str) -> Tuple[str, Optional[str], AuthInfo]
+    def get_url_rev_and_auth(cls, url: str) -> Tuple[str, Optional[str], AuthInfo]:
         # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it
         url, rev, user_pass = super().get_url_rev_and_auth(url)
-        if url.startswith('ssh://'):
-            url = 'svn+' + url
+        if url.startswith("ssh://"):
+            url = "svn+" + url
         return url, rev, user_pass
 
     @staticmethod
-    def make_rev_args(username, password):
-        # type: (Optional[str], Optional[HiddenText]) -> CommandArgs
-        extra_args = []  # type: CommandArgs
+    def make_rev_args(
+        username: Optional[str], password: Optional[HiddenText]
+    ) -> CommandArgs:
+        extra_args: CommandArgs = []
         if username:
-            extra_args += ['--username', username]
+            extra_args += ["--username", username]
         if password:
-            extra_args += ['--password', password]
+            extra_args += ["--password", password]
 
         return extra_args
 
     @classmethod
-    def get_remote_url(cls, location):
-        # type: (str) -> str
-        # In cases where the source is in a subdirectory, not alongside
-        # setup.py we have to look up in the location until we find a real
-        # setup.py
+    def get_remote_url(cls, location: str) -> str:
+        # In cases where the source is in a subdirectory, we have to look up in
+        # the location until we find a valid project root.
         orig_location = location
-        while not os.path.exists(os.path.join(location, 'setup.py')):
+        while not is_installable_dir(location):
             last_location = location
             location = os.path.dirname(location)
             if location == last_location:
                 # We've traversed up to the root of the filesystem without
-                # finding setup.py
+                # finding a Python project.
                 logger.warning(
-                    "Could not find setup.py for directory %s (tried all "
+                    "Could not find Python project for directory %s (tried all "
                     "parent directories)",
                     orig_location,
                 )
@@ -136,29 +130,27 @@ def get_remote_url(cls, location):
         return url
 
     @classmethod
-    def _get_svn_url_rev(cls, location):
+    def _get_svn_url_rev(cls, location: str) -> Tuple[Optional[str], int]:
         from pip._internal.exceptions import InstallationError
 
-        entries_path = os.path.join(location, cls.dirname, 'entries')
+        entries_path = os.path.join(location, cls.dirname, "entries")
         if os.path.exists(entries_path):
             with open(entries_path) as f:
                 data = f.read()
         else:  # subversion >= 1.7 does not have the 'entries' file
-            data = ''
-
-        if (data.startswith('8') or
-                data.startswith('9') or
-                data.startswith('10')):
-            data = list(map(str.splitlines, data.split('\n\x0c\n')))
-            del data[0][0]  # get rid of the '8'
-            url = data[0][3]
-            revs = [int(d[9]) for d in data if len(d) > 9 and d[9]] + [0]
-        elif data.startswith(' 9 and d[9]] + [0]
+        elif data.startswith(" bool:
         """Always assume the versions don't match"""
         return False
 
-    def __init__(self, use_interactive=None):
-        # type: (bool) -> None
+    def __init__(self, use_interactive: bool = None) -> None:
         if use_interactive is None:
             use_interactive = is_console_interactive()
         self.use_interactive = use_interactive
@@ -203,12 +194,11 @@ def __init__(self, use_interactive=None):
         # Special value definitions:
         #   None: Not evaluated yet.
         #   Empty tuple: Could not parse version.
-        self._vcs_version = None  # type: Optional[Tuple[int, ...]]
+        self._vcs_version: Optional[Tuple[int, ...]] = None
 
         super().__init__()
 
-    def call_vcs_version(self):
-        # type: () -> Tuple[int, ...]
+    def call_vcs_version(self) -> Tuple[int, ...]:
         """Query the version of the currently installed Subversion client.
 
         :return: A tuple containing the parts of the version information or
@@ -222,15 +212,13 @@ def call_vcs_version(self):
         #      compiled Mar 28 2018, 08:49:13 on x86_64-pc-linux-gnu
         #   svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)
         #      compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2
-        version_prefix = 'svn, version '
-        version = self.run_command(
-            ['--version'], show_stdout=False, stdout_only=True
-        )
+        version_prefix = "svn, version "
+        version = self.run_command(["--version"], show_stdout=False, stdout_only=True)
         if not version.startswith(version_prefix):
             return ()
 
-        version = version[len(version_prefix):].split()[0]
-        version_list = version.partition('-')[0].split('.')
+        version = version[len(version_prefix) :].split()[0]
+        version_list = version.partition("-")[0].split(".")
         try:
             parsed_version = tuple(map(int, version_list))
         except ValueError:
@@ -238,8 +226,7 @@ def call_vcs_version(self):
 
         return parsed_version
 
-    def get_vcs_version(self):
-        # type: () -> Tuple[int, ...]
+    def get_vcs_version(self) -> Tuple[int, ...]:
         """Return the version of the currently installed Subversion client.
 
         If the version of the Subversion client has already been queried,
@@ -259,15 +246,13 @@ def get_vcs_version(self):
         self._vcs_version = vcs_version
         return vcs_version
 
-    def get_remote_call_options(self):
-        # type: () -> CommandArgs
+    def get_remote_call_options(self) -> CommandArgs:
         """Return options to be used on calls to Subversion that contact the server.
 
         These options are applicable for the following ``svn`` subcommands used
         in this class.
 
             - checkout
-            - export
             - switch
             - update
 
@@ -276,7 +261,7 @@ def get_remote_call_options(self):
         if not self.use_interactive:
             # --non-interactive switch is available since Subversion 0.14.4.
             # Subversion < 1.8 runs in interactive mode by default.
-            return ['--non-interactive']
+            return ["--non-interactive"]
 
         svn_version = self.get_vcs_version()
         # By default, Subversion >= 1.8 runs in non-interactive mode if
@@ -288,54 +273,49 @@ def get_remote_call_options(self):
         # SVN 1.7, pip should continue to support SVN 1.7. Therefore, pip
         # can't safely add the option if the SVN version is < 1.8 (or unknown).
         if svn_version >= (1, 8):
-            return ['--force-interactive']
+            return ["--force-interactive"]
 
         return []
 
-    def export(self, location, url):
-        # type: (str, HiddenText) -> None
-        """Export the svn repository at the url to the destination location"""
-        url, rev_options = self.get_url_rev_options(url)
-
-        logger.info('Exporting svn repository %s to %s', url, location)
-        with indent_log():
-            if os.path.exists(location):
-                # Subversion doesn't like to check out over an existing
-                # directory --force fixes this, but was only added in svn 1.5
-                rmtree(location)
-            cmd_args = make_command(
-                'export', self.get_remote_call_options(),
-                rev_options.to_args(), url, location,
-            )
-            self.run_command(cmd_args, show_stdout=False)
-
-    def fetch_new(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def fetch_new(
+        self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
+    ) -> None:
         rev_display = rev_options.to_display()
         logger.info(
-            'Checking out %s%s to %s',
+            "Checking out %s%s to %s",
             url,
             rev_display,
             display_path(dest),
         )
+        if verbosity <= 0:
+            flag = "--quiet"
+        else:
+            flag = ""
         cmd_args = make_command(
-            'checkout', '-q', self.get_remote_call_options(),
-            rev_options.to_args(), url, dest,
+            "checkout",
+            flag,
+            self.get_remote_call_options(),
+            rev_options.to_args(),
+            url,
+            dest,
         )
         self.run_command(cmd_args)
 
-    def switch(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
         cmd_args = make_command(
-            'switch', self.get_remote_call_options(), rev_options.to_args(),
-            url, dest,
+            "switch",
+            self.get_remote_call_options(),
+            rev_options.to_args(),
+            url,
+            dest,
         )
         self.run_command(cmd_args)
 
-    def update(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def update(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
         cmd_args = make_command(
-            'update', self.get_remote_call_options(), rev_options.to_args(),
+            "update",
+            self.get_remote_call_options(),
+            rev_options.to_args(),
             dest,
         )
         self.run_command(cmd_args)
diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py
index 218de58a6d4..02bbf68e7ad 100644
--- a/src/pip/_internal/vcs/versioncontrol.py
+++ b/src/pip/_internal/vcs/versioncontrol.py
@@ -5,62 +5,68 @@
 import shutil
 import sys
 import urllib.parse
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+)
 
-from pip._vendor import pkg_resources
-
+from pip._internal.cli.spinners import SpinnerInterface
 from pip._internal.exceptions import BadCommand, InstallationError
 from pip._internal.utils.misc import (
+    HiddenText,
     ask_path_exists,
     backup_dir,
     display_path,
     hide_url,
     hide_value,
+    is_installable_dir,
     rmtree,
 )
-from pip._internal.utils.subprocess import call_subprocess, make_command
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from pip._internal.utils.subprocess import (
+    CommandArgs,
+    call_subprocess,
+    format_command_args,
+    make_command,
+)
 from pip._internal.utils.urls import get_url_scheme
 
-if MYPY_CHECK_RUNNING:
-    from typing import (
-        Any,
-        Dict,
-        Iterable,
-        Iterator,
-        List,
-        Mapping,
-        Optional,
-        Tuple,
-        Type,
-        Union,
-    )
-
-    from pip._internal.cli.spinners import SpinnerInterface
-    from pip._internal.utils.misc import HiddenText
-    from pip._internal.utils.subprocess import CommandArgs
-
-    AuthInfo = Tuple[Optional[str], Optional[str]]
+if TYPE_CHECKING:
+    # Literal was introduced in Python 3.8.
+    #
+    # TODO: Remove `if TYPE_CHECKING` when dropping support for Python 3.7.
+    from typing import Literal
 
 
-__all__ = ['vcs']
+__all__ = ["vcs"]
 
 
 logger = logging.getLogger(__name__)
 
+AuthInfo = Tuple[Optional[str], Optional[str]]
 
-def is_url(name):
-    # type: (str) -> bool
+
+def is_url(name: str) -> bool:
     """
     Return true if the name looks like a URL.
     """
     scheme = get_url_scheme(name)
     if scheme is None:
         return False
-    return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
+    return scheme in ["http", "https", "file", "ftp"] + vcs.all_schemes
 
 
-def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None):
-    # type: (str, str, str, Optional[str]) -> str
+def make_vcs_requirement_url(
+    repo_url: str, rev: str, project_name: str, subdir: Optional[str] = None
+) -> str:
     """
     Return the URL for a VCS requirement.
 
@@ -68,31 +74,32 @@ def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None):
       repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+").
       project_name: the (unescaped) project name.
     """
-    egg_project_name = pkg_resources.to_filename(project_name)
-    req = f'{repo_url}@{rev}#egg={egg_project_name}'
+    egg_project_name = project_name.replace("-", "_")
+    req = f"{repo_url}@{rev}#egg={egg_project_name}"
     if subdir:
-        req += f'&subdirectory={subdir}'
+        req += f"&subdirectory={subdir}"
 
     return req
 
 
-def find_path_to_setup_from_repo_root(location, repo_root):
-    # type: (str, str) -> Optional[str]
+def find_path_to_project_root_from_repo_root(
+    location: str, repo_root: str
+) -> Optional[str]:
     """
-    Find the path to `setup.py` by searching up the filesystem from `location`.
-    Return the path to `setup.py` relative to `repo_root`.
-    Return None if `setup.py` is in `repo_root` or cannot be found.
+    Find the the Python project's root by searching up the filesystem from
+    `location`. Return the path to project root relative to `repo_root`.
+    Return None if the project root is `repo_root`, or cannot be found.
     """
-    # find setup.py
+    # find project root.
     orig_location = location
-    while not os.path.exists(os.path.join(location, 'setup.py')):
+    while not is_installable_dir(location):
         last_location = location
         location = os.path.dirname(location)
         if location == last_location:
             # We've traversed up to the root of the filesystem without
-            # finding setup.py
+            # finding a Python project.
             logger.warning(
-                "Could not find setup.py for directory %s (tried all "
+                "Could not find a Python project for directory %s (tried all "
                 "parent directories)",
                 orig_location,
             )
@@ -108,6 +115,12 @@ class RemoteNotFoundError(Exception):
     pass
 
 
+class RemoteNotValidError(Exception):
+    def __init__(self, url: str):
+        super().__init__(url)
+        self.url = url
+
+
 class RevOptions:
 
     """
@@ -119,11 +132,10 @@ class RevOptions:
 
     def __init__(
         self,
-        vc_class,  # type: Type[VersionControl]
-        rev=None,  # type: Optional[str]
-        extra_args=None,  # type: Optional[CommandArgs]
-    ):
-        # type: (...) -> None
+        vc_class: Type["VersionControl"],
+        rev: Optional[str] = None,
+        extra_args: Optional[CommandArgs] = None,
+    ) -> None:
         """
         Args:
           vc_class: a VersionControl subclass.
@@ -136,26 +148,23 @@ def __init__(
         self.extra_args = extra_args
         self.rev = rev
         self.vc_class = vc_class
-        self.branch_name = None  # type: Optional[str]
+        self.branch_name: Optional[str] = None
 
-    def __repr__(self):
-        # type: () -> str
-        return f''
+    def __repr__(self) -> str:
+        return f""
 
     @property
-    def arg_rev(self):
-        # type: () -> Optional[str]
+    def arg_rev(self) -> Optional[str]:
         if self.rev is None:
             return self.vc_class.default_arg_rev
 
         return self.rev
 
-    def to_args(self):
-        # type: () -> CommandArgs
+    def to_args(self) -> CommandArgs:
         """
         Return the VCS-specific command arguments.
         """
-        args = []  # type: CommandArgs
+        args: CommandArgs = []
         rev = self.arg_rev
         if rev is not None:
             args += self.vc_class.get_base_rev_args(rev)
@@ -163,15 +172,13 @@ def to_args(self):
 
         return args
 
-    def to_display(self):
-        # type: () -> str
+    def to_display(self) -> str:
         if not self.rev:
-            return ''
+            return ""
 
-        return f' (to revision {self.rev})'
+        return f" (to revision {self.rev})"
 
-    def make_new(self, rev):
-        # type: (str) -> RevOptions
+    def make_new(self, rev: str) -> "RevOptions":
         """
         Make a copy of the current instance, but with a new rev.
 
@@ -182,54 +189,46 @@ def make_new(self, rev):
 
 
 class VcsSupport:
-    _registry = {}  # type: Dict[str, VersionControl]
-    schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn']
+    _registry: Dict[str, "VersionControl"] = {}
+    schemes = ["ssh", "git", "hg", "bzr", "sftp", "svn"]
 
-    def __init__(self):
-        # type: () -> None
+    def __init__(self) -> None:
         # Register more schemes with urlparse for various version control
         # systems
         urllib.parse.uses_netloc.extend(self.schemes)
         super().__init__()
 
-    def __iter__(self):
-        # type: () -> Iterator[str]
+    def __iter__(self) -> Iterator[str]:
         return self._registry.__iter__()
 
     @property
-    def backends(self):
-        # type: () -> List[VersionControl]
+    def backends(self) -> List["VersionControl"]:
         return list(self._registry.values())
 
     @property
-    def dirnames(self):
-        # type: () -> List[str]
+    def dirnames(self) -> List[str]:
         return [backend.dirname for backend in self.backends]
 
     @property
-    def all_schemes(self):
-        # type: () -> List[str]
-        schemes = []  # type: List[str]
+    def all_schemes(self) -> List[str]:
+        schemes: List[str] = []
         for backend in self.backends:
             schemes.extend(backend.schemes)
         return schemes
 
-    def register(self, cls):
-        # type: (Type[VersionControl]) -> None
-        if not hasattr(cls, 'name'):
-            logger.warning('Cannot register VCS %s', cls.__name__)
+    def register(self, cls: Type["VersionControl"]) -> None:
+        if not hasattr(cls, "name"):
+            logger.warning("Cannot register VCS %s", cls.__name__)
             return
         if cls.name not in self._registry:
             self._registry[cls.name] = cls()
-            logger.debug('Registered VCS backend: %s', cls.name)
+            logger.debug("Registered VCS backend: %s", cls.name)
 
-    def unregister(self, name):
-        # type: (str) -> None
+    def unregister(self, name: str) -> None:
         if name in self._registry:
             del self._registry[name]
 
-    def get_backend_for_dir(self, location):
-        # type: (str) -> Optional[VersionControl]
+    def get_backend_for_dir(self, location: str) -> Optional["VersionControl"]:
         """
         Return a VersionControl object if a repository of that type is found
         at the given directory.
@@ -239,8 +238,7 @@ def get_backend_for_dir(self, location):
             repo_path = vcs_backend.get_repository_root(location)
             if not repo_path:
                 continue
-            logger.debug('Determine that %s uses VCS: %s',
-                         location, vcs_backend.name)
+            logger.debug("Determine that %s uses VCS: %s", location, vcs_backend.name)
             vcs_backends[repo_path] = vcs_backend
 
         if not vcs_backends:
@@ -253,8 +251,7 @@ def get_backend_for_dir(self, location):
         inner_most_repo_path = max(vcs_backends, key=len)
         return vcs_backends[inner_most_repo_path]
 
-    def get_backend_for_scheme(self, scheme):
-        # type: (str) -> Optional[VersionControl]
+    def get_backend_for_scheme(self, scheme: str) -> Optional["VersionControl"]:
         """
         Return a VersionControl object or None.
         """
@@ -263,8 +260,7 @@ def get_backend_for_scheme(self, scheme):
                 return vcs_backend
         return None
 
-    def get_backend(self, name):
-        # type: (str) -> Optional[VersionControl]
+    def get_backend(self, name: str) -> Optional["VersionControl"]:
         """
         Return a VersionControl object or None.
         """
@@ -276,44 +272,40 @@ def get_backend(self, name):
 
 
 class VersionControl:
-    name = ''
-    dirname = ''
-    repo_name = ''
+    name = ""
+    dirname = ""
+    repo_name = ""
     # List of supported schemes for this Version Control
-    schemes = ()  # type: Tuple[str, ...]
+    schemes: Tuple[str, ...] = ()
     # Iterable of environment variable names to pass to call_subprocess().
-    unset_environ = ()  # type: Tuple[str, ...]
-    default_arg_rev = None  # type: Optional[str]
+    unset_environ: Tuple[str, ...] = ()
+    default_arg_rev: Optional[str] = None
 
     @classmethod
-    def should_add_vcs_url_prefix(cls, remote_url):
-        # type: (str) -> bool
+    def should_add_vcs_url_prefix(cls, remote_url: str) -> bool:
         """
         Return whether the vcs prefix (e.g. "git+") should be added to a
         repository's remote url when used in a requirement.
         """
-        return not remote_url.lower().startswith(f'{cls.name}:')
+        return not remote_url.lower().startswith(f"{cls.name}:")
 
     @classmethod
-    def get_subdirectory(cls, location):
-        # type: (str) -> Optional[str]
+    def get_subdirectory(cls, location: str) -> Optional[str]:
         """
-        Return the path to setup.py, relative to the repo root.
-        Return None if setup.py is in the repo root.
+        Return the path to Python project root, relative to the repo root.
+        Return None if the project root is in the repo root.
         """
         return None
 
     @classmethod
-    def get_requirement_revision(cls, repo_dir):
-        # type: (str) -> str
+    def get_requirement_revision(cls, repo_dir: str) -> str:
         """
         Return the revision string that should be used in a requirement.
         """
         return cls.get_revision(repo_dir)
 
     @classmethod
-    def get_src_requirement(cls, repo_dir, project_name):
-        # type: (str, str) -> str
+    def get_src_requirement(cls, repo_dir: str, project_name: str) -> str:
         """
         Return the requirement string to use to redownload the files
         currently at the given repository directory.
@@ -328,18 +320,16 @@ def get_src_requirement(cls, repo_dir, project_name):
         repo_url = cls.get_remote_url(repo_dir)
 
         if cls.should_add_vcs_url_prefix(repo_url):
-            repo_url = f'{cls.name}+{repo_url}'
+            repo_url = f"{cls.name}+{repo_url}"
 
         revision = cls.get_requirement_revision(repo_dir)
         subdir = cls.get_subdirectory(repo_dir)
-        req = make_vcs_requirement_url(repo_url, revision, project_name,
-                                       subdir=subdir)
+        req = make_vcs_requirement_url(repo_url, revision, project_name, subdir=subdir)
 
         return req
 
     @staticmethod
-    def get_base_rev_args(rev):
-        # type: (str) -> List[str]
+    def get_base_rev_args(rev: str) -> List[str]:
         """
         Return the base revision arguments for a vcs command.
 
@@ -348,8 +338,7 @@ def get_base_rev_args(rev):
         """
         raise NotImplementedError
 
-    def is_immutable_rev_checkout(self, url, dest):
-        # type: (str, str) -> bool
+    def is_immutable_rev_checkout(self, url: str, dest: str) -> bool:
         """
         Return true if the commit hash checked out at dest matches
         the revision in url.
@@ -363,8 +352,9 @@ def is_immutable_rev_checkout(self, url, dest):
         return False
 
     @classmethod
-    def make_rev_options(cls, rev=None, extra_args=None):
-        # type: (Optional[str], Optional[CommandArgs]) -> RevOptions
+    def make_rev_options(
+        cls, rev: Optional[str] = None, extra_args: Optional[CommandArgs] = None
+    ) -> RevOptions:
         """
         Return a RevOptions object.
 
@@ -375,28 +365,18 @@ def make_rev_options(cls, rev=None, extra_args=None):
         return RevOptions(cls, rev, extra_args=extra_args)
 
     @classmethod
-    def _is_local_repository(cls, repo):
-        # type: (str) -> bool
+    def _is_local_repository(cls, repo: str) -> bool:
         """
-           posix absolute paths start with os.path.sep,
-           win32 ones start with drive (like c:\\folder)
+        posix absolute paths start with os.path.sep,
+        win32 ones start with drive (like c:\\folder)
         """
         drive, tail = os.path.splitdrive(repo)
         return repo.startswith(os.path.sep) or bool(drive)
 
-    def export(self, location, url):
-        # type: (str, HiddenText) -> None
-        """
-        Export the repository at the url to the destination location
-        i.e. only download the files, without vcs informations
-
-        :param url: the repository URL starting with a vcs prefix.
-        """
-        raise NotImplementedError
-
     @classmethod
-    def get_netloc_and_auth(cls, netloc, scheme):
-        # type: (str, str) -> Tuple[str, Tuple[Optional[str], Optional[str]]]
+    def get_netloc_and_auth(
+        cls, netloc: str, scheme: str
+    ) -> Tuple[str, Tuple[Optional[str], Optional[str]]]:
         """
         Parse the repository URL's netloc, and return the new netloc to use
         along with auth information.
@@ -415,8 +395,7 @@ def get_netloc_and_auth(cls, netloc, scheme):
         return netloc, (None, None)
 
     @classmethod
-    def get_url_rev_and_auth(cls, url):
-        # type: (str) -> Tuple[str, Optional[str], AuthInfo]
+    def get_url_rev_and_auth(cls, url: str) -> Tuple[str, Optional[str], AuthInfo]:
         """
         Parse the repository URL to use, and return the URL, revision,
         and auth info to use.
@@ -424,44 +403,44 @@ def get_url_rev_and_auth(cls, url):
         Returns: (url, rev, (username, password)).
         """
         scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
-        if '+' not in scheme:
+        if "+" not in scheme:
             raise ValueError(
                 "Sorry, {!r} is a malformed VCS url. "
                 "The format is +://, "
                 "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url)
             )
         # Remove the vcs prefix.
-        scheme = scheme.split('+', 1)[1]
+        scheme = scheme.split("+", 1)[1]
         netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme)
         rev = None
-        if '@' in path:
-            path, rev = path.rsplit('@', 1)
+        if "@" in path:
+            path, rev = path.rsplit("@", 1)
             if not rev:
                 raise InstallationError(
                     "The URL {!r} has an empty revision (after @) "
                     "which is not supported. Include a revision after @ "
                     "or remove @ from the URL.".format(url)
                 )
-        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))
+        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ""))
         return url, rev, user_pass
 
     @staticmethod
-    def make_rev_args(username, password):
-        # type: (Optional[str], Optional[HiddenText]) -> CommandArgs
+    def make_rev_args(
+        username: Optional[str], password: Optional[HiddenText]
+    ) -> CommandArgs:
         """
         Return the RevOptions "extra arguments" to use in obtain().
         """
         return []
 
-    def get_url_rev_options(self, url):
-        # type: (HiddenText) -> Tuple[HiddenText, RevOptions]
+    def get_url_rev_options(self, url: HiddenText) -> Tuple[HiddenText, RevOptions]:
         """
-        Return the URL and RevOptions object to use in obtain() and in
-        some cases export(), as a tuple (url, rev_options).
+        Return the URL and RevOptions object to use in obtain(),
+        as a tuple (url, rev_options).
         """
         secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret)
         username, secret_password = user_pass
-        password = None  # type: Optional[HiddenText]
+        password: Optional[HiddenText] = None
         if secret_password is not None:
             password = hide_value(secret_password)
         extra_args = self.make_rev_args(username, password)
@@ -470,24 +449,23 @@ def get_url_rev_options(self, url):
         return hide_url(secret_url), rev_options
 
     @staticmethod
-    def normalize_url(url):
-        # type: (str) -> str
+    def normalize_url(url: str) -> str:
         """
         Normalize a URL for comparison by unquoting it and removing any
         trailing slash.
         """
-        return urllib.parse.unquote(url).rstrip('/')
+        return urllib.parse.unquote(url).rstrip("/")
 
     @classmethod
-    def compare_urls(cls, url1, url2):
-        # type: (str, str) -> bool
+    def compare_urls(cls, url1: str, url2: str) -> bool:
         """
         Compare two repo URLs for identity, ignoring incidental differences.
         """
-        return (cls.normalize_url(url1) == cls.normalize_url(url2))
+        return cls.normalize_url(url1) == cls.normalize_url(url2)
 
-    def fetch_new(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def fetch_new(
+        self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
+    ) -> None:
         """
         Fetch a revision from a repository, in the case that this is the
         first fetch from the repository.
@@ -495,11 +473,11 @@ def fetch_new(self, dest, url, rev_options):
         Args:
           dest: the directory to fetch the repository to.
           rev_options: a RevOptions object.
+          verbosity: verbosity level.
         """
         raise NotImplementedError
 
-    def switch(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def switch(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
         """
         Switch the repo at ``dest`` to point to ``URL``.
 
@@ -508,8 +486,7 @@ def switch(self, dest, url, rev_options):
         """
         raise NotImplementedError
 
-    def update(self, dest, url, rev_options):
-        # type: (str, HiddenText, RevOptions) -> None
+    def update(self, dest: str, url: HiddenText, rev_options: RevOptions) -> None:
         """
         Update an already-existing repo to the given ``rev_options``.
 
@@ -519,8 +496,7 @@ def update(self, dest, url, rev_options):
         raise NotImplementedError
 
     @classmethod
-    def is_commit_id_equal(cls, dest, name):
-        # type: (str, Optional[str]) -> bool
+    def is_commit_id_equal(cls, dest: str, name: Optional[str]) -> bool:
         """
         Return whether the id of the current commit equals the given name.
 
@@ -530,19 +506,19 @@ def is_commit_id_equal(cls, dest, name):
         """
         raise NotImplementedError
 
-    def obtain(self, dest, url):
-        # type: (str, HiddenText) -> None
+    def obtain(self, dest: str, url: HiddenText, verbosity: int) -> None:
         """
         Install or update in editable mode the package represented by this
         VersionControl object.
 
         :param dest: the repository directory in which to install or update.
         :param url: the repository URL starting with a vcs prefix.
+        :param verbosity: verbosity level.
         """
         url, rev_options = self.get_url_rev_options(url)
 
         if not os.path.exists(dest):
-            self.fetch_new(dest, url, rev_options)
+            self.fetch_new(dest, url, rev_options, verbosity=verbosity)
             return
 
         rev_display = rev_options.to_display()
@@ -550,73 +526,68 @@ def obtain(self, dest, url):
             existing_url = self.get_remote_url(dest)
             if self.compare_urls(existing_url, url.secret):
                 logger.debug(
-                    '%s in %s exists, and has correct URL (%s)',
+                    "%s in %s exists, and has correct URL (%s)",
                     self.repo_name.title(),
                     display_path(dest),
                     url,
                 )
                 if not self.is_commit_id_equal(dest, rev_options.rev):
                     logger.info(
-                        'Updating %s %s%s',
+                        "Updating %s %s%s",
                         display_path(dest),
                         self.repo_name,
                         rev_display,
                     )
                     self.update(dest, url, rev_options)
                 else:
-                    logger.info('Skipping because already up-to-date.')
+                    logger.info("Skipping because already up-to-date.")
                 return
 
             logger.warning(
-                '%s %s in %s exists with URL %s',
+                "%s %s in %s exists with URL %s",
                 self.name,
                 self.repo_name,
                 display_path(dest),
                 existing_url,
             )
-            prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ',
-                      ('s', 'i', 'w', 'b'))
+            prompt = ("(s)witch, (i)gnore, (w)ipe, (b)ackup ", ("s", "i", "w", "b"))
         else:
             logger.warning(
-                'Directory %s already exists, and is not a %s %s.',
+                "Directory %s already exists, and is not a %s %s.",
                 dest,
                 self.name,
                 self.repo_name,
             )
             # https://github.com/python/mypy/issues/1174
-            prompt = ('(i)gnore, (w)ipe, (b)ackup ',  # type: ignore
-                      ('i', 'w', 'b'))
+            prompt = ("(i)gnore, (w)ipe, (b)ackup ", ("i", "w", "b"))  # type: ignore
 
         logger.warning(
-            'The plan is to install the %s repository %s',
+            "The plan is to install the %s repository %s",
             self.name,
             url,
         )
-        response = ask_path_exists('What to do?  {}'.format(
-            prompt[0]), prompt[1])
+        response = ask_path_exists("What to do?  {}".format(prompt[0]), prompt[1])
 
-        if response == 'a':
+        if response == "a":
             sys.exit(-1)
 
-        if response == 'w':
-            logger.warning('Deleting %s', display_path(dest))
+        if response == "w":
+            logger.warning("Deleting %s", display_path(dest))
             rmtree(dest)
-            self.fetch_new(dest, url, rev_options)
+            self.fetch_new(dest, url, rev_options, verbosity=verbosity)
             return
 
-        if response == 'b':
+        if response == "b":
             dest_dir = backup_dir(dest)
-            logger.warning(
-                'Backing up %s to %s', display_path(dest), dest_dir,
-            )
+            logger.warning("Backing up %s to %s", display_path(dest), dest_dir)
             shutil.move(dest, dest_dir)
-            self.fetch_new(dest, url, rev_options)
+            self.fetch_new(dest, url, rev_options, verbosity=verbosity)
             return
 
         # Do nothing if the response is "i".
-        if response == 's':
+        if response == "s":
             logger.info(
-                'Switching %s %s to %s%s',
+                "Switching %s %s to %s%s",
                 self.repo_name,
                 display_path(dest),
                 url,
@@ -624,21 +595,20 @@ def obtain(self, dest, url):
             )
             self.switch(dest, url, rev_options)
 
-    def unpack(self, location, url):
-        # type: (str, HiddenText) -> None
+    def unpack(self, location: str, url: HiddenText, verbosity: int) -> None:
         """
         Clean up current location and download the url repository
         (and vcs infos) into location
 
         :param url: the repository URL starting with a vcs prefix.
+        :param verbosity: verbosity level.
         """
         if os.path.exists(location):
             rmtree(location)
-        self.obtain(location, url=url)
+        self.obtain(location, url=url, verbosity=verbosity)
 
     @classmethod
-    def get_remote_url(cls, location):
-        # type: (str) -> str
+    def get_remote_url(cls, location: str) -> str:
         """
         Return the url used at location
 
@@ -648,8 +618,7 @@ def get_remote_url(cls, location):
         raise NotImplementedError
 
     @classmethod
-    def get_revision(cls, location):
-        # type: (str) -> str
+    def get_revision(cls, location: str) -> str:
         """
         Return the current commit id of the files at the given location.
         """
@@ -658,55 +627,69 @@ def get_revision(cls, location):
     @classmethod
     def run_command(
         cls,
-        cmd,  # type: Union[List[str], CommandArgs]
-        show_stdout=True,  # type: bool
-        cwd=None,  # type: Optional[str]
-        on_returncode='raise',  # type: str
-        extra_ok_returncodes=None,  # type: Optional[Iterable[int]]
-        command_desc=None,  # type: Optional[str]
-        extra_environ=None,  # type: Optional[Mapping[str, Any]]
-        spinner=None,  # type: Optional[SpinnerInterface]
-        log_failed_cmd=True,  # type: bool
-        stdout_only=False,  # type: bool
-    ):
-        # type: (...) -> str
+        cmd: Union[List[str], CommandArgs],
+        show_stdout: bool = True,
+        cwd: Optional[str] = None,
+        on_returncode: 'Literal["raise", "warn", "ignore"]' = "raise",
+        extra_ok_returncodes: Optional[Iterable[int]] = None,
+        command_desc: Optional[str] = None,
+        extra_environ: Optional[Mapping[str, Any]] = None,
+        spinner: Optional[SpinnerInterface] = None,
+        log_failed_cmd: bool = True,
+        stdout_only: bool = False,
+    ) -> str:
         """
         Run a VCS subcommand
         This is simply a wrapper around call_subprocess that adds the VCS
         command name, and checks that the VCS is available
         """
         cmd = make_command(cls.name, *cmd)
+        if command_desc is None:
+            command_desc = format_command_args(cmd)
         try:
-            return call_subprocess(cmd, show_stdout, cwd,
-                                   on_returncode=on_returncode,
-                                   extra_ok_returncodes=extra_ok_returncodes,
-                                   command_desc=command_desc,
-                                   extra_environ=extra_environ,
-                                   unset_environ=cls.unset_environ,
-                                   spinner=spinner,
-                                   log_failed_cmd=log_failed_cmd,
-                                   stdout_only=stdout_only)
+            return call_subprocess(
+                cmd,
+                show_stdout,
+                cwd,
+                on_returncode=on_returncode,
+                extra_ok_returncodes=extra_ok_returncodes,
+                command_desc=command_desc,
+                extra_environ=extra_environ,
+                unset_environ=cls.unset_environ,
+                spinner=spinner,
+                log_failed_cmd=log_failed_cmd,
+                stdout_only=stdout_only,
+            )
         except FileNotFoundError:
             # errno.ENOENT = no such file or directory
             # In other words, the VCS executable isn't available
             raise BadCommand(
-                'Cannot find command {cls.name!r} - do you have '
-                '{cls.name!r} installed and in your '
-                'PATH?'.format(**locals()))
+                f"Cannot find command {cls.name!r} - do you have "
+                f"{cls.name!r} installed and in your PATH?"
+            )
+        except PermissionError:
+            # errno.EACCES = Permission denied
+            # This error occurs, for instance, when the command is installed
+            # only for another user. So, the current user don't have
+            # permission to call the other user command.
+            raise BadCommand(
+                f"No permission to execute {cls.name!r} - install it "
+                f"locally, globally (ask admin), or check your PATH. "
+                f"See possible solutions at "
+                f"https://pip.pypa.io/en/latest/reference/pip_freeze/"
+                f"#fixing-permission-denied."
+            )
 
     @classmethod
-    def is_repository_directory(cls, path):
-        # type: (str) -> bool
+    def is_repository_directory(cls, path: str) -> bool:
         """
         Return whether a directory path is a repository directory.
         """
-        logger.debug('Checking in %s for %s (%s)...',
-                     path, cls.dirname, cls.name)
+        logger.debug("Checking in %s for %s (%s)...", path, cls.dirname, cls.name)
         return os.path.exists(os.path.join(path, cls.dirname))
 
     @classmethod
-    def get_repository_root(cls, location):
-        # type: (str) -> Optional[str]
+    def get_repository_root(cls, location: str) -> Optional[str]:
         """
         Return the "root" (top-level) directory controlled by the vcs,
         or `None` if the directory is not in any.
diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py
index c23ee1ba764..d0663443b22 100644
--- a/src/pip/_internal/wheel_builder.py
+++ b/src/pip/_internal/wheel_builder.py
@@ -5,43 +5,37 @@
 import os.path
 import re
 import shutil
-import zipfile
+from typing import Any, Callable, Iterable, List, Optional, Tuple
 
 from pip._vendor.packaging.utils import canonicalize_name, canonicalize_version
 from pip._vendor.packaging.version import InvalidVersion, Version
-from pip._vendor.pkg_resources import Distribution
 
+from pip._internal.cache import WheelCache
 from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel
+from pip._internal.metadata import FilesystemWheel, get_wheel_distribution
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
 from pip._internal.operations.build.wheel import build_wheel_pep517
+from pip._internal.operations.build.wheel_editable import build_wheel_editable
 from pip._internal.operations.build.wheel_legacy import build_wheel_legacy
+from pip._internal.req.req_install import InstallRequirement
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import ensure_dir, hash_file, is_wheel_installed
 from pip._internal.utils.setuptools_build import make_setuptools_clean_args
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory
-from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 from pip._internal.utils.urls import path_to_url
-from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel
 from pip._internal.vcs import vcs
 
-if MYPY_CHECK_RUNNING:
-    from typing import Any, Callable, Iterable, List, Optional, Tuple
-
-    from pip._internal.cache import WheelCache
-    from pip._internal.req.req_install import InstallRequirement
-
-    BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
-    BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]]
-
 logger = logging.getLogger(__name__)
 
-_egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.IGNORECASE)
+_egg_info_re = re.compile(r"([a-z0-9_.]+)-([a-z0-9_.!+-]+)", re.IGNORECASE)
+
+BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
+BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]]
 
 
-def _contains_egg_info(s):
-    # type: (str) -> bool
+def _contains_egg_info(s: str) -> bool:
     """Determine whether the string looks like an egg_info.
 
     :param s: The string to parse. E.g. foo-2.1
@@ -50,11 +44,10 @@ def _contains_egg_info(s):
 
 
 def _should_build(
-    req,  # type: InstallRequirement
-    need_wheel,  # type: bool
-    check_binary_allowed,  # type: BinaryAllowedPredicate
-):
-    # type: (...) -> bool
+    req: InstallRequirement,
+    need_wheel: bool,
+    check_binary_allowed: BinaryAllowedPredicate,
+) -> bool:
     """Return whether an InstallRequirement should be built into a wheel."""
     if req.constraint:
         # never build requirements that are merely constraints
@@ -62,7 +55,8 @@ def _should_build(
     if req.is_wheel:
         if need_wheel:
             logger.info(
-                'Skipping %s, due to already being wheel.', req.name,
+                "Skipping %s, due to already being wheel.",
+                req.name,
             )
         return False
 
@@ -73,16 +67,20 @@ def _should_build(
     # From this point, this concerns the pip install command only
     # (need_wheel=False).
 
-    if req.editable or not req.source_dir:
+    if not req.source_dir:
         return False
 
+    if req.editable:
+        # we only build PEP 660 editable requirements
+        return req.supports_pyproject_editable()
+
     if req.use_pep517:
         return True
 
     if not check_binary_allowed(req):
         logger.info(
-            "Skipping wheel build for %s, due to binaries "
-            "being disabled for it.", req.name,
+            "Skipping wheel build for %s, due to binaries being disabled for it.",
+            req.name,
         )
         return False
 
@@ -90,7 +88,8 @@ def _should_build(
         # we don't build legacy requirements if wheel is not installed
         logger.info(
             "Using legacy 'setup.py install' for %s, "
-            "since package 'wheel' is not installed.", req.name,
+            "since package 'wheel' is not installed.",
+            req.name,
         )
         return False
 
@@ -98,28 +97,23 @@ def _should_build(
 
 
 def should_build_for_wheel_command(
-    req,  # type: InstallRequirement
-):
-    # type: (...) -> bool
-    return _should_build(
-        req, need_wheel=True, check_binary_allowed=_always_true
-    )
+    req: InstallRequirement,
+) -> bool:
+    return _should_build(req, need_wheel=True, check_binary_allowed=_always_true)
 
 
 def should_build_for_install_command(
-    req,  # type: InstallRequirement
-    check_binary_allowed,  # type: BinaryAllowedPredicate
-):
-    # type: (...) -> bool
+    req: InstallRequirement,
+    check_binary_allowed: BinaryAllowedPredicate,
+) -> bool:
     return _should_build(
         req, need_wheel=False, check_binary_allowed=check_binary_allowed
     )
 
 
 def _should_cache(
-    req,  # type: InstallRequirement
-):
-    # type: (...) -> Optional[bool]
+    req: InstallRequirement,
+) -> Optional[bool]:
     """
     Return whether a built InstallRequirement can be stored in the persistent
     wheel cache, assuming the wheel cache is available, and _should_build()
@@ -150,10 +144,9 @@ def _should_cache(
 
 
 def _get_cache_dir(
-    req,  # type: InstallRequirement
-    wheel_cache,  # type: WheelCache
-):
-    # type: (...) -> str
+    req: InstallRequirement,
+    wheel_cache: WheelCache,
+) -> str:
     """Return the persistent or temporary cache directory where the built
     wheel need to be stored.
     """
@@ -166,103 +159,112 @@ def _get_cache_dir(
     return cache_dir
 
 
-def _always_true(_):
-    # type: (Any) -> bool
+def _always_true(_: Any) -> bool:
     return True
 
 
-def _get_metadata_version(dist):
-    # type: (Distribution) -> Optional[Version]
-    for line in dist.get_metadata_lines(dist.PKG_INFO):
-        if line.lower().startswith("metadata-version:"):
-            value = line.split(":", 1)[-1].strip()
-            try:
-                return Version(value)
-            except InvalidVersion:
-                msg = "Invalid Metadata-Version: {}".format(value)
-                raise UnsupportedWheel(msg)
-    raise UnsupportedWheel("Missing Metadata-Version")
-
-
-def _verify_one(req, wheel_path):
-    # type: (InstallRequirement, str) -> None
-    canonical_name = canonicalize_name(req.name)
+def _verify_one(req: InstallRequirement, wheel_path: str) -> None:
+    canonical_name = canonicalize_name(req.name or "")
     w = Wheel(os.path.basename(wheel_path))
     if canonicalize_name(w.name) != canonical_name:
         raise InvalidWheelFilename(
             "Wheel has unexpected file name: expected {!r}, "
             "got {!r}".format(canonical_name, w.name),
         )
-    with zipfile.ZipFile(wheel_path, allowZip64=True) as zf:
-        dist = pkg_resources_distribution_for_wheel(
-            zf, canonical_name, wheel_path,
-        )
-    if canonicalize_version(dist.version) != canonicalize_version(w.version):
+    dist = get_wheel_distribution(FilesystemWheel(wheel_path), canonical_name)
+    dist_verstr = str(dist.version)
+    if canonicalize_version(dist_verstr) != canonicalize_version(w.version):
         raise InvalidWheelFilename(
             "Wheel has unexpected file name: expected {!r}, "
-            "got {!r}".format(dist.version, w.version),
+            "got {!r}".format(dist_verstr, w.version),
         )
-    if (_get_metadata_version(dist) >= Version("1.2")
-            and not isinstance(dist.parsed_version, Version)):
+    metadata_version_value = dist.metadata_version
+    if metadata_version_value is None:
+        raise UnsupportedWheel("Missing Metadata-Version")
+    try:
+        metadata_version = Version(metadata_version_value)
+    except InvalidVersion:
+        msg = f"Invalid Metadata-Version: {metadata_version_value}"
+        raise UnsupportedWheel(msg)
+    if metadata_version >= Version("1.2") and not isinstance(dist.version, Version):
         raise UnsupportedWheel(
             "Metadata 1.2 mandates PEP 440 version, "
-            "but {!r} is not".format(dist.version)
+            "but {!r} is not".format(dist_verstr)
         )
 
 
 def _build_one(
-    req,  # type: InstallRequirement
-    output_dir,  # type: str
-    verify,  # type: bool
-    build_options,  # type: List[str]
-    global_options,  # type: List[str]
-):
-    # type: (...) -> Optional[str]
+    req: InstallRequirement,
+    output_dir: str,
+    verify: bool,
+    build_options: List[str],
+    global_options: List[str],
+    editable: bool,
+) -> Optional[str]:
     """Build one wheel.
 
     :return: The filename of the built wheel, or None if the build failed.
     """
+    artifact = "editable" if editable else "wheel"
     try:
         ensure_dir(output_dir)
     except OSError as e:
         logger.warning(
-            "Building wheel for %s failed: %s",
-            req.name, e,
+            "Building %s for %s failed: %s",
+            artifact,
+            req.name,
+            e,
         )
         return None
 
     # Install build deps into temporary directory (PEP 518)
     with req.build_env:
         wheel_path = _build_one_inside_env(
-            req, output_dir, build_options, global_options
+            req, output_dir, build_options, global_options, editable
         )
     if wheel_path and verify:
         try:
             _verify_one(req, wheel_path)
         except (InvalidWheelFilename, UnsupportedWheel) as e:
-            logger.warning("Built wheel for %s is invalid: %s", req.name, e)
+            logger.warning("Built %s for %s is invalid: %s", artifact, req.name, e)
             return None
     return wheel_path
 
 
 def _build_one_inside_env(
-    req,  # type: InstallRequirement
-    output_dir,  # type: str
-    build_options,  # type: List[str]
-    global_options,  # type: List[str]
-):
-    # type: (...) -> Optional[str]
+    req: InstallRequirement,
+    output_dir: str,
+    build_options: List[str],
+    global_options: List[str],
+    editable: bool,
+) -> Optional[str]:
     with TempDirectory(kind="wheel") as temp_dir:
         assert req.name
         if req.use_pep517:
             assert req.metadata_directory
-            wheel_path = build_wheel_pep517(
-                name=req.name,
-                backend=req.pep517_backend,
-                metadata_directory=req.metadata_directory,
-                build_options=build_options,
-                tempd=temp_dir.path,
-            )
+            assert req.pep517_backend
+            if global_options:
+                logger.warning(
+                    "Ignoring --global-option when building %s using PEP 517", req.name
+                )
+            if build_options:
+                logger.warning(
+                    "Ignoring --build-option when building %s using PEP 517", req.name
+                )
+            if editable:
+                wheel_path = build_wheel_editable(
+                    name=req.name,
+                    backend=req.pep517_backend,
+                    metadata_directory=req.metadata_directory,
+                    tempd=temp_dir.path,
+                )
+            else:
+                wheel_path = build_wheel_pep517(
+                    name=req.name,
+                    backend=req.pep517_backend,
+                    metadata_directory=req.metadata_directory,
+                    tempd=temp_dir.path,
+                )
         else:
             wheel_path = build_wheel_legacy(
                 name=req.name,
@@ -279,16 +281,20 @@ def _build_one_inside_env(
             try:
                 wheel_hash, length = hash_file(wheel_path)
                 shutil.move(wheel_path, dest_path)
-                logger.info('Created wheel for %s: '
-                            'filename=%s size=%d sha256=%s',
-                            req.name, wheel_name, length,
-                            wheel_hash.hexdigest())
-                logger.info('Stored in directory: %s', output_dir)
+                logger.info(
+                    "Created wheel for %s: filename=%s size=%d sha256=%s",
+                    req.name,
+                    wheel_name,
+                    length,
+                    wheel_hash.hexdigest(),
+                )
+                logger.info("Stored in directory: %s", output_dir)
                 return dest_path
             except Exception as e:
                 logger.warning(
                     "Building wheel for %s failed: %s",
-                    req.name, e,
+                    req.name,
+                    e,
                 )
         # Ignore return, we can't do anything else useful.
         if not req.use_pep517:
@@ -296,30 +302,30 @@ def _build_one_inside_env(
         return None
 
 
-def _clean_one_legacy(req, global_options):
-    # type: (InstallRequirement, List[str]) -> bool
+def _clean_one_legacy(req: InstallRequirement, global_options: List[str]) -> bool:
     clean_args = make_setuptools_clean_args(
         req.setup_py_path,
         global_options=global_options,
     )
 
-    logger.info('Running setup.py clean for %s', req.name)
+    logger.info("Running setup.py clean for %s", req.name)
     try:
-        call_subprocess(clean_args, cwd=req.source_dir)
+        call_subprocess(
+            clean_args, command_desc="python setup.py clean", cwd=req.source_dir
+        )
         return True
     except Exception:
-        logger.error('Failed cleaning build dir for %s', req.name)
+        logger.error("Failed cleaning build dir for %s", req.name)
         return False
 
 
 def build(
-    requirements,  # type: Iterable[InstallRequirement]
-    wheel_cache,  # type: WheelCache
-    verify,  # type: bool
-    build_options,  # type: List[str]
-    global_options,  # type: List[str]
-):
-    # type: (...) -> BuildResult
+    requirements: Iterable[InstallRequirement],
+    wheel_cache: WheelCache,
+    verify: bool,
+    build_options: List[str],
+    global_options: List[str],
+) -> BuildResult:
     """Build wheels.
 
     :return: The list of InstallRequirement that succeeded to build and
@@ -330,16 +336,22 @@ def build(
 
     # Build the wheels.
     logger.info(
-        'Building wheels for collected packages: %s',
-        ', '.join(req.name for req in requirements),  # type: ignore
+        "Building wheels for collected packages: %s",
+        ", ".join(req.name for req in requirements),  # type: ignore
     )
 
     with indent_log():
         build_successes, build_failures = [], []
         for req in requirements:
+            assert req.name
             cache_dir = _get_cache_dir(req, wheel_cache)
             wheel_file = _build_one(
-                req, cache_dir, verify, build_options, global_options
+                req,
+                cache_dir,
+                verify,
+                build_options,
+                global_options,
+                req.editable and req.permit_editable_wheels,
             )
             if wheel_file:
                 # Update the link for this.
@@ -353,13 +365,13 @@ def build(
     # notify success/failure
     if build_successes:
         logger.info(
-            'Successfully built %s',
-            ' '.join([req.name for req in build_successes]),  # type: ignore
+            "Successfully built %s",
+            " ".join([req.name for req in build_successes]),  # type: ignore
         )
     if build_failures:
         logger.info(
-            'Failed to build %s',
-            ' '.join([req.name for req in build_failures]),  # type: ignore
+            "Failed to build %s",
+            " ".join([req.name for req in build_failures]),  # type: ignore
         )
     # Return a list of requirements that failed to build
     return build_successes, build_failures
diff --git a/src/pip/_vendor/README.rst b/src/pip/_vendor/README.rst
index 6699d72c2f3..26904ca251a 100644
--- a/src/pip/_vendor/README.rst
+++ b/src/pip/_vendor/README.rst
@@ -16,7 +16,7 @@ Vendoring Policy
   pure Python.
 * Any modifications made to libraries **MUST** be noted in
   ``pip/_vendor/README.rst`` and their corresponding patches **MUST** be
-  included ``tools/automation/vendoring/patches``.
+  included ``tools/vendoring/patches``.
 * Vendored libraries should have corresponding ``vendored()`` entries in
   ``pip/_vendor/__init__.py``.
 
@@ -100,7 +100,8 @@ Modifications
 
 * ``setuptools`` is completely stripped to only keep ``pkg_resources``.
 * ``pkg_resources`` has been modified to import its dependencies from
-  ``pip._vendor``.
+  ``pip._vendor``, and to use the vendored copy of ``platformdirs``
+  rather than ``appdirs``.
 * ``packaging`` has been modified to import its dependencies from
   ``pip._vendor``.
 * ``html5lib`` has been modified to import six from ``pip._vendor``, to prefer
@@ -111,14 +112,14 @@ Modifications
 * ``requests`` has been modified to import its other dependencies from
   ``pip._vendor`` and to *not* load ``simplejson`` (all platforms) and
   ``pyopenssl`` (Windows).
-
+* ``platformdirs`` has been modified to import its submodules from ``pip._vendor.platformdirs``.
 
 Automatic Vendoring
 ===================
 
-Vendoring is automated via the ``vendoring`` tool from the content of
+Vendoring is automated via the `vendoring `_ tool from the content of
 ``pip/_vendor/vendor.txt`` and the different patches in
-``tools/automation/vendoring/patches``.
+``tools/vendoring/patches``.
 Launch it via ``vendoring sync . -v`` (requires ``vendoring>=0.2.2``).
 
 
diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py
index c3db83ff6aa..3843cb09955 100644
--- a/src/pip/_vendor/__init__.py
+++ b/src/pip/_vendor/__init__.py
@@ -58,11 +58,9 @@ def vendored(modulename):
     sys.path[:] = glob.glob(os.path.join(WHEEL_DIR, "*.whl")) + sys.path
 
     # Actually alias all of our vendored dependencies.
-    vendored("appdirs")
     vendored("cachecontrol")
     vendored("certifi")
     vendored("colorama")
-    vendored("contextlib2")
     vendored("distlib")
     vendored("distro")
     vendored("html5lib")
@@ -75,8 +73,8 @@ def vendored(modulename):
     vendored("packaging.specifiers")
     vendored("pep517")
     vendored("pkg_resources")
+    vendored("platformdirs")
     vendored("progress")
-    vendored("retrying")
     vendored("requests")
     vendored("requests.exceptions")
     vendored("requests.packages")
@@ -108,7 +106,6 @@ def vendored(modulename):
     vendored("requests.packages.urllib3.util.timeout")
     vendored("requests.packages.urllib3.util.url")
     vendored("resolvelib")
-    vendored("toml")
-    vendored("toml.encoder")
-    vendored("toml.decoder")
+    vendored("tenacity")
+    vendored("tomli")
     vendored("urllib3")
diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py
deleted file mode 100644
index 33a3b77410c..00000000000
--- a/src/pip/_vendor/appdirs.py
+++ /dev/null
@@ -1,633 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2005-2010 ActiveState Software Inc.
-# Copyright (c) 2013 Eddy Petrișor
-
-"""Utilities for determining application-specific dirs.
-
-See  for details and usage.
-"""
-# Dev Notes:
-# - MSDN on where to store app data files:
-#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
-# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html
-# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html
-
-__version__ = "1.4.4"
-__version_info__ = tuple(int(segment) for segment in __version__.split("."))
-
-
-import sys
-import os
-
-PY3 = sys.version_info[0] == 3
-
-if PY3:
-    unicode = str
-
-if sys.platform.startswith('java'):
-    import platform
-    os_name = platform.java_ver()[3][0]
-    if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc.
-        system = 'win32'
-    elif os_name.startswith('Mac'): # "Mac OS X", etc.
-        system = 'darwin'
-    else: # "Linux", "SunOS", "FreeBSD", etc.
-        # Setting this to "linux2" is not ideal, but only Windows or Mac
-        # are actually checked for and the rest of the module expects
-        # *sys.platform* style strings.
-        system = 'linux2'
-elif sys.platform == 'cli' and os.name == 'nt':
-    # Detect Windows in IronPython to match pip._internal.utils.compat.WINDOWS
-    # Discussion: 
-    system = 'win32'
-else:
-    system = sys.platform
-
-
-
-def user_data_dir(appname=None, appauthor=None, version=None, roaming=False):
-    r"""Return full path to the user-specific data dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "roaming" (boolean, default False) can be set True to use the Windows
-            roaming appdata directory. That means that for users on a Windows
-            network setup for roaming profiles, this user data will be
-            sync'd on login. See
-            
-            for a discussion of issues.
-
-    Typical user data directories are:
-        Mac OS X:               ~/Library/Application Support/  # or ~/.config/, if the other does not exist
-        Unix:                   ~/.local/share/    # or in $XDG_DATA_HOME, if defined
-        Win XP (not roaming):   C:\Documents and Settings\\Application Data\\
-        Win XP (roaming):       C:\Documents and Settings\\Local Settings\Application Data\\
-        Win 7  (not roaming):   C:\Users\\AppData\Local\\
-        Win 7  (roaming):       C:\Users\\AppData\Roaming\\
-
-    For Unix, we follow the XDG spec and support $XDG_DATA_HOME.
-    That means, by default "~/.local/share/".
-    """
-    if system == "win32":
-        if appauthor is None:
-            appauthor = appname
-        const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA"
-        path = os.path.normpath(_get_win_folder(const))
-        if appname:
-            if appauthor is not False:
-                path = os.path.join(path, appauthor, appname)
-            else:
-                path = os.path.join(path, appname)
-    elif system == 'darwin':
-        path = os.path.expanduser('~/Library/Application Support/')
-        if appname:
-            path = os.path.join(path, appname)
-    else:
-        path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share"))
-        if appname:
-            path = os.path.join(path, appname)
-    if appname and version:
-        path = os.path.join(path, version)
-    return path
-
-
-def site_data_dir(appname=None, appauthor=None, version=None, multipath=False):
-    r"""Return full path to the user-shared data dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "multipath" is an optional parameter only applicable to *nix
-            which indicates that the entire list of data dirs should be
-            returned. By default, the first item from XDG_DATA_DIRS is
-            returned, or '/usr/local/share/',
-            if XDG_DATA_DIRS is not set
-
-    Typical site data directories are:
-        Mac OS X:   /Library/Application Support/
-        Unix:       /usr/local/share/ or /usr/share/
-        Win XP:     C:\Documents and Settings\All Users\Application Data\\
-        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
-        Win 7:      C:\ProgramData\\   # Hidden, but writeable on Win 7.
-
-    For Unix, this is using the $XDG_DATA_DIRS[0] default.
-
-    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
-    """
-    if system == "win32":
-        if appauthor is None:
-            appauthor = appname
-        path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA"))
-        if appname:
-            if appauthor is not False:
-                path = os.path.join(path, appauthor, appname)
-            else:
-                path = os.path.join(path, appname)
-    elif system == 'darwin':
-        path = os.path.expanduser('/Library/Application Support')
-        if appname:
-            path = os.path.join(path, appname)
-    else:
-        # XDG default for $XDG_DATA_DIRS
-        # only first, if multipath is False
-        path = os.getenv('XDG_DATA_DIRS',
-                         os.pathsep.join(['/usr/local/share', '/usr/share']))
-        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)]
-        if appname:
-            if version:
-                appname = os.path.join(appname, version)
-            pathlist = [os.path.join(x, appname) for x in pathlist]
-
-        if multipath:
-            path = os.pathsep.join(pathlist)
-        else:
-            path = pathlist[0]
-        return path
-
-    if appname and version:
-        path = os.path.join(path, version)
-    return path
-
-
-def user_config_dir(appname=None, appauthor=None, version=None, roaming=False):
-    r"""Return full path to the user-specific config dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "roaming" (boolean, default False) can be set True to use the Windows
-            roaming appdata directory. That means that for users on a Windows
-            network setup for roaming profiles, this user data will be
-            sync'd on login. See
-            
-            for a discussion of issues.
-
-    Typical user config directories are:
-        Mac OS X:               same as user_data_dir
-        Unix:                   ~/.config/     # or in $XDG_CONFIG_HOME, if defined
-        Win *:                  same as user_data_dir
-
-    For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME.
-    That means, by default "~/.config/".
-    """
-    if system in ["win32", "darwin"]:
-        path = user_data_dir(appname, appauthor, None, roaming)
-    else:
-        path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config"))
-        if appname:
-            path = os.path.join(path, appname)
-    if appname and version:
-        path = os.path.join(path, version)
-    return path
-
-
-# for the discussion regarding site_config_dir locations
-# see 
-def site_config_dir(appname=None, appauthor=None, version=None, multipath=False):
-    r"""Return full path to the user-shared data dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "multipath" is an optional parameter only applicable to *nix
-            which indicates that the entire list of config dirs should be
-            returned. By default, the first item from XDG_CONFIG_DIRS is
-            returned, or '/etc/xdg/', if XDG_CONFIG_DIRS is not set
-
-    Typical site config directories are:
-        Mac OS X:   same as site_data_dir
-        Unix:       /etc/xdg/ or $XDG_CONFIG_DIRS[i]/ for each value in
-                    $XDG_CONFIG_DIRS
-        Win *:      same as site_data_dir
-        Vista:      (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.)
-
-    For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False
-
-    WARNING: Do not use this on Windows. See the Vista-Fail note above for why.
-    """
-    if system in ["win32", "darwin"]:
-        path = site_data_dir(appname, appauthor)
-        if appname and version:
-            path = os.path.join(path, version)
-    else:
-        # XDG default for $XDG_CONFIG_DIRS (missing or empty)
-        # see 
-        # only first, if multipath is False
-        path = os.getenv('XDG_CONFIG_DIRS') or '/etc/xdg'
-        pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) if x]
-        if appname:
-            if version:
-                appname = os.path.join(appname, version)
-            pathlist = [os.path.join(x, appname) for x in pathlist]
-
-        if multipath:
-            path = os.pathsep.join(pathlist)
-        else:
-            path = pathlist[0]
-    return path
-
-
-def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True):
-    r"""Return full path to the user-specific cache dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "opinion" (boolean) can be False to disable the appending of
-            "Cache" to the base app data dir for Windows. See
-            discussion below.
-
-    Typical user cache directories are:
-        Mac OS X:   ~/Library/Caches/
-        Unix:       ~/.cache/ (XDG default)
-        Win XP:     C:\Documents and Settings\\Local Settings\Application Data\\\Cache
-        Vista:      C:\Users\\AppData\Local\\\Cache
-
-    On Windows the only suggestion in the MSDN docs is that local settings go in
-    the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming
-    app data dir (the default returned by `user_data_dir` above). Apps typically
-    put cache data somewhere *under* the given dir here. Some examples:
-        ...\Mozilla\Firefox\Profiles\\Cache
-        ...\Acme\SuperApp\Cache\1.0
-    OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value.
-    This can be disabled with the `opinion=False` option.
-    """
-    if system == "win32":
-        if appauthor is None:
-            appauthor = appname
-        path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA"))
-        # When using Python 2, return paths as bytes on Windows like we do on
-        # other operating systems. See helper function docs for more details.
-        if not PY3 and isinstance(path, unicode):
-            path = _win_path_to_bytes(path)
-        if appname:
-            if appauthor is not False:
-                path = os.path.join(path, appauthor, appname)
-            else:
-                path = os.path.join(path, appname)
-            if opinion:
-                path = os.path.join(path, "Cache")
-    elif system == 'darwin':
-        path = os.path.expanduser('~/Library/Caches')
-        if appname:
-            path = os.path.join(path, appname)
-    else:
-        path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
-        if appname:
-            path = os.path.join(path, appname)
-    if appname and version:
-        path = os.path.join(path, version)
-    return path
-
-
-def user_state_dir(appname=None, appauthor=None, version=None, roaming=False):
-    r"""Return full path to the user-specific state dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "roaming" (boolean, default False) can be set True to use the Windows
-            roaming appdata directory. That means that for users on a Windows
-            network setup for roaming profiles, this user data will be
-            sync'd on login. See
-            
-            for a discussion of issues.
-
-    Typical user state directories are:
-        Mac OS X:  same as user_data_dir
-        Unix:      ~/.local/state/   # or in $XDG_STATE_HOME, if defined
-        Win *:     same as user_data_dir
-
-    For Unix, we follow this Debian proposal 
-    to extend the XDG spec and support $XDG_STATE_HOME.
-
-    That means, by default "~/.local/state/".
-    """
-    if system in ["win32", "darwin"]:
-        path = user_data_dir(appname, appauthor, None, roaming)
-    else:
-        path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state"))
-        if appname:
-            path = os.path.join(path, appname)
-    if appname and version:
-        path = os.path.join(path, version)
-    return path
-
-
-def user_log_dir(appname=None, appauthor=None, version=None, opinion=True):
-    r"""Return full path to the user-specific log dir for this application.
-
-        "appname" is the name of application.
-            If None, just the system directory is returned.
-        "appauthor" (only used on Windows) is the name of the
-            appauthor or distributing body for this application. Typically
-            it is the owning company name. This falls back to appname. You may
-            pass False to disable it.
-        "version" is an optional version path element to append to the
-            path. You might want to use this if you want multiple versions
-            of your app to be able to run independently. If used, this
-            would typically be ".".
-            Only applied when appname is present.
-        "opinion" (boolean) can be False to disable the appending of
-            "Logs" to the base app data dir for Windows, and "log" to the
-            base cache dir for Unix. See discussion below.
-
-    Typical user log directories are:
-        Mac OS X:   ~/Library/Logs/
-        Unix:       ~/.cache//log  # or under $XDG_CACHE_HOME if defined
-        Win XP:     C:\Documents and Settings\\Local Settings\Application Data\\\Logs
-        Vista:      C:\Users\\AppData\Local\\\Logs
-
-    On Windows the only suggestion in the MSDN docs is that local settings
-    go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in
-    examples of what some windows apps use for a logs dir.)
-
-    OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA`
-    value for Windows and appends "log" to the user cache dir for Unix.
-    This can be disabled with the `opinion=False` option.
-    """
-    if system == "darwin":
-        path = os.path.join(
-            os.path.expanduser('~/Library/Logs'),
-            appname)
-    elif system == "win32":
-        path = user_data_dir(appname, appauthor, version)
-        version = False
-        if opinion:
-            path = os.path.join(path, "Logs")
-    else:
-        path = user_cache_dir(appname, appauthor, version)
-        version = False
-        if opinion:
-            path = os.path.join(path, "log")
-    if appname and version:
-        path = os.path.join(path, version)
-    return path
-
-
-class AppDirs(object):
-    """Convenience wrapper for getting application dirs."""
-    def __init__(self, appname=None, appauthor=None, version=None,
-            roaming=False, multipath=False):
-        self.appname = appname
-        self.appauthor = appauthor
-        self.version = version
-        self.roaming = roaming
-        self.multipath = multipath
-
-    @property
-    def user_data_dir(self):
-        return user_data_dir(self.appname, self.appauthor,
-                             version=self.version, roaming=self.roaming)
-
-    @property
-    def site_data_dir(self):
-        return site_data_dir(self.appname, self.appauthor,
-                             version=self.version, multipath=self.multipath)
-
-    @property
-    def user_config_dir(self):
-        return user_config_dir(self.appname, self.appauthor,
-                               version=self.version, roaming=self.roaming)
-
-    @property
-    def site_config_dir(self):
-        return site_config_dir(self.appname, self.appauthor,
-                             version=self.version, multipath=self.multipath)
-
-    @property
-    def user_cache_dir(self):
-        return user_cache_dir(self.appname, self.appauthor,
-                              version=self.version)
-
-    @property
-    def user_state_dir(self):
-        return user_state_dir(self.appname, self.appauthor,
-                              version=self.version)
-
-    @property
-    def user_log_dir(self):
-        return user_log_dir(self.appname, self.appauthor,
-                            version=self.version)
-
-
-#---- internal support stuff
-
-def _get_win_folder_from_registry(csidl_name):
-    """This is a fallback technique at best. I'm not sure if using the
-    registry for this guarantees us the correct answer for all CSIDL_*
-    names.
-    """
-    if PY3:
-      import winreg as _winreg
-    else:
-      import _winreg
-
-    shell_folder_name = {
-        "CSIDL_APPDATA": "AppData",
-        "CSIDL_COMMON_APPDATA": "Common AppData",
-        "CSIDL_LOCAL_APPDATA": "Local AppData",
-    }[csidl_name]
-
-    key = _winreg.OpenKey(
-        _winreg.HKEY_CURRENT_USER,
-        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
-    )
-    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
-    return dir
-
-
-def _get_win_folder_with_pywin32(csidl_name):
-    from win32com.shell import shellcon, shell
-    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
-    # Try to make this a unicode path because SHGetFolderPath does
-    # not return unicode strings when there is unicode data in the
-    # path.
-    try:
-        dir = unicode(dir)
-
-        # Downgrade to short path name if have highbit chars. See
-        # .
-        has_high_char = False
-        for c in dir:
-            if ord(c) > 255:
-                has_high_char = True
-                break
-        if has_high_char:
-            try:
-                import win32api
-                dir = win32api.GetShortPathName(dir)
-            except ImportError:
-                pass
-    except UnicodeError:
-        pass
-    return dir
-
-
-def _get_win_folder_with_ctypes(csidl_name):
-    import ctypes
-
-    csidl_const = {
-        "CSIDL_APPDATA": 26,
-        "CSIDL_COMMON_APPDATA": 35,
-        "CSIDL_LOCAL_APPDATA": 28,
-    }[csidl_name]
-
-    buf = ctypes.create_unicode_buffer(1024)
-    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
-
-    # Downgrade to short path name if have highbit chars. See
-    # .
-    has_high_char = False
-    for c in buf:
-        if ord(c) > 255:
-            has_high_char = True
-            break
-    if has_high_char:
-        buf2 = ctypes.create_unicode_buffer(1024)
-        if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
-            buf = buf2
-
-    return buf.value
-
-def _get_win_folder_with_jna(csidl_name):
-    import array
-    from com.sun import jna
-    from com.sun.jna.platform import win32
-
-    buf_size = win32.WinDef.MAX_PATH * 2
-    buf = array.zeros('c', buf_size)
-    shell = win32.Shell32.INSTANCE
-    shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf)
-    dir = jna.Native.toString(buf.tostring()).rstrip("\0")
-
-    # Downgrade to short path name if have highbit chars. See
-    # .
-    has_high_char = False
-    for c in dir:
-        if ord(c) > 255:
-            has_high_char = True
-            break
-    if has_high_char:
-        buf = array.zeros('c', buf_size)
-        kernel = win32.Kernel32.INSTANCE
-        if kernel.GetShortPathName(dir, buf, buf_size):
-            dir = jna.Native.toString(buf.tostring()).rstrip("\0")
-
-    return dir
-
-if system == "win32":
-    try:
-        from ctypes import windll
-        _get_win_folder = _get_win_folder_with_ctypes
-    except ImportError:
-        try:
-            import com.sun.jna
-            _get_win_folder = _get_win_folder_with_jna
-        except ImportError:
-            _get_win_folder = _get_win_folder_from_registry
-
-
-def _win_path_to_bytes(path):
-    """Encode Windows paths to bytes. Only used on Python 2.
-
-    Motivation is to be consistent with other operating systems where paths
-    are also returned as bytes. This avoids problems mixing bytes and Unicode
-    elsewhere in the codebase. For more details and discussion see
-    .
-
-    If encoding using ASCII and MBCS fails, return the original Unicode path.
-    """
-    for encoding in ('ASCII', 'MBCS'):
-        try:
-            return path.encode(encoding)
-        except (UnicodeEncodeError, LookupError):
-            pass
-    return path
-
-
-#---- self test code
-
-if __name__ == "__main__":
-    appname = "MyApp"
-    appauthor = "MyCompany"
-
-    props = ("user_data_dir",
-             "user_config_dir",
-             "user_cache_dir",
-             "user_state_dir",
-             "user_log_dir",
-             "site_data_dir",
-             "site_config_dir")
-
-    print("-- app dirs %s --" % __version__)
-
-    print("-- app dirs (with optional 'version')")
-    dirs = AppDirs(appname, appauthor, version="1.0")
-    for prop in props:
-        print("%s: %s" % (prop, getattr(dirs, prop)))
-
-    print("\n-- app dirs (without optional 'version')")
-    dirs = AppDirs(appname, appauthor)
-    for prop in props:
-        print("%s: %s" % (prop, getattr(dirs, prop)))
-
-    print("\n-- app dirs (without optional 'appauthor')")
-    dirs = AppDirs(appname)
-    for prop in props:
-        print("%s: %s" % (prop, getattr(dirs, prop)))
-
-    print("\n-- app dirs (with disabled 'appauthor')")
-    dirs = AppDirs(appname, appauthor=False)
-    for prop in props:
-        print("%s: %s" % (prop, getattr(dirs, prop)))
diff --git a/src/pip/_vendor/cachecontrol/LICENSE.txt b/src/pip/_vendor/cachecontrol/LICENSE.txt
index 1ed31ac3688..d8b3b56d361 100644
--- a/src/pip/_vendor/cachecontrol/LICENSE.txt
+++ b/src/pip/_vendor/cachecontrol/LICENSE.txt
@@ -1,4 +1,4 @@
-Copyright 2015 Eric Larson
+Copyright 2012-2021  Eric Larson
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -8,8 +8,6 @@ You may obtain a copy of the License at
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied.
-
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py
index a1bbbbe3bff..8435d628d20 100644
--- a/src/pip/_vendor/cachecontrol/__init__.py
+++ b/src/pip/_vendor/cachecontrol/__init__.py
@@ -1,11 +1,18 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 """CacheControl import Interface.
 
 Make it easy to import from cachecontrol without long namespaces.
 """
 __author__ = "Eric Larson"
 __email__ = "eric@ionrock.org"
-__version__ = "0.12.6"
+__version__ = "0.12.10"
 
 from .wrapper import CacheControl
 from .adapter import CacheControlAdapter
 from .controller import CacheController
+
+import logging
+logging.getLogger(__name__).addHandler(logging.NullHandler())
diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py
index f1e0ad94a1c..4266b5ee92a 100644
--- a/src/pip/_vendor/cachecontrol/_cmd.py
+++ b/src/pip/_vendor/cachecontrol/_cmd.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import logging
 
 from pip._vendor import requests
diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py
index 815650e81fe..94c75e1a05b 100644
--- a/src/pip/_vendor/cachecontrol/adapter.py
+++ b/src/pip/_vendor/cachecontrol/adapter.py
@@ -1,16 +1,20 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import types
 import functools
 import zlib
 
 from pip._vendor.requests.adapters import HTTPAdapter
 
-from .controller import CacheController
+from .controller import CacheController, PERMANENT_REDIRECT_STATUSES
 from .cache import DictCache
 from .filewrapper import CallbackFileWrapper
 
 
 class CacheControlAdapter(HTTPAdapter):
-    invalidating_methods = {"PUT", "DELETE"}
+    invalidating_methods = {"PUT", "PATCH", "DELETE"}
 
     def __init__(
         self,
@@ -93,7 +97,7 @@ def build_response(
                 response = cached_response
 
             # We always cache the 301 responses
-            elif response.status == 301:
+            elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
                 self.controller.cache_response(request, response)
             else:
                 # Wrap the response file with a wrapper that will cache the
diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py
index 94e07732d91..44e4309d20d 100644
--- a/src/pip/_vendor/cachecontrol/cache.py
+++ b/src/pip/_vendor/cachecontrol/cache.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 """
 The cache object API for implementing caches. The default is a thread
 safe in-memory dictionary.
@@ -10,7 +14,7 @@ class BaseCache(object):
     def get(self, key):
         raise NotImplementedError()
 
-    def set(self, key, value):
+    def set(self, key, value, expires=None):
         raise NotImplementedError()
 
     def delete(self, key):
@@ -29,7 +33,7 @@ def __init__(self, init_dict=None):
     def get(self, key):
         return self.data.get(key, None)
 
-    def set(self, key, value):
+    def set(self, key, value, expires=None):
         with self.lock:
             self.data.update({key: value})
 
diff --git a/src/pip/_vendor/cachecontrol/caches/__init__.py b/src/pip/_vendor/cachecontrol/caches/__init__.py
index 0e1658fa5e5..44becd68439 100644
--- a/src/pip/_vendor/cachecontrol/caches/__init__.py
+++ b/src/pip/_vendor/cachecontrol/caches/__init__.py
@@ -1,2 +1,6 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from .file_cache import FileCache  # noqa
 from .redis_cache import RedisCache  # noqa
diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py
index 607b9452428..6cd1106f880 100644
--- a/src/pip/_vendor/cachecontrol/caches/file_cache.py
+++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import hashlib
 import os
 from textwrap import dedent
@@ -114,7 +118,7 @@ def get(self, key):
         except FileNotFoundError:
             return None
 
-    def set(self, key, value):
+    def set(self, key, value, expires=None):
         name = self._fn(key)
 
         # Make sure the directory exists
diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py
index ed705ce7df6..720b507c523 100644
--- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py
+++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from __future__ import division
 
 from datetime import datetime
diff --git a/src/pip/_vendor/cachecontrol/compat.py b/src/pip/_vendor/cachecontrol/compat.py
index 33b5aed0a32..ccec9379dba 100644
--- a/src/pip/_vendor/cachecontrol/compat.py
+++ b/src/pip/_vendor/cachecontrol/compat.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 try:
     from urllib.parse import urljoin
 except ImportError:
@@ -9,7 +13,6 @@
 except ImportError:
     import pickle
 
-
 # Handle the case where the requests module has been patched to not have
 # urllib3 bundled as part of its source.
 try:
diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py
index dafe55ca70c..d7e73802818 100644
--- a/src/pip/_vendor/cachecontrol/controller.py
+++ b/src/pip/_vendor/cachecontrol/controller.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 """
 The httplib2 algorithms ported for use with requests.
 """
@@ -17,6 +21,8 @@
 
 URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
 
+PERMANENT_REDIRECT_STATUSES = (301, 308)
+
 
 def parse_uri(uri):
     """Parses a URI using the regex given in Appendix B of RFC 3986.
@@ -37,7 +43,7 @@ def __init__(
         self.cache = DictCache() if cache is None else cache
         self.cache_etags = cache_etags
         self.serializer = serializer or Serializer()
-        self.cacheable_status_codes = status_codes or (200, 203, 300, 301)
+        self.cacheable_status_codes = status_codes or (200, 203, 300, 301, 308)
 
     @classmethod
     def _urlnorm(cls, uri):
@@ -147,17 +153,18 @@ def cached_request(self, request):
             logger.warning("Cache entry deserialization failed, entry ignored")
             return False
 
-        # If we have a cached 301, return it immediately. We don't
-        # need to test our response for other headers b/c it is
+        # If we have a cached permanent redirect, return it immediately. We
+        # don't need to test our response for other headers b/c it is
         # intrinsically "cacheable" as it is Permanent.
+        #
         # See:
         #   https://tools.ietf.org/html/rfc7231#section-6.4.2
         #
         # Client can try to refresh the value by repeating the request
         # with cache busting headers as usual (ie no-cache).
-        if resp.status == 301:
+        if int(resp.status) in PERMANENT_REDIRECT_STATUSES:
             msg = (
-                'Returning cached "301 Moved Permanently" response '
+                "Returning cached permanent redirect response "
                 "(ignoring date and etag information)"
             )
             logger.debug(msg)
@@ -261,6 +268,11 @@ def cache_response(self, request, response, body=None, status_codes=None):
 
         response_headers = CaseInsensitiveDict(response.headers)
 
+        if "date" in response_headers:
+            date = calendar.timegm(parsedate_tz(response_headers["date"]))
+        else:
+            date = 0
+
         # If we've been given a body, our response has a Content-Length, that
         # Content-Length is valid then we can check to see if the body we've
         # been given matches the expected size, and if it doesn't we'll just
@@ -304,35 +316,62 @@ def cache_response(self, request, response, body=None, status_codes=None):
 
         # If we've been given an etag, then keep the response
         if self.cache_etags and "etag" in response_headers:
+            expires_time = 0
+            if response_headers.get("expires"):
+                expires = parsedate_tz(response_headers["expires"])
+                if expires is not None:
+                    expires_time = calendar.timegm(expires) - date
+
+            expires_time = max(expires_time, 14 * 86400)
+
+            logger.debug("etag object cached for {0} seconds".format(expires_time))
             logger.debug("Caching due to etag")
             self.cache.set(
-                cache_url, self.serializer.dumps(request, response, body=body)
+                cache_url,
+                self.serializer.dumps(request, response, body),
+                expires=expires_time,
             )
 
-        # Add to the cache any 301s. We do this before looking that
-        # the Date headers.
-        elif response.status == 301:
-            logger.debug("Caching permanant redirect")
-            self.cache.set(cache_url, self.serializer.dumps(request, response))
+        # Add to the cache any permanent redirects. We do this before looking
+        # that the Date headers.
+        elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
+            logger.debug("Caching permanent redirect")
+            self.cache.set(cache_url, self.serializer.dumps(request, response, b""))
 
         # Add to the cache if the response headers demand it. If there
         # is no date header then we can't do anything about expiring
         # the cache.
         elif "date" in response_headers:
+            date = calendar.timegm(parsedate_tz(response_headers["date"]))
             # cache when there is a max-age > 0
             if "max-age" in cc and cc["max-age"] > 0:
                 logger.debug("Caching b/c date exists and max-age > 0")
+                expires_time = cc["max-age"]
                 self.cache.set(
-                    cache_url, self.serializer.dumps(request, response, body=body)
+                    cache_url,
+                    self.serializer.dumps(request, response, body),
+                    expires=expires_time,
                 )
 
             # If the request can expire, it means we should cache it
             # in the meantime.
             elif "expires" in response_headers:
                 if response_headers["expires"]:
-                    logger.debug("Caching b/c of expires header")
+                    expires = parsedate_tz(response_headers["expires"])
+                    if expires is not None:
+                        expires_time = calendar.timegm(expires) - date
+                    else:
+                        expires_time = None
+
+                    logger.debug(
+                        "Caching b/c of expires header. expires in {0} seconds".format(
+                            expires_time
+                        )
+                    )
                     self.cache.set(
-                        cache_url, self.serializer.dumps(request, response, body=body)
+                        cache_url,
+                        self.serializer.dumps(request, response, body=body),
+                        expires=expires_time,
                     )
 
     def update_cached_response(self, request, response):
diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py
index 30ed4c5a62a..f5ed5f6f6ec 100644
--- a/src/pip/_vendor/cachecontrol/filewrapper.py
+++ b/src/pip/_vendor/cachecontrol/filewrapper.py
@@ -1,4 +1,9 @@
-from io import BytesIO
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from tempfile import NamedTemporaryFile
+import mmap
 
 
 class CallbackFileWrapper(object):
@@ -11,10 +16,17 @@ class CallbackFileWrapper(object):
 
     This class uses members with a double underscore (__) leading prefix so as
     not to accidentally shadow an attribute.
+
+    The data is stored in a temporary file until it is all available.  As long
+    as the temporary files directory is disk-based (sometimes it's a
+    memory-backed-``tmpfs`` on Linux), data will be unloaded to disk if memory
+    pressure is high.  For small files the disk usually won't be used at all,
+    it'll all be in the filesystem memory cache, so there should be no
+    performance impact.
     """
 
     def __init__(self, fp, callback):
-        self.__buf = BytesIO()
+        self.__buf = NamedTemporaryFile("rb+", delete=True)
         self.__fp = fp
         self.__callback = callback
 
@@ -49,7 +61,19 @@ def __is_fp_closed(self):
 
     def _close(self):
         if self.__callback:
-            self.__callback(self.__buf.getvalue())
+            if self.__buf.tell() == 0:
+                # Empty file:
+                result = b""
+            else:
+                # Return the data without actually loading it into memory,
+                # relying on Python's buffer API and mmap(). mmap() just gives
+                # a view directly into the filesystem's memory cache, so it
+                # doesn't result in duplicate memory use.
+                self.__buf.seek(0, 0)
+                result = memoryview(
+                    mmap.mmap(self.__buf.fileno(), 0, access=mmap.ACCESS_READ)
+                )
+            self.__callback(result)
 
         # We assign this to None here, because otherwise we can get into
         # really tricky problems where the CPython interpreter dead locks
@@ -58,9 +82,16 @@ def _close(self):
         # and allows the garbage collector to do it's thing normally.
         self.__callback = None
 
+        # Closing the temporary file releases memory and frees disk space.
+        # Important when caching big files.
+        self.__buf.close()
+
     def read(self, amt=None):
         data = self.__fp.read(amt)
-        self.__buf.write(data)
+        if data:
+            # We may be dealing with b'', a sign that things are over:
+            # it's passed e.g. after we've already closed self.__buf.
+            self.__buf.write(data)
         if self.__is_fp_closed():
             self._close()
 
diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py
index 6c0e9790d5d..ebe4a96f589 100644
--- a/src/pip/_vendor/cachecontrol/heuristics.py
+++ b/src/pip/_vendor/cachecontrol/heuristics.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import calendar
 import time
 
diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py
index 3b6ec2de1c1..b075df18682 100644
--- a/src/pip/_vendor/cachecontrol/serialize.py
+++ b/src/pip/_vendor/cachecontrol/serialize.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import base64
 import io
 import json
@@ -17,24 +21,18 @@ def _b64_decode_str(s):
     return _b64_decode_bytes(s).decode("utf8")
 
 
-class Serializer(object):
+_default_body_read = object()
+
 
+class Serializer(object):
     def dumps(self, request, response, body=None):
         response_headers = CaseInsensitiveDict(response.headers)
 
         if body is None:
+            # When a body isn't passed in, we'll read the response. We
+            # also update the response with a new file handler to be
+            # sure it acts as though it was never read.
             body = response.read(decode_content=False)
-
-            # NOTE: 99% sure this is dead code. I'm only leaving it
-            #       here b/c I don't have a test yet to prove
-            #       it. Basically, before using
-            #       `cachecontrol.filewrapper.CallbackFileWrapper`,
-            #       this made an effort to reset the file handle. The
-            #       `CallbackFileWrapper` short circuits this code by
-            #       setting the body as the content is consumed, the
-            #       result being a `body` argument is *always* passed
-            #       into cache_response, and in turn,
-            #       `Serializer.dump`.
             response._fp = io.BytesIO(body)
 
         # NOTE: This is all a bit weird, but it's really important that on
diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py
index d8e6fc6a9e3..b6ee7f20398 100644
--- a/src/pip/_vendor/cachecontrol/wrapper.py
+++ b/src/pip/_vendor/cachecontrol/wrapper.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from .adapter import CacheControlAdapter
 from .cache import DictCache
 
diff --git a/src/pip/_vendor/certifi/LICENSE b/src/pip/_vendor/certifi/LICENSE
index 802b53ff11e..c2fda9a2642 100644
--- a/src/pip/_vendor/certifi/LICENSE
+++ b/src/pip/_vendor/certifi/LICENSE
@@ -1,4 +1,4 @@
-This packge contains a modified version of ca-bundle.crt:
+This package contains a modified version of ca-bundle.crt:
 
 ca-bundle.crt -- Bundle of CA Root Certificates
 
diff --git a/src/pip/_vendor/certifi/__init__.py b/src/pip/_vendor/certifi/__init__.py
index 4e5133b261d..8db1a0e5544 100644
--- a/src/pip/_vendor/certifi/__init__.py
+++ b/src/pip/_vendor/certifi/__init__.py
@@ -1,3 +1,3 @@
 from .core import contents, where
 
-__version__ = "2020.11.08"
+__version__ = "2021.10.08"
diff --git a/src/pip/_vendor/certifi/cacert.pem b/src/pip/_vendor/certifi/cacert.pem
index a1072085ce5..6d0ccc0d1c9 100644
--- a/src/pip/_vendor/certifi/cacert.pem
+++ b/src/pip/_vendor/certifi/cacert.pem
@@ -155,112 +155,6 @@ eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0tHuu2guQOHXvgR1m
 0vdXcDazv/wor3ElhVsT/h5/WrQ8
 -----END CERTIFICATE-----
 
-# Issuer: CN=GeoTrust Global CA O=GeoTrust Inc.
-# Subject: CN=GeoTrust Global CA O=GeoTrust Inc.
-# Label: "GeoTrust Global CA"
-# Serial: 144470
-# MD5 Fingerprint: f7:75:ab:29:fb:51:4e:b7:77:5e:ff:05:3c:99:8e:f5
-# SHA1 Fingerprint: de:28:f4:a4:ff:e5:b9:2f:a3:c5:03:d1:a3:49:a7:f9:96:2a:82:12
-# SHA256 Fingerprint: ff:85:6a:2d:25:1d:cd:88:d3:66:56:f4:50:12:67:98:cf:ab:aa:de:40:79:9c:72:2d:e4:d2:b5:db:36:a7:3a
------BEGIN CERTIFICATE-----
-MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT
-MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i
-YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG
-EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg
-R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9
-9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq
-fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv
-iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU
-1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+
-bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW
-MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA
-ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l
-uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn
-Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS
-tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF
-PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un
-hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV
-5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw==
------END CERTIFICATE-----
-
-# Issuer: CN=GeoTrust Universal CA O=GeoTrust Inc.
-# Subject: CN=GeoTrust Universal CA O=GeoTrust Inc.
-# Label: "GeoTrust Universal CA"
-# Serial: 1
-# MD5 Fingerprint: 92:65:58:8b:a2:1a:31:72:73:68:5c:b4:a5:7a:07:48
-# SHA1 Fingerprint: e6:21:f3:35:43:79:05:9a:4b:68:30:9d:8a:2f:74:22:15:87:ec:79
-# SHA256 Fingerprint: a0:45:9b:9f:63:b2:25:59:f5:fa:5d:4c:6d:b3:f9:f7:2f:f1:93:42:03:35:78:f0:73:bf:1d:1b:46:cb:b9:12
------BEGIN CERTIFICATE-----
-MIIFaDCCA1CgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMQswCQYDVQQGEwJVUzEW
-MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEeMBwGA1UEAxMVR2VvVHJ1c3QgVW5pdmVy
-c2FsIENBMB4XDTA0MDMwNDA1MDAwMFoXDTI5MDMwNDA1MDAwMFowRTELMAkGA1UE
-BhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xHjAcBgNVBAMTFUdlb1RydXN0
-IFVuaXZlcnNhbCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKYV
-VaCjxuAfjJ0hUNfBvitbtaSeodlyWL0AG0y/YckUHUWCq8YdgNY96xCcOq9tJPi8
-cQGeBvV8Xx7BDlXKg5pZMK4ZyzBIle0iN430SppyZj6tlcDgFgDgEB8rMQ7XlFTT
-QjOgNB0eRXbdT8oYN+yFFXoZCPzVx5zw8qkuEKmS5j1YPakWaDwvdSEYfyh3peFh
-F7em6fgemdtzbvQKoiFs7tqqhZJmr/Z6a4LauiIINQ/PQvE1+mrufislzDoR5G2v
-c7J2Ha3QsnhnGqQ5HFELZ1aD/ThdDc7d8Lsrlh/eezJS/R27tQahsiFepdaVaH/w
-mZ7cRQg+59IJDTWU3YBOU5fXtQlEIGQWFwMCTFMNaN7VqnJNk22CDtucvc+081xd
-VHppCZbW2xHBjXWotM85yM48vCR85mLK4b19p71XZQvk/iXttmkQ3CgaRr0BHdCX
-teGYO8A3ZNY9lO4L4fUorgtWv3GLIylBjobFS1J72HGrH4oVpjuDWtdYAVHGTEHZ
-f9hBZ3KiKN9gg6meyHv8U3NyWfWTehd2Ds735VzZC1U0oqpbtWpU5xPKV+yXbfRe
-Bi9Fi1jUIxaS5BZuKGNZMN9QAZxjiRqf2xeUgnA3wySemkfWWspOqGmJch+RbNt+
-nhutxx9z3SxPGWX9f5NAEC7S8O08ni4oPmkmM8V7AgMBAAGjYzBhMA8GA1UdEwEB
-/wQFMAMBAf8wHQYDVR0OBBYEFNq7LqqwDLiIJlF0XG0D08DYj3rWMB8GA1UdIwQY
-MBaAFNq7LqqwDLiIJlF0XG0D08DYj3rWMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
-9w0BAQUFAAOCAgEAMXjmx7XfuJRAyXHEqDXsRh3ChfMoWIawC/yOsjmPRFWrZIRc
-aanQmjg8+uUfNeVE44B5lGiku8SfPeE0zTBGi1QrlaXv9z+ZhP015s8xxtxqv6fX
-IwjhmF7DWgh2qaavdy+3YL1ERmrvl/9zlcGO6JP7/TG37FcREUWbMPEaiDnBTzyn
-ANXH/KttgCJwpQzgXQQpAvvLoJHRfNbDflDVnVi+QTjruXU8FdmbyUqDWcDaU/0z
-uzYYm4UPFd3uLax2k7nZAY1IEKj79TiG8dsKxr2EoyNB3tZ3b4XUhRxQ4K5RirqN
-Pnbiucon8l+f725ZDQbYKxek0nxru18UGkiPGkzns0ccjkxFKyDuSN/n3QmOGKja
-QI2SJhFTYXNd673nxE0pN2HrrDktZy4W1vUAg4WhzH92xH3kt0tm7wNFYGm2DFKW
-koRepqO1pD4r2czYG0eq8kTaT/kD6PAUyz/zg97QwVTjt+gKN02LIFkDMBmhLMi9
-ER/frslKxfMnZmaGrGiR/9nmUxwPi1xpZQomyB40w11Re9epnAahNt3ViZS82eQt
-DF4JbAiXfKM9fJP/P6EUp8+1Xevb2xzEdt+Iub1FBZUbrvxGakyvSOPOrg/Sfuvm
-bJxPgWp6ZKy7PtXny3YuxadIwVyQD8vIP/rmMuGNG2+k5o7Y+SlIis5z/iw=
------END CERTIFICATE-----
-
-# Issuer: CN=GeoTrust Universal CA 2 O=GeoTrust Inc.
-# Subject: CN=GeoTrust Universal CA 2 O=GeoTrust Inc.
-# Label: "GeoTrust Universal CA 2"
-# Serial: 1
-# MD5 Fingerprint: 34:fc:b8:d0:36:db:9e:14:b3:c2:f2:db:8f:e4:94:c7
-# SHA1 Fingerprint: 37:9a:19:7b:41:85:45:35:0c:a6:03:69:f3:3c:2e:af:47:4f:20:79
-# SHA256 Fingerprint: a0:23:4f:3b:c8:52:7c:a5:62:8e:ec:81:ad:5d:69:89:5d:a5:68:0d:c9:1d:1c:b8:47:7f:33:f8:78:b9:5b:0b
------BEGIN CERTIFICATE-----
-MIIFbDCCA1SgAwIBAgIBATANBgkqhkiG9w0BAQUFADBHMQswCQYDVQQGEwJVUzEW
-MBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1c3QgVW5pdmVy
-c2FsIENBIDIwHhcNMDQwMzA0MDUwMDAwWhcNMjkwMzA0MDUwMDAwWjBHMQswCQYD
-VQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEgMB4GA1UEAxMXR2VvVHJ1
-c3QgVW5pdmVyc2FsIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
-AQCzVFLByT7y2dyxUxpZKeexw0Uo5dfR7cXFS6GqdHtXr0om/Nj1XqduGdt0DE81
-WzILAePb63p3NeqqWuDW6KFXlPCQo3RWlEQwAx5cTiuFJnSCegx2oG9NzkEtoBUG
-FF+3Qs17j1hhNNwqCPkuwwGmIkQcTAeC5lvO0Ep8BNMZcyfwqph/Lq9O64ceJHdq
-XbboW0W63MOhBW9Wjo8QJqVJwy7XQYci4E+GymC16qFjwAGXEHm9ADwSbSsVsaxL
-se4YuU6W3Nx2/zu+z18DwPw76L5GG//aQMJS9/7jOvdqdzXQ2o3rXhhqMcceujwb
-KNZrVMaqW9eiLBsZzKIC9ptZvTdrhrVtgrrY6slWvKk2WP0+GfPtDCapkzj4T8Fd
-IgbQl+rhrcZV4IErKIM6+vR7IVEAvlI4zs1meaj0gVbi0IMJR1FbUGrP20gaXT73
-y/Zl92zxlfgCOzJWgjl6W70viRu/obTo/3+NjN8D8WBOWBFM66M/ECuDmgFz2ZRt
-hAAnZqzwcEAJQpKtT5MNYQlRJNiS1QuUYbKHsu3/mjX/hVTK7URDrBs8FmtISgoc
-QIgfksILAAX/8sgCSqSqqcyZlpwvWOB94b67B9xfBHJcMTTD7F8t4D1kkCLm0ey4
-Lt1ZrtmhN79UNdxzMk+MBB4zsslG8dhcyFVQyWi9qLo2CQIDAQABo2MwYTAPBgNV
-HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAfBgNV
-HSMEGDAWgBR281Xh+qQ2+/CfXGJx7Tz0RzgQKzAOBgNVHQ8BAf8EBAMCAYYwDQYJ
-KoZIhvcNAQEFBQADggIBAGbBxiPz2eAubl/oz66wsCVNK/g7WJtAJDday6sWSf+z
-dXkzoS9tcBc0kf5nfo/sm+VegqlVHy/c1FEHEv6sFj4sNcZj/NwQ6w2jqtB8zNHQ
-L1EuxBRa3ugZ4T7GzKQp5y6EqgYweHZUcyiYWTjgAA1i00J9IZ+uPTqM1fp3DRgr
-Fg5fNuH8KrUwJM/gYwx7WBr+mbpCErGR9Hxo4sjoryzqyX6uuyo9DRXcNJW2GHSo
-ag/HtPQTxORb7QrSpJdMKu0vbBKJPfEncKpqA1Ihn0CoZ1Dy81of398j9tx4TuaY
-T1U6U+Pv8vSfx3zYWK8pIpe44L2RLrB27FcRz+8pRPPphXpgY+RdM4kX2TGq2tbz
-GDVyz4crL2MjhF2EjD9XoIj8mZEoJmmZ1I+XRL6O1UixpCgp8RW04eWe3fiPpm8m
-1wk8OhwRDqZsN/etRIcsKMfYdIKz0G9KV7s1KSegi+ghp4dkNl3M2Basx7InQJJV
-OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH
-6aLcr34YEoP9VhdBLtUpgn2Z9DH2canPLAEnpQW5qrJITirvn5NSUZU8UnOOVkwX
-QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS
------END CERTIFICATE-----
-
 # Issuer: CN=AAA Certificate Services O=Comodo CA Limited
 # Subject: CN=AAA Certificate Services O=Comodo CA Limited
 # Label: "Comodo AAA Services root"
@@ -294,48 +188,6 @@ l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3
 smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg==
 -----END CERTIFICATE-----
 
-# Issuer: CN=QuoVadis Root Certification Authority O=QuoVadis Limited OU=Root Certification Authority
-# Subject: CN=QuoVadis Root Certification Authority O=QuoVadis Limited OU=Root Certification Authority
-# Label: "QuoVadis Root CA"
-# Serial: 985026699
-# MD5 Fingerprint: 27:de:36:fe:72:b7:00:03:00:9d:f4:f0:1e:6c:04:24
-# SHA1 Fingerprint: de:3f:40:bd:50:93:d3:9b:6c:60:f6:da:bc:07:62:01:00:89:76:c9
-# SHA256 Fingerprint: a4:5e:de:3b:bb:f0:9c:8a:e1:5c:72:ef:c0:72:68:d6:93:a2:1c:99:6f:d5:1e:67:ca:07:94:60:fd:6d:88:73
------BEGIN CERTIFICATE-----
-MIIF0DCCBLigAwIBAgIEOrZQizANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJC
-TTEZMBcGA1UEChMQUXVvVmFkaXMgTGltaXRlZDElMCMGA1UECxMcUm9vdCBDZXJ0
-aWZpY2F0aW9uIEF1dGhvcml0eTEuMCwGA1UEAxMlUXVvVmFkaXMgUm9vdCBDZXJ0
-aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wMTAzMTkxODMzMzNaFw0yMTAzMTcxODMz
-MzNaMH8xCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMSUw
-IwYDVQQLExxSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MS4wLAYDVQQDEyVR
-dW9WYWRpcyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG
-9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv2G1lVO6V/z68mcLOhrfEYBklbTRvM16z/Yp
-li4kVEAkOPcahdxYTMukJ0KX0J+DisPkBgNbAKVRHnAEdOLB1Dqr1607BxgFjv2D
-rOpm2RgbaIr1VxqYuvXtdj182d6UajtLF8HVj71lODqV0D1VNk7feVcxKh7YWWVJ
-WCCYfqtffp/p1k3sg3Spx2zY7ilKhSoGFPlU5tPaZQeLYzcS19Dsw3sgQUSj7cug
-F+FxZc4dZjH3dgEZyH0DWLaVSR2mEiboxgx24ONmy+pdpibu5cxfvWenAScOospU
-xbF6lR1xHkopigPcakXBpBlebzbNw6Kwt/5cOOJSvPhEQ+aQuwIDAQABo4ICUjCC
-Ak4wPQYIKwYBBQUHAQEEMTAvMC0GCCsGAQUFBzABhiFodHRwczovL29jc3AucXVv
-dmFkaXNvZmZzaG9yZS5jb20wDwYDVR0TAQH/BAUwAwEB/zCCARoGA1UdIASCAREw
-ggENMIIBCQYJKwYBBAG+WAABMIH7MIHUBggrBgEFBQcCAjCBxxqBxFJlbGlhbmNl
-IG9uIHRoZSBRdW9WYWRpcyBSb290IENlcnRpZmljYXRlIGJ5IGFueSBwYXJ0eSBh
-c3N1bWVzIGFjY2VwdGFuY2Ugb2YgdGhlIHRoZW4gYXBwbGljYWJsZSBzdGFuZGFy
-ZCB0ZXJtcyBhbmQgY29uZGl0aW9ucyBvZiB1c2UsIGNlcnRpZmljYXRpb24gcHJh
-Y3RpY2VzLCBhbmQgdGhlIFF1b1ZhZGlzIENlcnRpZmljYXRlIFBvbGljeS4wIgYI
-KwYBBQUHAgEWFmh0dHA6Ly93d3cucXVvdmFkaXMuYm0wHQYDVR0OBBYEFItLbe3T
-KbkGGew5Oanwl4Rqy+/fMIGuBgNVHSMEgaYwgaOAFItLbe3TKbkGGew5Oanwl4Rq
-y+/foYGEpIGBMH8xCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1p
-dGVkMSUwIwYDVQQLExxSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MS4wLAYD
-VQQDEyVRdW9WYWRpcyBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggQ6tlCL
-MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOCAQEAitQUtf70mpKnGdSk
-fnIYj9lofFIk3WdvOXrEql494liwTXCYhGHoG+NpGA7O+0dQoE7/8CQfvbLO9Sf8
-7C9TqnN7Az10buYWnuulLsS/VidQK2K6vkscPFVcQR0kvoIgR13VRH56FmjffU1R
-cHhXHTMe/QKZnAzNCgVPx7uOpHX6Sm2xgI4JVrmcGmD+XcHXetwReNDWXcG31a0y
-mQM6isxUJTkxgXsTIlG6Rmyhu576BGxJJnSP0nPrzDCi5upZIof4l/UO/erMkqQW
-xFIY6iHOsfHmhIHluqmGKPJDWl0Snawe2ajlCmqnf6CHKc/yiU3U7MXi5nrQNiOK
-SnQ2+Q==
------END CERTIFICATE-----
-
 # Issuer: CN=QuoVadis Root CA 2 O=QuoVadis Limited
 # Subject: CN=QuoVadis Root CA 2 O=QuoVadis Limited
 # Label: "QuoVadis Root CA 2"
@@ -451,33 +303,6 @@ JRDL8Try2frbSVa7pv6nQTXD4IhhyYjH3zYQIphZ6rBK+1YWc26sTfcioU+tHXot
 RSflMMFe8toTyyVCUZVHA4xsIcx0Qu1T/zOLjw9XARYvz6buyXAiFL39vmwLAw==
 -----END CERTIFICATE-----
 
-# Issuer: CN=Sonera Class2 CA O=Sonera
-# Subject: CN=Sonera Class2 CA O=Sonera
-# Label: "Sonera Class 2 Root CA"
-# Serial: 29
-# MD5 Fingerprint: a3:ec:75:0f:2e:88:df:fa:48:01:4e:0b:5c:48:6f:fb
-# SHA1 Fingerprint: 37:f7:6d:e6:07:7c:90:c5:b1:3e:93:1a:b7:41:10:b4:f2:e4:9a:27
-# SHA256 Fingerprint: 79:08:b4:03:14:c1:38:10:0b:51:8d:07:35:80:7f:fb:fc:f8:51:8a:00:95:33:71:05:ba:38:6b:15:3d:d9:27
------BEGIN CERTIFICATE-----
-MIIDIDCCAgigAwIBAgIBHTANBgkqhkiG9w0BAQUFADA5MQswCQYDVQQGEwJGSTEP
-MA0GA1UEChMGU29uZXJhMRkwFwYDVQQDExBTb25lcmEgQ2xhc3MyIENBMB4XDTAx
-MDQwNjA3Mjk0MFoXDTIxMDQwNjA3Mjk0MFowOTELMAkGA1UEBhMCRkkxDzANBgNV
-BAoTBlNvbmVyYTEZMBcGA1UEAxMQU29uZXJhIENsYXNzMiBDQTCCASIwDQYJKoZI
-hvcNAQEBBQADggEPADCCAQoCggEBAJAXSjWdyvANlsdE+hY3/Ei9vX+ALTU74W+o
-Z6m/AxxNjG8yR9VBaKQTBME1DJqEQ/xcHf+Js+gXGM2RX/uJ4+q/Tl18GybTdXnt
-5oTjV+WtKcT0OijnpXuENmmz/V52vaMtmdOQTiMofRhj8VQ7Jp12W5dCsv+u8E7s
-3TmVToMGf+dJQMjFAbJUWmYdPfz56TwKnoG4cPABi+QjVHzIrviQHgCWctRUz2Ej
-vOr7nQKV0ba5cTppCD8PtOFCx4j1P5iop7oc4HFx71hXgVB6XGt0Rg6DA5jDjqhu
-8nYybieDwnPz3BjotJPqdURrBGAgcVeHnfO+oJAjPYok4doh28MCAwEAAaMzMDEw
-DwYDVR0TAQH/BAUwAwEB/zARBgNVHQ4ECgQISqCqWITTXjwwCwYDVR0PBAQDAgEG
-MA0GCSqGSIb3DQEBBQUAA4IBAQBazof5FnIVV0sd2ZvnoiYw7JNn39Yt0jSv9zil
-zqsWuasvfDXLrNAPtEwr/IDva4yRXzZ299uzGxnq9LIR/WFxRL8oszodv7ND6J+/
-3DEIcbCdjdY0RzKQxmUk96BKfARzjzlvF4xytb1LyHr4e4PDKE6cCepnP7JnBBvD
-FNr450kkkdAdavphOe9r5yF1BgfYErQhIHBCcYHaPJo2vqZbDWpsmh+Re/n570K6
-Tk6ezAyNlNzZRZxe7EJQY670XcSxEtzKO6gunRRaBXW37Ndj4ro1tgQIkejanZz2
-ZrUYrAqmVCY0M9IbwdR/GjqOC6oybtv8TyWf2TLHllpwrN9M
------END CERTIFICATE-----
-
 # Issuer: CN=XRamp Global Certification Authority O=XRamp Security Services Inc OU=www.xrampsecurity.com
 # Subject: CN=XRamp Global Certification Authority O=XRamp Security Services Inc OU=www.xrampsecurity.com
 # Label: "XRamp Global CA Root"
@@ -776,104 +601,6 @@ hAhm0sQ2fac+EPyI4NSA5QC9qvNOBqN6avlicuMJT+ubDgEj8Z+7fNzcbBGXJbLy
 tGMU0gYqZ4yD9c7qB9iaah7s5Aq7KkzrCWA5zspi2C5u
 -----END CERTIFICATE-----
 
-# Issuer: CN=GeoTrust Primary Certification Authority O=GeoTrust Inc.
-# Subject: CN=GeoTrust Primary Certification Authority O=GeoTrust Inc.
-# Label: "GeoTrust Primary Certification Authority"
-# Serial: 32798226551256963324313806436981982369
-# MD5 Fingerprint: 02:26:c3:01:5e:08:30:37:43:a9:d0:7d:cf:37:e6:bf
-# SHA1 Fingerprint: 32:3c:11:8e:1b:f7:b8:b6:52:54:e2:e2:10:0d:d6:02:90:37:f0:96
-# SHA256 Fingerprint: 37:d5:10:06:c5:12:ea:ab:62:64:21:f1:ec:8c:92:01:3f:c5:f8:2a:e9:8e:e5:33:eb:46:19:b8:de:b4:d0:6c
------BEGIN CERTIFICATE-----
-MIIDfDCCAmSgAwIBAgIQGKy1av1pthU6Y2yv2vrEoTANBgkqhkiG9w0BAQUFADBY
-MQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjExMC8GA1UEAxMo
-R2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjEx
-MjcwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMFgxCzAJBgNVBAYTAlVTMRYwFAYDVQQK
-Ew1HZW9UcnVzdCBJbmMuMTEwLwYDVQQDEyhHZW9UcnVzdCBQcmltYXJ5IENlcnRp
-ZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAvrgVe//UfH1nrYNke8hCUy3f9oQIIGHWAVlqnEQRr+92/ZV+zmEwu3qDXwK9
-AWbK7hWNb6EwnL2hhZ6UOvNWiAAxz9juapYC2e0DjPt1befquFUWBRaa9OBesYjA
-ZIVcFU2Ix7e64HXprQU9nceJSOC7KMgD4TCTZF5SwFlwIjVXiIrxlQqD17wxcwE0
-7e9GceBrAqg1cmuXm2bgyxx5X9gaBGgeRwLmnWDiNpcB3841kt++Z8dtd1k7j53W
-kBWUvEI0EME5+bEnPn7WinXFsq+W06Lem+SYvn3h6YGttm/81w7a4DSwDRp35+MI
-mO9Y+pyEtzavwt+s0vQQBnBxNQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4G
-A1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQULNVQQZcVi/CPNmFbSvtr2ZnJM5IwDQYJ
-KoZIhvcNAQEFBQADggEBAFpwfyzdtzRP9YZRqSa+S7iq8XEN3GHHoOo0Hnp3DwQ1
-6CePbJC/kRYkRj5KTs4rFtULUh38H2eiAkUxT87z+gOneZ1TatnaYzr4gNfTmeGl
-4b7UVXGYNTq+k+qurUKykG/g/CFNNWMziUnWm07Kx+dOCQD32sfvmWKZd7aVIl6K
-oKv0uHiYyjgZmclynnjNS6yvGaBzEi38wkG6gZHaFloxt/m0cYASSJlyc1pZU8Fj
-UjPtp8nSOQJw+uCxQmYpqptR7TBUIhRf2asdweSU8Pj1K/fqynhG1riR/aYNKxoU
-AT6A8EKglQdebc3MS6RFjasS6LPeWuWgfOgPIh1a6Vk=
------END CERTIFICATE-----
-
-# Issuer: CN=thawte Primary Root CA O=thawte, Inc. OU=Certification Services Division/(c) 2006 thawte, Inc. - For authorized use only
-# Subject: CN=thawte Primary Root CA O=thawte, Inc. OU=Certification Services Division/(c) 2006 thawte, Inc. - For authorized use only
-# Label: "thawte Primary Root CA"
-# Serial: 69529181992039203566298953787712940909
-# MD5 Fingerprint: 8c:ca:dc:0b:22:ce:f5:be:72:ac:41:1a:11:a8:d8:12
-# SHA1 Fingerprint: 91:c6:d6:ee:3e:8a:c8:63:84:e5:48:c2:99:29:5c:75:6c:81:7b:81
-# SHA256 Fingerprint: 8d:72:2f:81:a9:c1:13:c0:79:1d:f1:36:a2:96:6d:b2:6c:95:0a:97:1d:b4:6b:41:99:f4:ea:54:b7:8b:fb:9f
------BEGIN CERTIFICATE-----
-MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB
-qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf
-Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw
-MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV
-BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw
-NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j
-LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG
-A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl
-IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG
-SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs
-W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta
-3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk
-6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6
-Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J
-NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA
-MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP
-r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU
-DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz
-YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX
-xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2
-/qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/
-LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7
-jVaMaA==
------END CERTIFICATE-----
-
-# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G5 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2006 VeriSign, Inc. - For authorized use only
-# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G5 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2006 VeriSign, Inc. - For authorized use only
-# Label: "VeriSign Class 3 Public Primary Certification Authority - G5"
-# Serial: 33037644167568058970164719475676101450
-# MD5 Fingerprint: cb:17:e4:31:67:3e:e2:09:fe:45:57:93:f3:0a:fa:1c
-# SHA1 Fingerprint: 4e:b6:d5:78:49:9b:1c:cf:5f:58:1e:ad:56:be:3d:9b:67:44:a5:e5
-# SHA256 Fingerprint: 9a:cf:ab:7e:43:c8:d8:80:d0:6b:26:2a:94:de:ee:e4:b4:65:99:89:c3:d0:ca:f1:9b:af:64:05:e4:1a:b7:df
------BEGIN CERTIFICATE-----
-MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB
-yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL
-ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp
-U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW
-ZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0
-aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL
-MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW
-ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln
-biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp
-U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y
-aXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1
-nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex
-t0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz
-SdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG
-BO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+
-rCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/
-NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E
-BAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH
-BgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy
-aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv
-MzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE
-p6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y
-5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK
-WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ
-4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N
-hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq
------END CERTIFICATE-----
-
 # Issuer: CN=SecureTrust CA O=SecureTrust Corporation
 # Subject: CN=SecureTrust CA O=SecureTrust Corporation
 # Label: "SecureTrust CA"
@@ -1151,185 +878,6 @@ i/nDhDwTqn6Sm1dTk/pwwpEOMfmbZ13pljheX7NzTogVZ96edhBiIL5VaZVDADlN
 9u6wWk5JRFRYX0KD
 -----END CERTIFICATE-----
 
-# Issuer: CN=GeoTrust Primary Certification Authority - G3 O=GeoTrust Inc. OU=(c) 2008 GeoTrust Inc. - For authorized use only
-# Subject: CN=GeoTrust Primary Certification Authority - G3 O=GeoTrust Inc. OU=(c) 2008 GeoTrust Inc. - For authorized use only
-# Label: "GeoTrust Primary Certification Authority - G3"
-# Serial: 28809105769928564313984085209975885599
-# MD5 Fingerprint: b5:e8:34:36:c9:10:44:58:48:70:6d:2e:83:d4:b8:05
-# SHA1 Fingerprint: 03:9e:ed:b8:0b:e7:a0:3c:69:53:89:3b:20:d2:d9:32:3a:4c:2a:fd
-# SHA256 Fingerprint: b4:78:b8:12:25:0d:f8:78:63:5c:2a:a7:ec:7d:15:5e:aa:62:5e:e8:29:16:e2:cd:29:43:61:88:6c:d1:fb:d4
------BEGIN CERTIFICATE-----
-MIID/jCCAuagAwIBAgIQFaxulBmyeUtB9iepwxgPHzANBgkqhkiG9w0BAQsFADCB
-mDELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsT
-MChjKSAyMDA4IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s
-eTE2MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhv
-cml0eSAtIEczMB4XDTA4MDQwMjAwMDAwMFoXDTM3MTIwMTIzNTk1OVowgZgxCzAJ
-BgNVBAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykg
-MjAwOCBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0
-BgNVBAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg
-LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANziXmJYHTNXOTIz
-+uvLh4yn1ErdBojqZI4xmKU4kB6Yzy5jK/BGvESyiaHAKAxJcCGVn2TAppMSAmUm
-hsalifD614SgcK9PGpc/BkTVyetyEH3kMSj7HGHmKAdEc5IiaacDiGydY8hS2pgn
-5whMcD60yRLBxWeDXTPzAxHsatBT4tG6NmCUgLthY2xbF37fQJQeqw3CIShwiP/W
-JmxsYAQlTlV+fe+/lEjetx3dcI0FX4ilm/LC7urRQEFtYjgdVgbFA0dRIBn8exAL
-DmKudlW/X3e+PkkBUz2YJQN2JFodtNuJ6nnltrM7P7pMKEF/BqxqjsHQ9gUdfeZC
-huOl1UcCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw
-HQYDVR0OBBYEFMR5yo6hTgMdHNxr2zFblD4/MH8tMA0GCSqGSIb3DQEBCwUAA4IB
-AQAtxRPPVoB7eni9n64smefv2t+UXglpp+duaIy9cr5HqQ6XErhK8WTTOd8lNNTB
-zU6B8A8ExCSzNJbGpqow32hhc9f5joWJ7w5elShKKiePEI4ufIbEAp7aDHdlDkQN
-kv39sxY2+hENHYwOB4lqKVb3cvTdFZx3NWZXqxNT2I7BQMXXExZacse3aQHEerGD
-AWh9jUGhlBjBJVz88P6DAod8DQ3PLghcSkANPuyBYeYk28rgDi0Hsj5W3I31QYUH
-SJsMC8tJP33st/3LjWeJGqvtux6jAAgIFyqCXDFdRootD4abdNlF+9RAsXqqaC2G
-spki4cErx5z481+oghLrGREt
------END CERTIFICATE-----
-
-# Issuer: CN=thawte Primary Root CA - G2 O=thawte, Inc. OU=(c) 2007 thawte, Inc. - For authorized use only
-# Subject: CN=thawte Primary Root CA - G2 O=thawte, Inc. OU=(c) 2007 thawte, Inc. - For authorized use only
-# Label: "thawte Primary Root CA - G2"
-# Serial: 71758320672825410020661621085256472406
-# MD5 Fingerprint: 74:9d:ea:60:24:c4:fd:22:53:3e:cc:3a:72:d9:29:4f
-# SHA1 Fingerprint: aa:db:bc:22:23:8f:c4:01:a1:27:bb:38:dd:f4:1d:db:08:9e:f0:12
-# SHA256 Fingerprint: a4:31:0d:50:af:18:a6:44:71:90:37:2a:86:af:af:8b:95:1f:fb:43:1d:83:7f:1e:56:88:b4:59:71:ed:15:57
------BEGIN CERTIFICATE-----
-MIICiDCCAg2gAwIBAgIQNfwmXNmET8k9Jj1Xm67XVjAKBggqhkjOPQQDAzCBhDEL
-MAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjE4MDYGA1UECxMvKGMp
-IDIwMDcgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAi
-BgNVBAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMjAeFw0wNzExMDUwMDAw
-MDBaFw0zODAxMTgyMzU5NTlaMIGEMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhh
-d3RlLCBJbmMuMTgwNgYDVQQLEy8oYykgMjAwNyB0aGF3dGUsIEluYy4gLSBGb3Ig
-YXV0aG9yaXplZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9v
-dCBDQSAtIEcyMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEotWcgnuVnfFSeIf+iha/
-BebfowJPDQfGAFG6DAJSLSKkQjnE/o/qycG+1E3/n3qe4rF8mq2nhglzh9HnmuN6
-papu+7qzcMBniKI11KOasf2twu8x+qi58/sIxpHR+ymVo0IwQDAPBgNVHRMBAf8E
-BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUmtgAMADna3+FGO6Lts6K
-DPgR4bswCgYIKoZIzj0EAwMDaQAwZgIxAN344FdHW6fmCsO99YCKlzUNG4k8VIZ3
-KMqh9HneteY4sPBlcIx/AlTCv//YoT7ZzwIxAMSNlPzcU9LcnXgWHxUzI1NS41ox
-XZ3Krr0TKUQNJ1uo52icEvdYPy5yAlejj6EULg==
------END CERTIFICATE-----
-
-# Issuer: CN=thawte Primary Root CA - G3 O=thawte, Inc. OU=Certification Services Division/(c) 2008 thawte, Inc. - For authorized use only
-# Subject: CN=thawte Primary Root CA - G3 O=thawte, Inc. OU=Certification Services Division/(c) 2008 thawte, Inc. - For authorized use only
-# Label: "thawte Primary Root CA - G3"
-# Serial: 127614157056681299805556476275995414779
-# MD5 Fingerprint: fb:1b:5d:43:8a:94:cd:44:c6:76:f2:43:4b:47:e7:31
-# SHA1 Fingerprint: f1:8b:53:8d:1b:e9:03:b6:a6:f0:56:43:5b:17:15:89:ca:f3:6b:f2
-# SHA256 Fingerprint: 4b:03:f4:58:07:ad:70:f2:1b:fc:2c:ae:71:c9:fd:e4:60:4c:06:4c:f5:ff:b6:86:ba:e5:db:aa:d7:fd:d3:4c
------BEGIN CERTIFICATE-----
-MIIEKjCCAxKgAwIBAgIQYAGXt0an6rS0mtZLL/eQ+zANBgkqhkiG9w0BAQsFADCB
-rjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf
-Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw
-MDggdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxJDAiBgNV
-BAMTG3RoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EgLSBHMzAeFw0wODA0MDIwMDAwMDBa
-Fw0zNzEyMDEyMzU5NTlaMIGuMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMdGhhd3Rl
-LCBJbmMuMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9uIFNlcnZpY2VzIERpdmlzaW9u
-MTgwNgYDVQQLEy8oYykgMjAwOCB0aGF3dGUsIEluYy4gLSBGb3IgYXV0aG9yaXpl
-ZCB1c2Ugb25seTEkMCIGA1UEAxMbdGhhd3RlIFByaW1hcnkgUm9vdCBDQSAtIEcz
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsr8nLPvb2FvdeHsbnndm
-gcs+vHyu86YnmjSjaDFxODNi5PNxZnmxqWWjpYvVj2AtP0LMqmsywCPLLEHd5N/8
-YZzic7IilRFDGF/Eth9XbAoFWCLINkw6fKXRz4aviKdEAhN0cXMKQlkC+BsUa0Lf
-b1+6a4KinVvnSr0eAXLbS3ToO39/fR8EtCab4LRarEc9VbjXsCZSKAExQGbY2SS9
-9irY7CFJXJv2eul/VTV+lmuNk5Mny5K76qxAwJ/C+IDPXfRa3M50hqY+bAtTyr2S
-zhkGcuYMXDhpxwTWvGzOW/b3aJzcJRVIiKHpqfiYnODz1TEoYRFsZ5aNOZnLwkUk
-OQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNV
-HQ4EFgQUrWyqlGCc7eT/+j4KdCtjA/e2Wb8wDQYJKoZIhvcNAQELBQADggEBABpA
-2JVlrAmSicY59BDlqQ5mU1143vokkbvnRFHfxhY0Cu9qRFHqKweKA3rD6z8KLFIW
-oCtDuSWQP3CpMyVtRRooOyfPqsMpQhvfO0zAMzRbQYi/aytlryjvsvXDqmbOe1bu
-t8jLZ8HJnBoYuMTDSQPxYA5QzUbF83d597YV4Djbxy8ooAw/dyZ02SUS2jHaGh7c
-KUGRIjxpp7sC8rZcJwOJ9Abqm+RyguOhCcHpABnTPtRwa7pxpqpYrvS76Wy274fM
-m7v/OeZWYdMKp8RcTGB7BXcmer/YB1IsYvdwY9k5vG8cwnncdimvzsUsZAReiDZu
-MdRAGmI0Nj81Aa6sY6A=
------END CERTIFICATE-----
-
-# Issuer: CN=GeoTrust Primary Certification Authority - G2 O=GeoTrust Inc. OU=(c) 2007 GeoTrust Inc. - For authorized use only
-# Subject: CN=GeoTrust Primary Certification Authority - G2 O=GeoTrust Inc. OU=(c) 2007 GeoTrust Inc. - For authorized use only
-# Label: "GeoTrust Primary Certification Authority - G2"
-# Serial: 80682863203381065782177908751794619243
-# MD5 Fingerprint: 01:5e:d8:6b:bd:6f:3d:8e:a1:31:f8:12:e0:98:73:6a
-# SHA1 Fingerprint: 8d:17:84:d5:37:f3:03:7d:ec:70:fe:57:8b:51:9a:99:e6:10:d7:b0
-# SHA256 Fingerprint: 5e:db:7a:c4:3b:82:a0:6a:87:61:e8:d7:be:49:79:eb:f2:61:1f:7d:d7:9b:f9:1c:1c:6b:56:6a:21:9e:d7:66
------BEGIN CERTIFICATE-----
-MIICrjCCAjWgAwIBAgIQPLL0SAoA4v7rJDteYD7DazAKBggqhkjOPQQDAzCBmDEL
-MAkGA1UEBhMCVVMxFjAUBgNVBAoTDUdlb1RydXN0IEluYy4xOTA3BgNVBAsTMChj
-KSAyMDA3IEdlb1RydXN0IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTE2
-MDQGA1UEAxMtR2VvVHJ1c3QgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0
-eSAtIEcyMB4XDTA3MTEwNTAwMDAwMFoXDTM4MDExODIzNTk1OVowgZgxCzAJBgNV
-BAYTAlVTMRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMTkwNwYDVQQLEzAoYykgMjAw
-NyBHZW9UcnVzdCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxNjA0BgNV
-BAMTLUdlb1RydXN0IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH
-MjB2MBAGByqGSM49AgEGBSuBBAAiA2IABBWx6P0DFUPlrOuHNxFi79KDNlJ9RVcL
-So17VDs6bl8VAsBQps8lL33KSLjHUGMcKiEIfJo22Av+0SbFWDEwKCXzXV2juLal
-tJLtbCyf691DiaI8S0iRHVDsJt/WYC69IaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO
-BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFBVfNVdRVfslsq0DafwBo/q+EVXVMAoG
-CCqGSM49BAMDA2cAMGQCMGSWWaboCd6LuvpaiIjwH5HTRqjySkwCY/tsXzjbLkGT
-qQ7mndwxHLKgpxgceeHHNgIwOlavmnRs9vuD4DPTCF+hnMJbn0bWtsuRBmOiBucz
-rD6ogRLQy7rQkgu2npaqBA+K
------END CERTIFICATE-----
-
-# Issuer: CN=VeriSign Universal Root Certification Authority O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2008 VeriSign, Inc. - For authorized use only
-# Subject: CN=VeriSign Universal Root Certification Authority O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2008 VeriSign, Inc. - For authorized use only
-# Label: "VeriSign Universal Root Certification Authority"
-# Serial: 85209574734084581917763752644031726877
-# MD5 Fingerprint: 8e:ad:b5:01:aa:4d:81:e4:8c:1d:d1:e1:14:00:95:19
-# SHA1 Fingerprint: 36:79:ca:35:66:87:72:30:4d:30:a5:fb:87:3b:0f:a7:7b:b7:0d:54
-# SHA256 Fingerprint: 23:99:56:11:27:a5:71:25:de:8c:ef:ea:61:0d:df:2f:a0:78:b5:c8:06:7f:4e:82:82:90:bf:b8:60:e8:4b:3c
------BEGIN CERTIFICATE-----
-MIIEuTCCA6GgAwIBAgIQQBrEZCGzEyEDDrvkEhrFHTANBgkqhkiG9w0BAQsFADCB
-vTELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL
-ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwOCBWZXJp
-U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MTgwNgYDVQQDEy9W
-ZXJpU2lnbiBVbml2ZXJzYWwgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe
-Fw0wODA0MDIwMDAwMDBaFw0zNzEyMDEyMzU5NTlaMIG9MQswCQYDVQQGEwJVUzEX
-MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlTaWduIFRydXN0
-IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAyMDA4IFZlcmlTaWduLCBJbmMuIC0gRm9y
-IGF1dGhvcml6ZWQgdXNlIG9ubHkxODA2BgNVBAMTL1ZlcmlTaWduIFVuaXZlcnNh
-bCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEF
-AAOCAQ8AMIIBCgKCAQEAx2E3XrEBNNti1xWb/1hajCMj1mCOkdeQmIN65lgZOIzF
-9uVkhbSicfvtvbnazU0AtMgtc6XHaXGVHzk8skQHnOgO+k1KxCHfKWGPMiJhgsWH
-H26MfF8WIFFE0XBPV+rjHOPMee5Y2A7Cs0WTwCznmhcrewA3ekEzeOEz4vMQGn+H
-LL729fdC4uW/h2KJXwBL38Xd5HVEMkE6HnFuacsLdUYI0crSK5XQz/u5QGtkjFdN
-/BMReYTtXlT2NJ8IAfMQJQYXStrxHXpma5hgZqTZ79IugvHw7wnqRMkVauIDbjPT
-rJ9VAMf2CGqUuV/c4DPxhGD5WycRtPwW8rtWaoAljQIDAQABo4GyMIGvMA8GA1Ud
-EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMG0GCCsGAQUFBwEMBGEwX6FdoFsw
-WTBXMFUWCWltYWdlL2dpZjAhMB8wBwYFKw4DAhoEFI/l0xqGrI2Oa8PPgGrUSBgs
-exkuMCUWI2h0dHA6Ly9sb2dvLnZlcmlzaWduLmNvbS92c2xvZ28uZ2lmMB0GA1Ud
-DgQWBBS2d/ppSEefUxLVwuoHMnYH0ZcHGTANBgkqhkiG9w0BAQsFAAOCAQEASvj4
-sAPmLGd75JR3Y8xuTPl9Dg3cyLk1uXBPY/ok+myDjEedO2Pzmvl2MpWRsXe8rJq+
-seQxIcaBlVZaDrHC1LGmWazxY8u4TB1ZkErvkBYoH1quEPuBUDgMbMzxPcP1Y+Oz
-4yHJJDnp/RVmRvQbEdBNc6N9Rvk97ahfYtTxP/jgdFcrGJ2BtMQo2pSXpXDrrB2+
-BxHw1dvd5Yzw1TKwg+ZX4o+/vqGqvz0dtdQ46tewXDpPaj+PwGZsY6rp2aQW9IHR
-lRQOfc2VNNnSj3BzgXucfr2YYdhFh5iQxeuGMMY1v/D/w1WIg0vvBZIGcfK4mJO3
-7M2CYfE45k+XmCpajQ==
------END CERTIFICATE-----
-
-# Issuer: CN=VeriSign Class 3 Public Primary Certification Authority - G4 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2007 VeriSign, Inc. - For authorized use only
-# Subject: CN=VeriSign Class 3 Public Primary Certification Authority - G4 O=VeriSign, Inc. OU=VeriSign Trust Network/(c) 2007 VeriSign, Inc. - For authorized use only
-# Label: "VeriSign Class 3 Public Primary Certification Authority - G4"
-# Serial: 63143484348153506665311985501458640051
-# MD5 Fingerprint: 3a:52:e1:e7:fd:6f:3a:e3:6f:f3:6f:99:1b:f9:22:41
-# SHA1 Fingerprint: 22:d5:d8:df:8f:02:31:d1:8d:f7:9d:b7:cf:8a:2d:64:c9:3f:6c:3a
-# SHA256 Fingerprint: 69:dd:d7:ea:90:bb:57:c9:3e:13:5d:c8:5e:a6:fc:d5:48:0b:60:32:39:bd:c4:54:fc:75:8b:2a:26:cf:7f:79
------BEGIN CERTIFICATE-----
-MIIDhDCCAwqgAwIBAgIQL4D+I4wOIg9IZxIokYesszAKBggqhkjOPQQDAzCByjEL
-MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW
-ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2ln
-biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp
-U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y
-aXR5IC0gRzQwHhcNMDcxMTA1MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCByjELMAkG
-A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZWZXJp
-U2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNyBWZXJpU2lnbiwg
-SW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJpU2ln
-biBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5
-IC0gRzQwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASnVnp8Utpkmw4tXNherJI9/gHm
-GUo9FANL+mAnINmDiWn6VMaaGF5VKmTeBvaNSjutEDxlPZCIBIngMGGzrl0Bp3ve
-fLK+ymVhAIau2o970ImtTR1ZmkGxvEeA3J5iw/mjgbIwga8wDwYDVR0TAQH/BAUw
-AwEB/zAOBgNVHQ8BAf8EBAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJ
-aW1hZ2UvZ2lmMCEwHzAHBgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYj
-aHR0cDovL2xvZ28udmVyaXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFLMW
-kf3upm7ktS5Jj4d4gYDs5bG1MAoGCCqGSM49BAMDA2gAMGUCMGYhDBgmYFo4e1ZC
-4Kf8NoRRkSAsdk1DPcQdhCPQrNZ8NQbOzWm9kA3bbEhCHQ6qQgIxAJw9SDkjOVga
-FRJZap7v1VmyHVIsmXHNxynfGyphe3HR3vPA5Q06Sqotp9iGKt0uEA==
------END CERTIFICATE-----
-
 # Issuer: CN=NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny O=NetLock Kft. OU=Tan\xfas\xedtv\xe1nykiad\xf3k (Certification Services)
 # Subject: CN=NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny O=NetLock Kft. OU=Tan\xfas\xedtv\xe1nykiad\xf3k (Certification Services)
 # Label: "NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny"
@@ -1565,105 +1113,6 @@ naM8THLCV8Sg1Mw4J87VBp6iSNnpn86CcDaTmjvfliHjWbcM2pE38P1ZWrOZyGls
 QyYBNWNgVYkDOnXYukrZVP/u3oDYLdE41V4tC5h9Pmzb/CaIxw==
 -----END CERTIFICATE-----
 
-# Issuer: CN=Chambers of Commerce Root - 2008 O=AC Camerfirma S.A.
-# Subject: CN=Chambers of Commerce Root - 2008 O=AC Camerfirma S.A.
-# Label: "Chambers of Commerce Root - 2008"
-# Serial: 11806822484801597146
-# MD5 Fingerprint: 5e:80:9e:84:5a:0e:65:0b:17:02:f3:55:18:2a:3e:d7
-# SHA1 Fingerprint: 78:6a:74:ac:76:ab:14:7f:9c:6a:30:50:ba:9e:a8:7e:fe:9a:ce:3c
-# SHA256 Fingerprint: 06:3e:4a:fa:c4:91:df:d3:32:f3:08:9b:85:42:e9:46:17:d8:93:d7:fe:94:4e:10:a7:93:7e:e2:9d:96:93:c0
------BEGIN CERTIFICATE-----
-MIIHTzCCBTegAwIBAgIJAKPaQn6ksa7aMA0GCSqGSIb3DQEBBQUAMIGuMQswCQYD
-VQQGEwJFVTFDMEEGA1UEBxM6TWFkcmlkIChzZWUgY3VycmVudCBhZGRyZXNzIGF0
-IHd3dy5jYW1lcmZpcm1hLmNvbS9hZGRyZXNzKTESMBAGA1UEBRMJQTgyNzQzMjg3
-MRswGQYDVQQKExJBQyBDYW1lcmZpcm1hIFMuQS4xKTAnBgNVBAMTIENoYW1iZXJz
-IG9mIENvbW1lcmNlIFJvb3QgLSAyMDA4MB4XDTA4MDgwMTEyMjk1MFoXDTM4MDcz
-MTEyMjk1MFowga4xCzAJBgNVBAYTAkVVMUMwQQYDVQQHEzpNYWRyaWQgKHNlZSBj
-dXJyZW50IGFkZHJlc3MgYXQgd3d3LmNhbWVyZmlybWEuY29tL2FkZHJlc3MpMRIw
-EAYDVQQFEwlBODI3NDMyODcxGzAZBgNVBAoTEkFDIENhbWVyZmlybWEgUy5BLjEp
-MCcGA1UEAxMgQ2hhbWJlcnMgb2YgQ29tbWVyY2UgUm9vdCAtIDIwMDgwggIiMA0G
-CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCvAMtwNyuAWko6bHiUfaN/Gh/2NdW9
-28sNRHI+JrKQUrpjOyhYb6WzbZSm891kDFX29ufyIiKAXuFixrYp4YFs8r/lfTJq
-VKAyGVn+H4vXPWCGhSRv4xGzdz4gljUha7MI2XAuZPeEklPWDrCQiorjh40G072Q
-DuKZoRuGDtqaCrsLYVAGUvGef3bsyw/QHg3PmTA9HMRFEFis1tPo1+XqxQEHd9ZR
-5gN/ikilTWh1uem8nk4ZcfUyS5xtYBkL+8ydddy/Js2Pk3g5eXNeJQ7KXOt3EgfL
-ZEFHcpOrUMPrCXZkNNI5t3YRCQ12RcSprj1qr7V9ZS+UWBDsXHyvfuK2GNnQm05a
-Sd+pZgvMPMZ4fKecHePOjlO+Bd5gD2vlGts/4+EhySnB8esHnFIbAURRPHsl18Tl
-UlRdJQfKFiC4reRB7noI/plvg6aRArBsNlVq5331lubKgdaX8ZSD6e2wsWsSaR6s
-+12pxZjptFtYer49okQ6Y1nUCyXeG0+95QGezdIp1Z8XGQpvvwyQ0wlf2eOKNcx5
-Wk0ZN5K3xMGtr/R5JJqyAQuxr1yW84Ay+1w9mPGgP0revq+ULtlVmhduYJ1jbLhj
-ya6BXBg14JC7vjxPNyK5fuvPnnchpj04gftI2jE9K+OJ9dC1vX7gUMQSibMjmhAx
-hduub+84Mxh2EQIDAQABo4IBbDCCAWgwEgYDVR0TAQH/BAgwBgEB/wIBDDAdBgNV
-HQ4EFgQU+SSsD7K1+HnA+mCIG8TZTQKeFxkwgeMGA1UdIwSB2zCB2IAU+SSsD7K1
-+HnA+mCIG8TZTQKeFxmhgbSkgbEwga4xCzAJBgNVBAYTAkVVMUMwQQYDVQQHEzpN
-YWRyaWQgKHNlZSBjdXJyZW50IGFkZHJlc3MgYXQgd3d3LmNhbWVyZmlybWEuY29t
-L2FkZHJlc3MpMRIwEAYDVQQFEwlBODI3NDMyODcxGzAZBgNVBAoTEkFDIENhbWVy
-ZmlybWEgUy5BLjEpMCcGA1UEAxMgQ2hhbWJlcnMgb2YgQ29tbWVyY2UgUm9vdCAt
-IDIwMDiCCQCj2kJ+pLGu2jAOBgNVHQ8BAf8EBAMCAQYwPQYDVR0gBDYwNDAyBgRV
-HSAAMCowKAYIKwYBBQUHAgEWHGh0dHA6Ly9wb2xpY3kuY2FtZXJmaXJtYS5jb20w
-DQYJKoZIhvcNAQEFBQADggIBAJASryI1wqM58C7e6bXpeHxIvj99RZJe6dqxGfwW
-PJ+0W2aeaufDuV2I6A+tzyMP3iU6XsxPpcG1Lawk0lgH3qLPaYRgM+gQDROpI9CF
-5Y57pp49chNyM/WqfcZjHwj0/gF/JM8rLFQJ3uIrbZLGOU8W6jx+ekbURWpGqOt1
-glanq6B8aBMz9p0w8G8nOSQjKpD9kCk18pPfNKXG9/jvjA9iSnyu0/VU+I22mlaH
-FoI6M6taIgj3grrqLuBHmrS1RaMFO9ncLkVAO+rcf+g769HsJtg1pDDFOqxXnrN2
-pSB7+R5KBWIBpih1YJeSDW4+TTdDDZIVnBgizVGZoCkaPF+KMjNbMMeJL0eYD6MD
-xvbxrN8y8NmBGuScvfaAFPDRLLmF9dijscilIeUcE5fuDr3fKanvNFNb0+RqE4QG
-tjICxFKuItLcsiFCGtpA8CnJ7AoMXOLQusxI0zcKzBIKinmwPQN/aUv0NCB9szTq
-jktk9T79syNnFQ0EuPAtwQlRPLJsFfClI9eDdOTlLsn+mCdCxqvGnrDQWzilm1De
-fhiYtUU79nm06PcaewaD+9CL2rvHvRirCG88gGtAPxkZumWK5r7VXNM21+9AUiRg
-OGcEMeyP84LG3rlV8zsxkVrctQgVrXYlCg17LofiDKYGvCYQbTed7N14jHyAxfDZ
-d0jQ
------END CERTIFICATE-----
-
-# Issuer: CN=Global Chambersign Root - 2008 O=AC Camerfirma S.A.
-# Subject: CN=Global Chambersign Root - 2008 O=AC Camerfirma S.A.
-# Label: "Global Chambersign Root - 2008"
-# Serial: 14541511773111788494
-# MD5 Fingerprint: 9e:80:ff:78:01:0c:2e:c1:36:bd:fe:96:90:6e:08:f3
-# SHA1 Fingerprint: 4a:bd:ee:ec:95:0d:35:9c:89:ae:c7:52:a1:2c:5b:29:f6:d6:aa:0c
-# SHA256 Fingerprint: 13:63:35:43:93:34:a7:69:80:16:a0:d3:24:de:72:28:4e:07:9d:7b:52:20:bb:8f:bd:74:78:16:ee:be:ba:ca
------BEGIN CERTIFICATE-----
-MIIHSTCCBTGgAwIBAgIJAMnN0+nVfSPOMA0GCSqGSIb3DQEBBQUAMIGsMQswCQYD
-VQQGEwJFVTFDMEEGA1UEBxM6TWFkcmlkIChzZWUgY3VycmVudCBhZGRyZXNzIGF0
-IHd3dy5jYW1lcmZpcm1hLmNvbS9hZGRyZXNzKTESMBAGA1UEBRMJQTgyNzQzMjg3
-MRswGQYDVQQKExJBQyBDYW1lcmZpcm1hIFMuQS4xJzAlBgNVBAMTHkdsb2JhbCBD
-aGFtYmVyc2lnbiBSb290IC0gMjAwODAeFw0wODA4MDExMjMxNDBaFw0zODA3MzEx
-MjMxNDBaMIGsMQswCQYDVQQGEwJFVTFDMEEGA1UEBxM6TWFkcmlkIChzZWUgY3Vy
-cmVudCBhZGRyZXNzIGF0IHd3dy5jYW1lcmZpcm1hLmNvbS9hZGRyZXNzKTESMBAG
-A1UEBRMJQTgyNzQzMjg3MRswGQYDVQQKExJBQyBDYW1lcmZpcm1hIFMuQS4xJzAl
-BgNVBAMTHkdsb2JhbCBDaGFtYmVyc2lnbiBSb290IC0gMjAwODCCAiIwDQYJKoZI
-hvcNAQEBBQADggIPADCCAgoCggIBAMDfVtPkOpt2RbQT2//BthmLN0EYlVJH6xed
-KYiONWwGMi5HYvNJBL99RDaxccy9Wglz1dmFRP+RVyXfXjaOcNFccUMd2drvXNL7
-G706tcuto8xEpw2uIRU/uXpbknXYpBI4iRmKt4DS4jJvVpyR1ogQC7N0ZJJ0YPP2
-zxhPYLIj0Mc7zmFLmY/CDNBAspjcDahOo7kKrmCgrUVSY7pmvWjg+b4aqIG7HkF4
-ddPB/gBVsIdU6CeQNR1MM62X/JcumIS/LMmjv9GYERTtY/jKmIhYF5ntRQOXfjyG
-HoiMvvKRhI9lNNgATH23MRdaKXoKGCQwoze1eqkBfSbW+Q6OWfH9GzO1KTsXO0G2
-Id3UwD2ln58fQ1DJu7xsepeY7s2MH/ucUa6LcL0nn3HAa6x9kGbo1106DbDVwo3V
-yJ2dwW3Q0L9R5OP4wzg2rtandeavhENdk5IMagfeOx2YItaswTXbo6Al/3K1dh3e
-beksZixShNBFks4c5eUzHdwHU1SjqoI7mjcv3N2gZOnm3b2u/GSFHTynyQbehP9r
-6GsaPMWis0L7iwk+XwhSx2LE1AVxv8Rk5Pihg+g+EpuoHtQ2TS9x9o0o9oOpE9Jh
-wZG7SMA0j0GMS0zbaRL/UJScIINZc+18ofLx/d33SdNDWKBWY8o9PeU1VlnpDsog
-zCtLkykPAgMBAAGjggFqMIIBZjASBgNVHRMBAf8ECDAGAQH/AgEMMB0GA1UdDgQW
-BBS5CcqcHtvTbDprru1U8VuTBjUuXjCB4QYDVR0jBIHZMIHWgBS5CcqcHtvTbDpr
-ru1U8VuTBjUuXqGBsqSBrzCBrDELMAkGA1UEBhMCRVUxQzBBBgNVBAcTOk1hZHJp
-ZCAoc2VlIGN1cnJlbnQgYWRkcmVzcyBhdCB3d3cuY2FtZXJmaXJtYS5jb20vYWRk
-cmVzcykxEjAQBgNVBAUTCUE4Mjc0MzI4NzEbMBkGA1UEChMSQUMgQ2FtZXJmaXJt
-YSBTLkEuMScwJQYDVQQDEx5HbG9iYWwgQ2hhbWJlcnNpZ24gUm9vdCAtIDIwMDiC
-CQDJzdPp1X0jzjAOBgNVHQ8BAf8EBAMCAQYwPQYDVR0gBDYwNDAyBgRVHSAAMCow
-KAYIKwYBBQUHAgEWHGh0dHA6Ly9wb2xpY3kuY2FtZXJmaXJtYS5jb20wDQYJKoZI
-hvcNAQEFBQADggIBAICIf3DekijZBZRG/5BXqfEv3xoNa/p8DhxJJHkn2EaqbylZ
-UohwEurdPfWbU1Rv4WCiqAm57OtZfMY18dwY6fFn5a+6ReAJ3spED8IXDneRRXoz
-X1+WLGiLwUePmJs9wOzL9dWCkoQ10b42OFZyMVtHLaoXpGNR6woBrX/sdZ7LoR/x
-fxKxueRkf2fWIyr0uDldmOghp+G9PUIadJpwr2hsUF1Jz//7Dl3mLEfXgTpZALVz
-a2Mg9jFFCDkO9HB+QHBaP9BrQql0PSgvAm11cpUJjUhjxsYjV5KTXjXBjfkK9yyd
-Yhz2rXzdpjEetrHHfoUm+qRqtdpjMNHvkzeyZi99Bffnt0uYlDXA2TopwZ2yUDMd
-SqlapskD7+3056huirRXhOukP9DuqqqHW2Pok+JrqNS4cnhrG+055F3Lm6qH1U9O
-AP7Zap88MQ8oAgF9mOinsKJknnn4SPIVqczmyETrP3iZ8ntxPjzxmKfFGBI/5rso
-M0LpRQp8bfKGeS/Fghl9CYl8slR2iK7ewfPM4W7bMdaTrpmg7yVqc5iJWzouE4ge
-v8CSlDQb4ye3ix5vQv/n6TebUB0tovkC7stYWDpxvGjjqsGvHCgfotwjZT+B6q6Z
-09gwzxMNTxXJhLynSC34MCN32EZLeW32jO06f2ARePTpm67VVMB0gNELQp/B
------END CERTIFICATE-----
-
 # Issuer: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc.
 # Subject: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc.
 # Label: "Go Daddy Root Certificate Authority - G2"
@@ -2075,35 +1524,6 @@ LysRJyU3eExRarDzzFhdFPFqSBX/wge2sY0PjlxQRrM9vwGYT7JZVEc+NHt4bVaT
 LnPqZih4zR0Uv6CPLy64Lo7yFIrM6bV8+2ydDKXhlg==
 -----END CERTIFICATE-----
 
-# Issuer: O=Trustis Limited OU=Trustis FPS Root CA
-# Subject: O=Trustis Limited OU=Trustis FPS Root CA
-# Label: "Trustis FPS Root CA"
-# Serial: 36053640375399034304724988975563710553
-# MD5 Fingerprint: 30:c9:e7:1e:6b:e6:14:eb:65:b2:16:69:20:31:67:4d
-# SHA1 Fingerprint: 3b:c0:38:0b:33:c3:f6:a6:0c:86:15:22:93:d9:df:f5:4b:81:c0:04
-# SHA256 Fingerprint: c1:b4:82:99:ab:a5:20:8f:e9:63:0a:ce:55:ca:68:a0:3e:da:5a:51:9c:88:02:a0:d3:a6:73:be:8f:8e:55:7d
------BEGIN CERTIFICATE-----
-MIIDZzCCAk+gAwIBAgIQGx+ttiD5JNM2a/fH8YygWTANBgkqhkiG9w0BAQUFADBF
-MQswCQYDVQQGEwJHQjEYMBYGA1UEChMPVHJ1c3RpcyBMaW1pdGVkMRwwGgYDVQQL
-ExNUcnVzdGlzIEZQUyBSb290IENBMB4XDTAzMTIyMzEyMTQwNloXDTI0MDEyMTEx
-MzY1NFowRTELMAkGA1UEBhMCR0IxGDAWBgNVBAoTD1RydXN0aXMgTGltaXRlZDEc
-MBoGA1UECxMTVHJ1c3RpcyBGUFMgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQAD
-ggEPADCCAQoCggEBAMVQe547NdDfxIzNjpvto8A2mfRC6qc+gIMPpqdZh8mQRUN+
-AOqGeSoDvT03mYlmt+WKVoaTnGhLaASMk5MCPjDSNzoiYYkchU59j9WvezX2fihH
-iTHcDnlkH5nSW7r+f2C/revnPDgpai/lkQtV/+xvWNUtyd5MZnGPDNcE2gfmHhjj
-vSkCqPoc4Vu5g6hBSLwacY3nYuUtsuvffM/bq1rKMfFMIvMFE/eC+XN5DL7XSxzA
-0RU8k0Fk0ea+IxciAIleH2ulrG6nS4zto3Lmr2NNL4XSFDWaLk6M6jKYKIahkQlB
-OrTh4/L68MkKokHdqeMDx4gVOxzUGpTXn2RZEm0CAwEAAaNTMFEwDwYDVR0TAQH/
-BAUwAwEB/zAfBgNVHSMEGDAWgBS6+nEleYtXQSUhhgtx67JkDoshZzAdBgNVHQ4E
-FgQUuvpxJXmLV0ElIYYLceuyZA6LIWcwDQYJKoZIhvcNAQEFBQADggEBAH5Y//01
-GX2cGE+esCu8jowU/yyg2kdbw++BLa8F6nRIW/M+TgfHbcWzk88iNVy2P3UnXwmW
-zaD+vkAMXBJV+JOCyinpXj9WV4s4NvdFGkwozZ5BuO1WTISkQMi4sKUraXAEasP4
-1BIy+Q7DsdwyhEQsb8tGD+pmQQ9P8Vilpg0ND2HepZ5dfWWhPBfnqFVO76DH7cZE
-f1T1o+CP8HxVIo8ptoGj4W1OLBuAZ+ytIJ8MYmHVl/9D7S3B2l0pKoU/rGXuhg8F
-jZBf3+6f9L/uHfuY5H+QK4R4EA5sSVPvFVtlRkpdr7r7OnIdzfYliB6XzCGcKQEN
-ZetX2fNXlrtIzYE=
------END CERTIFICATE-----
-
 # Issuer: CN=Buypass Class 2 Root CA O=Buypass AS-983163327
 # Subject: CN=Buypass Class 2 Root CA O=Buypass AS-983163327
 # Label: "Buypass Class 2 Root CA"
@@ -2965,46 +2385,6 @@ KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg
 xwy8p2Fp8fc74SrL+SvzZpA3
 -----END CERTIFICATE-----
 
-# Issuer: CN=Staat der Nederlanden Root CA - G3 O=Staat der Nederlanden
-# Subject: CN=Staat der Nederlanden Root CA - G3 O=Staat der Nederlanden
-# Label: "Staat der Nederlanden Root CA - G3"
-# Serial: 10003001
-# MD5 Fingerprint: 0b:46:67:07:db:10:2f:19:8c:35:50:60:d1:0b:f4:37
-# SHA1 Fingerprint: d8:eb:6b:41:51:92:59:e0:f3:e7:85:00:c0:3d:b6:88:97:c9:ee:fc
-# SHA256 Fingerprint: 3c:4f:b0:b9:5a:b8:b3:00:32:f4:32:b8:6f:53:5f:e1:72:c1:85:d0:fd:39:86:58:37:cf:36:18:7f:a6:f4:28
------BEGIN CERTIFICATE-----
-MIIFdDCCA1ygAwIBAgIEAJiiOTANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJO
-TDEeMBwGA1UECgwVU3RhYXQgZGVyIE5lZGVybGFuZGVuMSswKQYDVQQDDCJTdGFh
-dCBkZXIgTmVkZXJsYW5kZW4gUm9vdCBDQSAtIEczMB4XDTEzMTExNDExMjg0MloX
-DTI4MTExMzIzMDAwMFowWjELMAkGA1UEBhMCTkwxHjAcBgNVBAoMFVN0YWF0IGRl
-ciBOZWRlcmxhbmRlbjErMCkGA1UEAwwiU3RhYXQgZGVyIE5lZGVybGFuZGVuIFJv
-b3QgQ0EgLSBHMzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL4yolQP
-cPssXFnrbMSkUeiFKrPMSjTysF/zDsccPVMeiAho2G89rcKezIJnByeHaHE6n3WW
-IkYFsO2tx1ueKt6c/DrGlaf1F2cY5y9JCAxcz+bMNO14+1Cx3Gsy8KL+tjzk7FqX
-xz8ecAgwoNzFs21v0IJyEavSgWhZghe3eJJg+szeP4TrjTgzkApyI/o1zCZxMdFy
-KJLZWyNtZrVtB0LrpjPOktvA9mxjeM3KTj215VKb8b475lRgsGYeCasH/lSJEULR
-9yS6YHgamPfJEf0WwTUaVHXvQ9Plrk7O53vDxk5hUUurmkVLoR9BvUhTFXFkC4az
-5S6+zqQbwSmEorXLCCN2QyIkHxcE1G6cxvx/K2Ya7Irl1s9N9WMJtxU51nus6+N8
-6U78dULI7ViVDAZCopz35HCz33JvWjdAidiFpNfxC95DGdRKWCyMijmev4SH8RY7
-Ngzp07TKbBlBUgmhHbBqv4LvcFEhMtwFdozL92TkA1CvjJFnq8Xy7ljY3r735zHP
-bMk7ccHViLVlvMDoFxcHErVc0qsgk7TmgoNwNsXNo42ti+yjwUOH5kPiNL6VizXt
-BznaqB16nzaeErAMZRKQFWDZJkBE41ZgpRDUajz9QdwOWke275dhdU/Z/seyHdTt
-XUmzqWrLZoQT1Vyg3N9udwbRcXXIV2+vD3dbAgMBAAGjQjBAMA8GA1UdEwEB/wQF
-MAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRUrfrHkleuyjWcLhL75Lpd
-INyUVzANBgkqhkiG9w0BAQsFAAOCAgEAMJmdBTLIXg47mAE6iqTnB/d6+Oea31BD
-U5cqPco8R5gu4RV78ZLzYdqQJRZlwJ9UXQ4DO1t3ApyEtg2YXzTdO2PCwyiBwpwp
-LiniyMMB8jPqKqrMCQj3ZWfGzd/TtiunvczRDnBfuCPRy5FOCvTIeuXZYzbB1N/8
-Ipf3YF3qKS9Ysr1YvY2WTxB1v0h7PVGHoTx0IsL8B3+A3MSs/mrBcDCw6Y5p4ixp
-gZQJut3+TcCDjJRYwEYgr5wfAvg1VUkvRtTA8KCWAg8zxXHzniN9lLf9OtMJgwYh
-/WA9rjLA0u6NpvDntIJ8CsxwyXmA+P5M9zWEGYox+wrZ13+b8KKaa8MFSu1BYBQw
-0aoRQm7TIwIEC8Zl3d1Sd9qBa7Ko+gE4uZbqKmxnl4mUnrzhVNXkanjvSr0rmj1A
-fsbAddJu+2gw7OyLnflJNZoaLNmzlTnVHpL3prllL+U9bTpITAjc5CgSKL59NVzq
-4BZ+Extq1z7XnvwtdbLBFNUjA9tbbws+eC8N3jONFrdI54OagQ97wUNNVQQXOEpR
-1VmiiXTTn74eS9fGbbeIJG9gkaSChVtWQbzQRKtqE77RLFi3EjNYsjdj3BP1lB0/
-QFH1T/U67cjF68IeHRaVesd+QnGTbksVtzDfqu1XhUisHWrdOWnk4Xl4vs4Fv6EM
-94B7IWcnMFk=
------END CERTIFICATE-----
-
 # Issuer: CN=Staat der Nederlanden EV Root CA O=Staat der Nederlanden
 # Subject: CN=Staat der Nederlanden EV Root CA O=Staat der Nederlanden
 # Label: "Staat der Nederlanden EV Root CA"
@@ -4604,3 +3984,379 @@ AZKXRRJ+oPM+rRk6ct30UJMDEr5E0k9BpIycnR+j9sKS50gU/k6bpZFXrsY3crsC
 MGclCrEMXu6pY5Jv5ZAL/mYiykf9ijH3g/56vxC+GCsej/YpHpRZ744hN8tRmKVu
 Sw==
 -----END CERTIFICATE-----
+
+# Issuer: CN=NAVER Global Root Certification Authority O=NAVER BUSINESS PLATFORM Corp.
+# Subject: CN=NAVER Global Root Certification Authority O=NAVER BUSINESS PLATFORM Corp.
+# Label: "NAVER Global Root Certification Authority"
+# Serial: 9013692873798656336226253319739695165984492813
+# MD5 Fingerprint: c8:7e:41:f6:25:3b:f5:09:b3:17:e8:46:3d:bf:d0:9b
+# SHA1 Fingerprint: 8f:6b:f2:a9:27:4a:da:14:a0:c4:f4:8e:61:27:f9:c0:1e:78:5d:d1
+# SHA256 Fingerprint: 88:f4:38:dc:f8:ff:d1:fa:8f:42:91:15:ff:e5:f8:2a:e1:e0:6e:0c:70:c3:75:fa:ad:71:7b:34:a4:9e:72:65
+-----BEGIN CERTIFICATE-----
+MIIFojCCA4qgAwIBAgIUAZQwHqIL3fXFMyqxQ0Rx+NZQTQ0wDQYJKoZIhvcNAQEM
+BQAwaTELMAkGA1UEBhMCS1IxJjAkBgNVBAoMHU5BVkVSIEJVU0lORVNTIFBMQVRG
+T1JNIENvcnAuMTIwMAYDVQQDDClOQVZFUiBHbG9iYWwgUm9vdCBDZXJ0aWZpY2F0
+aW9uIEF1dGhvcml0eTAeFw0xNzA4MTgwODU4NDJaFw0zNzA4MTgyMzU5NTlaMGkx
+CzAJBgNVBAYTAktSMSYwJAYDVQQKDB1OQVZFUiBCVVNJTkVTUyBQTEFURk9STSBD
+b3JwLjEyMDAGA1UEAwwpTkFWRVIgR2xvYmFsIFJvb3QgQ2VydGlmaWNhdGlvbiBB
+dXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC21PGTXLVA
+iQqrDZBbUGOukJR0F0Vy1ntlWilLp1agS7gvQnXp2XskWjFlqxcX0TM62RHcQDaH
+38dq6SZeWYp34+hInDEW+j6RscrJo+KfziFTowI2MMtSAuXaMl3Dxeb57hHHi8lE
+HoSTGEq0n+USZGnQJoViAbbJAh2+g1G7XNr4rRVqmfeSVPc0W+m/6imBEtRTkZaz
+kVrd/pBzKPswRrXKCAfHcXLJZtM0l/aM9BhK4dA9WkW2aacp+yPOiNgSnABIqKYP
+szuSjXEOdMWLyEz59JuOuDxp7W87UC9Y7cSw0BwbagzivESq2M0UXZR4Yb8Obtoq
+vC8MC3GmsxY/nOb5zJ9TNeIDoKAYv7vxvvTWjIcNQvcGufFt7QSUqP620wbGQGHf
+nZ3zVHbOUzoBppJB7ASjjw2i1QnK1sua8e9DXcCrpUHPXFNwcMmIpi3Ua2FzUCaG
+YQ5fG8Ir4ozVu53BA0K6lNpfqbDKzE0K70dpAy8i+/Eozr9dUGWokG2zdLAIx6yo
+0es+nPxdGoMuK8u180SdOqcXYZaicdNwlhVNt0xz7hlcxVs+Qf6sdWA7G2POAN3a
+CJBitOUt7kinaxeZVL6HSuOpXgRM6xBtVNbv8ejyYhbLgGvtPe31HzClrkvJE+2K
+AQHJuFFYwGY6sWZLxNUxAmLpdIQM201GLQIDAQABo0IwQDAdBgNVHQ4EFgQU0p+I
+36HNLL3s9TsBAZMzJ7LrYEswDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
+Af8wDQYJKoZIhvcNAQEMBQADggIBADLKgLOdPVQG3dLSLvCkASELZ0jKbY7gyKoN
+qo0hV4/GPnrK21HUUrPUloSlWGB/5QuOH/XcChWB5Tu2tyIvCZwTFrFsDDUIbatj
+cu3cvuzHV+YwIHHW1xDBE1UBjCpD5EHxzzp6U5LOogMFDTjfArsQLtk70pt6wKGm
++LUx5vR1yblTmXVHIloUFcd4G7ad6Qz4G3bxhYTeodoS76TiEJd6eN4MUZeoIUCL
+hr0N8F5OSza7OyAfikJW4Qsav3vQIkMsRIz75Sq0bBwcupTgE34h5prCy8VCZLQe
+lHsIJchxzIdFV4XTnyliIoNRlwAYl3dqmJLJfGBs32x9SuRwTMKeuB330DTHD8z7
+p/8Dvq1wkNoL3chtl1+afwkyQf3NosxabUzyqkn+Zvjp2DXrDige7kgvOtB5CTh8
+piKCk5XQA76+AqAF3SAi428diDRgxuYKuQl1C/AH6GmWNcf7I4GOODm4RStDeKLR
+LBT/DShycpWbXgnbiUSYqqFJu3FS8r/2/yehNq+4tneI3TqkbZs0kNwUXTC/t+sX
+5Ie3cdCh13cV1ELX8vMxmV2b3RZtP+oGI/hGoiLtk/bdmuYqh7GYVPEi92tF4+KO
+dh2ajcQGjTa3FPOdVGm3jjzVpG2Tgbet9r1ke8LJaDmgkpzNNIaRkPpkUZ3+/uul
+9XXeifdy
+-----END CERTIFICATE-----
+
+# Issuer: CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS O=FNMT-RCM OU=Ceres
+# Subject: CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS O=FNMT-RCM OU=Ceres
+# Label: "AC RAIZ FNMT-RCM SERVIDORES SEGUROS"
+# Serial: 131542671362353147877283741781055151509
+# MD5 Fingerprint: 19:36:9c:52:03:2f:d2:d1:bb:23:cc:dd:1e:12:55:bb
+# SHA1 Fingerprint: 62:ff:d9:9e:c0:65:0d:03:ce:75:93:d2:ed:3f:2d:32:c9:e3:e5:4a
+# SHA256 Fingerprint: 55:41:53:b1:3d:2c:f9:dd:b7:53:bf:be:1a:4e:0a:e0:8d:0a:a4:18:70:58:fe:60:a2:b8:62:b2:e4:b8:7b:cb
+-----BEGIN CERTIFICATE-----
+MIICbjCCAfOgAwIBAgIQYvYybOXE42hcG2LdnC6dlTAKBggqhkjOPQQDAzB4MQsw
+CQYDVQQGEwJFUzERMA8GA1UECgwIRk5NVC1SQ00xDjAMBgNVBAsMBUNlcmVzMRgw
+FgYDVQRhDA9WQVRFUy1RMjgyNjAwNEoxLDAqBgNVBAMMI0FDIFJBSVogRk5NVC1S
+Q00gU0VSVklET1JFUyBTRUdVUk9TMB4XDTE4MTIyMDA5MzczM1oXDTQzMTIyMDA5
+MzczM1oweDELMAkGA1UEBhMCRVMxETAPBgNVBAoMCEZOTVQtUkNNMQ4wDAYDVQQL
+DAVDZXJlczEYMBYGA1UEYQwPVkFURVMtUTI4MjYwMDRKMSwwKgYDVQQDDCNBQyBS
+QUlaIEZOTVQtUkNNIFNFUlZJRE9SRVMgU0VHVVJPUzB2MBAGByqGSM49AgEGBSuB
+BAAiA2IABPa6V1PIyqvfNkpSIeSX0oNnnvBlUdBeh8dHsVnyV0ebAAKTRBdp20LH
+sbI6GA60XYyzZl2hNPk2LEnb80b8s0RpRBNm/dfF/a82Tc4DTQdxz69qBdKiQ1oK
+Um8BA06Oi6NCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD
+VR0OBBYEFAG5L++/EYZg8k/QQW6rcx/n0m5JMAoGCCqGSM49BAMDA2kAMGYCMQCu
+SuMrQMN0EfKVrRYj3k4MGuZdpSRea0R7/DjiT8ucRRcRTBQnJlU5dUoDzBOQn5IC
+MQD6SmxgiHPz7riYYqnOK8LZiqZwMR2vsJRM60/G49HzYqc8/5MuB1xJAWdpEgJy
+v+c=
+-----END CERTIFICATE-----
+
+# Issuer: CN=GlobalSign Root R46 O=GlobalSign nv-sa
+# Subject: CN=GlobalSign Root R46 O=GlobalSign nv-sa
+# Label: "GlobalSign Root R46"
+# Serial: 1552617688466950547958867513931858518042577
+# MD5 Fingerprint: c4:14:30:e4:fa:66:43:94:2a:6a:1b:24:5f:19:d0:ef
+# SHA1 Fingerprint: 53:a2:b0:4b:ca:6b:d6:45:e6:39:8a:8e:c4:0d:d2:bf:77:c3:a2:90
+# SHA256 Fingerprint: 4f:a3:12:6d:8d:3a:11:d1:c4:85:5a:4f:80:7c:ba:d6:cf:91:9d:3a:5a:88:b0:3b:ea:2c:63:72:d9:3c:40:c9
+-----BEGIN CERTIFICATE-----
+MIIFWjCCA0KgAwIBAgISEdK7udcjGJ5AXwqdLdDfJWfRMA0GCSqGSIb3DQEBDAUA
+MEYxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYD
+VQQDExNHbG9iYWxTaWduIFJvb3QgUjQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMy
+MDAwMDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYt
+c2ExHDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQCsrHQy6LNl5brtQyYdpokNRbopiLKkHWPd08EsCVeJ
+OaFV6Wc0dwxu5FUdUiXSE2te4R2pt32JMl8Nnp8semNgQB+msLZ4j5lUlghYruQG
+vGIFAha/r6gjA7aUD7xubMLL1aa7DOn2wQL7Id5m3RerdELv8HQvJfTqa1VbkNud
+316HCkD7rRlr+/fKYIje2sGP1q7Vf9Q8g+7XFkyDRTNrJ9CG0Bwta/OrffGFqfUo
+0q3v84RLHIf8E6M6cqJaESvWJ3En7YEtbWaBkoe0G1h6zD8K+kZPTXhc+CtI4wSE
+y132tGqzZfxCnlEmIyDLPRT5ge1lFgBPGmSXZgjPjHvjK8Cd+RTyG/FWaha/LIWF
+zXg4mutCagI0GIMXTpRW+LaCtfOW3T3zvn8gdz57GSNrLNRyc0NXfeD412lPFzYE
++cCQYDdF3uYM2HSNrpyibXRdQr4G9dlkbgIQrImwTDsHTUB+JMWKmIJ5jqSngiCN
+I/onccnfxkF0oE32kRbcRoxfKWMxWXEM2G/CtjJ9++ZdU6Z+Ffy7dXxd7Pj2Fxzs
+x2sZy/N78CsHpdlseVR2bJ0cpm4O6XkMqCNqo98bMDGfsVR7/mrLZqrcZdCinkqa
+ByFrgY/bxFn63iLABJzjqls2k+g9vXqhnQt2sQvHnf3PmKgGwvgqo6GDoLclcqUC
+4wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV
+HQ4EFgQUA1yrc4GHqMywptWU4jaWSf8FmSwwDQYJKoZIhvcNAQEMBQADggIBAHx4
+7PYCLLtbfpIrXTncvtgdokIzTfnvpCo7RGkerNlFo048p9gkUbJUHJNOxO97k4Vg
+JuoJSOD1u8fpaNK7ajFxzHmuEajwmf3lH7wvqMxX63bEIaZHU1VNaL8FpO7XJqti
+2kM3S+LGteWygxk6x9PbTZ4IevPuzz5i+6zoYMzRx6Fcg0XERczzF2sUyQQCPtIk
+pnnpHs6i58FZFZ8d4kuaPp92CC1r2LpXFNqD6v6MVenQTqnMdzGxRBF6XLE+0xRF
+FRhiJBPSy03OXIPBNvIQtQ6IbbjhVp+J3pZmOUdkLG5NrmJ7v2B0GbhWrJKsFjLt
+rWhV/pi60zTe9Mlhww6G9kuEYO4Ne7UyWHmRVSyBQ7N0H3qqJZ4d16GLuc1CLgSk
+ZoNNiTW2bKg2SnkheCLQQrzRQDGQob4Ez8pn7fXwgNNgyYMqIgXQBztSvwyeqiv5
+u+YfjyW6hY0XHgL+XVAEV8/+LbzvXMAaq7afJMbfc2hIkCwU9D9SGuTSyxTDYWnP
+4vkYxboznxSjBF25cfe1lNj2M8FawTSLfJvdkzrnE6JwYZ+vj+vYxXX4M2bUdGc6
+N3ec592kD3ZDZopD8p/7DEJ4Y9HiD2971KE9dJeFt0g5QdYg/NA6s/rob8SKunE3
+vouXsXgxT7PntgMTzlSdriVZzH81Xwj3QEUxeCp6
+-----END CERTIFICATE-----
+
+# Issuer: CN=GlobalSign Root E46 O=GlobalSign nv-sa
+# Subject: CN=GlobalSign Root E46 O=GlobalSign nv-sa
+# Label: "GlobalSign Root E46"
+# Serial: 1552617690338932563915843282459653771421763
+# MD5 Fingerprint: b5:b8:66:ed:de:08:83:e3:c9:e2:01:34:06:ac:51:6f
+# SHA1 Fingerprint: 39:b4:6c:d5:fe:80:06:eb:e2:2f:4a:bb:08:33:a0:af:db:b9:dd:84
+# SHA256 Fingerprint: cb:b9:c4:4d:84:b8:04:3e:10:50:ea:31:a6:9f:51:49:55:d7:bf:d2:e2:c6:b4:93:01:01:9a:d6:1d:9f:50:58
+-----BEGIN CERTIFICATE-----
+MIICCzCCAZGgAwIBAgISEdK7ujNu1LzmJGjFDYQdmOhDMAoGCCqGSM49BAMDMEYx
+CzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYDVQQD
+ExNHbG9iYWxTaWduIFJvb3QgRTQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMyMDAw
+MDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2Ex
+HDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA
+IgNiAAScDrHPt+ieUnd1NPqlRqetMhkytAepJ8qUuwzSChDH2omwlwxwEwkBjtjq
+R+q+soArzfwoDdusvKSGN+1wCAB16pMLey5SnCNoIwZD7JIvU4Tb+0cUB+hflGdd
+yXqBPCCjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud
+DgQWBBQxCpCPtsad0kRLgLWi5h+xEk8blTAKBggqhkjOPQQDAwNoADBlAjEA31SQ
+7Zvvi5QCkxeCmb6zniz2C5GMn0oUsfZkvLtoURMMA/cVi4RguYv/Uo7njLwcAjA8
++RHUjE7AwWHCFUyqqx0LMV87HOIAl0Qx5v5zli/altP+CAezNIm8BZ/3Hobui3A=
+-----END CERTIFICATE-----
+
+# Issuer: CN=GLOBALTRUST 2020 O=e-commerce monitoring GmbH
+# Subject: CN=GLOBALTRUST 2020 O=e-commerce monitoring GmbH
+# Label: "GLOBALTRUST 2020"
+# Serial: 109160994242082918454945253
+# MD5 Fingerprint: 8a:c7:6f:cb:6d:e3:cc:a2:f1:7c:83:fa:0e:78:d7:e8
+# SHA1 Fingerprint: d0:67:c1:13:51:01:0c:aa:d0:c7:6a:65:37:31:16:26:4f:53:71:a2
+# SHA256 Fingerprint: 9a:29:6a:51:82:d1:d4:51:a2:e3:7f:43:9b:74:da:af:a2:67:52:33:29:f9:0f:9a:0d:20:07:c3:34:e2:3c:9a
+-----BEGIN CERTIFICATE-----
+MIIFgjCCA2qgAwIBAgILWku9WvtPilv6ZeUwDQYJKoZIhvcNAQELBQAwTTELMAkG
+A1UEBhMCQVQxIzAhBgNVBAoTGmUtY29tbWVyY2UgbW9uaXRvcmluZyBHbWJIMRkw
+FwYDVQQDExBHTE9CQUxUUlVTVCAyMDIwMB4XDTIwMDIxMDAwMDAwMFoXDTQwMDYx
+MDAwMDAwMFowTTELMAkGA1UEBhMCQVQxIzAhBgNVBAoTGmUtY29tbWVyY2UgbW9u
+aXRvcmluZyBHbWJIMRkwFwYDVQQDExBHTE9CQUxUUlVTVCAyMDIwMIICIjANBgkq
+hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAri5WrRsc7/aVj6B3GyvTY4+ETUWiD59b
+RatZe1E0+eyLinjF3WuvvcTfk0Uev5E4C64OFudBc/jbu9G4UeDLgztzOG53ig9Z
+YybNpyrOVPu44sB8R85gfD+yc/LAGbaKkoc1DZAoouQVBGM+uq/ufF7MpotQsjj3
+QWPKzv9pj2gOlTblzLmMCcpL3TGQlsjMH/1WljTbjhzqLL6FLmPdqqmV0/0plRPw
+yJiT2S0WR5ARg6I6IqIoV6Lr/sCMKKCmfecqQjuCgGOlYx8ZzHyyZqjC0203b+J+
+BlHZRYQfEs4kUmSFC0iAToexIiIwquuuvuAC4EDosEKAA1GqtH6qRNdDYfOiaxaJ
+SaSjpCuKAsR49GiKweR6NrFvG5Ybd0mN1MkGco/PU+PcF4UgStyYJ9ORJitHHmkH
+r96i5OTUawuzXnzUJIBHKWk7buis/UDr2O1xcSvy6Fgd60GXIsUf1DnQJ4+H4xj0
+4KlGDfV0OoIu0G4skaMxXDtG6nsEEFZegB31pWXogvziB4xiRfUg3kZwhqG8k9Me
+dKZssCz3AwyIDMvUclOGvGBG85hqwvG/Q/lwIHfKN0F5VVJjjVsSn8VoxIidrPIw
+q7ejMZdnrY8XD2zHc+0klGvIg5rQmjdJBKuxFshsSUktq6HQjJLyQUp5ISXbY9e2
+nKd+Qmn7OmMCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC
+AQYwHQYDVR0OBBYEFNwuH9FhN3nkq9XVsxJxaD1qaJwiMB8GA1UdIwQYMBaAFNwu
+H9FhN3nkq9XVsxJxaD1qaJwiMA0GCSqGSIb3DQEBCwUAA4ICAQCR8EICaEDuw2jA
+VC/f7GLDw56KoDEoqoOOpFaWEhCGVrqXctJUMHytGdUdaG/7FELYjQ7ztdGl4wJC
+XtzoRlgHNQIw4Lx0SsFDKv/bGtCwr2zD/cuz9X9tAy5ZVp0tLTWMstZDFyySCstd
+6IwPS3BD0IL/qMy/pJTAvoe9iuOTe8aPmxadJ2W8esVCgmxcB9CpwYhgROmYhRZf
++I/KARDOJcP5YBugxZfD0yyIMaK9MOzQ0MAS8cE54+X1+NZK3TTN+2/BT+MAi1bi
+kvcoskJ3ciNnxz8RFbLEAwW+uxF7Cr+obuf/WEPPm2eggAe2HcqtbepBEX4tdJP7
+wry+UUTF72glJ4DjyKDUEuzZpTcdN3y0kcra1LGWge9oXHYQSa9+pTeAsRxSvTOB
+TI/53WXZFM2KJVj04sWDpQmQ1GwUY7VA3+vA/MRYfg0UFodUJ25W5HCEuGwyEn6C
+MUO+1918oa2u1qsgEu8KwxCMSZY13At1XrFP1U80DhEgB3VDRemjEdqso5nCtnkn
+4rnvyOL2NSl6dPrFf4IFYqYK6miyeUcGbvJXqBUzxvd4Sj1Ce2t+/vdG6tHrju+I
+aFvowdlxfv1k7/9nR4hYJS8+hge9+6jlgqispdNpQ80xiEmEU5LAsTkbOYMBMMTy
+qfrQA71yN2BWHzZ8vTmR9W0Nv3vXkg==
+-----END CERTIFICATE-----
+
+# Issuer: CN=ANF Secure Server Root CA O=ANF Autoridad de Certificacion OU=ANF CA Raiz
+# Subject: CN=ANF Secure Server Root CA O=ANF Autoridad de Certificacion OU=ANF CA Raiz
+# Label: "ANF Secure Server Root CA"
+# Serial: 996390341000653745
+# MD5 Fingerprint: 26:a6:44:5a:d9:af:4e:2f:b2:1d:b6:65:b0:4e:e8:96
+# SHA1 Fingerprint: 5b:6e:68:d0:cc:15:b6:a0:5f:1e:c1:5f:ae:02:fc:6b:2f:5d:6f:74
+# SHA256 Fingerprint: fb:8f:ec:75:91:69:b9:10:6b:1e:51:16:44:c6:18:c5:13:04:37:3f:6c:06:43:08:8d:8b:ef:fd:1b:99:75:99
+-----BEGIN CERTIFICATE-----
+MIIF7zCCA9egAwIBAgIIDdPjvGz5a7EwDQYJKoZIhvcNAQELBQAwgYQxEjAQBgNV
+BAUTCUc2MzI4NzUxMDELMAkGA1UEBhMCRVMxJzAlBgNVBAoTHkFORiBBdXRvcmlk
+YWQgZGUgQ2VydGlmaWNhY2lvbjEUMBIGA1UECxMLQU5GIENBIFJhaXoxIjAgBgNV
+BAMTGUFORiBTZWN1cmUgU2VydmVyIFJvb3QgQ0EwHhcNMTkwOTA0MTAwMDM4WhcN
+MzkwODMwMTAwMDM4WjCBhDESMBAGA1UEBRMJRzYzMjg3NTEwMQswCQYDVQQGEwJF
+UzEnMCUGA1UEChMeQU5GIEF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uMRQwEgYD
+VQQLEwtBTkYgQ0EgUmFpejEiMCAGA1UEAxMZQU5GIFNlY3VyZSBTZXJ2ZXIgUm9v
+dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANvrayvmZFSVgpCj
+cqQZAZ2cC4Ffc0m6p6zzBE57lgvsEeBbphzOG9INgxwruJ4dfkUyYA8H6XdYfp9q
+yGFOtibBTI3/TO80sh9l2Ll49a2pcbnvT1gdpd50IJeh7WhM3pIXS7yr/2WanvtH
+2Vdy8wmhrnZEE26cLUQ5vPnHO6RYPUG9tMJJo8gN0pcvB2VSAKduyK9o7PQUlrZX
+H1bDOZ8rbeTzPvY1ZNoMHKGESy9LS+IsJJ1tk0DrtSOOMspvRdOoiXsezx76W0OL
+zc2oD2rKDF65nkeP8Nm2CgtYZRczuSPkdxl9y0oukntPLxB3sY0vaJxizOBQ+OyR
+p1RMVwnVdmPF6GUe7m1qzwmd+nxPrWAI/VaZDxUse6mAq4xhj0oHdkLePfTdsiQz
+W7i1o0TJrH93PB0j7IKppuLIBkwC/qxcmZkLLxCKpvR/1Yd0DVlJRfbwcVw5Kda/
+SiOL9V8BY9KHcyi1Swr1+KuCLH5zJTIdC2MKF4EA/7Z2Xue0sUDKIbvVgFHlSFJn
+LNJhiQcND85Cd8BEc5xEUKDbEAotlRyBr+Qc5RQe8TZBAQIvfXOn3kLMTOmJDVb3
+n5HUA8ZsyY/b2BzgQJhdZpmYgG4t/wHFzstGH6wCxkPmrqKEPMVOHj1tyRRM4y5B
+u8o5vzY8KhmqQYdOpc5LMnndkEl/AgMBAAGjYzBhMB8GA1UdIwQYMBaAFJxf0Gxj
+o1+TypOYCK2Mh6UsXME3MB0GA1UdDgQWBBScX9BsY6Nfk8qTmAitjIelLFzBNzAO
+BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC
+AgEATh65isagmD9uw2nAalxJUqzLK114OMHVVISfk/CHGT0sZonrDUL8zPB1hT+L
+9IBdeeUXZ701guLyPI59WzbLWoAAKfLOKyzxj6ptBZNscsdW699QIyjlRRA96Gej
+rw5VD5AJYu9LWaL2U/HANeQvwSS9eS9OICI7/RogsKQOLHDtdD+4E5UGUcjohybK
+pFtqFiGS3XNgnhAY3jyB6ugYw3yJ8otQPr0R4hUDqDZ9MwFsSBXXiJCZBMXM5gf0
+vPSQ7RPi6ovDj6MzD8EpTBNO2hVWcXNyglD2mjN8orGoGjR0ZVzO0eurU+AagNjq
+OknkJjCb5RyKqKkVMoaZkgoQI1YS4PbOTOK7vtuNknMBZi9iPrJyJ0U27U1W45eZ
+/zo1PqVUSlJZS2Db7v54EX9K3BR5YLZrZAPbFYPhor72I5dQ8AkzNqdxliXzuUJ9
+2zg/LFis6ELhDtjTO0wugumDLmsx2d1Hhk9tl5EuT+IocTUW0fJz/iUrB0ckYyfI
++PbZa/wSMVYIwFNCr5zQM378BvAxRAMU8Vjq8moNqRGyg77FGr8H6lnco4g175x2
+MjxNBiLOFeXdntiP2t7SxDnlF4HPOEfrf4htWRvfn0IUrn7PqLBmZdo3r5+qPeoo
+tt7VMVgWglvquxl1AnMaykgaIZOQCo6ThKd9OyMYkomgjaw=
+-----END CERTIFICATE-----
+
+# Issuer: CN=Certum EC-384 CA O=Asseco Data Systems S.A. OU=Certum Certification Authority
+# Subject: CN=Certum EC-384 CA O=Asseco Data Systems S.A. OU=Certum Certification Authority
+# Label: "Certum EC-384 CA"
+# Serial: 160250656287871593594747141429395092468
+# MD5 Fingerprint: b6:65:b3:96:60:97:12:a1:ec:4e:e1:3d:a3:c6:c9:f1
+# SHA1 Fingerprint: f3:3e:78:3c:ac:df:f4:a2:cc:ac:67:55:69:56:d7:e5:16:3c:e1:ed
+# SHA256 Fingerprint: 6b:32:80:85:62:53:18:aa:50:d1:73:c9:8d:8b:da:09:d5:7e:27:41:3d:11:4c:f7:87:a0:f5:d0:6c:03:0c:f6
+-----BEGIN CERTIFICATE-----
+MIICZTCCAeugAwIBAgIQeI8nXIESUiClBNAt3bpz9DAKBggqhkjOPQQDAzB0MQsw
+CQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEuMScw
+JQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAXBgNVBAMT
+EENlcnR1bSBFQy0zODQgQ0EwHhcNMTgwMzI2MDcyNDU0WhcNNDMwMzI2MDcyNDU0
+WjB0MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBT
+LkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAX
+BgNVBAMTEENlcnR1bSBFQy0zODQgQ0EwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATE
+KI6rGFtqvm5kN2PkzeyrOvfMobgOgknXhimfoZTy42B4mIF4Bk3y7JoOV2CDn7Tm
+Fy8as10CW4kjPMIRBSqniBMY81CE1700LCeJVf/OTOffph8oxPBUw7l8t1Ot68Kj
+QjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI0GZnQkdjrzife81r1HfS+8
+EF9LMA4GA1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNoADBlAjADVS2m5hjEfO/J
+UG7BJw+ch69u1RsIGL2SKcHvlJF40jocVYli5RsJHrpka/F2tNQCMQC0QoSZ/6vn
+nvuRlydd3LBbMHHOXjgaatkl5+r3YZJW+OraNsKHZZYuciUvf9/DE8k=
+-----END CERTIFICATE-----
+
+# Issuer: CN=Certum Trusted Root CA O=Asseco Data Systems S.A. OU=Certum Certification Authority
+# Subject: CN=Certum Trusted Root CA O=Asseco Data Systems S.A. OU=Certum Certification Authority
+# Label: "Certum Trusted Root CA"
+# Serial: 40870380103424195783807378461123655149
+# MD5 Fingerprint: 51:e1:c2:e7:fe:4c:84:af:59:0e:2f:f4:54:6f:ea:29
+# SHA1 Fingerprint: c8:83:44:c0:18:ae:9f:cc:f1:87:b7:8f:22:d1:c5:d7:45:84:ba:e5
+# SHA256 Fingerprint: fe:76:96:57:38:55:77:3e:37:a9:5e:7a:d4:d9:cc:96:c3:01:57:c1:5d:31:76:5b:a9:b1:57:04:e1:ae:78:fd
+-----BEGIN CERTIFICATE-----
+MIIFwDCCA6igAwIBAgIQHr9ZULjJgDdMBvfrVU+17TANBgkqhkiG9w0BAQ0FADB6
+MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEu
+MScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxHzAdBgNV
+BAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwHhcNMTgwMzE2MTIxMDEzWhcNNDMw
+MzE2MTIxMDEzWjB6MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEg
+U3lzdGVtcyBTLkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRo
+b3JpdHkxHzAdBgNVBAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwggIiMA0GCSqG
+SIb3DQEBAQUAA4ICDwAwggIKAoICAQDRLY67tzbqbTeRn06TpwXkKQMlzhyC93yZ
+n0EGze2jusDbCSzBfN8pfktlL5On1AFrAygYo9idBcEq2EXxkd7fO9CAAozPOA/q
+p1x4EaTByIVcJdPTsuclzxFUl6s1wB52HO8AU5853BSlLCIls3Jy/I2z5T4IHhQq
+NwuIPMqw9MjCoa68wb4pZ1Xi/K1ZXP69VyywkI3C7Te2fJmItdUDmj0VDT06qKhF
+8JVOJVkdzZhpu9PMMsmN74H+rX2Ju7pgE8pllWeg8xn2A1bUatMn4qGtg/BKEiJ3
+HAVz4hlxQsDsdUaakFjgao4rpUYwBI4Zshfjvqm6f1bxJAPXsiEodg42MEx51UGa
+mqi4NboMOvJEGyCI98Ul1z3G4z5D3Yf+xOr1Uz5MZf87Sst4WmsXXw3Hw09Omiqi
+7VdNIuJGmj8PkTQkfVXjjJU30xrwCSss0smNtA0Aq2cpKNgB9RkEth2+dv5yXMSF
+ytKAQd8FqKPVhJBPC/PgP5sZ0jeJP/J7UhyM9uH3PAeXjA6iWYEMspA90+NZRu0P
+qafegGtaqge2Gcu8V/OXIXoMsSt0Puvap2ctTMSYnjYJdmZm/Bo/6khUHL4wvYBQ
+v3y1zgD2DGHZ5yQD4OMBgQ692IU0iL2yNqh7XAjlRICMb/gv1SHKHRzQ+8S1h9E6
+Tsd2tTVItQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSM+xx1
+vALTn04uSNn5YFSqxLNP+jAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQENBQAD
+ggIBAEii1QALLtA/vBzVtVRJHlpr9OTy4EA34MwUe7nJ+jW1dReTagVphZzNTxl4
+WxmB82M+w85bj/UvXgF2Ez8sALnNllI5SW0ETsXpD4YN4fqzX4IS8TrOZgYkNCvo
+zMrnadyHncI013nR03e4qllY/p0m+jiGPp2Kh2RX5Rc64vmNueMzeMGQ2Ljdt4NR
+5MTMI9UGfOZR0800McD2RrsLrfw9EAUqO0qRJe6M1ISHgCq8CYyqOhNf6DR5UMEQ
+GfnTKB7U0VEwKbOukGfWHwpjscWpxkIxYxeU72nLL/qMFH3EQxiJ2fAyQOaA4kZf
+5ePBAFmo+eggvIksDkc0C+pXwlM2/KfUrzHN/gLldfq5Jwn58/U7yn2fqSLLiMmq
+0Uc9NneoWWRrJ8/vJ8HjJLWG965+Mk2weWjROeiQWMODvA8s1pfrzgzhIMfatz7D
+P78v3DSk+yshzWePS/Tj6tQ/50+6uaWTRRxmHyH6ZF5v4HaUMst19W7l9o/HuKTM
+qJZ9ZPskWkoDbGs4xugDQ5r3V7mzKWmTOPQD8rv7gmsHINFSH5pkAnuYZttcTVoP
+0ISVoDwUQwbKytu4QTbaakRnh6+v40URFWkIsr4WOZckbxJF0WddCajJFdr60qZf
+E2Efv4WstK2tBZQIgx51F9NxO5NQI1mg7TyRVJ12AMXDuDjb
+-----END CERTIFICATE-----
+
+# Issuer: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique
+# Subject: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique
+# Label: "TunTrust Root CA"
+# Serial: 108534058042236574382096126452369648152337120275
+# MD5 Fingerprint: 85:13:b9:90:5b:36:5c:b6:5e:b8:5a:f8:e0:31:57:b4
+# SHA1 Fingerprint: cf:e9:70:84:0f:e0:73:0f:9d:f6:0c:7f:2c:4b:ee:20:46:34:9c:bb
+# SHA256 Fingerprint: 2e:44:10:2a:b5:8c:b8:54:19:45:1c:8e:19:d9:ac:f3:66:2c:af:bc:61:4b:6a:53:96:0a:30:f7:d0:e2:eb:41
+-----BEGIN CERTIFICATE-----
+MIIFszCCA5ugAwIBAgIUEwLV4kBMkkaGFmddtLu7sms+/BMwDQYJKoZIhvcNAQEL
+BQAwYTELMAkGA1UEBhMCVE4xNzA1BgNVBAoMLkFnZW5jZSBOYXRpb25hbGUgZGUg
+Q2VydGlmaWNhdGlvbiBFbGVjdHJvbmlxdWUxGTAXBgNVBAMMEFR1blRydXN0IFJv
+b3QgQ0EwHhcNMTkwNDI2MDg1NzU2WhcNNDQwNDI2MDg1NzU2WjBhMQswCQYDVQQG
+EwJUTjE3MDUGA1UECgwuQWdlbmNlIE5hdGlvbmFsZSBkZSBDZXJ0aWZpY2F0aW9u
+IEVsZWN0cm9uaXF1ZTEZMBcGA1UEAwwQVHVuVHJ1c3QgUm9vdCBDQTCCAiIwDQYJ
+KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMPN0/y9BFPdDCA61YguBUtB9YOCfvdZ
+n56eY+hz2vYGqU8ftPkLHzmMmiDQfgbU7DTZhrx1W4eI8NLZ1KMKsmwb60ksPqxd
+2JQDoOw05TDENX37Jk0bbjBU2PWARZw5rZzJJQRNmpA+TkBuimvNKWfGzC3gdOgF
+VwpIUPp6Q9p+7FuaDmJ2/uqdHYVy7BG7NegfJ7/Boce7SBbdVtfMTqDhuazb1YMZ
+GoXRlJfXyqNlC/M4+QKu3fZnz8k/9YosRxqZbwUN/dAdgjH8KcwAWJeRTIAAHDOF
+li/LQcKLEITDCSSJH7UP2dl3RxiSlGBcx5kDPP73lad9UKGAwqmDrViWVSHbhlnU
+r8a83YFuB9tgYv7sEG7aaAH0gxupPqJbI9dkxt/con3YS7qC0lH4Zr8GRuR5KiY2
+eY8fTpkdso8MDhz/yV3A/ZAQprE38806JG60hZC/gLkMjNWb1sjxVj8agIl6qeIb
+MlEsPvLfe/ZdeikZjuXIvTZxi11Mwh0/rViizz1wTaZQmCXcI/m4WEEIcb9PuISg
+jwBUFfyRbVinljvrS5YnzWuioYasDXxU5mZMZl+QviGaAkYt5IPCgLnPSz7ofzwB
+7I9ezX/SKEIBlYrilz0QIX32nRzFNKHsLA4KUiwSVXAkPcvCFDVDXSdOvsC9qnyW
+5/yeYa1E0wCXAgMBAAGjYzBhMB0GA1UdDgQWBBQGmpsfU33x9aTI04Y+oXNZtPdE
+ITAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFAaamx9TffH1pMjThj6hc1m0
+90QhMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAqgVutt0Vyb+z
+xiD2BkewhpMl0425yAA/l/VSJ4hxyXT968pk21vvHl26v9Hr7lxpuhbI87mP0zYu
+QEkHDVneixCwSQXi/5E/S7fdAo74gShczNxtr18UnH1YeA32gAm56Q6XKRm4t+v4
+FstVEuTGfbvE7Pi1HE4+Z7/FXxttbUcoqgRYYdZ2vyJ/0Adqp2RT8JeNnYA/u8EH
+22Wv5psymsNUk8QcCMNE+3tjEUPRahphanltkE8pjkcFwRJpadbGNjHh/PqAulxP
+xOu3Mqz4dWEX1xAZufHSCe96Qp1bWgvUxpVOKs7/B9dPfhgGiPEZtdmYu65xxBzn
+dFlY7wyJz4sfdZMaBBSSSFCp61cpABbjNhzI+L/wM9VBD8TMPN3pM0MBkRArHtG5
+Xc0yGYuPjCB31yLEQtyEFpslbei0VXF/sHyz03FJuc9SpAQ/3D2gu68zngowYI7b
+nV2UqL1g52KAdoGDDIzMMEZJ4gzSqK/rYXHv5yJiqfdcZGyfFoxnNidF9Ql7v/YQ
+CvGwjVRDjAS6oz/v4jXH+XTgbzRB0L9zZVcg+ZtnemZoJE6AZb0QmQZZ8mWvuMZH
+u/2QeItBcy6vVR/cO5JyboTT0GFMDcx2V+IthSIVNg3rAZ3r2OvEhJn7wAzMMujj
+d9qDRIueVSjAi1jTkD5OGwDxFa2DK5o=
+-----END CERTIFICATE-----
+
+# Issuer: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA
+# Subject: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA
+# Label: "HARICA TLS RSA Root CA 2021"
+# Serial: 76817823531813593706434026085292783742
+# MD5 Fingerprint: 65:47:9b:58:86:dd:2c:f0:fc:a2:84:1f:1e:96:c4:91
+# SHA1 Fingerprint: 02:2d:05:82:fa:88:ce:14:0c:06:79:de:7f:14:10:e9:45:d7:a5:6d
+# SHA256 Fingerprint: d9:5d:0e:8e:da:79:52:5b:f9:be:b1:1b:14:d2:10:0d:32:94:98:5f:0c:62:d9:fa:bd:9c:d9:99:ec:cb:7b:1d
+-----BEGIN CERTIFICATE-----
+MIIFpDCCA4ygAwIBAgIQOcqTHO9D88aOk8f0ZIk4fjANBgkqhkiG9w0BAQsFADBs
+MQswCQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl
+c2VhcmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBSU0Eg
+Um9vdCBDQSAyMDIxMB4XDTIxMDIxOTEwNTUzOFoXDTQ1MDIxMzEwNTUzN1owbDEL
+MAkGA1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl
+YXJjaCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgUlNBIFJv
+b3QgQ0EgMjAyMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIvC569l
+mwVnlskNJLnQDmT8zuIkGCyEf3dRywQRNrhe7Wlxp57kJQmXZ8FHws+RFjZiPTgE
+4VGC/6zStGndLuwRo0Xua2s7TL+MjaQenRG56Tj5eg4MmOIjHdFOY9TnuEFE+2uv
+a9of08WRiFukiZLRgeaMOVig1mlDqa2YUlhu2wr7a89o+uOkXjpFc5gH6l8Cct4M
+pbOfrqkdtx2z/IpZ525yZa31MJQjB/OCFks1mJxTuy/K5FrZx40d/JiZ+yykgmvw
+Kh+OC19xXFyuQnspiYHLA6OZyoieC0AJQTPb5lh6/a6ZcMBaD9YThnEvdmn8kN3b
+LW7R8pv1GmuebxWMevBLKKAiOIAkbDakO/IwkfN4E8/BPzWr8R0RI7VDIp4BkrcY
+AuUR0YLbFQDMYTfBKnya4dC6s1BG7oKsnTH4+yPiAwBIcKMJJnkVU2DzOFytOOqB
+AGMUuTNe3QvboEUHGjMJ+E20pwKmafTCWQWIZYVWrkvL4N48fS0ayOn7H6NhStYq
+E613TBoYm5EPWNgGVMWX+Ko/IIqmhaZ39qb8HOLubpQzKoNQhArlT4b4UEV4AIHr
+W2jjJo3Me1xR9BQsQL4aYB16cmEdH2MtiKrOokWQCPxrvrNQKlr9qEgYRtaQQJKQ
+CoReaDH46+0N0x3GfZkYVVYnZS6NRcUk7M7jAgMBAAGjQjBAMA8GA1UdEwEB/wQF
+MAMBAf8wHQYDVR0OBBYEFApII6ZgpJIKM+qTW8VX6iVNvRLuMA4GA1UdDwEB/wQE
+AwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAPpBIqm5iFSVmewzVjIuJndftTgfvnNAU
+X15QvWiWkKQUEapobQk1OUAJ2vQJLDSle1mESSmXdMgHHkdt8s4cUCbjnj1AUz/3
+f5Z2EMVGpdAgS1D0NTsY9FVqQRtHBmg8uwkIYtlfVUKqrFOFrJVWNlar5AWMxaja
+H6NpvVMPxP/cyuN+8kyIhkdGGvMA9YCRotxDQpSbIPDRzbLrLFPCU3hKTwSUQZqP
+JzLB5UkZv/HywouoCjkxKLR9YjYsTewfM7Z+d21+UPCfDtcRj88YxeMn/ibvBZ3P
+zzfF0HvaO7AWhAw6k9a+F9sPPg4ZeAnHqQJyIkv3N3a6dcSFA1pj1bF1BcK5vZSt
+jBWZp5N99sXzqnTPBIWUmAD04vnKJGW/4GKvyMX6ssmeVkjaef2WdhW+o45WxLM0
+/L5H9MG0qPzVMIho7suuyWPEdr6sOBjhXlzPrjoiUevRi7PzKzMHVIf6tLITe7pT
+BGIBnfHAT+7hOtSLIBD6Alfm78ELt5BGnBkpjNxvoEppaZS3JGWg/6w/zgH7IS79
+aPib8qXPMThcFarmlwDB31qlpzmq6YR/PFGoOtmUW4y/Twhx5duoXNTSpv4Ao8YW
+xw/ogM4cKGR0GQjTQuPOAF1/sdwTsOEFy9EgqoZ0njnnkf3/W9b3raYvAwtt41dU
+63ZTGI0RmLo=
+-----END CERTIFICATE-----
+
+# Issuer: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA
+# Subject: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA
+# Label: "HARICA TLS ECC Root CA 2021"
+# Serial: 137515985548005187474074462014555733966
+# MD5 Fingerprint: ae:f7:4c:e5:66:35:d1:b7:9b:8c:22:93:74:d3:4b:b0
+# SHA1 Fingerprint: bc:b0:c1:9d:e9:98:92:70:19:38:57:e9:8d:a7:b4:5d:6e:ee:01:48
+# SHA256 Fingerprint: 3f:99:cc:47:4a:cf:ce:4d:fe:d5:87:94:66:5e:47:8d:15:47:73:9f:2e:78:0f:1b:b4:ca:9b:13:30:97:d4:01
+-----BEGIN CERTIFICATE-----
+MIICVDCCAdugAwIBAgIQZ3SdjXfYO2rbIvT/WeK/zjAKBggqhkjOPQQDAzBsMQsw
+CQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2Vh
+cmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBFQ0MgUm9v
+dCBDQSAyMDIxMB4XDTIxMDIxOTExMDExMFoXDTQ1MDIxMzExMDEwOVowbDELMAkG
+A1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj
+aCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgRUNDIFJvb3Qg
+Q0EgMjAyMTB2MBAGByqGSM49AgEGBSuBBAAiA2IABDgI/rGgltJ6rK9JOtDA4MM7
+KKrxcm1lAEeIhPyaJmuqS7psBAqIXhfyVYf8MLA04jRYVxqEU+kw2anylnTDUR9Y
+STHMmE5gEYd103KUkE+bECUqqHgtvpBBWJAVcqeht6NCMEAwDwYDVR0TAQH/BAUw
+AwEB/zAdBgNVHQ4EFgQUyRtTgRL+BNUW0aq8mm+3oJUZbsowDgYDVR0PAQH/BAQD
+AgGGMAoGCCqGSM49BAMDA2cAMGQCMBHervjcToiwqfAircJRQO9gcS3ujwLEXQNw
+SaSS6sUUiHCm0w2wqsosQJz76YJumgIwK0eaB8bRwoF8yguWGEEbo/QwCZ61IygN
+nxS2PFOiTAZpffpskcYqSUXm7LcT4Tps
+-----END CERTIFICATE-----
diff --git a/src/pip/_vendor/certifi/core.py b/src/pip/_vendor/certifi/core.py
index 8987449f6b5..b8140cf1ae7 100644
--- a/src/pip/_vendor/certifi/core.py
+++ b/src/pip/_vendor/certifi/core.py
@@ -8,7 +8,21 @@
 """
 import os
 
+
+class _PipPatchedCertificate(Exception):
+    pass
+
+
 try:
+    # Return a certificate file on disk for a standalone pip zipapp running in
+    # an isolated build environment to use. Passing --cert to the standalone
+    # pip does not work since requests calls where() unconditionally on import.
+    _PIP_STANDALONE_CERT = os.environ.get("_PIP_STANDALONE_CERT")
+    if _PIP_STANDALONE_CERT:
+        def where():
+            return _PIP_STANDALONE_CERT
+        raise _PipPatchedCertificate()
+
     from importlib.resources import path as get_path, read_text
 
     _CACERT_CTX = None
@@ -38,6 +52,8 @@ def where():
 
         return _CACERT_PATH
 
+except _PipPatchedCertificate:
+    pass
 
 except ImportError:
     # This fallback will work for Python versions prior to 3.7 that lack the
diff --git a/src/pip/_vendor/chardet/__init__.py b/src/pip/_vendor/chardet/__init__.py
index 0f9f820ef6e..80ad2546d79 100644
--- a/src/pip/_vendor/chardet/__init__.py
+++ b/src/pip/_vendor/chardet/__init__.py
@@ -16,11 +16,14 @@
 ######################### END LICENSE BLOCK #########################
 
 
-from .compat import PY2, PY3
 from .universaldetector import UniversalDetector
+from .enums import InputState
 from .version import __version__, VERSION
 
 
+__all__ = ['UniversalDetector', 'detect', 'detect_all', '__version__', 'VERSION']
+
+
 def detect(byte_str):
     """
     Detect the encoding of the given byte string.
@@ -31,9 +34,50 @@ def detect(byte_str):
     if not isinstance(byte_str, bytearray):
         if not isinstance(byte_str, bytes):
             raise TypeError('Expected object of type bytes or bytearray, got: '
-                            '{0}'.format(type(byte_str)))
+                            '{}'.format(type(byte_str)))
         else:
             byte_str = bytearray(byte_str)
     detector = UniversalDetector()
     detector.feed(byte_str)
     return detector.close()
+
+
+def detect_all(byte_str):
+    """
+    Detect all the possible encodings of the given byte string.
+
+    :param byte_str:     The byte sequence to examine.
+    :type byte_str:      ``bytes`` or ``bytearray``
+    """
+    if not isinstance(byte_str, bytearray):
+        if not isinstance(byte_str, bytes):
+            raise TypeError('Expected object of type bytes or bytearray, got: '
+                            '{}'.format(type(byte_str)))
+        else:
+            byte_str = bytearray(byte_str)
+
+    detector = UniversalDetector()
+    detector.feed(byte_str)
+    detector.close()
+
+    if detector._input_state == InputState.HIGH_BYTE:
+        results = []
+        for prober in detector._charset_probers:
+            if prober.get_confidence() > detector.MINIMUM_THRESHOLD:
+                charset_name = prober.charset_name
+                lower_charset_name = prober.charset_name.lower()
+                # Use Windows encoding name instead of ISO-8859 if we saw any
+                # extra Windows-specific bytes
+                if lower_charset_name.startswith('iso-8859'):
+                    if detector._has_win_bytes:
+                        charset_name = detector.ISO_WIN_MAP.get(lower_charset_name,
+                                                            charset_name)
+                results.append({
+                    'encoding': charset_name,
+                    'confidence': prober.get_confidence(),
+                    'language': prober.language,
+                })
+        if len(results) > 0:
+            return sorted(results, key=lambda result: -result['confidence'])
+
+    return [detector.result]
diff --git a/src/pip/_vendor/chardet/charsetgroupprober.py b/src/pip/_vendor/chardet/charsetgroupprober.py
index 8b3738efd8e..5812cef0b59 100644
--- a/src/pip/_vendor/chardet/charsetgroupprober.py
+++ b/src/pip/_vendor/chardet/charsetgroupprober.py
@@ -73,6 +73,7 @@ def feed(self, byte_str):
                 continue
             if state == ProbingState.FOUND_IT:
                 self._best_guess_prober = prober
+                self._state = ProbingState.FOUND_IT
                 return self.state
             elif state == ProbingState.NOT_ME:
                 prober.active = False
diff --git a/src/pip/_vendor/chardet/cli/chardetect.py b/src/pip/_vendor/chardet/cli/chardetect.py
index c61136b639e..6d6f93aabd4 100644
--- a/src/pip/_vendor/chardet/cli/chardetect.py
+++ b/src/pip/_vendor/chardet/cli/chardetect.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
 Script which takes one or more file paths and reports on their detected
 encodings
@@ -45,10 +44,10 @@ def description_of(lines, name='stdin'):
     if PY2:
         name = name.decode(sys.getfilesystemencoding(), 'ignore')
     if result['encoding']:
-        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
+        return '{}: {} with confidence {}'.format(name, result['encoding'],
                                                      result['confidence'])
     else:
-        return '{0}: no result'.format(name)
+        return '{}: no result'.format(name)
 
 
 def main(argv=None):
@@ -69,7 +68,7 @@ def main(argv=None):
                         type=argparse.FileType('rb'), nargs='*',
                         default=[sys.stdin if PY2 else sys.stdin.buffer])
     parser.add_argument('--version', action='version',
-                        version='%(prog)s {0}'.format(__version__))
+                        version='%(prog)s {}'.format(__version__))
     args = parser.parse_args(argv)
 
     for f in args.input:
diff --git a/src/pip/_vendor/chardet/compat.py b/src/pip/_vendor/chardet/compat.py
index ddd74687c02..8941572b3e6 100644
--- a/src/pip/_vendor/chardet/compat.py
+++ b/src/pip/_vendor/chardet/compat.py
@@ -25,10 +25,12 @@
 if sys.version_info < (3, 0):
     PY2 = True
     PY3 = False
-    base_str = (str, unicode)
+    string_types = (str, unicode)
     text_type = unicode
+    iteritems = dict.iteritems
 else:
     PY2 = False
     PY3 = True
-    base_str = (bytes, str)
+    string_types = (bytes, str)
     text_type = str
+    iteritems = dict.items
diff --git a/src/pip/_vendor/chardet/langbulgarianmodel.py b/src/pip/_vendor/chardet/langbulgarianmodel.py
index 2aa4fb2e22f..e963a50979a 100644
--- a/src/pip/_vendor/chardet/langbulgarianmodel.py
+++ b/src/pip/_vendor/chardet/langbulgarianmodel.py
@@ -1,228 +1,4650 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-# 255: Control characters that usually does not exist in any text
-# 254: Carriage/Return
-# 253: symbol (punctuation) that does not belong to word
-# 252: 0 - 9
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
 
-# Character Mapping Table:
-# this table is modified base on win1251BulgarianCharToOrderMap, so
-# only number <64 is sure valid
 
-Latin5_BulgarianCharToOrderMap = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 77, 90, 99,100, 72,109,107,101, 79,185, 81,102, 76, 94, 82,  # 40
-110,186,108, 91, 74,119, 84, 96,111,187,115,253,253,253,253,253,  # 50
-253, 65, 69, 70, 66, 63, 68,112,103, 92,194,104, 95, 86, 87, 71,  # 60
-116,195, 85, 93, 97,113,196,197,198,199,200,253,253,253,253,253,  # 70
-194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,  # 80
-210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,  # 90
- 81,226,227,228,229,230,105,231,232,233,234,235,236, 45,237,238,  # a0
- 31, 32, 35, 43, 37, 44, 55, 47, 40, 59, 33, 46, 38, 36, 41, 30,  # b0
- 39, 28, 34, 51, 48, 49, 53, 50, 54, 57, 61,239, 67,240, 60, 56,  # c0
-  1, 18,  9, 20, 11,  3, 23, 15,  2, 26, 12, 10, 14,  6,  4, 13,  # d0
-  7,  8,  5, 19, 29, 25, 22, 21, 27, 24, 17, 75, 52,241, 42, 16,  # e0
- 62,242,243,244, 58,245, 98,246,247,248,249,250,251, 91,252,253,  # f0
-)
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
 
-win1251BulgarianCharToOrderMap = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 77, 90, 99,100, 72,109,107,101, 79,185, 81,102, 76, 94, 82,  # 40
-110,186,108, 91, 74,119, 84, 96,111,187,115,253,253,253,253,253,  # 50
-253, 65, 69, 70, 66, 63, 68,112,103, 92,194,104, 95, 86, 87, 71,  # 60
-116,195, 85, 93, 97,113,196,197,198,199,200,253,253,253,253,253,  # 70
-206,207,208,209,210,211,212,213,120,214,215,216,217,218,219,220,  # 80
-221, 78, 64, 83,121, 98,117,105,222,223,224,225,226,227,228,229,  # 90
- 88,230,231,232,233,122, 89,106,234,235,236,237,238, 45,239,240,  # a0
- 73, 80,118,114,241,242,243,244,245, 62, 58,246,247,248,249,250,  # b0
- 31, 32, 35, 43, 37, 44, 55, 47, 40, 59, 33, 46, 38, 36, 41, 30,  # c0
- 39, 28, 34, 51, 48, 49, 53, 50, 54, 57, 61,251, 67,252, 60, 56,  # d0
-  1, 18,  9, 20, 11,  3, 23, 15,  2, 26, 12, 10, 14,  6,  4, 13,  # e0
-  7,  8,  5, 19, 29, 25, 22, 21, 27, 24, 17, 75, 52,253, 42, 16,  # f0
-)
+BULGARIAN_LANG_MODEL = {
+    63: {  # 'e'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 1,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 1,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    45: {  # '\xad'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 1,  # 'М'
+        36: 0,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    31: {  # 'А'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 2,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 1,  # 'К'
+        46: 2,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 1,  # 'О'
+        30: 2,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 2,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 1,  # 'е'
+        23: 1,  # 'ж'
+        15: 2,  # 'з'
+        2: 0,  # 'и'
+        26: 2,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 0,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 1,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    32: {  # 'Б'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 2,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 1,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 2,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 2,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 1,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    35: {  # 'В'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 2,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 2,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    43: {  # 'Г'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 1,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    37: {  # 'Д'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 2,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    44: {  # 'Е'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 2,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 2,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 2,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 0,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 0,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 1,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    55: {  # 'Ж'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    47: {  # 'З'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 2,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 1,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    40: {  # 'И'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 2,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 2,  # 'Л'
+        38: 2,  # 'М'
+        36: 2,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 2,  # 'Я'
+        1: 1,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 1,  # 'е'
+        23: 0,  # 'ж'
+        15: 3,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 0,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 0,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    59: {  # 'Й'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 1,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    33: {  # 'К'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    46: {  # 'Л'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 2,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    38: {  # 'М'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 0,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    36: {  # 'Н'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 2,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 1,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 1,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 2,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    41: {  # 'О'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 1,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 1,  # 'Й'
+        33: 2,  # 'К'
+        46: 2,  # 'Л'
+        38: 2,  # 'М'
+        36: 2,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 1,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 0,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 0,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 2,  # 'ч'
+        27: 0,  # 'ш'
+        24: 2,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    30: {  # 'П'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 2,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    39: {  # 'Р'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 2,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 1,  # 'с'
+        5: 0,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    28: {  # 'С'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 3,  # 'А'
+        32: 2,  # 'Б'
+        35: 2,  # 'В'
+        43: 1,  # 'Г'
+        37: 2,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 2,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 1,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    34: {  # 'Т'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 2,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 2,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 1,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 1,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 3,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    51: {  # 'У'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 2,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 2,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 2,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 2,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 2,  # 'с'
+        5: 1,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    48: {  # 'Ф'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 2,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 1,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 2,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    49: {  # 'Х'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 1,  # 'П'
+        39: 1,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    53: {  # 'Ц'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 2,  # 'И'
+        59: 0,  # 'Й'
+        33: 2,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 2,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 1,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    50: {  # 'Ч'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 2,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 2,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    54: {  # 'Ш'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 1,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 1,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 2,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    57: {  # 'Щ'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 1,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 1,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 1,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    61: {  # 'Ъ'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 1,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 2,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 1,  # 'П'
+        39: 2,  # 'Р'
+        28: 1,  # 'С'
+        34: 1,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 1,  # 'Х'
+        53: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        54: 1,  # 'Ш'
+        57: 1,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 1,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    60: {  # 'Ю'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 1,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 0,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 0,  # 'е'
+        23: 2,  # 'ж'
+        15: 1,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 0,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    56: {  # 'Я'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 1,  # 'В'
+        43: 1,  # 'Г'
+        37: 1,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 1,  # 'Л'
+        38: 1,  # 'М'
+        36: 1,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 2,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 1,  # 'и'
+        26: 1,  # 'й'
+        12: 1,  # 'к'
+        10: 1,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 0,  # 'о'
+        13: 2,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    1: {  # 'а'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 1,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    18: {  # 'б'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 3,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 0,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 2,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 3,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    9: {  # 'в'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 1,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 0,  # 'в'
+        20: 2,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    20: {  # 'г'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    11: {  # 'д'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 2,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 1,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    3: {  # 'е'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 2,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    23: {  # 'ж'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    15: {  # 'з'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    2: {  # 'и'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 1,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 1,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 1,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 1,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    26: {  # 'й'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 2,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 2,  # 'з'
+        2: 1,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    12: {  # 'к'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 1,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 1,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 3,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    10: {  # 'л'
+        63: 1,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 1,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 1,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 3,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    14: {  # 'м'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 1,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 1,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    6: {  # 'н'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 1,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 2,  # 'б'
+        9: 2,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 2,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 3,  # 'ф'
+        25: 2,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 2,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    4: {  # 'о'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 2,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 3,  # 'и'
+        26: 3,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 2,  # 'у'
+        29: 3,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 3,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    13: {  # 'п'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 3,  # 'л'
+        14: 1,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    7: {  # 'р'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 3,  # 'е'
+        23: 3,  # 'ж'
+        15: 2,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 1,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 3,  # 'х'
+        22: 3,  # 'ц'
+        21: 2,  # 'ч'
+        27: 3,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    8: {  # 'с'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 2,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 2,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 2,  # 'ш'
+        24: 0,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    5: {  # 'т'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 2,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 3,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 2,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 3,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 2,  # 'ю'
+        16: 3,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    19: {  # 'у'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 2,  # 'и'
+        26: 2,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 2,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 3,  # 'ш'
+        24: 2,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    29: {  # 'ф'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 1,  # 'в'
+        20: 1,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 2,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 2,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    25: {  # 'х'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 2,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 1,  # 'п'
+        7: 3,  # 'р'
+        8: 1,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 1,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    22: {  # 'ц'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 2,  # 'в'
+        20: 1,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 1,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 2,  # 'к'
+        10: 1,  # 'л'
+        14: 1,  # 'м'
+        6: 1,  # 'н'
+        4: 2,  # 'о'
+        13: 1,  # 'п'
+        7: 1,  # 'р'
+        8: 1,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 1,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 0,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    21: {  # 'ч'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 1,  # 'б'
+        9: 3,  # 'в'
+        20: 1,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 1,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 2,  # 'р'
+        8: 0,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 1,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    27: {  # 'ш'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 2,  # 'в'
+        20: 0,  # 'г'
+        11: 1,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 3,  # 'к'
+        10: 2,  # 'л'
+        14: 1,  # 'м'
+        6: 3,  # 'н'
+        4: 2,  # 'о'
+        13: 2,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 1,  # 'т'
+        19: 2,  # 'у'
+        29: 1,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 1,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 2,  # 'ъ'
+        52: 1,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    24: {  # 'щ'
+        63: 1,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 3,  # 'а'
+        18: 0,  # 'б'
+        9: 1,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 3,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 3,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 2,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 1,  # 'р'
+        8: 0,  # 'с'
+        5: 2,  # 'т'
+        19: 3,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 1,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 2,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    17: {  # 'ъ'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 3,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 3,  # 'ж'
+        15: 3,  # 'з'
+        2: 1,  # 'и'
+        26: 2,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 3,  # 'о'
+        13: 3,  # 'п'
+        7: 3,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 2,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 2,  # 'ш'
+        24: 3,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 2,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    52: {  # 'ь'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 1,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 1,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 1,  # 'н'
+        4: 3,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 1,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 1,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 1,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    42: {  # 'ю'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 1,  # 'а'
+        18: 2,  # 'б'
+        9: 1,  # 'в'
+        20: 2,  # 'г'
+        11: 2,  # 'д'
+        3: 1,  # 'е'
+        23: 2,  # 'ж'
+        15: 2,  # 'з'
+        2: 1,  # 'и'
+        26: 1,  # 'й'
+        12: 2,  # 'к'
+        10: 2,  # 'л'
+        14: 2,  # 'м'
+        6: 2,  # 'н'
+        4: 1,  # 'о'
+        13: 1,  # 'п'
+        7: 2,  # 'р'
+        8: 2,  # 'с'
+        5: 2,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 1,  # 'х'
+        22: 2,  # 'ц'
+        21: 3,  # 'ч'
+        27: 1,  # 'ш'
+        24: 1,  # 'щ'
+        17: 1,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    16: {  # 'я'
+        63: 0,  # 'e'
+        45: 1,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 3,  # 'б'
+        9: 3,  # 'в'
+        20: 2,  # 'г'
+        11: 3,  # 'д'
+        3: 2,  # 'е'
+        23: 1,  # 'ж'
+        15: 2,  # 'з'
+        2: 1,  # 'и'
+        26: 2,  # 'й'
+        12: 3,  # 'к'
+        10: 3,  # 'л'
+        14: 3,  # 'м'
+        6: 3,  # 'н'
+        4: 1,  # 'о'
+        13: 2,  # 'п'
+        7: 2,  # 'р'
+        8: 3,  # 'с'
+        5: 3,  # 'т'
+        19: 1,  # 'у'
+        29: 1,  # 'ф'
+        25: 3,  # 'х'
+        22: 2,  # 'ц'
+        21: 1,  # 'ч'
+        27: 1,  # 'ш'
+        24: 2,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 1,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    58: {  # 'є'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+    62: {  # '№'
+        63: 0,  # 'e'
+        45: 0,  # '\xad'
+        31: 0,  # 'А'
+        32: 0,  # 'Б'
+        35: 0,  # 'В'
+        43: 0,  # 'Г'
+        37: 0,  # 'Д'
+        44: 0,  # 'Е'
+        55: 0,  # 'Ж'
+        47: 0,  # 'З'
+        40: 0,  # 'И'
+        59: 0,  # 'Й'
+        33: 0,  # 'К'
+        46: 0,  # 'Л'
+        38: 0,  # 'М'
+        36: 0,  # 'Н'
+        41: 0,  # 'О'
+        30: 0,  # 'П'
+        39: 0,  # 'Р'
+        28: 0,  # 'С'
+        34: 0,  # 'Т'
+        51: 0,  # 'У'
+        48: 0,  # 'Ф'
+        49: 0,  # 'Х'
+        53: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        54: 0,  # 'Ш'
+        57: 0,  # 'Щ'
+        61: 0,  # 'Ъ'
+        60: 0,  # 'Ю'
+        56: 0,  # 'Я'
+        1: 0,  # 'а'
+        18: 0,  # 'б'
+        9: 0,  # 'в'
+        20: 0,  # 'г'
+        11: 0,  # 'д'
+        3: 0,  # 'е'
+        23: 0,  # 'ж'
+        15: 0,  # 'з'
+        2: 0,  # 'и'
+        26: 0,  # 'й'
+        12: 0,  # 'к'
+        10: 0,  # 'л'
+        14: 0,  # 'м'
+        6: 0,  # 'н'
+        4: 0,  # 'о'
+        13: 0,  # 'п'
+        7: 0,  # 'р'
+        8: 0,  # 'с'
+        5: 0,  # 'т'
+        19: 0,  # 'у'
+        29: 0,  # 'ф'
+        25: 0,  # 'х'
+        22: 0,  # 'ц'
+        21: 0,  # 'ч'
+        27: 0,  # 'ш'
+        24: 0,  # 'щ'
+        17: 0,  # 'ъ'
+        52: 0,  # 'ь'
+        42: 0,  # 'ю'
+        16: 0,  # 'я'
+        58: 0,  # 'є'
+        62: 0,  # '№'
+    },
+}
 
-# Model Table:
-# total sequences: 100%
-# first 512 sequences: 96.9392%
-# first 1024 sequences:3.0618%
-# rest  sequences:     0.2992%
-# negative sequences:  0.0020%
-BulgarianLangModel = (
-0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,3,3,3,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,2,2,3,2,2,1,2,2,
-3,1,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,3,3,3,3,0,3,0,1,
-0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,3,3,3,3,3,3,0,3,1,0,
-0,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-3,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,1,3,2,3,3,3,3,3,3,3,3,0,3,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,2,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,1,3,2,3,3,3,3,3,3,3,3,0,3,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,2,3,2,2,1,3,3,3,3,2,2,2,1,1,2,0,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,2,3,2,2,3,3,1,1,2,3,3,2,3,3,3,3,2,1,2,0,2,0,3,0,0,
-0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,1,3,3,3,3,3,2,3,2,3,3,3,3,3,2,3,3,1,3,0,3,0,2,0,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,3,1,3,3,2,3,3,3,1,3,3,2,3,2,2,2,0,0,2,0,2,0,2,0,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,3,3,0,3,3,3,2,2,3,3,3,1,2,2,3,2,1,1,2,0,2,0,0,0,0,
-1,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,2,3,3,1,2,3,2,2,2,3,3,3,3,3,2,2,3,1,2,0,2,1,2,0,0,
-0,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,1,3,3,3,3,3,2,3,3,3,2,3,3,2,3,2,2,2,3,1,2,0,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,3,3,3,3,1,1,1,2,2,1,3,1,3,2,2,3,0,0,1,0,1,0,1,0,0,
-0,0,0,1,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,2,2,3,2,2,3,1,2,1,1,1,2,3,1,3,1,2,2,0,1,1,1,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,1,3,2,2,3,3,1,2,3,1,1,3,3,3,3,1,2,2,1,1,1,0,2,0,2,0,1,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,2,2,3,3,3,2,2,1,1,2,0,2,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,0,1,2,1,3,3,2,3,3,3,3,3,2,3,2,1,0,3,1,2,1,2,1,2,3,2,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,1,1,2,3,3,3,3,3,3,3,3,3,3,3,3,0,0,3,1,3,3,2,3,3,2,2,2,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,3,3,3,0,3,3,3,3,3,2,1,1,2,1,3,3,0,3,1,1,1,1,3,2,0,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,2,2,2,3,3,3,3,3,3,3,3,3,3,3,1,1,3,1,3,3,2,3,2,2,2,3,0,2,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,2,3,3,2,2,3,2,1,1,1,1,1,3,1,3,1,1,0,0,0,1,0,0,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,2,3,2,0,3,2,0,3,0,2,0,0,2,1,3,1,0,0,1,0,0,0,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,2,1,1,1,1,2,1,1,2,1,1,1,2,2,1,2,1,1,1,0,1,1,0,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,2,1,3,1,1,2,1,3,2,1,1,0,1,2,3,2,1,1,1,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,3,3,3,2,2,1,0,1,0,0,1,0,0,0,2,1,0,3,0,0,1,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,2,3,2,3,3,1,3,2,1,1,1,2,1,1,2,1,3,0,1,0,0,0,1,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,1,2,2,3,3,2,3,2,2,2,3,1,2,2,1,1,2,1,1,2,2,0,1,1,0,1,0,2,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,2,1,3,1,0,2,2,1,3,2,1,0,0,2,0,2,0,1,0,0,0,0,0,0,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,3,1,2,0,2,3,1,2,3,2,0,1,3,1,2,1,1,1,0,0,1,0,0,2,2,2,3,
-2,2,2,2,1,2,1,1,2,2,1,1,2,0,1,1,1,0,0,1,1,0,0,1,1,0,0,0,1,1,0,1,
-3,3,3,3,3,2,1,2,2,1,2,0,2,0,1,0,1,2,1,2,1,1,0,0,0,1,0,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,
-3,3,2,3,3,1,1,3,1,0,3,2,1,0,0,0,1,2,0,2,0,1,0,0,0,1,0,1,2,1,2,2,
-1,1,1,1,1,1,1,2,2,2,1,1,1,1,1,1,1,0,1,2,1,1,1,0,0,0,0,0,1,1,0,0,
-3,1,0,1,0,2,3,2,2,2,3,2,2,2,2,2,1,0,2,1,2,1,1,1,0,1,2,1,2,2,2,1,
-1,1,2,2,2,2,1,2,1,1,0,1,2,1,2,2,2,1,1,1,0,1,1,1,1,2,0,1,0,0,0,0,
-2,3,2,3,3,0,0,2,1,0,2,1,0,0,0,0,2,3,0,2,0,0,0,0,0,1,0,0,2,0,1,2,
-2,1,2,1,2,2,1,1,1,2,1,1,1,0,1,2,2,1,1,1,1,1,0,1,1,1,0,0,1,2,0,0,
-3,3,2,2,3,0,2,3,1,1,2,0,0,0,1,0,0,2,0,2,0,0,0,1,0,1,0,1,2,0,2,2,
-1,1,1,1,2,1,0,1,2,2,2,1,1,1,1,1,1,1,0,1,1,1,0,0,0,0,0,0,1,1,0,0,
-2,3,2,3,3,0,0,3,0,1,1,0,1,0,0,0,2,2,1,2,0,0,0,0,0,0,0,0,2,0,1,2,
-2,2,1,1,1,1,1,2,2,2,1,0,2,0,1,0,1,0,0,1,0,1,0,0,1,0,0,0,0,1,0,0,
-3,3,3,3,2,2,2,2,2,0,2,1,1,1,1,2,1,2,1,1,0,2,0,1,0,1,0,0,2,0,1,2,
-1,1,1,1,1,1,1,2,2,1,1,0,2,0,1,0,2,0,0,1,1,1,0,0,2,0,0,0,1,1,0,0,
-2,3,3,3,3,1,0,0,0,0,0,0,0,0,0,0,2,0,0,1,1,0,0,0,0,0,0,1,2,0,1,2,
-2,2,2,1,1,2,1,1,2,2,2,1,2,0,1,1,1,1,1,1,0,1,1,1,1,0,0,1,1,1,0,0,
-2,3,3,3,3,0,2,2,0,2,1,0,0,0,1,1,1,2,0,2,0,0,0,3,0,0,0,0,2,0,2,2,
-1,1,1,2,1,2,1,1,2,2,2,1,2,0,1,1,1,0,1,1,1,1,0,2,1,0,0,0,1,1,0,0,
-2,3,3,3,3,0,2,1,0,0,2,0,0,0,0,0,1,2,0,2,0,0,0,0,0,0,0,0,2,0,1,2,
-1,1,1,2,1,1,1,1,2,2,2,0,1,0,1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,1,0,0,
-3,3,2,2,3,0,1,0,1,0,0,0,0,0,0,0,1,1,0,3,0,0,0,0,0,0,0,0,1,0,2,2,
-1,1,1,1,1,2,1,1,2,2,1,2,2,1,0,1,1,1,1,1,0,1,0,0,1,0,0,0,1,1,0,0,
-3,1,0,1,0,2,2,2,2,3,2,1,1,1,2,3,0,0,1,0,2,1,1,0,1,1,1,1,2,1,1,1,
-1,2,2,1,2,1,2,2,1,1,0,1,2,1,2,2,1,1,1,0,0,1,1,1,2,1,0,1,0,0,0,0,
-2,1,0,1,0,3,1,2,2,2,2,1,2,2,1,1,1,0,2,1,2,2,1,1,2,1,1,0,2,1,1,1,
-1,2,2,2,2,2,2,2,1,2,0,1,1,0,2,1,1,1,1,1,0,0,1,1,1,1,0,1,0,0,0,0,
-2,1,1,1,1,2,2,2,2,1,2,2,2,1,2,2,1,1,2,1,2,3,2,2,1,1,1,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,3,2,0,1,2,0,1,2,1,1,0,1,0,1,2,1,2,0,0,0,1,1,0,0,0,1,0,0,2,
-1,1,0,0,1,1,0,1,1,1,1,0,2,0,1,1,1,0,0,1,1,0,0,0,0,1,0,0,0,1,0,0,
-2,0,0,0,0,1,2,2,2,2,2,2,2,1,2,1,1,1,1,1,1,1,0,1,1,1,1,1,2,1,1,1,
-1,2,2,2,2,1,1,2,1,2,1,1,1,0,2,1,2,1,1,1,0,2,1,1,1,1,0,1,0,0,0,0,
-3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,
-1,1,0,1,0,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,3,2,0,0,0,0,1,0,0,0,0,0,0,1,1,0,2,0,0,0,0,0,0,0,0,1,0,1,2,
-1,1,1,1,1,1,0,0,2,2,2,2,2,0,1,1,0,1,1,1,1,1,0,0,1,0,0,0,1,1,0,1,
-2,3,1,2,1,0,1,1,0,2,2,2,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,1,2,
-1,1,1,1,2,1,1,1,1,1,1,1,1,0,1,1,0,1,0,1,0,1,0,0,1,0,0,0,0,1,0,0,
-2,2,2,2,2,0,0,2,0,0,2,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,2,0,2,2,
-1,1,1,1,1,0,0,1,2,1,1,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,
-1,2,2,2,2,0,0,2,0,1,1,0,0,0,1,0,0,2,0,2,0,0,0,0,0,0,0,0,0,0,1,1,
-0,0,0,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-1,2,2,3,2,0,0,1,0,0,1,0,0,0,0,0,0,1,0,2,0,0,0,1,0,0,0,0,0,0,0,2,
-1,1,0,0,1,0,0,0,1,1,0,0,1,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,
-2,1,2,2,2,1,2,1,2,2,1,1,2,1,1,1,0,1,1,1,1,2,0,1,0,1,1,1,1,0,1,1,
-1,1,2,1,1,1,1,1,1,0,0,1,2,1,1,1,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,
-1,0,0,1,3,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,2,1,0,0,1,0,2,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,2,0,0,1,
-0,2,0,1,0,0,1,1,2,0,1,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,
-1,2,2,2,2,0,1,1,0,2,1,0,1,1,1,0,0,1,0,2,0,1,0,0,0,0,0,0,0,0,0,1,
-0,1,0,0,1,0,0,0,1,1,0,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,2,2,0,0,1,0,0,0,1,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,
-0,1,0,1,1,1,0,0,1,1,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,
-2,0,1,0,0,1,2,1,1,1,1,1,1,2,2,1,0,0,1,0,1,0,0,0,0,1,1,1,1,0,0,0,
-1,1,2,1,1,1,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,1,2,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,
-0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
-0,1,1,0,1,1,1,0,0,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,
-1,0,1,0,0,1,1,1,1,1,1,1,1,1,1,1,0,0,1,0,2,0,0,2,0,1,0,0,1,0,0,1,
-1,1,0,0,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,
-1,1,1,1,1,1,1,2,0,0,0,0,0,0,2,1,0,1,1,0,0,1,1,1,0,1,0,0,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,1,1,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-)
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
 
-Latin5BulgarianModel = {
-  'char_to_order_map': Latin5_BulgarianCharToOrderMap,
-  'precedence_matrix': BulgarianLangModel,
-  'typical_positive_ratio': 0.969392,
-  'keep_english_letter': False,
-  'charset_name': "ISO-8859-5",
-  'language': 'Bulgairan',
+# Character Mapping Table(s):
+ISO_8859_5_BULGARIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 77,  # 'A'
+     66: 90,  # 'B'
+     67: 99,  # 'C'
+     68: 100,  # 'D'
+     69: 72,  # 'E'
+     70: 109,  # 'F'
+     71: 107,  # 'G'
+     72: 101,  # 'H'
+     73: 79,  # 'I'
+     74: 185,  # 'J'
+     75: 81,  # 'K'
+     76: 102,  # 'L'
+     77: 76,  # 'M'
+     78: 94,  # 'N'
+     79: 82,  # 'O'
+     80: 110,  # 'P'
+     81: 186,  # 'Q'
+     82: 108,  # 'R'
+     83: 91,  # 'S'
+     84: 74,  # 'T'
+     85: 119,  # 'U'
+     86: 84,  # 'V'
+     87: 96,  # 'W'
+     88: 111,  # 'X'
+     89: 187,  # 'Y'
+     90: 115,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 65,  # 'a'
+     98: 69,  # 'b'
+     99: 70,  # 'c'
+     100: 66,  # 'd'
+     101: 63,  # 'e'
+     102: 68,  # 'f'
+     103: 112,  # 'g'
+     104: 103,  # 'h'
+     105: 92,  # 'i'
+     106: 194,  # 'j'
+     107: 104,  # 'k'
+     108: 95,  # 'l'
+     109: 86,  # 'm'
+     110: 87,  # 'n'
+     111: 71,  # 'o'
+     112: 116,  # 'p'
+     113: 195,  # 'q'
+     114: 85,  # 'r'
+     115: 93,  # 's'
+     116: 97,  # 't'
+     117: 113,  # 'u'
+     118: 196,  # 'v'
+     119: 197,  # 'w'
+     120: 198,  # 'x'
+     121: 199,  # 'y'
+     122: 200,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 194,  # '\x80'
+     129: 195,  # '\x81'
+     130: 196,  # '\x82'
+     131: 197,  # '\x83'
+     132: 198,  # '\x84'
+     133: 199,  # '\x85'
+     134: 200,  # '\x86'
+     135: 201,  # '\x87'
+     136: 202,  # '\x88'
+     137: 203,  # '\x89'
+     138: 204,  # '\x8a'
+     139: 205,  # '\x8b'
+     140: 206,  # '\x8c'
+     141: 207,  # '\x8d'
+     142: 208,  # '\x8e'
+     143: 209,  # '\x8f'
+     144: 210,  # '\x90'
+     145: 211,  # '\x91'
+     146: 212,  # '\x92'
+     147: 213,  # '\x93'
+     148: 214,  # '\x94'
+     149: 215,  # '\x95'
+     150: 216,  # '\x96'
+     151: 217,  # '\x97'
+     152: 218,  # '\x98'
+     153: 219,  # '\x99'
+     154: 220,  # '\x9a'
+     155: 221,  # '\x9b'
+     156: 222,  # '\x9c'
+     157: 223,  # '\x9d'
+     158: 224,  # '\x9e'
+     159: 225,  # '\x9f'
+     160: 81,  # '\xa0'
+     161: 226,  # 'Ё'
+     162: 227,  # 'Ђ'
+     163: 228,  # 'Ѓ'
+     164: 229,  # 'Є'
+     165: 230,  # 'Ѕ'
+     166: 105,  # 'І'
+     167: 231,  # 'Ї'
+     168: 232,  # 'Ј'
+     169: 233,  # 'Љ'
+     170: 234,  # 'Њ'
+     171: 235,  # 'Ћ'
+     172: 236,  # 'Ќ'
+     173: 45,  # '\xad'
+     174: 237,  # 'Ў'
+     175: 238,  # 'Џ'
+     176: 31,  # 'А'
+     177: 32,  # 'Б'
+     178: 35,  # 'В'
+     179: 43,  # 'Г'
+     180: 37,  # 'Д'
+     181: 44,  # 'Е'
+     182: 55,  # 'Ж'
+     183: 47,  # 'З'
+     184: 40,  # 'И'
+     185: 59,  # 'Й'
+     186: 33,  # 'К'
+     187: 46,  # 'Л'
+     188: 38,  # 'М'
+     189: 36,  # 'Н'
+     190: 41,  # 'О'
+     191: 30,  # 'П'
+     192: 39,  # 'Р'
+     193: 28,  # 'С'
+     194: 34,  # 'Т'
+     195: 51,  # 'У'
+     196: 48,  # 'Ф'
+     197: 49,  # 'Х'
+     198: 53,  # 'Ц'
+     199: 50,  # 'Ч'
+     200: 54,  # 'Ш'
+     201: 57,  # 'Щ'
+     202: 61,  # 'Ъ'
+     203: 239,  # 'Ы'
+     204: 67,  # 'Ь'
+     205: 240,  # 'Э'
+     206: 60,  # 'Ю'
+     207: 56,  # 'Я'
+     208: 1,  # 'а'
+     209: 18,  # 'б'
+     210: 9,  # 'в'
+     211: 20,  # 'г'
+     212: 11,  # 'д'
+     213: 3,  # 'е'
+     214: 23,  # 'ж'
+     215: 15,  # 'з'
+     216: 2,  # 'и'
+     217: 26,  # 'й'
+     218: 12,  # 'к'
+     219: 10,  # 'л'
+     220: 14,  # 'м'
+     221: 6,  # 'н'
+     222: 4,  # 'о'
+     223: 13,  # 'п'
+     224: 7,  # 'р'
+     225: 8,  # 'с'
+     226: 5,  # 'т'
+     227: 19,  # 'у'
+     228: 29,  # 'ф'
+     229: 25,  # 'х'
+     230: 22,  # 'ц'
+     231: 21,  # 'ч'
+     232: 27,  # 'ш'
+     233: 24,  # 'щ'
+     234: 17,  # 'ъ'
+     235: 75,  # 'ы'
+     236: 52,  # 'ь'
+     237: 241,  # 'э'
+     238: 42,  # 'ю'
+     239: 16,  # 'я'
+     240: 62,  # '№'
+     241: 242,  # 'ё'
+     242: 243,  # 'ђ'
+     243: 244,  # 'ѓ'
+     244: 58,  # 'є'
+     245: 245,  # 'ѕ'
+     246: 98,  # 'і'
+     247: 246,  # 'ї'
+     248: 247,  # 'ј'
+     249: 248,  # 'љ'
+     250: 249,  # 'њ'
+     251: 250,  # 'ћ'
+     252: 251,  # 'ќ'
+     253: 91,  # '§'
+     254: 252,  # 'ў'
+     255: 253,  # 'џ'
 }
 
-Win1251BulgarianModel = {
-  'char_to_order_map': win1251BulgarianCharToOrderMap,
-  'precedence_matrix': BulgarianLangModel,
-  'typical_positive_ratio': 0.969392,
-  'keep_english_letter': False,
-  'charset_name': "windows-1251",
-  'language': 'Bulgarian',
+ISO_8859_5_BULGARIAN_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-5',
+                                                    language='Bulgarian',
+                                                    char_to_order_map=ISO_8859_5_BULGARIAN_CHAR_TO_ORDER,
+                                                    language_model=BULGARIAN_LANG_MODEL,
+                                                    typical_positive_ratio=0.969392,
+                                                    keep_ascii_letters=False,
+                                                    alphabet='АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя')
+
+WINDOWS_1251_BULGARIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 77,  # 'A'
+     66: 90,  # 'B'
+     67: 99,  # 'C'
+     68: 100,  # 'D'
+     69: 72,  # 'E'
+     70: 109,  # 'F'
+     71: 107,  # 'G'
+     72: 101,  # 'H'
+     73: 79,  # 'I'
+     74: 185,  # 'J'
+     75: 81,  # 'K'
+     76: 102,  # 'L'
+     77: 76,  # 'M'
+     78: 94,  # 'N'
+     79: 82,  # 'O'
+     80: 110,  # 'P'
+     81: 186,  # 'Q'
+     82: 108,  # 'R'
+     83: 91,  # 'S'
+     84: 74,  # 'T'
+     85: 119,  # 'U'
+     86: 84,  # 'V'
+     87: 96,  # 'W'
+     88: 111,  # 'X'
+     89: 187,  # 'Y'
+     90: 115,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 65,  # 'a'
+     98: 69,  # 'b'
+     99: 70,  # 'c'
+     100: 66,  # 'd'
+     101: 63,  # 'e'
+     102: 68,  # 'f'
+     103: 112,  # 'g'
+     104: 103,  # 'h'
+     105: 92,  # 'i'
+     106: 194,  # 'j'
+     107: 104,  # 'k'
+     108: 95,  # 'l'
+     109: 86,  # 'm'
+     110: 87,  # 'n'
+     111: 71,  # 'o'
+     112: 116,  # 'p'
+     113: 195,  # 'q'
+     114: 85,  # 'r'
+     115: 93,  # 's'
+     116: 97,  # 't'
+     117: 113,  # 'u'
+     118: 196,  # 'v'
+     119: 197,  # 'w'
+     120: 198,  # 'x'
+     121: 199,  # 'y'
+     122: 200,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 206,  # 'Ђ'
+     129: 207,  # 'Ѓ'
+     130: 208,  # '‚'
+     131: 209,  # 'ѓ'
+     132: 210,  # '„'
+     133: 211,  # '…'
+     134: 212,  # '†'
+     135: 213,  # '‡'
+     136: 120,  # '€'
+     137: 214,  # '‰'
+     138: 215,  # 'Љ'
+     139: 216,  # '‹'
+     140: 217,  # 'Њ'
+     141: 218,  # 'Ќ'
+     142: 219,  # 'Ћ'
+     143: 220,  # 'Џ'
+     144: 221,  # 'ђ'
+     145: 78,  # '‘'
+     146: 64,  # '’'
+     147: 83,  # '“'
+     148: 121,  # '”'
+     149: 98,  # '•'
+     150: 117,  # '–'
+     151: 105,  # '—'
+     152: 222,  # None
+     153: 223,  # '™'
+     154: 224,  # 'љ'
+     155: 225,  # '›'
+     156: 226,  # 'њ'
+     157: 227,  # 'ќ'
+     158: 228,  # 'ћ'
+     159: 229,  # 'џ'
+     160: 88,  # '\xa0'
+     161: 230,  # 'Ў'
+     162: 231,  # 'ў'
+     163: 232,  # 'Ј'
+     164: 233,  # '¤'
+     165: 122,  # 'Ґ'
+     166: 89,  # '¦'
+     167: 106,  # '§'
+     168: 234,  # 'Ё'
+     169: 235,  # '©'
+     170: 236,  # 'Є'
+     171: 237,  # '«'
+     172: 238,  # '¬'
+     173: 45,  # '\xad'
+     174: 239,  # '®'
+     175: 240,  # 'Ї'
+     176: 73,  # '°'
+     177: 80,  # '±'
+     178: 118,  # 'І'
+     179: 114,  # 'і'
+     180: 241,  # 'ґ'
+     181: 242,  # 'µ'
+     182: 243,  # '¶'
+     183: 244,  # '·'
+     184: 245,  # 'ё'
+     185: 62,  # '№'
+     186: 58,  # 'є'
+     187: 246,  # '»'
+     188: 247,  # 'ј'
+     189: 248,  # 'Ѕ'
+     190: 249,  # 'ѕ'
+     191: 250,  # 'ї'
+     192: 31,  # 'А'
+     193: 32,  # 'Б'
+     194: 35,  # 'В'
+     195: 43,  # 'Г'
+     196: 37,  # 'Д'
+     197: 44,  # 'Е'
+     198: 55,  # 'Ж'
+     199: 47,  # 'З'
+     200: 40,  # 'И'
+     201: 59,  # 'Й'
+     202: 33,  # 'К'
+     203: 46,  # 'Л'
+     204: 38,  # 'М'
+     205: 36,  # 'Н'
+     206: 41,  # 'О'
+     207: 30,  # 'П'
+     208: 39,  # 'Р'
+     209: 28,  # 'С'
+     210: 34,  # 'Т'
+     211: 51,  # 'У'
+     212: 48,  # 'Ф'
+     213: 49,  # 'Х'
+     214: 53,  # 'Ц'
+     215: 50,  # 'Ч'
+     216: 54,  # 'Ш'
+     217: 57,  # 'Щ'
+     218: 61,  # 'Ъ'
+     219: 251,  # 'Ы'
+     220: 67,  # 'Ь'
+     221: 252,  # 'Э'
+     222: 60,  # 'Ю'
+     223: 56,  # 'Я'
+     224: 1,  # 'а'
+     225: 18,  # 'б'
+     226: 9,  # 'в'
+     227: 20,  # 'г'
+     228: 11,  # 'д'
+     229: 3,  # 'е'
+     230: 23,  # 'ж'
+     231: 15,  # 'з'
+     232: 2,  # 'и'
+     233: 26,  # 'й'
+     234: 12,  # 'к'
+     235: 10,  # 'л'
+     236: 14,  # 'м'
+     237: 6,  # 'н'
+     238: 4,  # 'о'
+     239: 13,  # 'п'
+     240: 7,  # 'р'
+     241: 8,  # 'с'
+     242: 5,  # 'т'
+     243: 19,  # 'у'
+     244: 29,  # 'ф'
+     245: 25,  # 'х'
+     246: 22,  # 'ц'
+     247: 21,  # 'ч'
+     248: 27,  # 'ш'
+     249: 24,  # 'щ'
+     250: 17,  # 'ъ'
+     251: 75,  # 'ы'
+     252: 52,  # 'ь'
+     253: 253,  # 'э'
+     254: 42,  # 'ю'
+     255: 16,  # 'я'
 }
+
+WINDOWS_1251_BULGARIAN_MODEL = SingleByteCharSetModel(charset_name='windows-1251',
+                                                      language='Bulgarian',
+                                                      char_to_order_map=WINDOWS_1251_BULGARIAN_CHAR_TO_ORDER,
+                                                      language_model=BULGARIAN_LANG_MODEL,
+                                                      typical_positive_ratio=0.969392,
+                                                      keep_ascii_letters=False,
+                                                      alphabet='АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя')
+
diff --git a/src/pip/_vendor/chardet/langcyrillicmodel.py b/src/pip/_vendor/chardet/langcyrillicmodel.py
deleted file mode 100644
index e5f9a1fd19c..00000000000
--- a/src/pip/_vendor/chardet/langcyrillicmodel.py
+++ /dev/null
@@ -1,333 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-# KOI8-R language model
-# Character Mapping Table:
-KOI8R_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154,  # 40
-155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253,  # 50
-253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69,  # 60
- 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253,  # 70
-191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,  # 80
-207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,  # 90
-223,224,225, 68,226,227,228,229,230,231,232,233,234,235,236,237,  # a0
-238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,  # b0
- 27,  3, 21, 28, 13,  2, 39, 19, 26,  4, 23, 11,  8, 12,  5,  1,  # c0
- 15, 16,  9,  7,  6, 14, 24, 10, 17, 18, 20, 25, 30, 29, 22, 54,  # d0
- 59, 37, 44, 58, 41, 48, 53, 46, 55, 42, 60, 36, 49, 38, 31, 34,  # e0
- 35, 43, 45, 32, 40, 52, 56, 33, 61, 62, 51, 57, 47, 63, 50, 70,  # f0
-)
-
-win1251_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154,  # 40
-155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253,  # 50
-253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69,  # 60
- 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253,  # 70
-191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
-207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
-223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
-239,240,241,242,243,244,245,246, 68,247,248,249,250,251,252,253,
- 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
- 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
-  3, 21, 10, 19, 13,  2, 24, 20,  4, 23, 11,  8, 12,  5,  1, 15,
-  9,  7,  6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
-)
-
-latin5_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154,  # 40
-155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253,  # 50
-253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69,  # 60
- 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253,  # 70
-191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
-207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
-223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
- 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
- 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
-  3, 21, 10, 19, 13,  2, 24, 20,  4, 23, 11,  8, 12,  5,  1, 15,
-  9,  7,  6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
-239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255,
-)
-
-macCyrillic_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154,  # 40
-155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253,  # 50
-253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69,  # 60
- 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253,  # 70
- 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
- 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
-191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
-207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
-223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
-239,240,241,242,243,244,245,246,247,248,249,250,251,252, 68, 16,
-  3, 21, 10, 19, 13,  2, 24, 20,  4, 23, 11,  8, 12,  5,  1, 15,
-  9,  7,  6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27,255,
-)
-
-IBM855_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154,  # 40
-155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253,  # 50
-253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69,  # 60
- 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253,  # 70
-191,192,193,194, 68,195,196,197,198,199,200,201,202,203,204,205,
-206,207,208,209,210,211,212,213,214,215,216,217, 27, 59, 54, 70,
-  3, 37, 21, 44, 28, 58, 13, 41,  2, 48, 39, 53, 19, 46,218,219,
-220,221,222,223,224, 26, 55,  4, 42,225,226,227,228, 23, 60,229,
-230,231,232,233,234,235, 11, 36,236,237,238,239,240,241,242,243,
-  8, 49, 12, 38,  5, 31,  1, 34, 15,244,245,246,247, 35, 16,248,
- 43,  9, 45,  7, 32,  6, 40, 14, 52, 24, 56, 10, 33, 17, 61,249,
-250, 18, 62, 20, 51, 25, 57, 30, 47, 29, 63, 22, 50,251,252,255,
-)
-
-IBM866_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154,  # 40
-155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253,  # 50
-253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69,  # 60
- 67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253,  # 70
- 37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
- 45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
-  3, 21, 10, 19, 13,  2, 24, 20,  4, 23, 11,  8, 12,  5,  1, 15,
-191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
-207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
-223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
-  9,  7,  6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
-239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255,
-)
-
-# Model Table:
-# total sequences: 100%
-# first 512 sequences: 97.6601%
-# first 1024 sequences: 2.3389%
-# rest  sequences:      0.1237%
-# negative sequences:   0.0009%
-RussianLangModel = (
-0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,3,3,3,3,1,3,3,3,2,3,2,3,3,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,2,2,2,2,2,0,0,2,
-3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,2,3,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,2,2,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,2,3,3,1,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,2,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,3,3,3,3,3,3,3,3,3,3,3,2,1,
-0,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,3,3,2,1,
-0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,2,2,2,3,1,3,3,1,3,3,3,3,2,2,3,0,2,2,2,3,3,2,1,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,3,3,3,3,3,2,2,3,2,3,3,3,2,1,2,2,0,1,2,2,2,2,2,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,3,0,2,2,3,3,2,1,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,3,3,1,2,3,2,2,3,2,3,3,3,3,2,2,3,0,3,2,2,3,1,1,1,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,2,3,3,3,3,2,2,2,0,3,3,3,2,2,2,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,3,3,3,3,2,3,2,2,0,1,3,2,1,2,2,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,2,1,1,3,0,1,1,1,1,2,1,1,0,2,2,2,1,2,0,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,3,3,2,2,2,2,1,3,2,3,2,3,2,1,2,2,0,1,1,2,1,2,1,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,2,3,3,3,2,2,2,2,0,2,2,2,2,3,1,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-3,2,3,2,2,3,3,3,3,3,3,3,3,3,1,3,2,0,0,3,3,3,3,2,3,3,3,3,2,3,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,3,3,3,3,2,2,3,3,0,2,1,0,3,2,3,2,3,0,0,1,2,0,0,1,0,1,2,1,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,3,0,2,3,3,3,3,2,3,3,3,3,1,2,2,0,0,2,3,2,2,2,3,2,3,2,2,3,0,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,2,3,0,2,3,2,3,0,1,2,3,3,2,0,2,3,0,0,2,3,2,2,0,1,3,1,3,2,2,1,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,3,0,2,3,3,3,3,3,3,3,3,2,1,3,2,0,0,2,2,3,3,3,2,3,3,0,2,2,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,2,3,3,2,2,2,3,3,0,0,1,1,1,1,1,2,0,0,1,1,1,1,0,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,2,3,3,3,3,3,3,3,0,3,2,3,3,2,3,2,0,2,1,0,1,1,0,1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,3,3,3,2,2,2,2,3,1,3,2,3,1,1,2,1,0,2,2,2,2,1,3,1,0,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-2,2,3,3,3,3,3,1,2,2,1,3,1,0,3,0,0,3,0,0,0,1,1,0,1,2,1,0,0,0,0,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,2,2,1,1,3,3,3,2,2,1,2,2,3,1,1,2,0,0,2,2,1,3,0,0,2,1,1,2,1,1,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,2,3,3,3,3,1,2,2,2,1,2,1,3,3,1,1,2,1,2,1,2,2,0,2,0,0,1,1,0,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,3,3,3,3,2,1,3,2,2,3,2,0,3,2,0,3,0,1,0,1,1,0,0,1,1,1,1,0,1,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,2,3,3,3,2,2,2,3,3,1,2,1,2,1,0,1,0,1,1,0,1,0,0,2,1,1,1,0,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
-3,1,1,2,1,2,3,3,2,2,1,2,2,3,0,2,1,0,0,2,2,3,2,1,2,2,2,2,2,3,1,0,
-0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,1,1,0,1,1,2,2,1,1,3,0,0,1,3,1,1,1,0,0,0,1,0,1,1,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,1,3,3,3,2,0,0,0,2,1,0,1,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,1,0,0,2,3,2,2,2,1,2,2,2,1,2,1,0,0,1,1,1,0,2,0,1,1,1,0,0,1,1,
-1,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,
-2,3,3,3,3,0,0,0,0,1,0,0,0,0,3,0,1,2,1,0,0,0,0,0,0,0,1,1,0,0,1,1,
-1,0,1,0,1,2,0,0,1,1,2,1,0,1,1,1,1,0,1,1,1,1,0,1,0,0,1,0,0,1,1,0,
-2,2,3,2,2,2,3,1,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,0,1,0,1,1,1,0,2,1,
-1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,1,0,1,1,0,
-3,3,3,2,2,2,2,3,2,2,1,1,2,2,2,2,1,1,3,1,2,1,2,0,0,1,1,0,1,0,2,1,
-1,1,1,1,1,2,1,0,1,1,1,1,0,1,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,1,1,0,
-2,0,0,1,0,3,2,2,2,2,1,2,1,2,1,2,0,0,0,2,1,2,2,1,1,2,2,0,1,1,0,2,
-1,1,1,1,1,0,1,1,1,2,1,1,1,2,1,0,1,2,1,1,1,1,0,1,1,1,0,0,1,0,0,1,
-1,3,2,2,2,1,1,1,2,3,0,0,0,0,2,0,2,2,1,0,0,0,0,0,0,1,0,0,0,0,1,1,
-1,0,1,1,0,1,0,1,1,0,1,1,0,2,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,
-2,3,2,3,2,1,2,2,2,2,1,0,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,2,1,
-1,1,2,1,0,2,0,0,1,0,1,0,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0,
-3,0,0,1,0,2,2,2,3,2,2,2,2,2,2,2,0,0,0,2,1,2,1,1,1,2,2,0,0,0,1,2,
-1,1,1,1,1,0,1,2,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1,
-2,3,2,3,3,2,0,1,1,1,0,0,1,0,2,0,1,1,3,1,0,0,0,0,0,0,0,1,0,0,2,1,
-1,1,1,1,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,1,1,0,1,0,0,0,0,0,0,1,0,
-2,3,3,3,3,1,2,2,2,2,0,1,1,0,2,1,1,1,2,1,0,1,1,0,0,1,0,1,0,0,2,0,
-0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,3,3,2,0,0,1,1,2,2,1,0,0,2,0,1,1,3,0,0,1,0,0,0,0,0,1,0,1,2,1,
-1,1,2,0,1,1,1,0,1,0,1,1,0,1,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,1,1,0,
-1,3,2,3,2,1,0,0,2,2,2,0,1,0,2,0,1,1,1,0,1,0,0,0,3,0,1,1,0,0,2,1,
-1,1,1,0,1,1,0,0,0,0,1,1,0,1,0,0,2,1,1,0,1,0,0,0,1,0,1,0,0,1,1,0,
-3,1,2,1,1,2,2,2,2,2,2,1,2,2,1,1,0,0,0,2,2,2,0,0,0,1,2,1,0,1,0,1,
-2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,2,1,1,1,0,1,0,1,1,0,1,1,1,0,0,1,
-3,0,0,0,0,2,0,1,1,1,1,1,1,1,0,1,0,0,0,1,1,1,0,1,0,1,1,0,0,1,0,1,
-1,1,0,0,1,0,0,0,1,0,1,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,1,0,0,0,1,
-1,3,3,2,2,0,0,0,2,2,0,0,0,1,2,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,2,1,
-0,1,1,0,0,1,1,0,0,0,1,1,0,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,
-2,3,2,3,2,0,0,0,0,1,1,0,0,0,2,0,2,0,2,0,0,0,0,0,1,0,0,1,0,0,1,1,
-1,1,2,0,1,2,1,0,1,1,2,1,1,1,1,1,2,1,1,0,1,0,0,1,1,1,1,1,0,1,1,0,
-1,3,2,2,2,1,0,0,2,2,1,0,1,2,2,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,
-0,0,1,1,0,1,1,0,0,1,1,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,1,0,2,3,1,2,2,2,2,2,2,1,1,0,0,0,1,0,1,0,2,1,1,1,0,0,0,0,1,
-1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
-2,0,2,0,0,1,0,3,2,1,2,1,2,2,0,1,0,0,0,2,1,0,0,2,1,1,1,1,0,2,0,2,
-2,1,1,1,1,1,1,1,1,1,1,1,1,2,1,0,1,1,1,1,0,0,0,1,1,1,1,0,1,0,0,1,
-1,2,2,2,2,1,0,0,1,0,0,0,0,0,2,0,1,1,1,1,0,0,0,0,1,0,1,2,0,0,2,0,
-1,0,1,1,1,2,1,0,1,0,1,1,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,1,0,1,1,0,
-2,1,2,2,2,0,3,0,1,1,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-0,0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,
-1,2,2,3,2,2,0,0,1,1,2,0,1,2,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,
-0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,
-2,2,1,1,2,1,2,2,2,2,2,1,2,2,0,1,0,0,0,1,2,2,2,1,2,1,1,1,1,1,2,1,
-1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,0,1,1,1,0,0,0,0,1,1,1,0,1,1,0,0,1,
-1,2,2,2,2,0,1,0,2,2,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,
-0,0,1,0,0,1,0,0,0,0,1,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,2,2,2,2,0,0,0,2,2,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,
-0,1,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,2,2,2,2,0,0,0,0,1,0,0,1,1,2,0,0,0,0,1,0,1,0,0,1,0,0,2,0,0,0,1,
-0,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,
-1,2,2,2,1,1,2,0,2,1,1,1,1,0,2,2,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,
-0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-1,0,2,1,2,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,
-0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,
-1,0,0,0,0,2,0,1,2,1,0,1,1,1,0,1,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,1,
-0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,
-2,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-1,1,1,0,1,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-1,1,0,1,1,0,1,0,1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,1,0,1,1,0,1,0,0,0,
-0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,
-)
-
-Koi8rModel = {
-  'char_to_order_map': KOI8R_char_to_order_map,
-  'precedence_matrix': RussianLangModel,
-  'typical_positive_ratio': 0.976601,
-  'keep_english_letter': False,
-  'charset_name': "KOI8-R",
-  'language': 'Russian',
-}
-
-Win1251CyrillicModel = {
-  'char_to_order_map': win1251_char_to_order_map,
-  'precedence_matrix': RussianLangModel,
-  'typical_positive_ratio': 0.976601,
-  'keep_english_letter': False,
-  'charset_name': "windows-1251",
-  'language': 'Russian',
-}
-
-Latin5CyrillicModel = {
-  'char_to_order_map': latin5_char_to_order_map,
-  'precedence_matrix': RussianLangModel,
-  'typical_positive_ratio': 0.976601,
-  'keep_english_letter': False,
-  'charset_name': "ISO-8859-5",
-  'language': 'Russian',
-}
-
-MacCyrillicModel = {
-  'char_to_order_map': macCyrillic_char_to_order_map,
-  'precedence_matrix': RussianLangModel,
-  'typical_positive_ratio': 0.976601,
-  'keep_english_letter': False,
-  'charset_name': "MacCyrillic",
-  'language': 'Russian',
-}
-
-Ibm866Model = {
-  'char_to_order_map': IBM866_char_to_order_map,
-  'precedence_matrix': RussianLangModel,
-  'typical_positive_ratio': 0.976601,
-  'keep_english_letter': False,
-  'charset_name': "IBM866",
-  'language': 'Russian',
-}
-
-Ibm855Model = {
-  'char_to_order_map': IBM855_char_to_order_map,
-  'precedence_matrix': RussianLangModel,
-  'typical_positive_ratio': 0.976601,
-  'keep_english_letter': False,
-  'charset_name': "IBM855",
-  'language': 'Russian',
-}
diff --git a/src/pip/_vendor/chardet/langgreekmodel.py b/src/pip/_vendor/chardet/langgreekmodel.py
index 533222166cc..d99528ede73 100644
--- a/src/pip/_vendor/chardet/langgreekmodel.py
+++ b/src/pip/_vendor/chardet/langgreekmodel.py
@@ -1,225 +1,4398 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-# 255: Control characters that usually does not exist in any text
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
+
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+GREEK_LANG_MODEL = {
+    60: {  # 'e'
+        60: 2,  # 'e'
+        55: 1,  # 'o'
+        58: 2,  # 't'
+        36: 1,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    55: {  # 'o'
+        60: 0,  # 'e'
+        55: 2,  # 'o'
+        58: 2,  # 't'
+        36: 1,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 1,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    58: {  # 't'
+        60: 2,  # 'e'
+        55: 1,  # 'o'
+        58: 1,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 1,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    36: {  # '·'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    61: {  # 'Ά'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 1,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 1,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    46: {  # 'Έ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 2,  # 'β'
+        20: 2,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 2,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 1,  # 'σ'
+        2: 2,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    54: {  # 'Ό'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 2,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    31: {  # 'Α'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 2,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 1,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 2,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 2,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 1,  # 'θ'
+        5: 0,  # 'ι'
+        11: 2,  # 'κ'
+        16: 3,  # 'λ'
+        10: 2,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 2,  # 'ς'
+        7: 2,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    51: {  # 'Β'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 1,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 2,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    43: {  # 'Γ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 1,  # 'Α'
+        51: 0,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 1,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 1,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    41: {  # 'Δ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 1,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    34: {  # 'Ε'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 2,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 1,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 2,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 3,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 1,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 1,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 2,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 2,  # 'τ'
+        12: 2,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 1,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    40: {  # 'Η'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    52: {  # 'Θ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 1,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    47: {  # 'Ι'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 1,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 2,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 2,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 1,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 1,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    44: {  # 'Κ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 1,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 1,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    53: {  # 'Λ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 1,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    38: {  # 'Μ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 2,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    49: {  # 'Ν'
+        60: 2,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 1,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 1,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    59: {  # 'Ξ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 1,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    39: {  # 'Ο'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 1,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 2,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 2,  # 'Φ'
+        50: 2,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 2,  # 'κ'
+        16: 2,  # 'λ'
+        10: 2,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 2,  # 'υ'
+        28: 1,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    35: {  # 'Π'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 2,  # 'Λ'
+        38: 1,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 1,  # 'έ'
+        22: 1,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    48: {  # 'Ρ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 1,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 1,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    37: {  # 'Σ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 1,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 2,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 2,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 2,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 2,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 2,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    33: {  # 'Τ'
+        60: 0,  # 'e'
+        55: 1,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 2,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 2,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 1,  # 'Τ'
+        45: 1,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 2,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 2,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    45: {  # 'Υ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 2,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 2,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 2,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 2,  # 'Π'
+        48: 1,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    56: {  # 'Φ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 1,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 1,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 2,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 1,  # 'ύ'
+        27: 1,  # 'ώ'
+    },
+    50: {  # 'Χ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 1,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 2,  # 'Ε'
+        40: 2,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 2,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 1,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 1,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 1,  # 'Χ'
+        57: 1,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 2,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    57: {  # 'Ω'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 1,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 1,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 2,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 2,  # 'Ρ'
+        37: 2,  # 'Σ'
+        33: 2,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 2,  # 'ρ'
+        14: 2,  # 'ς'
+        7: 2,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    17: {  # 'ά'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 3,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 3,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    18: {  # 'έ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 3,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    22: {  # 'ή'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 1,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    15: {  # 'ί'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 3,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 1,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    1: {  # 'α'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 0,  # 'ή'
+        15: 3,  # 'ί'
+        1: 0,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 2,  # 'ε'
+        32: 3,  # 'ζ'
+        13: 1,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    29: {  # 'β'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 2,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 3,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 2,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    20: {  # 'γ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    21: {  # 'δ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    3: {  # 'ε'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 3,  # 'ί'
+        1: 2,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 2,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 2,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    32: {  # 'ζ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 2,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 1,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    13: {  # 'η'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    25: {  # 'θ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 1,  # 'λ'
+        10: 3,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    5: {  # 'ι'
+        60: 0,  # 'e'
+        55: 1,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 0,  # 'ί'
+        1: 3,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    11: {  # 'κ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 2,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    16: {  # 'λ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 1,  # 'β'
+        20: 2,  # 'γ'
+        21: 1,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 2,  # 'κ'
+        16: 3,  # 'λ'
+        10: 2,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    10: {  # 'μ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 1,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 3,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 2,  # 'υ'
+        28: 3,  # 'φ'
+        23: 0,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    6: {  # 'ν'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 1,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    30: {  # 'ξ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 2,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 2,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 1,  # 'ώ'
+    },
+    4: {  # 'ο'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 2,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 1,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    9: {  # 'π'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 3,  # 'λ'
+        10: 0,  # 'μ'
+        6: 2,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 2,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    8: {  # 'ρ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 1,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 2,  # 'π'
+        8: 2,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 2,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    14: {  # 'ς'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 2,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 0,  # 'θ'
+        5: 0,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 0,  # 'τ'
+        12: 0,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    7: {  # 'σ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 3,  # 'β'
+        20: 0,  # 'γ'
+        21: 2,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 3,  # 'θ'
+        5: 3,  # 'ι'
+        11: 3,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 3,  # 'φ'
+        23: 3,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    2: {  # 'τ'
+        60: 0,  # 'e'
+        55: 2,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 3,  # 'ι'
+        11: 2,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 2,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    12: {  # 'υ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 2,  # 'ί'
+        1: 3,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 2,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 2,  # 'η'
+        25: 3,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 3,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 2,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    28: {  # 'φ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 3,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 0,  # 'μ'
+        6: 1,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 1,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 2,  # 'ύ'
+        27: 2,  # 'ώ'
+    },
+    23: {  # 'χ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 3,  # 'ά'
+        18: 2,  # 'έ'
+        22: 3,  # 'ή'
+        15: 3,  # 'ί'
+        1: 3,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 2,  # 'θ'
+        5: 3,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 2,  # 'μ'
+        6: 3,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 0,  # 'π'
+        8: 3,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 3,  # 'τ'
+        12: 3,  # 'υ'
+        28: 0,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 3,  # 'ω'
+        19: 3,  # 'ό'
+        26: 3,  # 'ύ'
+        27: 3,  # 'ώ'
+    },
+    42: {  # 'ψ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 2,  # 'ά'
+        18: 2,  # 'έ'
+        22: 1,  # 'ή'
+        15: 2,  # 'ί'
+        1: 2,  # 'α'
+        29: 0,  # 'β'
+        20: 0,  # 'γ'
+        21: 0,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 3,  # 'η'
+        25: 0,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 0,  # 'λ'
+        10: 0,  # 'μ'
+        6: 0,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 0,  # 'π'
+        8: 0,  # 'ρ'
+        14: 0,  # 'ς'
+        7: 0,  # 'σ'
+        2: 2,  # 'τ'
+        12: 1,  # 'υ'
+        28: 0,  # 'φ'
+        23: 0,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    24: {  # 'ω'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 1,  # 'ά'
+        18: 0,  # 'έ'
+        22: 2,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 2,  # 'β'
+        20: 3,  # 'γ'
+        21: 2,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 0,  # 'η'
+        25: 3,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 0,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 2,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    19: {  # 'ό'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 3,  # 'β'
+        20: 3,  # 'γ'
+        21: 3,  # 'δ'
+        3: 1,  # 'ε'
+        32: 2,  # 'ζ'
+        13: 2,  # 'η'
+        25: 2,  # 'θ'
+        5: 2,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 1,  # 'ξ'
+        4: 2,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 3,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    26: {  # 'ύ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 2,  # 'α'
+        29: 2,  # 'β'
+        20: 2,  # 'γ'
+        21: 1,  # 'δ'
+        3: 3,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 2,  # 'η'
+        25: 3,  # 'θ'
+        5: 0,  # 'ι'
+        11: 3,  # 'κ'
+        16: 3,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 2,  # 'ξ'
+        4: 3,  # 'ο'
+        9: 3,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 2,  # 'φ'
+        23: 2,  # 'χ'
+        42: 2,  # 'ψ'
+        24: 2,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+    27: {  # 'ώ'
+        60: 0,  # 'e'
+        55: 0,  # 'o'
+        58: 0,  # 't'
+        36: 0,  # '·'
+        61: 0,  # 'Ά'
+        46: 0,  # 'Έ'
+        54: 0,  # 'Ό'
+        31: 0,  # 'Α'
+        51: 0,  # 'Β'
+        43: 0,  # 'Γ'
+        41: 0,  # 'Δ'
+        34: 0,  # 'Ε'
+        40: 0,  # 'Η'
+        52: 0,  # 'Θ'
+        47: 0,  # 'Ι'
+        44: 0,  # 'Κ'
+        53: 0,  # 'Λ'
+        38: 0,  # 'Μ'
+        49: 0,  # 'Ν'
+        59: 0,  # 'Ξ'
+        39: 0,  # 'Ο'
+        35: 0,  # 'Π'
+        48: 0,  # 'Ρ'
+        37: 0,  # 'Σ'
+        33: 0,  # 'Τ'
+        45: 0,  # 'Υ'
+        56: 0,  # 'Φ'
+        50: 0,  # 'Χ'
+        57: 0,  # 'Ω'
+        17: 0,  # 'ά'
+        18: 0,  # 'έ'
+        22: 0,  # 'ή'
+        15: 0,  # 'ί'
+        1: 0,  # 'α'
+        29: 1,  # 'β'
+        20: 0,  # 'γ'
+        21: 3,  # 'δ'
+        3: 0,  # 'ε'
+        32: 0,  # 'ζ'
+        13: 1,  # 'η'
+        25: 2,  # 'θ'
+        5: 2,  # 'ι'
+        11: 0,  # 'κ'
+        16: 2,  # 'λ'
+        10: 3,  # 'μ'
+        6: 3,  # 'ν'
+        30: 1,  # 'ξ'
+        4: 0,  # 'ο'
+        9: 2,  # 'π'
+        8: 3,  # 'ρ'
+        14: 3,  # 'ς'
+        7: 3,  # 'σ'
+        2: 3,  # 'τ'
+        12: 0,  # 'υ'
+        28: 1,  # 'φ'
+        23: 1,  # 'χ'
+        42: 0,  # 'ψ'
+        24: 0,  # 'ω'
+        19: 0,  # 'ό'
+        26: 0,  # 'ύ'
+        27: 0,  # 'ώ'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
+# 251: Control characters
 
-# Character Mapping Table:
-Latin7_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 82,100,104, 94, 98,101,116,102,111,187,117, 92, 88,113, 85,  # 40
- 79,118,105, 83, 67,114,119, 95, 99,109,188,253,253,253,253,253,  # 50
-253, 72, 70, 80, 81, 60, 96, 93, 89, 68,120, 97, 77, 86, 69, 55,  # 60
- 78,115, 65, 66, 58, 76,106,103, 87,107,112,253,253,253,253,253,  # 70
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 80
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 90
-253,233, 90,253,253,253,253,253,253,253,253,253,253, 74,253,253,  # a0
-253,253,253,253,247,248, 61, 36, 46, 71, 73,253, 54,253,108,123,  # b0
-110, 31, 51, 43, 41, 34, 91, 40, 52, 47, 44, 53, 38, 49, 59, 39,  # c0
- 35, 48,250, 37, 33, 45, 56, 50, 84, 57,120,121, 17, 18, 22, 15,  # d0
-124,  1, 29, 20, 21,  3, 32, 13, 25,  5, 11, 16, 10,  6, 30,  4,  # e0
-  9,  8, 14,  7,  2, 12, 28, 23, 42, 24, 64, 75, 19, 26, 27,253,  # f0
-)
-
-win1253_char_to_order_map = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 82,100,104, 94, 98,101,116,102,111,187,117, 92, 88,113, 85,  # 40
- 79,118,105, 83, 67,114,119, 95, 99,109,188,253,253,253,253,253,  # 50
-253, 72, 70, 80, 81, 60, 96, 93, 89, 68,120, 97, 77, 86, 69, 55,  # 60
- 78,115, 65, 66, 58, 76,106,103, 87,107,112,253,253,253,253,253,  # 70
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 80
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 90
-253,233, 61,253,253,253,253,253,253,253,253,253,253, 74,253,253,  # a0
-253,253,253,253,247,253,253, 36, 46, 71, 73,253, 54,253,108,123,  # b0
-110, 31, 51, 43, 41, 34, 91, 40, 52, 47, 44, 53, 38, 49, 59, 39,  # c0
- 35, 48,250, 37, 33, 45, 56, 50, 84, 57,120,121, 17, 18, 22, 15,  # d0
-124,  1, 29, 20, 21,  3, 32, 13, 25,  5, 11, 16, 10,  6, 30,  4,  # e0
-  9,  8, 14,  7,  2, 12, 28, 23, 42, 24, 64, 75, 19, 26, 27,253,  # f0
-)
+# Character Mapping Table(s):
+WINDOWS_1253_GREEK_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 82,  # 'A'
+     66: 100,  # 'B'
+     67: 104,  # 'C'
+     68: 94,  # 'D'
+     69: 98,  # 'E'
+     70: 101,  # 'F'
+     71: 116,  # 'G'
+     72: 102,  # 'H'
+     73: 111,  # 'I'
+     74: 187,  # 'J'
+     75: 117,  # 'K'
+     76: 92,  # 'L'
+     77: 88,  # 'M'
+     78: 113,  # 'N'
+     79: 85,  # 'O'
+     80: 79,  # 'P'
+     81: 118,  # 'Q'
+     82: 105,  # 'R'
+     83: 83,  # 'S'
+     84: 67,  # 'T'
+     85: 114,  # 'U'
+     86: 119,  # 'V'
+     87: 95,  # 'W'
+     88: 99,  # 'X'
+     89: 109,  # 'Y'
+     90: 188,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 72,  # 'a'
+     98: 70,  # 'b'
+     99: 80,  # 'c'
+     100: 81,  # 'd'
+     101: 60,  # 'e'
+     102: 96,  # 'f'
+     103: 93,  # 'g'
+     104: 89,  # 'h'
+     105: 68,  # 'i'
+     106: 120,  # 'j'
+     107: 97,  # 'k'
+     108: 77,  # 'l'
+     109: 86,  # 'm'
+     110: 69,  # 'n'
+     111: 55,  # 'o'
+     112: 78,  # 'p'
+     113: 115,  # 'q'
+     114: 65,  # 'r'
+     115: 66,  # 's'
+     116: 58,  # 't'
+     117: 76,  # 'u'
+     118: 106,  # 'v'
+     119: 103,  # 'w'
+     120: 87,  # 'x'
+     121: 107,  # 'y'
+     122: 112,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 255,  # '€'
+     129: 255,  # None
+     130: 255,  # '‚'
+     131: 255,  # 'ƒ'
+     132: 255,  # '„'
+     133: 255,  # '…'
+     134: 255,  # '†'
+     135: 255,  # '‡'
+     136: 255,  # None
+     137: 255,  # '‰'
+     138: 255,  # None
+     139: 255,  # '‹'
+     140: 255,  # None
+     141: 255,  # None
+     142: 255,  # None
+     143: 255,  # None
+     144: 255,  # None
+     145: 255,  # '‘'
+     146: 255,  # '’'
+     147: 255,  # '“'
+     148: 255,  # '”'
+     149: 255,  # '•'
+     150: 255,  # '–'
+     151: 255,  # '—'
+     152: 255,  # None
+     153: 255,  # '™'
+     154: 255,  # None
+     155: 255,  # '›'
+     156: 255,  # None
+     157: 255,  # None
+     158: 255,  # None
+     159: 255,  # None
+     160: 253,  # '\xa0'
+     161: 233,  # '΅'
+     162: 61,  # 'Ά'
+     163: 253,  # '£'
+     164: 253,  # '¤'
+     165: 253,  # '¥'
+     166: 253,  # '¦'
+     167: 253,  # '§'
+     168: 253,  # '¨'
+     169: 253,  # '©'
+     170: 253,  # None
+     171: 253,  # '«'
+     172: 253,  # '¬'
+     173: 74,  # '\xad'
+     174: 253,  # '®'
+     175: 253,  # '―'
+     176: 253,  # '°'
+     177: 253,  # '±'
+     178: 253,  # '²'
+     179: 253,  # '³'
+     180: 247,  # '΄'
+     181: 253,  # 'µ'
+     182: 253,  # '¶'
+     183: 36,  # '·'
+     184: 46,  # 'Έ'
+     185: 71,  # 'Ή'
+     186: 73,  # 'Ί'
+     187: 253,  # '»'
+     188: 54,  # 'Ό'
+     189: 253,  # '½'
+     190: 108,  # 'Ύ'
+     191: 123,  # 'Ώ'
+     192: 110,  # 'ΐ'
+     193: 31,  # 'Α'
+     194: 51,  # 'Β'
+     195: 43,  # 'Γ'
+     196: 41,  # 'Δ'
+     197: 34,  # 'Ε'
+     198: 91,  # 'Ζ'
+     199: 40,  # 'Η'
+     200: 52,  # 'Θ'
+     201: 47,  # 'Ι'
+     202: 44,  # 'Κ'
+     203: 53,  # 'Λ'
+     204: 38,  # 'Μ'
+     205: 49,  # 'Ν'
+     206: 59,  # 'Ξ'
+     207: 39,  # 'Ο'
+     208: 35,  # 'Π'
+     209: 48,  # 'Ρ'
+     210: 250,  # None
+     211: 37,  # 'Σ'
+     212: 33,  # 'Τ'
+     213: 45,  # 'Υ'
+     214: 56,  # 'Φ'
+     215: 50,  # 'Χ'
+     216: 84,  # 'Ψ'
+     217: 57,  # 'Ω'
+     218: 120,  # 'Ϊ'
+     219: 121,  # 'Ϋ'
+     220: 17,  # 'ά'
+     221: 18,  # 'έ'
+     222: 22,  # 'ή'
+     223: 15,  # 'ί'
+     224: 124,  # 'ΰ'
+     225: 1,  # 'α'
+     226: 29,  # 'β'
+     227: 20,  # 'γ'
+     228: 21,  # 'δ'
+     229: 3,  # 'ε'
+     230: 32,  # 'ζ'
+     231: 13,  # 'η'
+     232: 25,  # 'θ'
+     233: 5,  # 'ι'
+     234: 11,  # 'κ'
+     235: 16,  # 'λ'
+     236: 10,  # 'μ'
+     237: 6,  # 'ν'
+     238: 30,  # 'ξ'
+     239: 4,  # 'ο'
+     240: 9,  # 'π'
+     241: 8,  # 'ρ'
+     242: 14,  # 'ς'
+     243: 7,  # 'σ'
+     244: 2,  # 'τ'
+     245: 12,  # 'υ'
+     246: 28,  # 'φ'
+     247: 23,  # 'χ'
+     248: 42,  # 'ψ'
+     249: 24,  # 'ω'
+     250: 64,  # 'ϊ'
+     251: 75,  # 'ϋ'
+     252: 19,  # 'ό'
+     253: 26,  # 'ύ'
+     254: 27,  # 'ώ'
+     255: 253,  # None
+}
 
-# Model Table:
-# total sequences: 100%
-# first 512 sequences: 98.2851%
-# first 1024 sequences:1.7001%
-# rest  sequences:     0.0359%
-# negative sequences:  0.0148%
-GreekLangModel = (
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,2,2,3,3,3,3,3,3,3,3,1,3,3,3,0,2,2,3,3,0,3,0,3,2,0,3,3,3,0,
-3,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,0,3,3,0,3,2,3,3,0,3,2,3,3,3,0,0,3,0,3,0,3,3,2,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,
-0,2,3,2,2,3,3,3,3,3,3,3,3,0,3,3,3,3,0,2,3,3,0,3,3,3,3,2,3,3,3,0,
-2,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,0,2,1,3,3,3,3,2,3,3,2,3,3,2,0,
-0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,0,3,3,3,3,3,3,0,3,3,0,3,3,3,3,3,3,3,3,3,3,0,3,2,3,3,0,
-2,0,1,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,2,3,0,0,0,0,3,3,0,3,1,3,3,3,0,3,3,0,3,3,3,3,0,0,0,0,
-2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,0,3,0,3,3,3,3,3,0,3,2,2,2,3,0,2,3,3,3,3,3,2,3,3,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,3,2,2,2,3,3,3,3,0,3,1,3,3,3,3,2,3,3,3,3,3,3,3,2,2,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,2,0,3,0,0,0,3,3,2,3,3,3,3,3,0,0,3,2,3,0,2,3,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,0,3,3,3,3,0,0,3,3,0,2,3,0,3,0,3,3,3,0,0,3,0,3,0,2,2,3,3,0,0,
-0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,2,0,3,2,3,3,3,3,0,3,3,3,3,3,0,3,3,2,3,2,3,3,2,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,2,3,2,3,3,3,3,3,3,0,2,3,2,3,2,2,2,3,2,3,3,2,3,0,2,2,2,3,0,
-2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,0,0,0,3,3,3,2,3,3,0,0,3,0,3,0,0,0,3,2,0,3,0,3,0,0,2,0,2,0,
-0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,0,3,3,3,3,3,3,0,3,3,0,3,0,0,0,3,3,0,3,3,3,0,0,1,2,3,0,
-3,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,2,0,0,3,2,2,3,3,0,3,3,3,3,3,2,1,3,0,3,2,3,3,2,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,3,0,2,3,3,3,3,3,3,0,0,3,0,3,0,0,0,3,3,0,3,2,3,0,0,3,3,3,0,
-3,0,0,0,2,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,0,3,3,3,3,3,3,0,0,3,0,3,0,0,0,3,2,0,3,2,3,0,0,3,2,3,0,
-2,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,1,2,2,3,3,3,3,3,3,0,2,3,0,3,0,0,0,3,3,0,3,0,2,0,0,2,3,1,0,
-2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,0,3,3,3,3,0,3,0,3,3,2,3,0,3,3,3,3,3,3,0,3,3,3,0,2,3,0,0,3,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,0,3,3,3,0,0,3,0,0,0,3,3,0,3,0,2,3,3,0,0,3,0,3,0,3,3,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,0,0,0,3,3,3,3,3,3,0,0,3,0,2,0,0,0,3,3,0,3,0,3,0,0,2,0,2,0,
-0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,3,0,3,0,2,0,3,2,0,3,2,3,2,3,0,0,3,2,3,2,3,3,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,0,0,2,3,3,3,3,3,0,0,0,3,0,2,1,0,0,3,2,2,2,0,3,0,0,2,2,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,0,3,3,3,2,0,3,0,3,0,3,3,0,2,1,2,3,3,0,0,3,0,3,0,3,3,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,3,3,3,0,3,3,3,3,3,3,0,2,3,0,3,0,0,0,2,1,0,2,2,3,0,0,2,2,2,0,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,3,0,0,2,3,3,3,2,3,0,0,1,3,0,2,0,0,0,0,3,0,1,0,2,0,0,1,1,1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,3,1,0,3,0,0,0,3,2,0,3,2,3,3,3,0,0,3,0,3,2,2,2,1,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,0,3,3,3,0,0,3,0,0,0,0,2,0,2,3,3,2,2,2,2,3,0,2,0,2,2,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,3,3,3,2,0,0,0,0,0,0,2,3,0,2,0,2,3,2,0,0,3,0,3,0,3,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,3,2,3,3,2,2,3,0,2,0,3,0,0,0,2,0,0,0,0,1,2,0,2,0,2,0,
-0,2,0,2,0,2,2,0,0,1,0,2,2,2,0,2,2,2,0,2,2,2,0,0,2,0,0,1,0,0,0,0,
-0,2,0,3,3,2,0,0,0,0,0,0,1,3,0,2,0,2,2,2,0,0,2,0,3,0,0,2,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,0,2,3,2,0,2,2,0,2,0,2,2,0,2,0,2,2,2,0,0,0,0,0,0,2,3,0,0,0,2,
-0,1,2,0,0,0,0,2,2,0,0,0,2,1,0,2,2,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,
-0,0,2,1,0,2,3,2,2,3,2,3,2,0,0,3,3,3,0,0,3,2,0,0,0,1,1,0,2,0,2,2,
-0,2,0,2,0,2,2,0,0,2,0,2,2,2,0,2,2,2,2,0,0,2,0,0,0,2,0,1,0,0,0,0,
-0,3,0,3,3,2,2,0,3,0,0,0,2,2,0,2,2,2,1,2,0,0,1,2,2,0,0,3,0,0,0,2,
-0,1,2,0,0,0,1,2,0,0,0,0,0,0,0,2,2,0,1,0,0,2,0,0,0,2,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,3,3,2,2,0,0,0,2,0,2,3,3,0,2,0,0,0,0,0,0,2,2,2,0,2,2,0,2,0,2,
-0,2,2,0,0,2,2,2,2,1,0,0,2,2,0,2,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,
-0,2,0,3,2,3,0,0,0,3,0,0,2,2,0,2,0,2,2,2,0,0,2,0,0,0,0,0,0,0,0,2,
-0,0,2,2,0,0,2,2,2,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,2,0,0,3,2,0,2,2,2,2,2,0,0,0,2,0,0,0,0,2,0,1,0,0,2,0,1,0,0,0,
-0,2,2,2,0,2,2,0,1,2,0,2,2,2,0,2,2,2,2,1,2,2,0,0,2,0,0,0,0,0,0,0,
-0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-0,2,0,2,0,2,2,0,0,0,0,1,2,1,0,0,2,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,3,2,3,0,0,2,0,0,0,2,2,0,2,0,0,0,1,0,0,2,0,2,0,2,2,0,0,0,0,
-0,0,2,0,0,0,0,2,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,
-0,2,2,3,2,2,0,0,0,0,0,0,1,3,0,2,0,2,2,0,0,0,1,0,2,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,0,2,0,3,2,0,2,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-0,0,2,0,0,0,0,1,1,0,0,2,1,2,0,2,2,0,1,0,0,1,0,0,0,2,0,0,0,0,0,0,
-0,3,0,2,2,2,0,0,2,0,0,0,2,0,0,0,2,3,0,2,0,0,0,0,0,0,2,2,0,0,0,2,
-0,1,2,0,0,0,1,2,2,1,0,0,0,2,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,1,2,0,2,2,0,2,0,0,2,0,0,0,0,1,2,1,0,2,1,0,0,0,0,0,0,0,0,0,0,
-0,0,2,0,0,0,3,1,2,2,0,2,0,0,0,0,2,0,0,0,2,0,0,3,0,0,0,0,2,2,2,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,1,0,2,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,2,
-0,2,2,0,0,2,2,2,2,2,0,1,2,0,0,0,2,2,0,1,0,2,0,0,2,2,0,0,0,0,0,0,
-0,0,0,0,1,0,0,0,0,0,0,0,3,0,0,2,0,0,0,0,0,0,0,0,2,0,2,0,0,0,0,2,
-0,1,2,0,0,0,0,2,2,1,0,1,0,1,0,2,2,2,1,0,0,0,0,0,0,1,0,0,0,0,0,0,
-0,2,0,1,2,0,0,0,0,0,0,0,0,0,0,2,0,0,2,2,0,0,0,0,1,0,0,0,0,0,0,2,
-0,2,2,0,0,0,0,2,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,
-0,2,2,2,2,0,0,0,3,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,1,
-0,0,2,0,0,0,0,1,2,0,0,0,0,0,0,2,2,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,
-0,2,0,2,2,2,0,0,2,0,0,0,0,0,0,0,2,2,2,0,0,0,2,0,0,0,0,0,0,0,0,2,
-0,0,1,0,0,0,0,2,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,
-0,3,0,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,2,
-0,0,2,0,0,0,0,2,2,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,2,0,2,2,1,0,0,0,0,0,0,2,0,0,2,0,2,2,2,0,0,0,0,0,0,2,0,0,0,0,2,
-0,0,2,0,0,2,0,2,2,0,0,0,0,2,0,2,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,
-0,0,3,0,0,0,2,2,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,2,0,0,0,0,0,
-0,2,2,2,2,2,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,
-0,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,2,2,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-0,2,0,0,0,2,0,0,0,0,0,1,0,0,0,0,2,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,2,0,0,0,
-0,2,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,2,0,2,0,0,0,
-0,0,0,0,0,0,0,0,2,1,0,0,0,0,0,0,2,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-)
+WINDOWS_1253_GREEK_MODEL = SingleByteCharSetModel(charset_name='windows-1253',
+                                                  language='Greek',
+                                                  char_to_order_map=WINDOWS_1253_GREEK_CHAR_TO_ORDER,
+                                                  language_model=GREEK_LANG_MODEL,
+                                                  typical_positive_ratio=0.982851,
+                                                  keep_ascii_letters=False,
+                                                  alphabet='ΆΈΉΊΌΎΏΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩάέήίαβγδεζηθικλμνξοπρςστυφχψωόύώ')
 
-Latin7GreekModel = {
-  'char_to_order_map': Latin7_char_to_order_map,
-  'precedence_matrix': GreekLangModel,
-  'typical_positive_ratio': 0.982851,
-  'keep_english_letter': False,
-  'charset_name': "ISO-8859-7",
-  'language': 'Greek',
+ISO_8859_7_GREEK_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 82,  # 'A'
+     66: 100,  # 'B'
+     67: 104,  # 'C'
+     68: 94,  # 'D'
+     69: 98,  # 'E'
+     70: 101,  # 'F'
+     71: 116,  # 'G'
+     72: 102,  # 'H'
+     73: 111,  # 'I'
+     74: 187,  # 'J'
+     75: 117,  # 'K'
+     76: 92,  # 'L'
+     77: 88,  # 'M'
+     78: 113,  # 'N'
+     79: 85,  # 'O'
+     80: 79,  # 'P'
+     81: 118,  # 'Q'
+     82: 105,  # 'R'
+     83: 83,  # 'S'
+     84: 67,  # 'T'
+     85: 114,  # 'U'
+     86: 119,  # 'V'
+     87: 95,  # 'W'
+     88: 99,  # 'X'
+     89: 109,  # 'Y'
+     90: 188,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 72,  # 'a'
+     98: 70,  # 'b'
+     99: 80,  # 'c'
+     100: 81,  # 'd'
+     101: 60,  # 'e'
+     102: 96,  # 'f'
+     103: 93,  # 'g'
+     104: 89,  # 'h'
+     105: 68,  # 'i'
+     106: 120,  # 'j'
+     107: 97,  # 'k'
+     108: 77,  # 'l'
+     109: 86,  # 'm'
+     110: 69,  # 'n'
+     111: 55,  # 'o'
+     112: 78,  # 'p'
+     113: 115,  # 'q'
+     114: 65,  # 'r'
+     115: 66,  # 's'
+     116: 58,  # 't'
+     117: 76,  # 'u'
+     118: 106,  # 'v'
+     119: 103,  # 'w'
+     120: 87,  # 'x'
+     121: 107,  # 'y'
+     122: 112,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 255,  # '\x80'
+     129: 255,  # '\x81'
+     130: 255,  # '\x82'
+     131: 255,  # '\x83'
+     132: 255,  # '\x84'
+     133: 255,  # '\x85'
+     134: 255,  # '\x86'
+     135: 255,  # '\x87'
+     136: 255,  # '\x88'
+     137: 255,  # '\x89'
+     138: 255,  # '\x8a'
+     139: 255,  # '\x8b'
+     140: 255,  # '\x8c'
+     141: 255,  # '\x8d'
+     142: 255,  # '\x8e'
+     143: 255,  # '\x8f'
+     144: 255,  # '\x90'
+     145: 255,  # '\x91'
+     146: 255,  # '\x92'
+     147: 255,  # '\x93'
+     148: 255,  # '\x94'
+     149: 255,  # '\x95'
+     150: 255,  # '\x96'
+     151: 255,  # '\x97'
+     152: 255,  # '\x98'
+     153: 255,  # '\x99'
+     154: 255,  # '\x9a'
+     155: 255,  # '\x9b'
+     156: 255,  # '\x9c'
+     157: 255,  # '\x9d'
+     158: 255,  # '\x9e'
+     159: 255,  # '\x9f'
+     160: 253,  # '\xa0'
+     161: 233,  # '‘'
+     162: 90,  # '’'
+     163: 253,  # '£'
+     164: 253,  # '€'
+     165: 253,  # '₯'
+     166: 253,  # '¦'
+     167: 253,  # '§'
+     168: 253,  # '¨'
+     169: 253,  # '©'
+     170: 253,  # 'ͺ'
+     171: 253,  # '«'
+     172: 253,  # '¬'
+     173: 74,  # '\xad'
+     174: 253,  # None
+     175: 253,  # '―'
+     176: 253,  # '°'
+     177: 253,  # '±'
+     178: 253,  # '²'
+     179: 253,  # '³'
+     180: 247,  # '΄'
+     181: 248,  # '΅'
+     182: 61,  # 'Ά'
+     183: 36,  # '·'
+     184: 46,  # 'Έ'
+     185: 71,  # 'Ή'
+     186: 73,  # 'Ί'
+     187: 253,  # '»'
+     188: 54,  # 'Ό'
+     189: 253,  # '½'
+     190: 108,  # 'Ύ'
+     191: 123,  # 'Ώ'
+     192: 110,  # 'ΐ'
+     193: 31,  # 'Α'
+     194: 51,  # 'Β'
+     195: 43,  # 'Γ'
+     196: 41,  # 'Δ'
+     197: 34,  # 'Ε'
+     198: 91,  # 'Ζ'
+     199: 40,  # 'Η'
+     200: 52,  # 'Θ'
+     201: 47,  # 'Ι'
+     202: 44,  # 'Κ'
+     203: 53,  # 'Λ'
+     204: 38,  # 'Μ'
+     205: 49,  # 'Ν'
+     206: 59,  # 'Ξ'
+     207: 39,  # 'Ο'
+     208: 35,  # 'Π'
+     209: 48,  # 'Ρ'
+     210: 250,  # None
+     211: 37,  # 'Σ'
+     212: 33,  # 'Τ'
+     213: 45,  # 'Υ'
+     214: 56,  # 'Φ'
+     215: 50,  # 'Χ'
+     216: 84,  # 'Ψ'
+     217: 57,  # 'Ω'
+     218: 120,  # 'Ϊ'
+     219: 121,  # 'Ϋ'
+     220: 17,  # 'ά'
+     221: 18,  # 'έ'
+     222: 22,  # 'ή'
+     223: 15,  # 'ί'
+     224: 124,  # 'ΰ'
+     225: 1,  # 'α'
+     226: 29,  # 'β'
+     227: 20,  # 'γ'
+     228: 21,  # 'δ'
+     229: 3,  # 'ε'
+     230: 32,  # 'ζ'
+     231: 13,  # 'η'
+     232: 25,  # 'θ'
+     233: 5,  # 'ι'
+     234: 11,  # 'κ'
+     235: 16,  # 'λ'
+     236: 10,  # 'μ'
+     237: 6,  # 'ν'
+     238: 30,  # 'ξ'
+     239: 4,  # 'ο'
+     240: 9,  # 'π'
+     241: 8,  # 'ρ'
+     242: 14,  # 'ς'
+     243: 7,  # 'σ'
+     244: 2,  # 'τ'
+     245: 12,  # 'υ'
+     246: 28,  # 'φ'
+     247: 23,  # 'χ'
+     248: 42,  # 'ψ'
+     249: 24,  # 'ω'
+     250: 64,  # 'ϊ'
+     251: 75,  # 'ϋ'
+     252: 19,  # 'ό'
+     253: 26,  # 'ύ'
+     254: 27,  # 'ώ'
+     255: 253,  # None
 }
 
-Win1253GreekModel = {
-  'char_to_order_map': win1253_char_to_order_map,
-  'precedence_matrix': GreekLangModel,
-  'typical_positive_ratio': 0.982851,
-  'keep_english_letter': False,
-  'charset_name': "windows-1253",
-  'language': 'Greek',
-}
+ISO_8859_7_GREEK_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-7',
+                                                language='Greek',
+                                                char_to_order_map=ISO_8859_7_GREEK_CHAR_TO_ORDER,
+                                                language_model=GREEK_LANG_MODEL,
+                                                typical_positive_ratio=0.982851,
+                                                keep_ascii_letters=False,
+                                                alphabet='ΆΈΉΊΌΎΏΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩάέήίαβγδεζηθικλμνξοπρςστυφχψωόύώ')
+
diff --git a/src/pip/_vendor/chardet/langhebrewmodel.py b/src/pip/_vendor/chardet/langhebrewmodel.py
index 58f4c875ec9..484c652a48e 100644
--- a/src/pip/_vendor/chardet/langhebrewmodel.py
+++ b/src/pip/_vendor/chardet/langhebrewmodel.py
@@ -1,200 +1,4383 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-#          Simon Montagu
-# Portions created by the Initial Developer are Copyright (C) 2005
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#   Shoshannah Forbes - original C code (?)
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-# 255: Control characters that usually does not exist in any text
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
+
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+HEBREW_LANG_MODEL = {
+    50: {  # 'a'
+        50: 0,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 2,  # 'l'
+        54: 2,  # 'n'
+        49: 0,  # 'o'
+        51: 2,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 1,  # 'ק'
+        7: 0,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    60: {  # 'c'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 0,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 0,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    61: {  # 'd'
+        50: 1,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 2,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 0,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 1,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    42: {  # 'e'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 2,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 2,  # 'l'
+        54: 2,  # 'n'
+        49: 1,  # 'o'
+        51: 2,  # 'r'
+        43: 2,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 1,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    53: {  # 'i'
+        50: 1,  # 'a'
+        60: 2,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 0,  # 'i'
+        56: 1,  # 'l'
+        54: 2,  # 'n'
+        49: 2,  # 'o'
+        51: 1,  # 'r'
+        43: 2,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    56: {  # 'l'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 2,  # 'e'
+        53: 2,  # 'i'
+        56: 2,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    54: {  # 'n'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    49: {  # 'o'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 2,  # 'n'
+        49: 1,  # 'o'
+        51: 2,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    51: {  # 'r'
+        50: 2,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 2,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 2,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    43: {  # 's'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 0,  # 'd'
+        42: 2,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 2,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    44: {  # 't'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 0,  # 'd'
+        42: 2,  # 'e'
+        53: 2,  # 'i'
+        56: 1,  # 'l'
+        54: 0,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 2,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    63: {  # 'u'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 0,  # 'o'
+        51: 1,  # 'r'
+        43: 2,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    34: {  # '\xa0'
+        50: 1,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 0,  # 'e'
+        53: 1,  # 'i'
+        56: 0,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 2,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    55: {  # '´'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 1,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 2,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 1,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    48: {  # '¼'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    39: {  # '½'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    57: {  # '¾'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    30: {  # 'ְ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 2,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 1,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 2,  # 'ע'
+        26: 0,  # 'ף'
+        18: 2,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    59: {  # 'ֱ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 1,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 0,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    41: {  # 'ֲ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 0,  # 'ם'
+        6: 2,  # 'מ'
+        23: 0,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    33: {  # 'ִ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 2,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    37: {  # 'ֵ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 1,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    36: {  # 'ֶ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 1,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    31: {  # 'ַ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 2,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    29: {  # 'ָ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 2,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    35: {  # 'ֹ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 2,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 2,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    62: {  # 'ֻ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 2,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 1,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    28: {  # 'ּ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 3,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 1,  # 'ֲ'
+        33: 3,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 3,  # 'ַ'
+        29: 3,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 2,  # 'ׁ'
+        45: 1,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 2,  # 'ב'
+        20: 1,  # 'ג'
+        16: 2,  # 'ד'
+        3: 1,  # 'ה'
+        2: 2,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 2,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 2,  # 'ל'
+        11: 1,  # 'ם'
+        6: 2,  # 'מ'
+        23: 1,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 2,  # 'ר'
+        10: 2,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    38: {  # 'ׁ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 2,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    45: {  # 'ׂ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 2,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 0,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    9: {  # 'א'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 2,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 2,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    8: {  # 'ב'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 3,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 1,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    20: {  # 'ג'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 2,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    16: {  # 'ד'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 1,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    3: {  # 'ה'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 1,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 3,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 0,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    2: {  # 'ו'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 3,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 3,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 3,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    24: {  # 'ז'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 1,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 2,  # 'ב'
+        20: 2,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 2,  # 'ח'
+        22: 1,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 2,  # 'נ'
+        19: 1,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 1,  # 'ש'
+        5: 2,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    14: {  # 'ח'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 1,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 2,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 1,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    22: {  # 'ט'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 1,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 3,  # 'ר'
+        10: 2,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    1: {  # 'י'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 3,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    25: {  # 'ך'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 1,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    15: {  # 'כ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 3,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 2,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 2,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    4: {  # 'ל'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 3,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    11: {  # 'ם'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    6: {  # 'מ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 0,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    23: {  # 'ן'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 1,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 1,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    12: {  # 'נ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    19: {  # 'ס'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 2,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 1,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 3,  # 'ף'
+        18: 3,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 1,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    13: {  # 'ע'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 1,  # 'ֱ'
+        41: 2,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 1,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 2,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 2,  # 'ע'
+        26: 1,  # 'ף'
+        18: 2,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    26: {  # 'ף'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 1,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    18: {  # 'פ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 2,  # 'ב'
+        20: 3,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 2,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 2,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    27: {  # 'ץ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 1,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 0,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    21: {  # 'צ'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 1,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 1,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 0,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    17: {  # 'ק'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 1,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 2,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 1,  # 'ך'
+        15: 1,  # 'כ'
+        4: 3,  # 'ל'
+        11: 2,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 2,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 2,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    7: {  # 'ר'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 2,  # '´'
+        48: 1,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 1,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 2,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 3,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 3,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 3,  # 'ץ'
+        21: 3,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    10: {  # 'ש'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 1,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 1,  # 'ִ'
+        37: 1,  # 'ֵ'
+        36: 1,  # 'ֶ'
+        31: 1,  # 'ַ'
+        29: 1,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 3,  # 'ׁ'
+        45: 2,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 3,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 3,  # 'ט'
+        1: 3,  # 'י'
+        25: 3,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 2,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 1,  # '…'
+    },
+    5: {  # 'ת'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 1,  # '\xa0'
+        55: 0,  # '´'
+        48: 1,  # '¼'
+        39: 1,  # '½'
+        57: 0,  # '¾'
+        30: 2,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 2,  # 'ִ'
+        37: 2,  # 'ֵ'
+        36: 2,  # 'ֶ'
+        31: 2,  # 'ַ'
+        29: 2,  # 'ָ'
+        35: 1,  # 'ֹ'
+        62: 1,  # 'ֻ'
+        28: 2,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 3,  # 'א'
+        8: 3,  # 'ב'
+        20: 3,  # 'ג'
+        16: 2,  # 'ד'
+        3: 3,  # 'ה'
+        2: 3,  # 'ו'
+        24: 2,  # 'ז'
+        14: 3,  # 'ח'
+        22: 2,  # 'ט'
+        1: 3,  # 'י'
+        25: 2,  # 'ך'
+        15: 3,  # 'כ'
+        4: 3,  # 'ל'
+        11: 3,  # 'ם'
+        6: 3,  # 'מ'
+        23: 3,  # 'ן'
+        12: 3,  # 'נ'
+        19: 2,  # 'ס'
+        13: 3,  # 'ע'
+        26: 2,  # 'ף'
+        18: 3,  # 'פ'
+        27: 1,  # 'ץ'
+        21: 2,  # 'צ'
+        17: 3,  # 'ק'
+        7: 3,  # 'ר'
+        10: 3,  # 'ש'
+        5: 3,  # 'ת'
+        32: 1,  # '–'
+        52: 1,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+    32: {  # '–'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 1,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 1,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    52: {  # '’'
+        50: 1,  # 'a'
+        60: 0,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 1,  # 'r'
+        43: 2,  # 's'
+        44: 2,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 1,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    47: {  # '“'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 1,  # 'l'
+        54: 1,  # 'n'
+        49: 1,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 1,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 2,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 1,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 1,  # 'ח'
+        22: 1,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 1,  # 'ס'
+        13: 1,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 1,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    46: {  # '”'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 1,  # 'ב'
+        20: 1,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 1,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 0,  # '†'
+        40: 0,  # '…'
+    },
+    58: {  # '†'
+        50: 0,  # 'a'
+        60: 0,  # 'c'
+        61: 0,  # 'd'
+        42: 0,  # 'e'
+        53: 0,  # 'i'
+        56: 0,  # 'l'
+        54: 0,  # 'n'
+        49: 0,  # 'o'
+        51: 0,  # 'r'
+        43: 0,  # 's'
+        44: 0,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 0,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 0,  # 'ה'
+        2: 0,  # 'ו'
+        24: 0,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 0,  # 'י'
+        25: 0,  # 'ך'
+        15: 0,  # 'כ'
+        4: 0,  # 'ל'
+        11: 0,  # 'ם'
+        6: 0,  # 'מ'
+        23: 0,  # 'ן'
+        12: 0,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 0,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 0,  # 'ר'
+        10: 0,  # 'ש'
+        5: 0,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 0,  # '”'
+        58: 2,  # '†'
+        40: 0,  # '…'
+    },
+    40: {  # '…'
+        50: 1,  # 'a'
+        60: 1,  # 'c'
+        61: 1,  # 'd'
+        42: 1,  # 'e'
+        53: 1,  # 'i'
+        56: 0,  # 'l'
+        54: 1,  # 'n'
+        49: 0,  # 'o'
+        51: 1,  # 'r'
+        43: 1,  # 's'
+        44: 1,  # 't'
+        63: 0,  # 'u'
+        34: 0,  # '\xa0'
+        55: 0,  # '´'
+        48: 0,  # '¼'
+        39: 0,  # '½'
+        57: 0,  # '¾'
+        30: 0,  # 'ְ'
+        59: 0,  # 'ֱ'
+        41: 0,  # 'ֲ'
+        33: 0,  # 'ִ'
+        37: 0,  # 'ֵ'
+        36: 0,  # 'ֶ'
+        31: 0,  # 'ַ'
+        29: 0,  # 'ָ'
+        35: 0,  # 'ֹ'
+        62: 0,  # 'ֻ'
+        28: 0,  # 'ּ'
+        38: 0,  # 'ׁ'
+        45: 0,  # 'ׂ'
+        9: 1,  # 'א'
+        8: 0,  # 'ב'
+        20: 0,  # 'ג'
+        16: 0,  # 'ד'
+        3: 1,  # 'ה'
+        2: 1,  # 'ו'
+        24: 1,  # 'ז'
+        14: 0,  # 'ח'
+        22: 0,  # 'ט'
+        1: 1,  # 'י'
+        25: 0,  # 'ך'
+        15: 1,  # 'כ'
+        4: 1,  # 'ל'
+        11: 0,  # 'ם'
+        6: 1,  # 'מ'
+        23: 0,  # 'ן'
+        12: 1,  # 'נ'
+        19: 0,  # 'ס'
+        13: 0,  # 'ע'
+        26: 0,  # 'ף'
+        18: 1,  # 'פ'
+        27: 0,  # 'ץ'
+        21: 0,  # 'צ'
+        17: 0,  # 'ק'
+        7: 1,  # 'ר'
+        10: 1,  # 'ש'
+        5: 1,  # 'ת'
+        32: 0,  # '–'
+        52: 0,  # '’'
+        47: 0,  # '“'
+        46: 1,  # '”'
+        58: 0,  # '†'
+        40: 2,  # '…'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
+# 251: Control characters
 
-# Windows-1255 language model
-# Character Mapping Table:
-WIN1255_CHAR_TO_ORDER_MAP = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 69, 91, 79, 80, 92, 89, 97, 90, 68,111,112, 82, 73, 95, 85,  # 40
- 78,121, 86, 71, 67,102,107, 84,114,103,115,253,253,253,253,253,  # 50
-253, 50, 74, 60, 61, 42, 76, 70, 64, 53,105, 93, 56, 65, 54, 49,  # 60
- 66,110, 51, 43, 44, 63, 81, 77, 98, 75,108,253,253,253,253,253,  # 70
-124,202,203,204,205, 40, 58,206,207,208,209,210,211,212,213,214,
-215, 83, 52, 47, 46, 72, 32, 94,216,113,217,109,218,219,220,221,
- 34,116,222,118,100,223,224,117,119,104,125,225,226, 87, 99,227,
-106,122,123,228, 55,229,230,101,231,232,120,233, 48, 39, 57,234,
- 30, 59, 41, 88, 33, 37, 36, 31, 29, 35,235, 62, 28,236,126,237,
-238, 38, 45,239,240,241,242,243,127,244,245,246,247,248,249,250,
-  9,  8, 20, 16,  3,  2, 24, 14, 22,  1, 25, 15,  4, 11,  6, 23,
- 12, 19, 13, 26, 18, 27, 21, 17,  7, 10,  5,251,252,128, 96,253,
-)
+# Character Mapping Table(s):
+WINDOWS_1255_HEBREW_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 69,  # 'A'
+     66: 91,  # 'B'
+     67: 79,  # 'C'
+     68: 80,  # 'D'
+     69: 92,  # 'E'
+     70: 89,  # 'F'
+     71: 97,  # 'G'
+     72: 90,  # 'H'
+     73: 68,  # 'I'
+     74: 111,  # 'J'
+     75: 112,  # 'K'
+     76: 82,  # 'L'
+     77: 73,  # 'M'
+     78: 95,  # 'N'
+     79: 85,  # 'O'
+     80: 78,  # 'P'
+     81: 121,  # 'Q'
+     82: 86,  # 'R'
+     83: 71,  # 'S'
+     84: 67,  # 'T'
+     85: 102,  # 'U'
+     86: 107,  # 'V'
+     87: 84,  # 'W'
+     88: 114,  # 'X'
+     89: 103,  # 'Y'
+     90: 115,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 50,  # 'a'
+     98: 74,  # 'b'
+     99: 60,  # 'c'
+     100: 61,  # 'd'
+     101: 42,  # 'e'
+     102: 76,  # 'f'
+     103: 70,  # 'g'
+     104: 64,  # 'h'
+     105: 53,  # 'i'
+     106: 105,  # 'j'
+     107: 93,  # 'k'
+     108: 56,  # 'l'
+     109: 65,  # 'm'
+     110: 54,  # 'n'
+     111: 49,  # 'o'
+     112: 66,  # 'p'
+     113: 110,  # 'q'
+     114: 51,  # 'r'
+     115: 43,  # 's'
+     116: 44,  # 't'
+     117: 63,  # 'u'
+     118: 81,  # 'v'
+     119: 77,  # 'w'
+     120: 98,  # 'x'
+     121: 75,  # 'y'
+     122: 108,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 124,  # '€'
+     129: 202,  # None
+     130: 203,  # '‚'
+     131: 204,  # 'ƒ'
+     132: 205,  # '„'
+     133: 40,  # '…'
+     134: 58,  # '†'
+     135: 206,  # '‡'
+     136: 207,  # 'ˆ'
+     137: 208,  # '‰'
+     138: 209,  # None
+     139: 210,  # '‹'
+     140: 211,  # None
+     141: 212,  # None
+     142: 213,  # None
+     143: 214,  # None
+     144: 215,  # None
+     145: 83,  # '‘'
+     146: 52,  # '’'
+     147: 47,  # '“'
+     148: 46,  # '”'
+     149: 72,  # '•'
+     150: 32,  # '–'
+     151: 94,  # '—'
+     152: 216,  # '˜'
+     153: 113,  # '™'
+     154: 217,  # None
+     155: 109,  # '›'
+     156: 218,  # None
+     157: 219,  # None
+     158: 220,  # None
+     159: 221,  # None
+     160: 34,  # '\xa0'
+     161: 116,  # '¡'
+     162: 222,  # '¢'
+     163: 118,  # '£'
+     164: 100,  # '₪'
+     165: 223,  # '¥'
+     166: 224,  # '¦'
+     167: 117,  # '§'
+     168: 119,  # '¨'
+     169: 104,  # '©'
+     170: 125,  # '×'
+     171: 225,  # '«'
+     172: 226,  # '¬'
+     173: 87,  # '\xad'
+     174: 99,  # '®'
+     175: 227,  # '¯'
+     176: 106,  # '°'
+     177: 122,  # '±'
+     178: 123,  # '²'
+     179: 228,  # '³'
+     180: 55,  # '´'
+     181: 229,  # 'µ'
+     182: 230,  # '¶'
+     183: 101,  # '·'
+     184: 231,  # '¸'
+     185: 232,  # '¹'
+     186: 120,  # '÷'
+     187: 233,  # '»'
+     188: 48,  # '¼'
+     189: 39,  # '½'
+     190: 57,  # '¾'
+     191: 234,  # '¿'
+     192: 30,  # 'ְ'
+     193: 59,  # 'ֱ'
+     194: 41,  # 'ֲ'
+     195: 88,  # 'ֳ'
+     196: 33,  # 'ִ'
+     197: 37,  # 'ֵ'
+     198: 36,  # 'ֶ'
+     199: 31,  # 'ַ'
+     200: 29,  # 'ָ'
+     201: 35,  # 'ֹ'
+     202: 235,  # None
+     203: 62,  # 'ֻ'
+     204: 28,  # 'ּ'
+     205: 236,  # 'ֽ'
+     206: 126,  # '־'
+     207: 237,  # 'ֿ'
+     208: 238,  # '׀'
+     209: 38,  # 'ׁ'
+     210: 45,  # 'ׂ'
+     211: 239,  # '׃'
+     212: 240,  # 'װ'
+     213: 241,  # 'ױ'
+     214: 242,  # 'ײ'
+     215: 243,  # '׳'
+     216: 127,  # '״'
+     217: 244,  # None
+     218: 245,  # None
+     219: 246,  # None
+     220: 247,  # None
+     221: 248,  # None
+     222: 249,  # None
+     223: 250,  # None
+     224: 9,  # 'א'
+     225: 8,  # 'ב'
+     226: 20,  # 'ג'
+     227: 16,  # 'ד'
+     228: 3,  # 'ה'
+     229: 2,  # 'ו'
+     230: 24,  # 'ז'
+     231: 14,  # 'ח'
+     232: 22,  # 'ט'
+     233: 1,  # 'י'
+     234: 25,  # 'ך'
+     235: 15,  # 'כ'
+     236: 4,  # 'ל'
+     237: 11,  # 'ם'
+     238: 6,  # 'מ'
+     239: 23,  # 'ן'
+     240: 12,  # 'נ'
+     241: 19,  # 'ס'
+     242: 13,  # 'ע'
+     243: 26,  # 'ף'
+     244: 18,  # 'פ'
+     245: 27,  # 'ץ'
+     246: 21,  # 'צ'
+     247: 17,  # 'ק'
+     248: 7,  # 'ר'
+     249: 10,  # 'ש'
+     250: 5,  # 'ת'
+     251: 251,  # None
+     252: 252,  # None
+     253: 128,  # '\u200e'
+     254: 96,  # '\u200f'
+     255: 253,  # None
+}
 
-# Model Table:
-# total sequences: 100%
-# first 512 sequences: 98.4004%
-# first 1024 sequences: 1.5981%
-# rest  sequences:      0.087%
-# negative sequences:   0.0015%
-HEBREW_LANG_MODEL = (
-0,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,3,2,1,2,0,1,0,0,
-3,0,3,1,0,0,1,3,2,0,1,1,2,0,2,2,2,1,1,1,1,2,1,1,1,2,0,0,2,2,0,1,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,
-1,2,1,2,1,2,0,0,2,0,0,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,
-1,2,1,3,1,1,0,0,2,0,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,0,1,2,2,1,3,
-1,2,1,1,2,2,0,0,2,2,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,2,2,2,2,3,2,
-1,2,1,2,2,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,2,3,2,2,3,2,2,2,1,2,2,2,2,
-1,2,1,1,2,2,0,1,2,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,0,2,2,2,2,2,
-0,2,0,2,2,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,0,2,2,2,
-0,2,1,2,2,2,0,0,2,1,0,0,0,0,1,0,1,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,2,1,2,3,2,2,2,
-1,2,1,2,2,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,
-3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,1,0,2,0,2,
-0,2,1,2,2,2,0,0,1,2,0,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,2,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,2,3,2,2,3,2,1,2,1,1,1,
-0,1,1,1,1,1,3,0,1,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,1,1,0,0,1,0,0,1,0,0,0,0,
-0,0,1,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,
-0,2,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,2,3,3,3,2,1,2,3,3,2,3,3,3,3,2,3,2,1,2,0,2,1,2,
-0,2,0,2,2,2,0,0,1,2,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,
-3,3,3,3,3,3,3,3,3,2,3,3,3,1,2,2,3,3,2,3,2,3,2,2,3,1,2,2,0,2,2,2,
-0,2,1,2,2,2,0,0,1,2,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,3,2,2,2,3,3,3,3,1,3,2,2,2,
-0,2,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,3,3,2,3,2,2,2,1,2,2,0,2,2,2,2,
-0,2,0,2,2,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,1,3,2,3,3,2,3,3,2,2,1,2,2,2,2,2,2,
-0,2,1,2,1,2,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,2,3,2,3,3,2,3,3,3,3,2,3,2,3,3,3,3,3,2,2,2,2,2,2,2,1,
-0,2,0,1,2,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,2,1,2,3,3,3,3,3,3,3,2,3,2,3,2,1,2,3,0,2,1,2,2,
-0,2,1,1,2,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,
-3,3,3,3,3,3,3,3,3,2,3,3,3,3,2,1,3,1,2,2,2,1,2,3,3,1,2,1,2,2,2,2,
-0,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,0,2,3,3,3,1,3,3,3,1,2,2,2,2,1,1,2,2,2,2,2,2,
-0,2,0,1,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,2,3,3,3,2,2,3,3,3,2,1,2,3,2,3,2,2,2,2,1,2,1,1,1,2,2,
-0,2,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0,
-1,0,1,0,0,0,0,0,2,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,2,3,3,2,3,1,2,2,2,2,3,2,3,1,1,2,2,1,2,2,1,1,0,2,2,2,2,
-0,1,0,1,2,2,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,
-3,0,0,1,1,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,2,0,
-0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,1,0,1,0,1,1,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-3,2,2,1,2,2,2,2,2,2,2,1,2,2,1,2,2,1,1,1,1,1,1,1,1,2,1,1,0,3,3,3,
-0,3,0,2,2,2,2,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-2,2,2,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,2,2,1,2,2,2,1,1,1,2,0,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,2,2,2,2,2,2,0,2,2,0,0,0,0,0,0,
-0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,2,1,0,2,1,0,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
-0,3,1,1,2,2,2,2,2,1,2,2,2,1,1,2,2,2,2,2,2,2,1,2,2,1,0,1,1,1,1,0,
-0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,2,1,1,1,1,2,1,1,2,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,
-0,0,2,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,
-2,1,1,2,2,2,2,2,2,2,2,2,2,2,1,2,2,2,2,2,1,2,1,2,1,1,1,1,0,0,0,0,
-0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,2,1,2,2,2,2,2,2,2,2,2,2,1,2,1,2,1,1,2,1,1,1,2,1,2,1,2,0,1,0,1,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,3,1,2,2,2,1,2,2,2,2,2,2,2,2,1,2,1,1,1,1,1,1,2,1,2,1,1,0,1,0,1,
-0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,
-0,2,0,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,1,1,1,1,1,1,1,0,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,2,0,1,1,1,0,1,0,0,0,1,1,0,1,1,0,0,0,0,0,1,1,0,0,
-0,1,1,1,2,1,2,2,2,0,2,0,2,0,1,1,2,1,1,1,1,2,1,0,1,1,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,1,0,0,0,0,0,1,0,1,2,2,0,1,0,0,1,1,2,2,1,2,0,2,0,0,0,1,2,0,1,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,2,0,2,1,2,0,2,0,0,1,1,1,1,1,1,0,1,0,0,0,1,0,0,1,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,1,0,0,0,0,0,1,0,2,1,1,0,1,0,0,1,1,1,2,2,0,0,1,0,0,0,1,0,0,1,
-1,1,2,1,0,1,1,1,0,1,0,1,1,1,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,2,2,1,
-0,2,0,1,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,1,0,0,1,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,1,1,1,1,1,1,1,1,2,1,0,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,1,1,0,1,1,0,1,0,0,0,1,1,0,1,
-2,0,1,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,1,1,1,0,1,0,0,1,1,2,1,1,2,0,1,0,0,0,1,1,0,1,
-1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,1,1,2,0,1,0,0,0,0,2,1,1,2,0,2,0,0,0,1,1,0,1,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,2,1,1,0,1,0,0,2,2,1,2,1,1,0,1,0,0,0,1,1,0,1,
-2,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,2,2,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,1,0,1,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,2,2,0,0,0,0,2,1,1,1,0,2,1,1,0,0,0,2,1,0,1,
-1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,1,1,2,0,1,0,0,1,1,0,2,1,1,0,1,0,0,0,1,1,0,1,
-2,2,1,1,1,0,1,1,0,1,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,2,1,1,0,1,0,0,1,1,0,1,2,1,0,2,0,0,0,1,1,0,1,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,
-0,1,0,0,2,0,2,1,1,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,1,0,1,1,2,0,1,0,0,1,1,1,0,1,0,0,1,0,0,0,1,0,0,1,
-1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,0,0,0,0,0,1,0,1,1,0,0,1,0,0,2,1,1,1,1,1,0,1,0,0,0,0,1,0,1,
-0,1,1,1,2,1,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,1,2,1,0,0,0,0,0,1,1,1,1,1,0,1,0,0,0,1,1,0,0,
-)
+WINDOWS_1255_HEBREW_MODEL = SingleByteCharSetModel(charset_name='windows-1255',
+                                                   language='Hebrew',
+                                                   char_to_order_map=WINDOWS_1255_HEBREW_CHAR_TO_ORDER,
+                                                   language_model=HEBREW_LANG_MODEL,
+                                                   typical_positive_ratio=0.984004,
+                                                   keep_ascii_letters=False,
+                                                   alphabet='אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ')
 
-Win1255HebrewModel = {
-  'char_to_order_map': WIN1255_CHAR_TO_ORDER_MAP,
-  'precedence_matrix': HEBREW_LANG_MODEL,
-  'typical_positive_ratio': 0.984004,
-  'keep_english_letter': False,
-  'charset_name': "windows-1255",
-  'language': 'Hebrew',
-}
diff --git a/src/pip/_vendor/chardet/langhungarianmodel.py b/src/pip/_vendor/chardet/langhungarianmodel.py
index bb7c095e1ea..bbc5cda6441 100644
--- a/src/pip/_vendor/chardet/langhungarianmodel.py
+++ b/src/pip/_vendor/chardet/langhungarianmodel.py
@@ -1,225 +1,4650 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-# 255: Control characters that usually does not exist in any text
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
+
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+HUNGARIAN_LANG_MODEL = {
+    28: {  # 'A'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 2,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 2,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 2,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 1,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 1,  # 'Á'
+        44: 0,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    40: {  # 'B'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 0,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 3,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    54: {  # 'C'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 0,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 3,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    45: {  # 'D'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 0,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 1,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    32: {  # 'E'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 2,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 1,  # 't'
+        21: 2,  # 'u'
+        19: 1,  # 'v'
+        62: 1,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    50: {  # 'F'
+        28: 1,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 0,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    49: {  # 'G'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 2,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    38: {  # 'H'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 0,  # 'D'
+        32: 1,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 1,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 1,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 0,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 2,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 2,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    39: {  # 'I'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 2,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 0,  # 'e'
+        27: 1,  # 'f'
+        12: 2,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    53: {  # 'J'
+        28: 2,  # 'A'
+        40: 0,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 1,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 0,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    36: {  # 'K'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 2,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    41: {  # 'L'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 1,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    34: {  # 'M'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 3,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    35: {  # 'N'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 2,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 2,  # 'Y'
+        52: 1,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    47: {  # 'O'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 2,  # 'K'
+        41: 2,  # 'L'
+        34: 2,  # 'M'
+        35: 2,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 2,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 1,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    46: {  # 'P'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 0,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 3,  # 'á'
+        15: 2,  # 'é'
+        30: 0,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    43: {  # 'R'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 2,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 2,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    33: {  # 'S'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 3,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 1,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 1,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    37: {  # 'T'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 1,  # 'S'
+        37: 2,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 2,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 1,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 0,  # 't'
+        21: 2,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 2,  # 'Á'
+        44: 2,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    57: {  # 'U'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 2,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    48: {  # 'V'
+        28: 2,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 0,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 2,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 2,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 2,  # 'o'
+        23: 0,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 2,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    55: {  # 'Y'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 1,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 2,  # 'Z'
+        2: 1,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 0,  # 'r'
+        5: 0,  # 's'
+        3: 0,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 1,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    52: {  # 'Z'
+        28: 2,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 2,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 2,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 2,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 1,  # 'U'
+        48: 1,  # 'V'
+        55: 1,  # 'Y'
+        52: 1,  # 'Z'
+        2: 1,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 1,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 1,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 2,  # 's'
+        3: 0,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 2,  # 'Á'
+        44: 1,  # 'É'
+        61: 1,  # 'Í'
+        58: 1,  # 'Ó'
+        59: 1,  # 'Ö'
+        60: 1,  # 'Ú'
+        63: 1,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    2: {  # 'a'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 2,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 2,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    18: {  # 'b'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 2,  # 's'
+        3: 1,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    26: {  # 'c'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 1,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 1,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 2,  # 't'
+        21: 2,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 2,  # 'á'
+        15: 2,  # 'é'
+        30: 2,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    17: {  # 'd'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 2,  # 'k'
+        6: 1,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    1: {  # 'e'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 3,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 2,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 2,  # 'u'
+        19: 3,  # 'v'
+        62: 2,  # 'x'
+        16: 2,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    27: {  # 'f'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 3,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 2,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 3,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    12: {  # 'g'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 2,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 2,  # 'k'
+        6: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 3,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    20: {  # 'h'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 3,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 2,  # 's'
+        3: 1,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 1,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    9: {  # 'i'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 3,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 2,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 3,  # 'ó'
+        24: 1,  # 'ö'
+        31: 2,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    22: {  # 'j'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 1,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 1,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    7: {  # 'k'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 2,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 2,  # 'ó'
+        24: 3,  # 'ö'
+        31: 1,  # 'ú'
+        29: 3,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    6: {  # 'l'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 1,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 3,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 3,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 3,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    13: {  # 'm'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 1,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 3,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 3,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    4: {  # 'n'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 1,  # 'x'
+        16: 3,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    8: {  # 'o'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 1,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 2,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    23: {  # 'p'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 3,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    10: {  # 'r'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 2,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    5: {  # 's'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 2,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    3: {  # 't'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 1,  # 'g'
+        20: 3,  # 'h'
+        9: 3,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 3,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 3,  # 'ú'
+        29: 3,  # 'ü'
+        42: 3,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    21: {  # 'u'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 2,  # 'b'
+        26: 2,  # 'c'
+        17: 3,  # 'd'
+        1: 2,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 1,  # 'u'
+        19: 3,  # 'v'
+        62: 1,  # 'x'
+        16: 1,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 2,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 0,  # 'ö'
+        31: 1,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    19: {  # 'v'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 2,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 2,  # 'ö'
+        31: 1,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    62: {  # 'x'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 0,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 1,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 1,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 1,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    16: {  # 'y'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 3,  # 'e'
+        27: 2,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 2,  # 'j'
+        7: 2,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 2,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 2,  # 'í'
+        25: 2,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 2,  # 'ü'
+        42: 1,  # 'ő'
+        56: 2,  # 'ű'
+    },
+    11: {  # 'z'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 3,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 3,  # 'd'
+        1: 3,  # 'e'
+        27: 1,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 3,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 3,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 3,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 3,  # 'á'
+        15: 3,  # 'é'
+        30: 3,  # 'í'
+        25: 3,  # 'ó'
+        24: 3,  # 'ö'
+        31: 2,  # 'ú'
+        29: 3,  # 'ü'
+        42: 2,  # 'ő'
+        56: 1,  # 'ű'
+    },
+    51: {  # 'Á'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 1,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    44: {  # 'É'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 1,  # 'E'
+        50: 0,  # 'F'
+        49: 2,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 2,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 2,  # 'R'
+        33: 2,  # 'S'
+        37: 2,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 3,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    61: {  # 'Í'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 0,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 2,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 0,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 0,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    58: {  # 'Ó'
+        28: 1,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 1,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 2,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 2,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 0,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 1,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    59: {  # 'Ö'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 1,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 0,  # 'b'
+        26: 1,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 0,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 2,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    60: {  # 'Ú'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 1,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 1,  # 'F'
+        49: 1,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 0,  # 'b'
+        26: 0,  # 'c'
+        17: 0,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 2,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 2,  # 'j'
+        7: 0,  # 'k'
+        6: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 0,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 0,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    63: {  # 'Ü'
+        28: 0,  # 'A'
+        40: 1,  # 'B'
+        54: 0,  # 'C'
+        45: 1,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 1,  # 'G'
+        38: 1,  # 'H'
+        39: 0,  # 'I'
+        53: 1,  # 'J'
+        36: 1,  # 'K'
+        41: 1,  # 'L'
+        34: 1,  # 'M'
+        35: 1,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 1,  # 'R'
+        33: 1,  # 'S'
+        37: 1,  # 'T'
+        57: 0,  # 'U'
+        48: 1,  # 'V'
+        55: 0,  # 'Y'
+        52: 1,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 0,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 0,  # 'f'
+        12: 1,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 0,  # 'j'
+        7: 0,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 1,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    14: {  # 'á'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 3,  # 'b'
+        26: 3,  # 'c'
+        17: 3,  # 'd'
+        1: 1,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 2,  # 'i'
+        22: 3,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 2,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 1,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 2,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    15: {  # 'é'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 3,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 3,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 3,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 0,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    30: {  # 'í'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 0,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 0,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 0,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 2,  # 's'
+        3: 3,  # 't'
+        21: 0,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    25: {  # 'ó'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 2,  # 'a'
+        18: 3,  # 'b'
+        26: 2,  # 'c'
+        17: 3,  # 'd'
+        1: 1,  # 'e'
+        27: 2,  # 'f'
+        12: 2,  # 'g'
+        20: 2,  # 'h'
+        9: 2,  # 'i'
+        22: 2,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        8: 1,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 1,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 1,  # 'ö'
+        31: 1,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    24: {  # 'ö'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 0,  # 'a'
+        18: 3,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 0,  # 'e'
+        27: 1,  # 'f'
+        12: 2,  # 'g'
+        20: 1,  # 'h'
+        9: 0,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        8: 0,  # 'o'
+        23: 2,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 3,  # 't'
+        21: 0,  # 'u'
+        19: 3,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 3,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    31: {  # 'ú'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 2,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 2,  # 'f'
+        12: 3,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 3,  # 'j'
+        7: 1,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 3,  # 'r'
+        5: 3,  # 's'
+        3: 2,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 1,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    29: {  # 'ü'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 3,  # 'g'
+        20: 2,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 3,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        8: 0,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 0,  # 'u'
+        19: 2,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 1,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    42: {  # 'ő'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 2,  # 'b'
+        26: 1,  # 'c'
+        17: 2,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 2,  # 'k'
+        6: 3,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        8: 1,  # 'o'
+        23: 1,  # 'p'
+        10: 2,  # 'r'
+        5: 2,  # 's'
+        3: 2,  # 't'
+        21: 1,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 1,  # 'é'
+        30: 1,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 1,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+    56: {  # 'ű'
+        28: 0,  # 'A'
+        40: 0,  # 'B'
+        54: 0,  # 'C'
+        45: 0,  # 'D'
+        32: 0,  # 'E'
+        50: 0,  # 'F'
+        49: 0,  # 'G'
+        38: 0,  # 'H'
+        39: 0,  # 'I'
+        53: 0,  # 'J'
+        36: 0,  # 'K'
+        41: 0,  # 'L'
+        34: 0,  # 'M'
+        35: 0,  # 'N'
+        47: 0,  # 'O'
+        46: 0,  # 'P'
+        43: 0,  # 'R'
+        33: 0,  # 'S'
+        37: 0,  # 'T'
+        57: 0,  # 'U'
+        48: 0,  # 'V'
+        55: 0,  # 'Y'
+        52: 0,  # 'Z'
+        2: 1,  # 'a'
+        18: 1,  # 'b'
+        26: 0,  # 'c'
+        17: 1,  # 'd'
+        1: 1,  # 'e'
+        27: 1,  # 'f'
+        12: 1,  # 'g'
+        20: 1,  # 'h'
+        9: 1,  # 'i'
+        22: 1,  # 'j'
+        7: 1,  # 'k'
+        6: 1,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        8: 0,  # 'o'
+        23: 0,  # 'p'
+        10: 1,  # 'r'
+        5: 1,  # 's'
+        3: 1,  # 't'
+        21: 0,  # 'u'
+        19: 1,  # 'v'
+        62: 0,  # 'x'
+        16: 0,  # 'y'
+        11: 2,  # 'z'
+        51: 0,  # 'Á'
+        44: 0,  # 'É'
+        61: 0,  # 'Í'
+        58: 0,  # 'Ó'
+        59: 0,  # 'Ö'
+        60: 0,  # 'Ú'
+        63: 0,  # 'Ü'
+        14: 0,  # 'á'
+        15: 0,  # 'é'
+        30: 0,  # 'í'
+        25: 0,  # 'ó'
+        24: 0,  # 'ö'
+        31: 0,  # 'ú'
+        29: 0,  # 'ü'
+        42: 0,  # 'ő'
+        56: 0,  # 'ű'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
+# 251: Control characters
 
-# Character Mapping Table:
-Latin2_HungarianCharToOrderMap = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 28, 40, 54, 45, 32, 50, 49, 38, 39, 53, 36, 41, 34, 35, 47,
- 46, 71, 43, 33, 37, 57, 48, 64, 68, 55, 52,253,253,253,253,253,
-253,  2, 18, 26, 17,  1, 27, 12, 20,  9, 22,  7,  6, 13,  4,  8,
- 23, 67, 10,  5,  3, 21, 19, 65, 62, 16, 11,253,253,253,253,253,
-159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,
-175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,
-191,192,193,194,195,196,197, 75,198,199,200,201,202,203,204,205,
- 79,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,
-221, 51, 81,222, 78,223,224,225,226, 44,227,228,229, 61,230,231,
-232,233,234, 58,235, 66, 59,236,237,238, 60, 69, 63,239,240,241,
- 82, 14, 74,242, 70, 80,243, 72,244, 15, 83, 77, 84, 30, 76, 85,
-245,246,247, 25, 73, 42, 24,248,249,250, 31, 56, 29,251,252,253,
-)
-
-win1250HungarianCharToOrderMap = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253, 28, 40, 54, 45, 32, 50, 49, 38, 39, 53, 36, 41, 34, 35, 47,
- 46, 72, 43, 33, 37, 57, 48, 64, 68, 55, 52,253,253,253,253,253,
-253,  2, 18, 26, 17,  1, 27, 12, 20,  9, 22,  7,  6, 13,  4,  8,
- 23, 67, 10,  5,  3, 21, 19, 65, 62, 16, 11,253,253,253,253,253,
-161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,
-177,178,179,180, 78,181, 69,182,183,184,185,186,187,188,189,190,
-191,192,193,194,195,196,197, 76,198,199,200,201,202,203,204,205,
- 81,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,
-221, 51, 83,222, 80,223,224,225,226, 44,227,228,229, 61,230,231,
-232,233,234, 58,235, 66, 59,236,237,238, 60, 70, 63,239,240,241,
- 84, 14, 75,242, 71, 82,243, 73,244, 15, 85, 79, 86, 30, 77, 87,
-245,246,247, 25, 74, 42, 24,248,249,250, 31, 56, 29,251,252,253,
-)
+# Character Mapping Table(s):
+WINDOWS_1250_HUNGARIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 28,  # 'A'
+     66: 40,  # 'B'
+     67: 54,  # 'C'
+     68: 45,  # 'D'
+     69: 32,  # 'E'
+     70: 50,  # 'F'
+     71: 49,  # 'G'
+     72: 38,  # 'H'
+     73: 39,  # 'I'
+     74: 53,  # 'J'
+     75: 36,  # 'K'
+     76: 41,  # 'L'
+     77: 34,  # 'M'
+     78: 35,  # 'N'
+     79: 47,  # 'O'
+     80: 46,  # 'P'
+     81: 72,  # 'Q'
+     82: 43,  # 'R'
+     83: 33,  # 'S'
+     84: 37,  # 'T'
+     85: 57,  # 'U'
+     86: 48,  # 'V'
+     87: 64,  # 'W'
+     88: 68,  # 'X'
+     89: 55,  # 'Y'
+     90: 52,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 2,  # 'a'
+     98: 18,  # 'b'
+     99: 26,  # 'c'
+     100: 17,  # 'd'
+     101: 1,  # 'e'
+     102: 27,  # 'f'
+     103: 12,  # 'g'
+     104: 20,  # 'h'
+     105: 9,  # 'i'
+     106: 22,  # 'j'
+     107: 7,  # 'k'
+     108: 6,  # 'l'
+     109: 13,  # 'm'
+     110: 4,  # 'n'
+     111: 8,  # 'o'
+     112: 23,  # 'p'
+     113: 67,  # 'q'
+     114: 10,  # 'r'
+     115: 5,  # 's'
+     116: 3,  # 't'
+     117: 21,  # 'u'
+     118: 19,  # 'v'
+     119: 65,  # 'w'
+     120: 62,  # 'x'
+     121: 16,  # 'y'
+     122: 11,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 161,  # '€'
+     129: 162,  # None
+     130: 163,  # '‚'
+     131: 164,  # None
+     132: 165,  # '„'
+     133: 166,  # '…'
+     134: 167,  # '†'
+     135: 168,  # '‡'
+     136: 169,  # None
+     137: 170,  # '‰'
+     138: 171,  # 'Š'
+     139: 172,  # '‹'
+     140: 173,  # 'Ś'
+     141: 174,  # 'Ť'
+     142: 175,  # 'Ž'
+     143: 176,  # 'Ź'
+     144: 177,  # None
+     145: 178,  # '‘'
+     146: 179,  # '’'
+     147: 180,  # '“'
+     148: 78,  # '”'
+     149: 181,  # '•'
+     150: 69,  # '–'
+     151: 182,  # '—'
+     152: 183,  # None
+     153: 184,  # '™'
+     154: 185,  # 'š'
+     155: 186,  # '›'
+     156: 187,  # 'ś'
+     157: 188,  # 'ť'
+     158: 189,  # 'ž'
+     159: 190,  # 'ź'
+     160: 191,  # '\xa0'
+     161: 192,  # 'ˇ'
+     162: 193,  # '˘'
+     163: 194,  # 'Ł'
+     164: 195,  # '¤'
+     165: 196,  # 'Ą'
+     166: 197,  # '¦'
+     167: 76,  # '§'
+     168: 198,  # '¨'
+     169: 199,  # '©'
+     170: 200,  # 'Ş'
+     171: 201,  # '«'
+     172: 202,  # '¬'
+     173: 203,  # '\xad'
+     174: 204,  # '®'
+     175: 205,  # 'Ż'
+     176: 81,  # '°'
+     177: 206,  # '±'
+     178: 207,  # '˛'
+     179: 208,  # 'ł'
+     180: 209,  # '´'
+     181: 210,  # 'µ'
+     182: 211,  # '¶'
+     183: 212,  # '·'
+     184: 213,  # '¸'
+     185: 214,  # 'ą'
+     186: 215,  # 'ş'
+     187: 216,  # '»'
+     188: 217,  # 'Ľ'
+     189: 218,  # '˝'
+     190: 219,  # 'ľ'
+     191: 220,  # 'ż'
+     192: 221,  # 'Ŕ'
+     193: 51,  # 'Á'
+     194: 83,  # 'Â'
+     195: 222,  # 'Ă'
+     196: 80,  # 'Ä'
+     197: 223,  # 'Ĺ'
+     198: 224,  # 'Ć'
+     199: 225,  # 'Ç'
+     200: 226,  # 'Č'
+     201: 44,  # 'É'
+     202: 227,  # 'Ę'
+     203: 228,  # 'Ë'
+     204: 229,  # 'Ě'
+     205: 61,  # 'Í'
+     206: 230,  # 'Î'
+     207: 231,  # 'Ď'
+     208: 232,  # 'Đ'
+     209: 233,  # 'Ń'
+     210: 234,  # 'Ň'
+     211: 58,  # 'Ó'
+     212: 235,  # 'Ô'
+     213: 66,  # 'Ő'
+     214: 59,  # 'Ö'
+     215: 236,  # '×'
+     216: 237,  # 'Ř'
+     217: 238,  # 'Ů'
+     218: 60,  # 'Ú'
+     219: 70,  # 'Ű'
+     220: 63,  # 'Ü'
+     221: 239,  # 'Ý'
+     222: 240,  # 'Ţ'
+     223: 241,  # 'ß'
+     224: 84,  # 'ŕ'
+     225: 14,  # 'á'
+     226: 75,  # 'â'
+     227: 242,  # 'ă'
+     228: 71,  # 'ä'
+     229: 82,  # 'ĺ'
+     230: 243,  # 'ć'
+     231: 73,  # 'ç'
+     232: 244,  # 'č'
+     233: 15,  # 'é'
+     234: 85,  # 'ę'
+     235: 79,  # 'ë'
+     236: 86,  # 'ě'
+     237: 30,  # 'í'
+     238: 77,  # 'î'
+     239: 87,  # 'ď'
+     240: 245,  # 'đ'
+     241: 246,  # 'ń'
+     242: 247,  # 'ň'
+     243: 25,  # 'ó'
+     244: 74,  # 'ô'
+     245: 42,  # 'ő'
+     246: 24,  # 'ö'
+     247: 248,  # '÷'
+     248: 249,  # 'ř'
+     249: 250,  # 'ů'
+     250: 31,  # 'ú'
+     251: 56,  # 'ű'
+     252: 29,  # 'ü'
+     253: 251,  # 'ý'
+     254: 252,  # 'ţ'
+     255: 253,  # '˙'
+}
 
-# Model Table:
-# total sequences: 100%
-# first 512 sequences: 94.7368%
-# first 1024 sequences:5.2623%
-# rest  sequences:     0.8894%
-# negative sequences:  0.0009%
-HungarianLangModel = (
-0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,2,3,3,1,1,2,2,2,2,2,1,2,
-3,2,2,3,3,3,3,3,2,3,3,3,3,3,3,1,2,3,3,3,3,2,3,3,1,1,3,3,0,1,1,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,
-3,2,1,3,3,3,3,3,2,3,3,3,3,3,1,1,2,3,3,3,3,3,3,3,1,1,3,2,0,1,1,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,1,1,2,3,3,3,1,3,3,3,3,3,1,3,3,2,2,0,3,2,3,
-0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,3,3,3,2,3,3,2,3,3,3,3,3,2,3,3,2,2,3,2,3,2,0,3,2,2,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,
-3,3,3,3,3,3,2,3,3,3,3,3,2,3,3,3,1,2,3,2,2,3,1,2,3,3,2,2,0,3,3,3,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,3,2,3,3,3,3,2,3,3,3,3,0,2,3,2,
-0,0,0,1,1,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,1,1,1,3,3,2,1,3,2,2,3,2,1,3,2,2,1,0,3,3,1,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,2,2,3,3,3,3,3,1,2,3,3,3,3,1,2,1,3,3,3,3,2,2,3,1,1,3,2,0,1,1,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,2,1,3,3,3,3,3,2,2,1,3,3,3,0,1,1,2,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,2,3,3,2,3,3,3,2,0,3,2,3,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,0,
-3,3,3,3,3,3,2,3,3,3,2,3,2,3,3,3,1,3,2,2,2,3,1,1,3,3,1,1,0,3,3,2,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,2,3,3,3,2,3,2,3,3,3,2,3,3,3,3,3,1,2,3,2,2,0,2,2,2,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,2,2,2,3,1,3,3,2,2,1,3,3,3,1,1,3,1,2,3,2,3,2,2,2,1,0,2,2,2,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,
-3,1,1,3,3,3,3,3,1,2,3,3,3,3,1,2,1,3,3,3,2,2,3,2,1,0,3,2,0,1,1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,1,3,3,3,3,3,1,2,3,3,3,3,1,1,0,3,3,3,3,0,2,3,0,0,2,1,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,2,2,3,3,2,2,2,2,3,3,0,1,2,3,2,3,2,2,3,2,1,2,0,2,2,2,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,
-3,3,3,3,3,3,1,2,3,3,3,2,1,2,3,3,2,2,2,3,2,3,3,1,3,3,1,1,0,2,3,2,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,1,2,2,2,2,3,3,3,1,1,1,3,3,1,1,3,1,1,3,2,1,2,3,1,1,0,2,2,2,
-0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,2,1,2,1,1,3,3,1,1,1,1,3,3,1,1,2,2,1,2,1,1,2,2,1,1,0,2,2,1,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,1,1,2,1,1,3,3,1,0,1,1,3,3,2,0,1,1,2,3,1,0,2,2,1,0,0,1,3,2,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,2,1,3,3,3,3,3,1,2,3,2,3,3,2,1,1,3,2,3,2,1,2,2,0,1,2,1,0,0,1,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,3,3,2,2,2,2,3,1,2,2,1,1,3,3,0,3,2,1,2,3,2,1,3,3,1,1,0,2,1,3,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,3,3,2,2,2,3,2,3,3,3,2,1,1,3,3,1,1,1,2,2,3,2,3,2,2,2,1,0,2,2,1,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-1,0,0,3,3,3,3,3,0,0,3,3,2,3,0,0,0,2,3,3,1,0,1,2,0,0,1,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,2,3,3,3,3,3,1,2,3,3,2,2,1,1,0,3,3,2,2,1,2,2,1,0,2,2,0,1,1,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,2,2,1,3,1,2,3,3,2,2,1,1,2,2,1,1,1,1,3,2,1,1,1,1,2,1,0,1,2,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
-2,3,3,1,1,1,1,1,3,3,3,0,1,1,3,3,1,1,1,1,1,2,2,0,3,1,1,2,0,2,1,1,
-0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,
-3,1,0,1,2,1,2,2,0,1,2,3,1,2,0,0,0,2,1,1,1,1,1,2,0,0,1,1,0,0,0,0,
-1,2,1,2,2,2,1,2,1,2,0,2,0,2,2,1,1,2,1,1,2,1,1,1,0,1,0,0,0,1,1,0,
-1,1,1,2,3,2,3,3,0,1,2,2,3,1,0,1,0,2,1,2,2,0,1,1,0,0,1,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,3,3,2,2,1,0,0,3,2,3,2,0,0,0,1,1,3,0,0,1,1,0,0,2,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,1,2,2,3,3,1,0,1,3,2,3,1,1,1,0,1,1,1,1,1,3,1,0,0,2,2,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,1,1,2,2,2,1,0,1,2,3,3,2,0,0,0,2,1,1,1,2,1,1,1,0,1,1,1,0,0,0,
-1,2,2,2,2,2,1,1,1,2,0,2,1,1,1,1,1,2,1,1,1,1,1,1,0,1,1,1,0,0,1,1,
-3,2,2,1,0,0,1,1,2,2,0,3,0,1,2,1,1,0,0,1,1,1,0,1,1,1,1,0,2,1,1,1,
-2,2,1,1,1,2,1,2,1,1,1,1,1,1,1,2,1,1,1,2,3,1,1,1,1,1,1,1,1,1,0,1,
-2,3,3,0,1,0,0,0,3,3,1,0,0,1,2,2,1,0,0,0,0,2,0,0,1,1,1,0,2,1,1,1,
-2,1,1,1,1,1,1,2,1,1,0,1,1,0,1,1,1,0,1,2,1,1,0,1,1,1,1,1,1,1,0,1,
-2,3,3,0,1,0,0,0,2,2,0,0,0,0,1,2,2,0,0,0,0,1,0,0,1,1,0,0,2,0,1,0,
-2,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,2,0,1,1,1,1,1,0,1,
-3,2,2,0,1,0,1,0,2,3,2,0,0,1,2,2,1,0,0,1,1,1,0,0,2,1,0,1,2,2,1,1,
-2,1,1,1,1,1,1,2,1,1,1,1,1,1,0,2,1,0,1,1,0,1,1,1,0,1,1,2,1,1,0,1,
-2,2,2,0,0,1,0,0,2,2,1,1,0,0,2,1,1,0,0,0,1,2,0,0,2,1,0,0,2,1,1,1,
-2,1,1,1,1,2,1,2,1,1,1,2,2,1,1,2,1,1,1,2,1,1,1,1,1,1,1,1,1,1,0,1,
-1,2,3,0,0,0,1,0,3,2,1,0,0,1,2,1,1,0,0,0,0,2,1,0,1,1,0,0,2,1,2,1,
-1,1,0,0,0,1,0,1,1,1,1,1,2,0,0,1,0,0,0,2,0,0,1,1,1,1,1,1,1,1,0,1,
-3,0,0,2,1,2,2,1,0,0,2,1,2,2,0,0,0,2,1,1,1,0,1,1,0,0,1,1,2,0,0,0,
-1,2,1,2,2,1,1,2,1,2,0,1,1,1,1,1,1,1,1,1,2,1,1,0,0,1,1,1,1,0,0,1,
-1,3,2,0,0,0,1,0,2,2,2,0,0,0,2,2,1,0,0,0,0,3,1,1,1,1,0,0,2,1,1,1,
-2,1,0,1,1,1,0,1,1,1,1,1,1,1,0,2,1,0,0,1,0,1,1,0,1,1,1,1,1,1,0,1,
-2,3,2,0,0,0,1,0,2,2,0,0,0,0,2,1,1,0,0,0,0,2,1,0,1,1,0,0,2,1,1,0,
-2,1,1,1,1,2,1,2,1,2,0,1,1,1,0,2,1,1,1,2,1,1,1,1,0,1,1,1,1,1,0,1,
-3,1,1,2,2,2,3,2,1,1,2,2,1,1,0,1,0,2,2,1,1,1,1,1,0,0,1,1,0,1,1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,0,0,0,0,0,2,2,0,0,0,0,2,2,1,0,0,0,1,1,0,0,1,2,0,0,2,1,1,1,
-2,2,1,1,1,2,1,2,1,1,0,1,1,1,1,2,1,1,1,2,1,1,1,1,0,1,2,1,1,1,0,1,
-1,0,0,1,2,3,2,1,0,0,2,0,1,1,0,0,0,1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,
-1,2,1,2,1,2,1,1,1,2,0,2,1,1,1,0,1,2,0,0,1,1,1,0,0,0,0,0,0,0,0,0,
-2,3,2,0,0,0,0,0,1,1,2,1,0,0,1,1,1,0,0,0,0,2,0,0,1,1,0,0,2,1,1,1,
-2,1,1,1,1,1,1,2,1,0,1,1,1,1,0,2,1,1,1,1,1,1,0,1,0,1,1,1,1,1,0,1,
-1,2,2,0,1,1,1,0,2,2,2,0,0,0,3,2,1,0,0,0,1,1,0,0,1,1,0,1,1,1,0,0,
-1,1,0,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,1,0,0,1,1,1,0,1,0,1,
-2,1,0,2,1,1,2,2,1,1,2,1,1,1,0,0,0,1,1,0,1,1,1,1,0,0,1,1,1,0,0,0,
-1,2,2,2,2,2,1,1,1,2,0,2,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0,0,0,1,0,
-1,2,3,0,0,0,1,0,2,2,0,0,0,0,2,2,0,0,0,0,0,1,0,0,1,0,0,0,2,0,1,0,
-2,1,1,1,1,1,0,2,0,0,0,1,2,1,1,1,1,0,1,2,0,1,0,1,0,1,1,1,0,1,0,1,
-2,2,2,0,0,0,1,0,2,1,2,0,0,0,1,1,2,0,0,0,0,1,0,0,1,1,0,0,2,1,0,1,
-2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,0,1,1,1,1,1,0,1,
-1,2,2,0,0,0,1,0,2,2,2,0,0,0,1,1,0,0,0,0,0,1,1,0,2,0,0,1,1,1,0,1,
-1,0,1,1,1,1,1,1,0,1,1,1,1,0,0,1,0,0,1,1,0,1,0,1,1,1,1,1,0,0,0,1,
-1,0,0,1,0,1,2,1,0,0,1,1,1,2,0,0,0,1,1,0,1,0,1,1,0,0,1,0,0,0,0,0,
-0,2,1,2,1,1,1,1,1,2,0,2,0,1,1,0,1,2,1,0,1,1,1,0,0,0,0,0,0,1,0,0,
-2,1,1,0,1,2,0,0,1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,2,1,0,1,
-2,2,1,1,1,1,1,2,1,1,0,1,1,1,1,2,1,1,1,2,1,1,0,1,0,1,1,1,1,1,0,1,
-1,2,2,0,0,0,0,0,1,1,0,0,0,0,2,1,0,0,0,0,0,2,0,0,2,2,0,0,2,0,0,1,
-2,1,1,1,1,1,1,1,0,1,1,0,1,1,0,1,0,0,0,1,1,1,1,0,0,1,1,1,1,0,0,1,
-1,1,2,0,0,3,1,0,2,1,1,1,0,0,1,1,1,0,0,0,1,1,0,0,0,1,0,0,1,0,1,0,
-1,2,1,0,1,1,1,2,1,1,0,1,1,1,1,1,0,0,0,1,1,1,1,1,0,1,0,0,0,1,0,0,
-2,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,2,0,0,0,
-2,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,1,1,2,1,1,0,0,1,1,1,1,1,0,1,
-2,1,1,1,2,1,1,1,0,1,1,2,1,0,0,0,0,1,1,1,1,0,1,0,0,0,0,1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,1,0,1,1,1,1,1,0,0,1,1,2,1,0,0,0,1,1,0,0,0,1,1,0,0,1,0,1,0,0,0,
-1,2,1,1,1,1,1,1,1,1,0,1,0,1,1,1,1,1,1,0,1,1,1,0,0,0,0,0,0,1,0,0,
-2,0,0,0,1,1,1,1,0,0,1,1,0,0,0,0,0,1,1,1,2,0,0,1,0,0,1,0,1,0,0,0,
-0,1,1,1,1,1,1,1,1,2,0,1,1,1,1,0,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,
-1,0,0,1,1,1,1,1,0,0,2,1,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,
-0,1,1,1,1,1,1,0,1,1,0,1,0,1,1,0,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,
-1,0,0,1,1,1,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-0,1,1,1,1,1,0,0,1,1,0,1,0,1,0,0,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,
-0,0,0,1,0,0,0,0,0,0,1,1,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,1,1,1,0,1,0,0,1,1,0,1,0,1,1,0,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,
-2,1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,0,0,1,0,0,1,0,1,0,1,1,1,0,0,1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,0,1,1,1,1,0,0,0,1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,
-0,1,1,1,1,1,1,0,1,1,0,1,0,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,
-)
+WINDOWS_1250_HUNGARIAN_MODEL = SingleByteCharSetModel(charset_name='windows-1250',
+                                                      language='Hungarian',
+                                                      char_to_order_map=WINDOWS_1250_HUNGARIAN_CHAR_TO_ORDER,
+                                                      language_model=HUNGARIAN_LANG_MODEL,
+                                                      typical_positive_ratio=0.947368,
+                                                      keep_ascii_letters=True,
+                                                      alphabet='ABCDEFGHIJKLMNOPRSTUVZabcdefghijklmnoprstuvzÁÉÍÓÖÚÜáéíóöúüŐőŰű')
 
-Latin2HungarianModel = {
-  'char_to_order_map': Latin2_HungarianCharToOrderMap,
-  'precedence_matrix': HungarianLangModel,
-  'typical_positive_ratio': 0.947368,
-  'keep_english_letter': True,
-  'charset_name': "ISO-8859-2",
-  'language': 'Hungarian',
+ISO_8859_2_HUNGARIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 28,  # 'A'
+     66: 40,  # 'B'
+     67: 54,  # 'C'
+     68: 45,  # 'D'
+     69: 32,  # 'E'
+     70: 50,  # 'F'
+     71: 49,  # 'G'
+     72: 38,  # 'H'
+     73: 39,  # 'I'
+     74: 53,  # 'J'
+     75: 36,  # 'K'
+     76: 41,  # 'L'
+     77: 34,  # 'M'
+     78: 35,  # 'N'
+     79: 47,  # 'O'
+     80: 46,  # 'P'
+     81: 71,  # 'Q'
+     82: 43,  # 'R'
+     83: 33,  # 'S'
+     84: 37,  # 'T'
+     85: 57,  # 'U'
+     86: 48,  # 'V'
+     87: 64,  # 'W'
+     88: 68,  # 'X'
+     89: 55,  # 'Y'
+     90: 52,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 2,  # 'a'
+     98: 18,  # 'b'
+     99: 26,  # 'c'
+     100: 17,  # 'd'
+     101: 1,  # 'e'
+     102: 27,  # 'f'
+     103: 12,  # 'g'
+     104: 20,  # 'h'
+     105: 9,  # 'i'
+     106: 22,  # 'j'
+     107: 7,  # 'k'
+     108: 6,  # 'l'
+     109: 13,  # 'm'
+     110: 4,  # 'n'
+     111: 8,  # 'o'
+     112: 23,  # 'p'
+     113: 67,  # 'q'
+     114: 10,  # 'r'
+     115: 5,  # 's'
+     116: 3,  # 't'
+     117: 21,  # 'u'
+     118: 19,  # 'v'
+     119: 65,  # 'w'
+     120: 62,  # 'x'
+     121: 16,  # 'y'
+     122: 11,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 159,  # '\x80'
+     129: 160,  # '\x81'
+     130: 161,  # '\x82'
+     131: 162,  # '\x83'
+     132: 163,  # '\x84'
+     133: 164,  # '\x85'
+     134: 165,  # '\x86'
+     135: 166,  # '\x87'
+     136: 167,  # '\x88'
+     137: 168,  # '\x89'
+     138: 169,  # '\x8a'
+     139: 170,  # '\x8b'
+     140: 171,  # '\x8c'
+     141: 172,  # '\x8d'
+     142: 173,  # '\x8e'
+     143: 174,  # '\x8f'
+     144: 175,  # '\x90'
+     145: 176,  # '\x91'
+     146: 177,  # '\x92'
+     147: 178,  # '\x93'
+     148: 179,  # '\x94'
+     149: 180,  # '\x95'
+     150: 181,  # '\x96'
+     151: 182,  # '\x97'
+     152: 183,  # '\x98'
+     153: 184,  # '\x99'
+     154: 185,  # '\x9a'
+     155: 186,  # '\x9b'
+     156: 187,  # '\x9c'
+     157: 188,  # '\x9d'
+     158: 189,  # '\x9e'
+     159: 190,  # '\x9f'
+     160: 191,  # '\xa0'
+     161: 192,  # 'Ą'
+     162: 193,  # '˘'
+     163: 194,  # 'Ł'
+     164: 195,  # '¤'
+     165: 196,  # 'Ľ'
+     166: 197,  # 'Ś'
+     167: 75,  # '§'
+     168: 198,  # '¨'
+     169: 199,  # 'Š'
+     170: 200,  # 'Ş'
+     171: 201,  # 'Ť'
+     172: 202,  # 'Ź'
+     173: 203,  # '\xad'
+     174: 204,  # 'Ž'
+     175: 205,  # 'Ż'
+     176: 79,  # '°'
+     177: 206,  # 'ą'
+     178: 207,  # '˛'
+     179: 208,  # 'ł'
+     180: 209,  # '´'
+     181: 210,  # 'ľ'
+     182: 211,  # 'ś'
+     183: 212,  # 'ˇ'
+     184: 213,  # '¸'
+     185: 214,  # 'š'
+     186: 215,  # 'ş'
+     187: 216,  # 'ť'
+     188: 217,  # 'ź'
+     189: 218,  # '˝'
+     190: 219,  # 'ž'
+     191: 220,  # 'ż'
+     192: 221,  # 'Ŕ'
+     193: 51,  # 'Á'
+     194: 81,  # 'Â'
+     195: 222,  # 'Ă'
+     196: 78,  # 'Ä'
+     197: 223,  # 'Ĺ'
+     198: 224,  # 'Ć'
+     199: 225,  # 'Ç'
+     200: 226,  # 'Č'
+     201: 44,  # 'É'
+     202: 227,  # 'Ę'
+     203: 228,  # 'Ë'
+     204: 229,  # 'Ě'
+     205: 61,  # 'Í'
+     206: 230,  # 'Î'
+     207: 231,  # 'Ď'
+     208: 232,  # 'Đ'
+     209: 233,  # 'Ń'
+     210: 234,  # 'Ň'
+     211: 58,  # 'Ó'
+     212: 235,  # 'Ô'
+     213: 66,  # 'Ő'
+     214: 59,  # 'Ö'
+     215: 236,  # '×'
+     216: 237,  # 'Ř'
+     217: 238,  # 'Ů'
+     218: 60,  # 'Ú'
+     219: 69,  # 'Ű'
+     220: 63,  # 'Ü'
+     221: 239,  # 'Ý'
+     222: 240,  # 'Ţ'
+     223: 241,  # 'ß'
+     224: 82,  # 'ŕ'
+     225: 14,  # 'á'
+     226: 74,  # 'â'
+     227: 242,  # 'ă'
+     228: 70,  # 'ä'
+     229: 80,  # 'ĺ'
+     230: 243,  # 'ć'
+     231: 72,  # 'ç'
+     232: 244,  # 'č'
+     233: 15,  # 'é'
+     234: 83,  # 'ę'
+     235: 77,  # 'ë'
+     236: 84,  # 'ě'
+     237: 30,  # 'í'
+     238: 76,  # 'î'
+     239: 85,  # 'ď'
+     240: 245,  # 'đ'
+     241: 246,  # 'ń'
+     242: 247,  # 'ň'
+     243: 25,  # 'ó'
+     244: 73,  # 'ô'
+     245: 42,  # 'ő'
+     246: 24,  # 'ö'
+     247: 248,  # '÷'
+     248: 249,  # 'ř'
+     249: 250,  # 'ů'
+     250: 31,  # 'ú'
+     251: 56,  # 'ű'
+     252: 29,  # 'ü'
+     253: 251,  # 'ý'
+     254: 252,  # 'ţ'
+     255: 253,  # '˙'
 }
 
-Win1250HungarianModel = {
-  'char_to_order_map': win1250HungarianCharToOrderMap,
-  'precedence_matrix': HungarianLangModel,
-  'typical_positive_ratio': 0.947368,
-  'keep_english_letter': True,
-  'charset_name': "windows-1250",
-  'language': 'Hungarian',
-}
+ISO_8859_2_HUNGARIAN_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-2',
+                                                    language='Hungarian',
+                                                    char_to_order_map=ISO_8859_2_HUNGARIAN_CHAR_TO_ORDER,
+                                                    language_model=HUNGARIAN_LANG_MODEL,
+                                                    typical_positive_ratio=0.947368,
+                                                    keep_ascii_letters=True,
+                                                    alphabet='ABCDEFGHIJKLMNOPRSTUVZabcdefghijklmnoprstuvzÁÉÍÓÖÚÜáéíóöúüŐőŰű')
+
diff --git a/src/pip/_vendor/chardet/langrussianmodel.py b/src/pip/_vendor/chardet/langrussianmodel.py
new file mode 100644
index 00000000000..5594452b55b
--- /dev/null
+++ b/src/pip/_vendor/chardet/langrussianmodel.py
@@ -0,0 +1,5718 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
+
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+RUSSIAN_LANG_MODEL = {
+    37: {  # 'А'
+        37: 0,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 2,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 1,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 0,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    44: {  # 'Б'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 2,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    33: {  # 'В'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 2,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 0,  # 'ю'
+        16: 1,  # 'я'
+    },
+    46: {  # 'Г'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    41: {  # 'Д'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 2,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 3,  # 'ж'
+        20: 1,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    48: {  # 'Е'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 2,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 2,  # 'Р'
+        32: 2,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 2,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 1,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 1,  # 'р'
+        7: 3,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    56: {  # 'Ж'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 1,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 2,  # 'ю'
+        16: 0,  # 'я'
+    },
+    51: {  # 'З'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 1,  # 'я'
+    },
+    42: {  # 'И'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 2,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 2,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 2,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 1,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    60: {  # 'Й'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 1,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    36: {  # 'К'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 2,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    49: {  # 'Л'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 0,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 0,  # 'м'
+        5: 1,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 2,  # 'ю'
+        16: 1,  # 'я'
+    },
+    38: {  # 'М'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    31: {  # 'Н'
+        37: 2,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 2,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    34: {  # 'О'
+        37: 0,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 2,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 1,  # 'З'
+        42: 1,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 2,  # 'Л'
+        38: 1,  # 'М'
+        31: 2,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 2,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 1,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 1,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    35: {  # 'П'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 2,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 1,  # 'с'
+        6: 1,  # 'т'
+        14: 2,  # 'у'
+        39: 1,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 2,  # 'я'
+    },
+    45: {  # 'Р'
+        37: 2,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 2,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 2,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 2,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 2,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 2,  # 'я'
+    },
+    32: {  # 'С'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 2,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 2,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 2,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    40: {  # 'Т'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 2,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 1,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 1,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    52: {  # 'У'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 1,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 1,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 2,  # 'и'
+        23: 1,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 1,  # 'н'
+        1: 2,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 0,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    53: {  # 'Ф'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 1,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    55: {  # 'Х'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 2,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 0,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    58: {  # 'Ц'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 1,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    50: {  # 'Ч'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 1,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 1,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 1,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 1,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    57: {  # 'Ш'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 1,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 2,  # 'о'
+        15: 2,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    63: {  # 'Щ'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 1,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 1,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 1,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    62: {  # 'Ы'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 1,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 0,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    61: {  # 'Ь'
+        37: 0,  # 'А'
+        44: 1,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 1,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 1,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 1,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 0,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    47: {  # 'Э'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 1,  # 'Й'
+        36: 1,  # 'К'
+        49: 1,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 1,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 2,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    59: {  # 'Ю'
+        37: 1,  # 'А'
+        44: 1,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 1,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 0,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 0,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 1,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    43: {  # 'Я'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 1,  # 'В'
+        46: 1,  # 'Г'
+        41: 0,  # 'Д'
+        48: 1,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 1,  # 'С'
+        40: 1,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 1,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 1,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 1,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 1,  # 'Ю'
+        43: 1,  # 'Я'
+        3: 0,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 0,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 1,  # 'й'
+        11: 1,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 1,  # 'п'
+        9: 1,  # 'р'
+        7: 1,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    3: {  # 'а'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 1,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    21: {  # 'б'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 1,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 2,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 3,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    10: {  # 'в'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    19: {  # 'г'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    13: {  # 'д'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 3,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    2: {  # 'е'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    24: {  # 'ж'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 1,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 0,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    20: {  # 'з'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 1,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    4: {  # 'и'
+        37: 1,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    23: {  # 'й'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 1,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 2,  # 'з'
+        4: 1,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 2,  # 'я'
+    },
+    11: {  # 'к'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 3,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 1,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    8: {  # 'л'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 3,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 1,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 1,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    12: {  # 'м'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    5: {  # 'н'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 3,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 2,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    1: {  # 'о'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 3,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    15: {  # 'п'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 3,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 0,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 1,  # 'ш'
+        29: 1,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 2,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 3,  # 'я'
+    },
+    9: {  # 'р'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 2,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 2,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    7: {  # 'с'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 1,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 2,  # 'ш'
+        29: 1,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    6: {  # 'т'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 2,  # 'щ'
+        54: 2,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 3,  # 'ь'
+        30: 2,  # 'э'
+        27: 2,  # 'ю'
+        16: 3,  # 'я'
+    },
+    14: {  # 'у'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 3,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 2,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 2,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 2,  # 'э'
+        27: 3,  # 'ю'
+        16: 2,  # 'я'
+    },
+    39: {  # 'ф'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 0,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 2,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 1,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 2,  # 'ы'
+        17: 1,  # 'ь'
+        30: 2,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    26: {  # 'х'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 3,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 1,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 1,  # 'п'
+        9: 3,  # 'р'
+        7: 2,  # 'с'
+        6: 2,  # 'т'
+        14: 2,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 1,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 1,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    28: {  # 'ц'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 1,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 2,  # 'к'
+        8: 1,  # 'л'
+        12: 1,  # 'м'
+        5: 1,  # 'н'
+        1: 3,  # 'о'
+        15: 0,  # 'п'
+        9: 1,  # 'р'
+        7: 0,  # 'с'
+        6: 1,  # 'т'
+        14: 3,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 1,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 3,  # 'ы'
+        17: 1,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    22: {  # 'ч'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 2,  # 'л'
+        12: 1,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 3,  # 'т'
+        14: 3,  # 'у'
+        39: 1,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 1,  # 'ч'
+        25: 2,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 3,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    25: {  # 'ш'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 1,  # 'б'
+        10: 2,  # 'в'
+        19: 1,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 2,  # 'м'
+        5: 3,  # 'н'
+        1: 3,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 1,  # 'с'
+        6: 2,  # 'т'
+        14: 3,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 1,  # 'ц'
+        22: 1,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 3,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 0,  # 'я'
+    },
+    29: {  # 'щ'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 3,  # 'а'
+        21: 0,  # 'б'
+        10: 1,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 3,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 3,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 1,  # 'м'
+        5: 2,  # 'н'
+        1: 1,  # 'о'
+        15: 0,  # 'п'
+        9: 2,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 2,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 2,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 0,  # 'я'
+    },
+    54: {  # 'ъ'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 0,  # 'б'
+        10: 0,  # 'в'
+        19: 0,  # 'г'
+        13: 0,  # 'д'
+        2: 2,  # 'е'
+        24: 0,  # 'ж'
+        20: 0,  # 'з'
+        4: 0,  # 'и'
+        23: 0,  # 'й'
+        11: 0,  # 'к'
+        8: 0,  # 'л'
+        12: 0,  # 'м'
+        5: 0,  # 'н'
+        1: 0,  # 'о'
+        15: 0,  # 'п'
+        9: 0,  # 'р'
+        7: 0,  # 'с'
+        6: 0,  # 'т'
+        14: 0,  # 'у'
+        39: 0,  # 'ф'
+        26: 0,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 0,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 1,  # 'ю'
+        16: 2,  # 'я'
+    },
+    18: {  # 'ы'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 3,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 2,  # 'и'
+        23: 3,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 1,  # 'о'
+        15: 3,  # 'п'
+        9: 3,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 0,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 3,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 0,  # 'ю'
+        16: 2,  # 'я'
+    },
+    17: {  # 'ь'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 2,  # 'б'
+        10: 2,  # 'в'
+        19: 2,  # 'г'
+        13: 2,  # 'д'
+        2: 3,  # 'е'
+        24: 1,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 0,  # 'й'
+        11: 3,  # 'к'
+        8: 0,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 2,  # 'о'
+        15: 2,  # 'п'
+        9: 1,  # 'р'
+        7: 3,  # 'с'
+        6: 2,  # 'т'
+        14: 0,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 3,  # 'ш'
+        29: 2,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 3,  # 'ю'
+        16: 3,  # 'я'
+    },
+    30: {  # 'э'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 1,  # 'М'
+        31: 1,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 1,  # 'Р'
+        32: 1,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 1,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 1,  # 'б'
+        10: 1,  # 'в'
+        19: 1,  # 'г'
+        13: 2,  # 'д'
+        2: 1,  # 'е'
+        24: 0,  # 'ж'
+        20: 1,  # 'з'
+        4: 0,  # 'и'
+        23: 2,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 2,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 2,  # 'ф'
+        26: 1,  # 'х'
+        28: 0,  # 'ц'
+        22: 0,  # 'ч'
+        25: 1,  # 'ш'
+        29: 0,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 1,  # 'ю'
+        16: 1,  # 'я'
+    },
+    27: {  # 'ю'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 2,  # 'а'
+        21: 3,  # 'б'
+        10: 1,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 1,  # 'е'
+        24: 2,  # 'ж'
+        20: 2,  # 'з'
+        4: 1,  # 'и'
+        23: 1,  # 'й'
+        11: 2,  # 'к'
+        8: 2,  # 'л'
+        12: 2,  # 'м'
+        5: 2,  # 'н'
+        1: 1,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 0,  # 'у'
+        39: 1,  # 'ф'
+        26: 2,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 1,  # 'э'
+        27: 2,  # 'ю'
+        16: 1,  # 'я'
+    },
+    16: {  # 'я'
+        37: 0,  # 'А'
+        44: 0,  # 'Б'
+        33: 0,  # 'В'
+        46: 0,  # 'Г'
+        41: 0,  # 'Д'
+        48: 0,  # 'Е'
+        56: 0,  # 'Ж'
+        51: 0,  # 'З'
+        42: 0,  # 'И'
+        60: 0,  # 'Й'
+        36: 0,  # 'К'
+        49: 0,  # 'Л'
+        38: 0,  # 'М'
+        31: 0,  # 'Н'
+        34: 0,  # 'О'
+        35: 0,  # 'П'
+        45: 0,  # 'Р'
+        32: 0,  # 'С'
+        40: 0,  # 'Т'
+        52: 0,  # 'У'
+        53: 0,  # 'Ф'
+        55: 0,  # 'Х'
+        58: 0,  # 'Ц'
+        50: 0,  # 'Ч'
+        57: 0,  # 'Ш'
+        63: 0,  # 'Щ'
+        62: 0,  # 'Ы'
+        61: 0,  # 'Ь'
+        47: 0,  # 'Э'
+        59: 0,  # 'Ю'
+        43: 0,  # 'Я'
+        3: 0,  # 'а'
+        21: 2,  # 'б'
+        10: 3,  # 'в'
+        19: 2,  # 'г'
+        13: 3,  # 'д'
+        2: 3,  # 'е'
+        24: 3,  # 'ж'
+        20: 3,  # 'з'
+        4: 2,  # 'и'
+        23: 2,  # 'й'
+        11: 3,  # 'к'
+        8: 3,  # 'л'
+        12: 3,  # 'м'
+        5: 3,  # 'н'
+        1: 0,  # 'о'
+        15: 2,  # 'п'
+        9: 2,  # 'р'
+        7: 3,  # 'с'
+        6: 3,  # 'т'
+        14: 1,  # 'у'
+        39: 1,  # 'ф'
+        26: 3,  # 'х'
+        28: 2,  # 'ц'
+        22: 2,  # 'ч'
+        25: 2,  # 'ш'
+        29: 3,  # 'щ'
+        54: 0,  # 'ъ'
+        18: 0,  # 'ы'
+        17: 0,  # 'ь'
+        30: 0,  # 'э'
+        27: 2,  # 'ю'
+        16: 2,  # 'я'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
+# 254: Carriage/Return
+# 253: symbol (punctuation) that does not belong to word
+# 252: 0 - 9
+# 251: Control characters
+
+# Character Mapping Table(s):
+IBM866_RUSSIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 142,  # 'A'
+     66: 143,  # 'B'
+     67: 144,  # 'C'
+     68: 145,  # 'D'
+     69: 146,  # 'E'
+     70: 147,  # 'F'
+     71: 148,  # 'G'
+     72: 149,  # 'H'
+     73: 150,  # 'I'
+     74: 151,  # 'J'
+     75: 152,  # 'K'
+     76: 74,  # 'L'
+     77: 153,  # 'M'
+     78: 75,  # 'N'
+     79: 154,  # 'O'
+     80: 155,  # 'P'
+     81: 156,  # 'Q'
+     82: 157,  # 'R'
+     83: 158,  # 'S'
+     84: 159,  # 'T'
+     85: 160,  # 'U'
+     86: 161,  # 'V'
+     87: 162,  # 'W'
+     88: 163,  # 'X'
+     89: 164,  # 'Y'
+     90: 165,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 71,  # 'a'
+     98: 172,  # 'b'
+     99: 66,  # 'c'
+     100: 173,  # 'd'
+     101: 65,  # 'e'
+     102: 174,  # 'f'
+     103: 76,  # 'g'
+     104: 175,  # 'h'
+     105: 64,  # 'i'
+     106: 176,  # 'j'
+     107: 177,  # 'k'
+     108: 77,  # 'l'
+     109: 72,  # 'm'
+     110: 178,  # 'n'
+     111: 69,  # 'o'
+     112: 67,  # 'p'
+     113: 179,  # 'q'
+     114: 78,  # 'r'
+     115: 73,  # 's'
+     116: 180,  # 't'
+     117: 181,  # 'u'
+     118: 79,  # 'v'
+     119: 182,  # 'w'
+     120: 183,  # 'x'
+     121: 184,  # 'y'
+     122: 185,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 37,  # 'А'
+     129: 44,  # 'Б'
+     130: 33,  # 'В'
+     131: 46,  # 'Г'
+     132: 41,  # 'Д'
+     133: 48,  # 'Е'
+     134: 56,  # 'Ж'
+     135: 51,  # 'З'
+     136: 42,  # 'И'
+     137: 60,  # 'Й'
+     138: 36,  # 'К'
+     139: 49,  # 'Л'
+     140: 38,  # 'М'
+     141: 31,  # 'Н'
+     142: 34,  # 'О'
+     143: 35,  # 'П'
+     144: 45,  # 'Р'
+     145: 32,  # 'С'
+     146: 40,  # 'Т'
+     147: 52,  # 'У'
+     148: 53,  # 'Ф'
+     149: 55,  # 'Х'
+     150: 58,  # 'Ц'
+     151: 50,  # 'Ч'
+     152: 57,  # 'Ш'
+     153: 63,  # 'Щ'
+     154: 70,  # 'Ъ'
+     155: 62,  # 'Ы'
+     156: 61,  # 'Ь'
+     157: 47,  # 'Э'
+     158: 59,  # 'Ю'
+     159: 43,  # 'Я'
+     160: 3,  # 'а'
+     161: 21,  # 'б'
+     162: 10,  # 'в'
+     163: 19,  # 'г'
+     164: 13,  # 'д'
+     165: 2,  # 'е'
+     166: 24,  # 'ж'
+     167: 20,  # 'з'
+     168: 4,  # 'и'
+     169: 23,  # 'й'
+     170: 11,  # 'к'
+     171: 8,  # 'л'
+     172: 12,  # 'м'
+     173: 5,  # 'н'
+     174: 1,  # 'о'
+     175: 15,  # 'п'
+     176: 191,  # '░'
+     177: 192,  # '▒'
+     178: 193,  # '▓'
+     179: 194,  # '│'
+     180: 195,  # '┤'
+     181: 196,  # '╡'
+     182: 197,  # '╢'
+     183: 198,  # '╖'
+     184: 199,  # '╕'
+     185: 200,  # '╣'
+     186: 201,  # '║'
+     187: 202,  # '╗'
+     188: 203,  # '╝'
+     189: 204,  # '╜'
+     190: 205,  # '╛'
+     191: 206,  # '┐'
+     192: 207,  # '└'
+     193: 208,  # '┴'
+     194: 209,  # '┬'
+     195: 210,  # '├'
+     196: 211,  # '─'
+     197: 212,  # '┼'
+     198: 213,  # '╞'
+     199: 214,  # '╟'
+     200: 215,  # '╚'
+     201: 216,  # '╔'
+     202: 217,  # '╩'
+     203: 218,  # '╦'
+     204: 219,  # '╠'
+     205: 220,  # '═'
+     206: 221,  # '╬'
+     207: 222,  # '╧'
+     208: 223,  # '╨'
+     209: 224,  # '╤'
+     210: 225,  # '╥'
+     211: 226,  # '╙'
+     212: 227,  # '╘'
+     213: 228,  # '╒'
+     214: 229,  # '╓'
+     215: 230,  # '╫'
+     216: 231,  # '╪'
+     217: 232,  # '┘'
+     218: 233,  # '┌'
+     219: 234,  # '█'
+     220: 235,  # '▄'
+     221: 236,  # '▌'
+     222: 237,  # '▐'
+     223: 238,  # '▀'
+     224: 9,  # 'р'
+     225: 7,  # 'с'
+     226: 6,  # 'т'
+     227: 14,  # 'у'
+     228: 39,  # 'ф'
+     229: 26,  # 'х'
+     230: 28,  # 'ц'
+     231: 22,  # 'ч'
+     232: 25,  # 'ш'
+     233: 29,  # 'щ'
+     234: 54,  # 'ъ'
+     235: 18,  # 'ы'
+     236: 17,  # 'ь'
+     237: 30,  # 'э'
+     238: 27,  # 'ю'
+     239: 16,  # 'я'
+     240: 239,  # 'Ё'
+     241: 68,  # 'ё'
+     242: 240,  # 'Є'
+     243: 241,  # 'є'
+     244: 242,  # 'Ї'
+     245: 243,  # 'ї'
+     246: 244,  # 'Ў'
+     247: 245,  # 'ў'
+     248: 246,  # '°'
+     249: 247,  # '∙'
+     250: 248,  # '·'
+     251: 249,  # '√'
+     252: 250,  # '№'
+     253: 251,  # '¤'
+     254: 252,  # '■'
+     255: 255,  # '\xa0'
+}
+
+IBM866_RUSSIAN_MODEL = SingleByteCharSetModel(charset_name='IBM866',
+                                              language='Russian',
+                                              char_to_order_map=IBM866_RUSSIAN_CHAR_TO_ORDER,
+                                              language_model=RUSSIAN_LANG_MODEL,
+                                              typical_positive_ratio=0.976601,
+                                              keep_ascii_letters=False,
+                                              alphabet='ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё')
+
+WINDOWS_1251_RUSSIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 142,  # 'A'
+     66: 143,  # 'B'
+     67: 144,  # 'C'
+     68: 145,  # 'D'
+     69: 146,  # 'E'
+     70: 147,  # 'F'
+     71: 148,  # 'G'
+     72: 149,  # 'H'
+     73: 150,  # 'I'
+     74: 151,  # 'J'
+     75: 152,  # 'K'
+     76: 74,  # 'L'
+     77: 153,  # 'M'
+     78: 75,  # 'N'
+     79: 154,  # 'O'
+     80: 155,  # 'P'
+     81: 156,  # 'Q'
+     82: 157,  # 'R'
+     83: 158,  # 'S'
+     84: 159,  # 'T'
+     85: 160,  # 'U'
+     86: 161,  # 'V'
+     87: 162,  # 'W'
+     88: 163,  # 'X'
+     89: 164,  # 'Y'
+     90: 165,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 71,  # 'a'
+     98: 172,  # 'b'
+     99: 66,  # 'c'
+     100: 173,  # 'd'
+     101: 65,  # 'e'
+     102: 174,  # 'f'
+     103: 76,  # 'g'
+     104: 175,  # 'h'
+     105: 64,  # 'i'
+     106: 176,  # 'j'
+     107: 177,  # 'k'
+     108: 77,  # 'l'
+     109: 72,  # 'm'
+     110: 178,  # 'n'
+     111: 69,  # 'o'
+     112: 67,  # 'p'
+     113: 179,  # 'q'
+     114: 78,  # 'r'
+     115: 73,  # 's'
+     116: 180,  # 't'
+     117: 181,  # 'u'
+     118: 79,  # 'v'
+     119: 182,  # 'w'
+     120: 183,  # 'x'
+     121: 184,  # 'y'
+     122: 185,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 191,  # 'Ђ'
+     129: 192,  # 'Ѓ'
+     130: 193,  # '‚'
+     131: 194,  # 'ѓ'
+     132: 195,  # '„'
+     133: 196,  # '…'
+     134: 197,  # '†'
+     135: 198,  # '‡'
+     136: 199,  # '€'
+     137: 200,  # '‰'
+     138: 201,  # 'Љ'
+     139: 202,  # '‹'
+     140: 203,  # 'Њ'
+     141: 204,  # 'Ќ'
+     142: 205,  # 'Ћ'
+     143: 206,  # 'Џ'
+     144: 207,  # 'ђ'
+     145: 208,  # '‘'
+     146: 209,  # '’'
+     147: 210,  # '“'
+     148: 211,  # '”'
+     149: 212,  # '•'
+     150: 213,  # '–'
+     151: 214,  # '—'
+     152: 215,  # None
+     153: 216,  # '™'
+     154: 217,  # 'љ'
+     155: 218,  # '›'
+     156: 219,  # 'њ'
+     157: 220,  # 'ќ'
+     158: 221,  # 'ћ'
+     159: 222,  # 'џ'
+     160: 223,  # '\xa0'
+     161: 224,  # 'Ў'
+     162: 225,  # 'ў'
+     163: 226,  # 'Ј'
+     164: 227,  # '¤'
+     165: 228,  # 'Ґ'
+     166: 229,  # '¦'
+     167: 230,  # '§'
+     168: 231,  # 'Ё'
+     169: 232,  # '©'
+     170: 233,  # 'Є'
+     171: 234,  # '«'
+     172: 235,  # '¬'
+     173: 236,  # '\xad'
+     174: 237,  # '®'
+     175: 238,  # 'Ї'
+     176: 239,  # '°'
+     177: 240,  # '±'
+     178: 241,  # 'І'
+     179: 242,  # 'і'
+     180: 243,  # 'ґ'
+     181: 244,  # 'µ'
+     182: 245,  # '¶'
+     183: 246,  # '·'
+     184: 68,  # 'ё'
+     185: 247,  # '№'
+     186: 248,  # 'є'
+     187: 249,  # '»'
+     188: 250,  # 'ј'
+     189: 251,  # 'Ѕ'
+     190: 252,  # 'ѕ'
+     191: 253,  # 'ї'
+     192: 37,  # 'А'
+     193: 44,  # 'Б'
+     194: 33,  # 'В'
+     195: 46,  # 'Г'
+     196: 41,  # 'Д'
+     197: 48,  # 'Е'
+     198: 56,  # 'Ж'
+     199: 51,  # 'З'
+     200: 42,  # 'И'
+     201: 60,  # 'Й'
+     202: 36,  # 'К'
+     203: 49,  # 'Л'
+     204: 38,  # 'М'
+     205: 31,  # 'Н'
+     206: 34,  # 'О'
+     207: 35,  # 'П'
+     208: 45,  # 'Р'
+     209: 32,  # 'С'
+     210: 40,  # 'Т'
+     211: 52,  # 'У'
+     212: 53,  # 'Ф'
+     213: 55,  # 'Х'
+     214: 58,  # 'Ц'
+     215: 50,  # 'Ч'
+     216: 57,  # 'Ш'
+     217: 63,  # 'Щ'
+     218: 70,  # 'Ъ'
+     219: 62,  # 'Ы'
+     220: 61,  # 'Ь'
+     221: 47,  # 'Э'
+     222: 59,  # 'Ю'
+     223: 43,  # 'Я'
+     224: 3,  # 'а'
+     225: 21,  # 'б'
+     226: 10,  # 'в'
+     227: 19,  # 'г'
+     228: 13,  # 'д'
+     229: 2,  # 'е'
+     230: 24,  # 'ж'
+     231: 20,  # 'з'
+     232: 4,  # 'и'
+     233: 23,  # 'й'
+     234: 11,  # 'к'
+     235: 8,  # 'л'
+     236: 12,  # 'м'
+     237: 5,  # 'н'
+     238: 1,  # 'о'
+     239: 15,  # 'п'
+     240: 9,  # 'р'
+     241: 7,  # 'с'
+     242: 6,  # 'т'
+     243: 14,  # 'у'
+     244: 39,  # 'ф'
+     245: 26,  # 'х'
+     246: 28,  # 'ц'
+     247: 22,  # 'ч'
+     248: 25,  # 'ш'
+     249: 29,  # 'щ'
+     250: 54,  # 'ъ'
+     251: 18,  # 'ы'
+     252: 17,  # 'ь'
+     253: 30,  # 'э'
+     254: 27,  # 'ю'
+     255: 16,  # 'я'
+}
+
+WINDOWS_1251_RUSSIAN_MODEL = SingleByteCharSetModel(charset_name='windows-1251',
+                                                    language='Russian',
+                                                    char_to_order_map=WINDOWS_1251_RUSSIAN_CHAR_TO_ORDER,
+                                                    language_model=RUSSIAN_LANG_MODEL,
+                                                    typical_positive_ratio=0.976601,
+                                                    keep_ascii_letters=False,
+                                                    alphabet='ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё')
+
+IBM855_RUSSIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 142,  # 'A'
+     66: 143,  # 'B'
+     67: 144,  # 'C'
+     68: 145,  # 'D'
+     69: 146,  # 'E'
+     70: 147,  # 'F'
+     71: 148,  # 'G'
+     72: 149,  # 'H'
+     73: 150,  # 'I'
+     74: 151,  # 'J'
+     75: 152,  # 'K'
+     76: 74,  # 'L'
+     77: 153,  # 'M'
+     78: 75,  # 'N'
+     79: 154,  # 'O'
+     80: 155,  # 'P'
+     81: 156,  # 'Q'
+     82: 157,  # 'R'
+     83: 158,  # 'S'
+     84: 159,  # 'T'
+     85: 160,  # 'U'
+     86: 161,  # 'V'
+     87: 162,  # 'W'
+     88: 163,  # 'X'
+     89: 164,  # 'Y'
+     90: 165,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 71,  # 'a'
+     98: 172,  # 'b'
+     99: 66,  # 'c'
+     100: 173,  # 'd'
+     101: 65,  # 'e'
+     102: 174,  # 'f'
+     103: 76,  # 'g'
+     104: 175,  # 'h'
+     105: 64,  # 'i'
+     106: 176,  # 'j'
+     107: 177,  # 'k'
+     108: 77,  # 'l'
+     109: 72,  # 'm'
+     110: 178,  # 'n'
+     111: 69,  # 'o'
+     112: 67,  # 'p'
+     113: 179,  # 'q'
+     114: 78,  # 'r'
+     115: 73,  # 's'
+     116: 180,  # 't'
+     117: 181,  # 'u'
+     118: 79,  # 'v'
+     119: 182,  # 'w'
+     120: 183,  # 'x'
+     121: 184,  # 'y'
+     122: 185,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 191,  # 'ђ'
+     129: 192,  # 'Ђ'
+     130: 193,  # 'ѓ'
+     131: 194,  # 'Ѓ'
+     132: 68,  # 'ё'
+     133: 195,  # 'Ё'
+     134: 196,  # 'є'
+     135: 197,  # 'Є'
+     136: 198,  # 'ѕ'
+     137: 199,  # 'Ѕ'
+     138: 200,  # 'і'
+     139: 201,  # 'І'
+     140: 202,  # 'ї'
+     141: 203,  # 'Ї'
+     142: 204,  # 'ј'
+     143: 205,  # 'Ј'
+     144: 206,  # 'љ'
+     145: 207,  # 'Љ'
+     146: 208,  # 'њ'
+     147: 209,  # 'Њ'
+     148: 210,  # 'ћ'
+     149: 211,  # 'Ћ'
+     150: 212,  # 'ќ'
+     151: 213,  # 'Ќ'
+     152: 214,  # 'ў'
+     153: 215,  # 'Ў'
+     154: 216,  # 'џ'
+     155: 217,  # 'Џ'
+     156: 27,  # 'ю'
+     157: 59,  # 'Ю'
+     158: 54,  # 'ъ'
+     159: 70,  # 'Ъ'
+     160: 3,  # 'а'
+     161: 37,  # 'А'
+     162: 21,  # 'б'
+     163: 44,  # 'Б'
+     164: 28,  # 'ц'
+     165: 58,  # 'Ц'
+     166: 13,  # 'д'
+     167: 41,  # 'Д'
+     168: 2,  # 'е'
+     169: 48,  # 'Е'
+     170: 39,  # 'ф'
+     171: 53,  # 'Ф'
+     172: 19,  # 'г'
+     173: 46,  # 'Г'
+     174: 218,  # '«'
+     175: 219,  # '»'
+     176: 220,  # '░'
+     177: 221,  # '▒'
+     178: 222,  # '▓'
+     179: 223,  # '│'
+     180: 224,  # '┤'
+     181: 26,  # 'х'
+     182: 55,  # 'Х'
+     183: 4,  # 'и'
+     184: 42,  # 'И'
+     185: 225,  # '╣'
+     186: 226,  # '║'
+     187: 227,  # '╗'
+     188: 228,  # '╝'
+     189: 23,  # 'й'
+     190: 60,  # 'Й'
+     191: 229,  # '┐'
+     192: 230,  # '└'
+     193: 231,  # '┴'
+     194: 232,  # '┬'
+     195: 233,  # '├'
+     196: 234,  # '─'
+     197: 235,  # '┼'
+     198: 11,  # 'к'
+     199: 36,  # 'К'
+     200: 236,  # '╚'
+     201: 237,  # '╔'
+     202: 238,  # '╩'
+     203: 239,  # '╦'
+     204: 240,  # '╠'
+     205: 241,  # '═'
+     206: 242,  # '╬'
+     207: 243,  # '¤'
+     208: 8,  # 'л'
+     209: 49,  # 'Л'
+     210: 12,  # 'м'
+     211: 38,  # 'М'
+     212: 5,  # 'н'
+     213: 31,  # 'Н'
+     214: 1,  # 'о'
+     215: 34,  # 'О'
+     216: 15,  # 'п'
+     217: 244,  # '┘'
+     218: 245,  # '┌'
+     219: 246,  # '█'
+     220: 247,  # '▄'
+     221: 35,  # 'П'
+     222: 16,  # 'я'
+     223: 248,  # '▀'
+     224: 43,  # 'Я'
+     225: 9,  # 'р'
+     226: 45,  # 'Р'
+     227: 7,  # 'с'
+     228: 32,  # 'С'
+     229: 6,  # 'т'
+     230: 40,  # 'Т'
+     231: 14,  # 'у'
+     232: 52,  # 'У'
+     233: 24,  # 'ж'
+     234: 56,  # 'Ж'
+     235: 10,  # 'в'
+     236: 33,  # 'В'
+     237: 17,  # 'ь'
+     238: 61,  # 'Ь'
+     239: 249,  # '№'
+     240: 250,  # '\xad'
+     241: 18,  # 'ы'
+     242: 62,  # 'Ы'
+     243: 20,  # 'з'
+     244: 51,  # 'З'
+     245: 25,  # 'ш'
+     246: 57,  # 'Ш'
+     247: 30,  # 'э'
+     248: 47,  # 'Э'
+     249: 29,  # 'щ'
+     250: 63,  # 'Щ'
+     251: 22,  # 'ч'
+     252: 50,  # 'Ч'
+     253: 251,  # '§'
+     254: 252,  # '■'
+     255: 255,  # '\xa0'
+}
+
+IBM855_RUSSIAN_MODEL = SingleByteCharSetModel(charset_name='IBM855',
+                                              language='Russian',
+                                              char_to_order_map=IBM855_RUSSIAN_CHAR_TO_ORDER,
+                                              language_model=RUSSIAN_LANG_MODEL,
+                                              typical_positive_ratio=0.976601,
+                                              keep_ascii_letters=False,
+                                              alphabet='ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё')
+
+KOI8_R_RUSSIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 142,  # 'A'
+     66: 143,  # 'B'
+     67: 144,  # 'C'
+     68: 145,  # 'D'
+     69: 146,  # 'E'
+     70: 147,  # 'F'
+     71: 148,  # 'G'
+     72: 149,  # 'H'
+     73: 150,  # 'I'
+     74: 151,  # 'J'
+     75: 152,  # 'K'
+     76: 74,  # 'L'
+     77: 153,  # 'M'
+     78: 75,  # 'N'
+     79: 154,  # 'O'
+     80: 155,  # 'P'
+     81: 156,  # 'Q'
+     82: 157,  # 'R'
+     83: 158,  # 'S'
+     84: 159,  # 'T'
+     85: 160,  # 'U'
+     86: 161,  # 'V'
+     87: 162,  # 'W'
+     88: 163,  # 'X'
+     89: 164,  # 'Y'
+     90: 165,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 71,  # 'a'
+     98: 172,  # 'b'
+     99: 66,  # 'c'
+     100: 173,  # 'd'
+     101: 65,  # 'e'
+     102: 174,  # 'f'
+     103: 76,  # 'g'
+     104: 175,  # 'h'
+     105: 64,  # 'i'
+     106: 176,  # 'j'
+     107: 177,  # 'k'
+     108: 77,  # 'l'
+     109: 72,  # 'm'
+     110: 178,  # 'n'
+     111: 69,  # 'o'
+     112: 67,  # 'p'
+     113: 179,  # 'q'
+     114: 78,  # 'r'
+     115: 73,  # 's'
+     116: 180,  # 't'
+     117: 181,  # 'u'
+     118: 79,  # 'v'
+     119: 182,  # 'w'
+     120: 183,  # 'x'
+     121: 184,  # 'y'
+     122: 185,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 191,  # '─'
+     129: 192,  # '│'
+     130: 193,  # '┌'
+     131: 194,  # '┐'
+     132: 195,  # '└'
+     133: 196,  # '┘'
+     134: 197,  # '├'
+     135: 198,  # '┤'
+     136: 199,  # '┬'
+     137: 200,  # '┴'
+     138: 201,  # '┼'
+     139: 202,  # '▀'
+     140: 203,  # '▄'
+     141: 204,  # '█'
+     142: 205,  # '▌'
+     143: 206,  # '▐'
+     144: 207,  # '░'
+     145: 208,  # '▒'
+     146: 209,  # '▓'
+     147: 210,  # '⌠'
+     148: 211,  # '■'
+     149: 212,  # '∙'
+     150: 213,  # '√'
+     151: 214,  # '≈'
+     152: 215,  # '≤'
+     153: 216,  # '≥'
+     154: 217,  # '\xa0'
+     155: 218,  # '⌡'
+     156: 219,  # '°'
+     157: 220,  # '²'
+     158: 221,  # '·'
+     159: 222,  # '÷'
+     160: 223,  # '═'
+     161: 224,  # '║'
+     162: 225,  # '╒'
+     163: 68,  # 'ё'
+     164: 226,  # '╓'
+     165: 227,  # '╔'
+     166: 228,  # '╕'
+     167: 229,  # '╖'
+     168: 230,  # '╗'
+     169: 231,  # '╘'
+     170: 232,  # '╙'
+     171: 233,  # '╚'
+     172: 234,  # '╛'
+     173: 235,  # '╜'
+     174: 236,  # '╝'
+     175: 237,  # '╞'
+     176: 238,  # '╟'
+     177: 239,  # '╠'
+     178: 240,  # '╡'
+     179: 241,  # 'Ё'
+     180: 242,  # '╢'
+     181: 243,  # '╣'
+     182: 244,  # '╤'
+     183: 245,  # '╥'
+     184: 246,  # '╦'
+     185: 247,  # '╧'
+     186: 248,  # '╨'
+     187: 249,  # '╩'
+     188: 250,  # '╪'
+     189: 251,  # '╫'
+     190: 252,  # '╬'
+     191: 253,  # '©'
+     192: 27,  # 'ю'
+     193: 3,  # 'а'
+     194: 21,  # 'б'
+     195: 28,  # 'ц'
+     196: 13,  # 'д'
+     197: 2,  # 'е'
+     198: 39,  # 'ф'
+     199: 19,  # 'г'
+     200: 26,  # 'х'
+     201: 4,  # 'и'
+     202: 23,  # 'й'
+     203: 11,  # 'к'
+     204: 8,  # 'л'
+     205: 12,  # 'м'
+     206: 5,  # 'н'
+     207: 1,  # 'о'
+     208: 15,  # 'п'
+     209: 16,  # 'я'
+     210: 9,  # 'р'
+     211: 7,  # 'с'
+     212: 6,  # 'т'
+     213: 14,  # 'у'
+     214: 24,  # 'ж'
+     215: 10,  # 'в'
+     216: 17,  # 'ь'
+     217: 18,  # 'ы'
+     218: 20,  # 'з'
+     219: 25,  # 'ш'
+     220: 30,  # 'э'
+     221: 29,  # 'щ'
+     222: 22,  # 'ч'
+     223: 54,  # 'ъ'
+     224: 59,  # 'Ю'
+     225: 37,  # 'А'
+     226: 44,  # 'Б'
+     227: 58,  # 'Ц'
+     228: 41,  # 'Д'
+     229: 48,  # 'Е'
+     230: 53,  # 'Ф'
+     231: 46,  # 'Г'
+     232: 55,  # 'Х'
+     233: 42,  # 'И'
+     234: 60,  # 'Й'
+     235: 36,  # 'К'
+     236: 49,  # 'Л'
+     237: 38,  # 'М'
+     238: 31,  # 'Н'
+     239: 34,  # 'О'
+     240: 35,  # 'П'
+     241: 43,  # 'Я'
+     242: 45,  # 'Р'
+     243: 32,  # 'С'
+     244: 40,  # 'Т'
+     245: 52,  # 'У'
+     246: 56,  # 'Ж'
+     247: 33,  # 'В'
+     248: 61,  # 'Ь'
+     249: 62,  # 'Ы'
+     250: 51,  # 'З'
+     251: 57,  # 'Ш'
+     252: 47,  # 'Э'
+     253: 63,  # 'Щ'
+     254: 50,  # 'Ч'
+     255: 70,  # 'Ъ'
+}
+
+KOI8_R_RUSSIAN_MODEL = SingleByteCharSetModel(charset_name='KOI8-R',
+                                              language='Russian',
+                                              char_to_order_map=KOI8_R_RUSSIAN_CHAR_TO_ORDER,
+                                              language_model=RUSSIAN_LANG_MODEL,
+                                              typical_positive_ratio=0.976601,
+                                              keep_ascii_letters=False,
+                                              alphabet='ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё')
+
+MACCYRILLIC_RUSSIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 142,  # 'A'
+     66: 143,  # 'B'
+     67: 144,  # 'C'
+     68: 145,  # 'D'
+     69: 146,  # 'E'
+     70: 147,  # 'F'
+     71: 148,  # 'G'
+     72: 149,  # 'H'
+     73: 150,  # 'I'
+     74: 151,  # 'J'
+     75: 152,  # 'K'
+     76: 74,  # 'L'
+     77: 153,  # 'M'
+     78: 75,  # 'N'
+     79: 154,  # 'O'
+     80: 155,  # 'P'
+     81: 156,  # 'Q'
+     82: 157,  # 'R'
+     83: 158,  # 'S'
+     84: 159,  # 'T'
+     85: 160,  # 'U'
+     86: 161,  # 'V'
+     87: 162,  # 'W'
+     88: 163,  # 'X'
+     89: 164,  # 'Y'
+     90: 165,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 71,  # 'a'
+     98: 172,  # 'b'
+     99: 66,  # 'c'
+     100: 173,  # 'd'
+     101: 65,  # 'e'
+     102: 174,  # 'f'
+     103: 76,  # 'g'
+     104: 175,  # 'h'
+     105: 64,  # 'i'
+     106: 176,  # 'j'
+     107: 177,  # 'k'
+     108: 77,  # 'l'
+     109: 72,  # 'm'
+     110: 178,  # 'n'
+     111: 69,  # 'o'
+     112: 67,  # 'p'
+     113: 179,  # 'q'
+     114: 78,  # 'r'
+     115: 73,  # 's'
+     116: 180,  # 't'
+     117: 181,  # 'u'
+     118: 79,  # 'v'
+     119: 182,  # 'w'
+     120: 183,  # 'x'
+     121: 184,  # 'y'
+     122: 185,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 37,  # 'А'
+     129: 44,  # 'Б'
+     130: 33,  # 'В'
+     131: 46,  # 'Г'
+     132: 41,  # 'Д'
+     133: 48,  # 'Е'
+     134: 56,  # 'Ж'
+     135: 51,  # 'З'
+     136: 42,  # 'И'
+     137: 60,  # 'Й'
+     138: 36,  # 'К'
+     139: 49,  # 'Л'
+     140: 38,  # 'М'
+     141: 31,  # 'Н'
+     142: 34,  # 'О'
+     143: 35,  # 'П'
+     144: 45,  # 'Р'
+     145: 32,  # 'С'
+     146: 40,  # 'Т'
+     147: 52,  # 'У'
+     148: 53,  # 'Ф'
+     149: 55,  # 'Х'
+     150: 58,  # 'Ц'
+     151: 50,  # 'Ч'
+     152: 57,  # 'Ш'
+     153: 63,  # 'Щ'
+     154: 70,  # 'Ъ'
+     155: 62,  # 'Ы'
+     156: 61,  # 'Ь'
+     157: 47,  # 'Э'
+     158: 59,  # 'Ю'
+     159: 43,  # 'Я'
+     160: 191,  # '†'
+     161: 192,  # '°'
+     162: 193,  # 'Ґ'
+     163: 194,  # '£'
+     164: 195,  # '§'
+     165: 196,  # '•'
+     166: 197,  # '¶'
+     167: 198,  # 'І'
+     168: 199,  # '®'
+     169: 200,  # '©'
+     170: 201,  # '™'
+     171: 202,  # 'Ђ'
+     172: 203,  # 'ђ'
+     173: 204,  # '≠'
+     174: 205,  # 'Ѓ'
+     175: 206,  # 'ѓ'
+     176: 207,  # '∞'
+     177: 208,  # '±'
+     178: 209,  # '≤'
+     179: 210,  # '≥'
+     180: 211,  # 'і'
+     181: 212,  # 'µ'
+     182: 213,  # 'ґ'
+     183: 214,  # 'Ј'
+     184: 215,  # 'Є'
+     185: 216,  # 'є'
+     186: 217,  # 'Ї'
+     187: 218,  # 'ї'
+     188: 219,  # 'Љ'
+     189: 220,  # 'љ'
+     190: 221,  # 'Њ'
+     191: 222,  # 'њ'
+     192: 223,  # 'ј'
+     193: 224,  # 'Ѕ'
+     194: 225,  # '¬'
+     195: 226,  # '√'
+     196: 227,  # 'ƒ'
+     197: 228,  # '≈'
+     198: 229,  # '∆'
+     199: 230,  # '«'
+     200: 231,  # '»'
+     201: 232,  # '…'
+     202: 233,  # '\xa0'
+     203: 234,  # 'Ћ'
+     204: 235,  # 'ћ'
+     205: 236,  # 'Ќ'
+     206: 237,  # 'ќ'
+     207: 238,  # 'ѕ'
+     208: 239,  # '–'
+     209: 240,  # '—'
+     210: 241,  # '“'
+     211: 242,  # '”'
+     212: 243,  # '‘'
+     213: 244,  # '’'
+     214: 245,  # '÷'
+     215: 246,  # '„'
+     216: 247,  # 'Ў'
+     217: 248,  # 'ў'
+     218: 249,  # 'Џ'
+     219: 250,  # 'џ'
+     220: 251,  # '№'
+     221: 252,  # 'Ё'
+     222: 68,  # 'ё'
+     223: 16,  # 'я'
+     224: 3,  # 'а'
+     225: 21,  # 'б'
+     226: 10,  # 'в'
+     227: 19,  # 'г'
+     228: 13,  # 'д'
+     229: 2,  # 'е'
+     230: 24,  # 'ж'
+     231: 20,  # 'з'
+     232: 4,  # 'и'
+     233: 23,  # 'й'
+     234: 11,  # 'к'
+     235: 8,  # 'л'
+     236: 12,  # 'м'
+     237: 5,  # 'н'
+     238: 1,  # 'о'
+     239: 15,  # 'п'
+     240: 9,  # 'р'
+     241: 7,  # 'с'
+     242: 6,  # 'т'
+     243: 14,  # 'у'
+     244: 39,  # 'ф'
+     245: 26,  # 'х'
+     246: 28,  # 'ц'
+     247: 22,  # 'ч'
+     248: 25,  # 'ш'
+     249: 29,  # 'щ'
+     250: 54,  # 'ъ'
+     251: 18,  # 'ы'
+     252: 17,  # 'ь'
+     253: 30,  # 'э'
+     254: 27,  # 'ю'
+     255: 255,  # '€'
+}
+
+MACCYRILLIC_RUSSIAN_MODEL = SingleByteCharSetModel(charset_name='MacCyrillic',
+                                                   language='Russian',
+                                                   char_to_order_map=MACCYRILLIC_RUSSIAN_CHAR_TO_ORDER,
+                                                   language_model=RUSSIAN_LANG_MODEL,
+                                                   typical_positive_ratio=0.976601,
+                                                   keep_ascii_letters=False,
+                                                   alphabet='ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё')
+
+ISO_8859_5_RUSSIAN_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 142,  # 'A'
+     66: 143,  # 'B'
+     67: 144,  # 'C'
+     68: 145,  # 'D'
+     69: 146,  # 'E'
+     70: 147,  # 'F'
+     71: 148,  # 'G'
+     72: 149,  # 'H'
+     73: 150,  # 'I'
+     74: 151,  # 'J'
+     75: 152,  # 'K'
+     76: 74,  # 'L'
+     77: 153,  # 'M'
+     78: 75,  # 'N'
+     79: 154,  # 'O'
+     80: 155,  # 'P'
+     81: 156,  # 'Q'
+     82: 157,  # 'R'
+     83: 158,  # 'S'
+     84: 159,  # 'T'
+     85: 160,  # 'U'
+     86: 161,  # 'V'
+     87: 162,  # 'W'
+     88: 163,  # 'X'
+     89: 164,  # 'Y'
+     90: 165,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 71,  # 'a'
+     98: 172,  # 'b'
+     99: 66,  # 'c'
+     100: 173,  # 'd'
+     101: 65,  # 'e'
+     102: 174,  # 'f'
+     103: 76,  # 'g'
+     104: 175,  # 'h'
+     105: 64,  # 'i'
+     106: 176,  # 'j'
+     107: 177,  # 'k'
+     108: 77,  # 'l'
+     109: 72,  # 'm'
+     110: 178,  # 'n'
+     111: 69,  # 'o'
+     112: 67,  # 'p'
+     113: 179,  # 'q'
+     114: 78,  # 'r'
+     115: 73,  # 's'
+     116: 180,  # 't'
+     117: 181,  # 'u'
+     118: 79,  # 'v'
+     119: 182,  # 'w'
+     120: 183,  # 'x'
+     121: 184,  # 'y'
+     122: 185,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 191,  # '\x80'
+     129: 192,  # '\x81'
+     130: 193,  # '\x82'
+     131: 194,  # '\x83'
+     132: 195,  # '\x84'
+     133: 196,  # '\x85'
+     134: 197,  # '\x86'
+     135: 198,  # '\x87'
+     136: 199,  # '\x88'
+     137: 200,  # '\x89'
+     138: 201,  # '\x8a'
+     139: 202,  # '\x8b'
+     140: 203,  # '\x8c'
+     141: 204,  # '\x8d'
+     142: 205,  # '\x8e'
+     143: 206,  # '\x8f'
+     144: 207,  # '\x90'
+     145: 208,  # '\x91'
+     146: 209,  # '\x92'
+     147: 210,  # '\x93'
+     148: 211,  # '\x94'
+     149: 212,  # '\x95'
+     150: 213,  # '\x96'
+     151: 214,  # '\x97'
+     152: 215,  # '\x98'
+     153: 216,  # '\x99'
+     154: 217,  # '\x9a'
+     155: 218,  # '\x9b'
+     156: 219,  # '\x9c'
+     157: 220,  # '\x9d'
+     158: 221,  # '\x9e'
+     159: 222,  # '\x9f'
+     160: 223,  # '\xa0'
+     161: 224,  # 'Ё'
+     162: 225,  # 'Ђ'
+     163: 226,  # 'Ѓ'
+     164: 227,  # 'Є'
+     165: 228,  # 'Ѕ'
+     166: 229,  # 'І'
+     167: 230,  # 'Ї'
+     168: 231,  # 'Ј'
+     169: 232,  # 'Љ'
+     170: 233,  # 'Њ'
+     171: 234,  # 'Ћ'
+     172: 235,  # 'Ќ'
+     173: 236,  # '\xad'
+     174: 237,  # 'Ў'
+     175: 238,  # 'Џ'
+     176: 37,  # 'А'
+     177: 44,  # 'Б'
+     178: 33,  # 'В'
+     179: 46,  # 'Г'
+     180: 41,  # 'Д'
+     181: 48,  # 'Е'
+     182: 56,  # 'Ж'
+     183: 51,  # 'З'
+     184: 42,  # 'И'
+     185: 60,  # 'Й'
+     186: 36,  # 'К'
+     187: 49,  # 'Л'
+     188: 38,  # 'М'
+     189: 31,  # 'Н'
+     190: 34,  # 'О'
+     191: 35,  # 'П'
+     192: 45,  # 'Р'
+     193: 32,  # 'С'
+     194: 40,  # 'Т'
+     195: 52,  # 'У'
+     196: 53,  # 'Ф'
+     197: 55,  # 'Х'
+     198: 58,  # 'Ц'
+     199: 50,  # 'Ч'
+     200: 57,  # 'Ш'
+     201: 63,  # 'Щ'
+     202: 70,  # 'Ъ'
+     203: 62,  # 'Ы'
+     204: 61,  # 'Ь'
+     205: 47,  # 'Э'
+     206: 59,  # 'Ю'
+     207: 43,  # 'Я'
+     208: 3,  # 'а'
+     209: 21,  # 'б'
+     210: 10,  # 'в'
+     211: 19,  # 'г'
+     212: 13,  # 'д'
+     213: 2,  # 'е'
+     214: 24,  # 'ж'
+     215: 20,  # 'з'
+     216: 4,  # 'и'
+     217: 23,  # 'й'
+     218: 11,  # 'к'
+     219: 8,  # 'л'
+     220: 12,  # 'м'
+     221: 5,  # 'н'
+     222: 1,  # 'о'
+     223: 15,  # 'п'
+     224: 9,  # 'р'
+     225: 7,  # 'с'
+     226: 6,  # 'т'
+     227: 14,  # 'у'
+     228: 39,  # 'ф'
+     229: 26,  # 'х'
+     230: 28,  # 'ц'
+     231: 22,  # 'ч'
+     232: 25,  # 'ш'
+     233: 29,  # 'щ'
+     234: 54,  # 'ъ'
+     235: 18,  # 'ы'
+     236: 17,  # 'ь'
+     237: 30,  # 'э'
+     238: 27,  # 'ю'
+     239: 16,  # 'я'
+     240: 239,  # '№'
+     241: 68,  # 'ё'
+     242: 240,  # 'ђ'
+     243: 241,  # 'ѓ'
+     244: 242,  # 'є'
+     245: 243,  # 'ѕ'
+     246: 244,  # 'і'
+     247: 245,  # 'ї'
+     248: 246,  # 'ј'
+     249: 247,  # 'љ'
+     250: 248,  # 'њ'
+     251: 249,  # 'ћ'
+     252: 250,  # 'ќ'
+     253: 251,  # '§'
+     254: 252,  # 'ў'
+     255: 255,  # 'џ'
+}
+
+ISO_8859_5_RUSSIAN_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-5',
+                                                  language='Russian',
+                                                  char_to_order_map=ISO_8859_5_RUSSIAN_CHAR_TO_ORDER,
+                                                  language_model=RUSSIAN_LANG_MODEL,
+                                                  typical_positive_ratio=0.976601,
+                                                  keep_ascii_letters=False,
+                                                  alphabet='ЁАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяё')
+
diff --git a/src/pip/_vendor/chardet/langthaimodel.py b/src/pip/_vendor/chardet/langthaimodel.py
index 15f94c2df02..9a37db57388 100644
--- a/src/pip/_vendor/chardet/langthaimodel.py
+++ b/src/pip/_vendor/chardet/langthaimodel.py
@@ -1,199 +1,4383 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-# 255: Control characters that usually does not exist in any text
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
+
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+THAI_LANG_MODEL = {
+    5: {  # 'ก'
+        5: 2,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 3,  # 'ฎ'
+        57: 2,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 2,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 3,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 1,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 2,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 3,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 3,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    30: {  # 'ข'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 2,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 2,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 2,  # '่'
+        7: 3,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    24: {  # 'ค'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 2,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 0,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 2,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 3,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    8: {  # 'ง'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 2,  # 'ง'
+        26: 2,  # 'จ'
+        52: 1,  # 'ฉ'
+        34: 2,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 3,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    26: {  # 'จ'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 0,  # 'ค'
+        8: 2,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    52: {  # 'ฉ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 3,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 1,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 1,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    34: {  # 'ช'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 1,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 1,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    51: {  # 'ซ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 1,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 1,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    47: {  # 'ญ'
+        5: 1,  # 'ก'
+        30: 1,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 3,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 2,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    58: {  # 'ฎ'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 1,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    57: {  # 'ฏ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    49: {  # 'ฐ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    53: {  # 'ฑ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    55: {  # 'ฒ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    43: {  # 'ณ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 3,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 3,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    20: {  # 'ด'
+        5: 2,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 3,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 2,  # 'า'
+        36: 2,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 2,  # 'ๆ'
+        37: 2,  # '็'
+        6: 1,  # '่'
+        7: 3,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    19: {  # 'ต'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 2,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 2,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 0,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 2,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    44: {  # 'ถ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    14: {  # 'ท'
+        5: 1,  # 'ก'
+        30: 1,  # 'ข'
+        24: 3,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 3,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 1,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 2,  # 'ว'
+        42: 3,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 0,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    48: {  # 'ธ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 2,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    3: {  # 'น'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 1,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 2,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 3,  # 'ธ'
+        3: 2,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 1,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 3,  # 'โ'
+        29: 3,  # 'ใ'
+        33: 3,  # 'ไ'
+        50: 2,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    17: {  # 'บ'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 1,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 2,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    25: {  # 'ป'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 1,  # 'ฎ'
+        57: 3,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 1,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 2,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 3,  # '็'
+        6: 1,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    39: {  # 'ผ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 1,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    62: {  # 'ฝ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 1,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 2,  # '่'
+        7: 1,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    31: {  # 'พ'
+        5: 1,  # 'ก'
+        30: 1,  # 'ข'
+        24: 1,  # 'ค'
+        8: 1,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 2,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 1,  # '็'
+        6: 0,  # '่'
+        7: 1,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    54: {  # 'ฟ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 2,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 1,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    45: {  # 'ภ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    9: {  # 'ม'
+        5: 2,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 2,  # 'ร'
+        61: 2,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 1,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 2,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    16: {  # 'ย'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 2,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 3,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 1,  # 'ึ'
+        27: 2,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 2,  # 'ๆ'
+        37: 1,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    2: {  # 'ร'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 2,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 3,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 3,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 2,  # 'น'
+        17: 2,  # 'บ'
+        25: 3,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 3,  # 'เ'
+        28: 3,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 3,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 3,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    61: {  # 'ฤ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    15: {  # 'ล'
+        5: 2,  # 'ก'
+        30: 3,  # 'ข'
+        24: 1,  # 'ค'
+        8: 3,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 3,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 3,  # 'อ'
+        63: 2,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 2,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    12: {  # 'ว'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 1,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 2,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 2,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    42: {  # 'ศ'
+        5: 1,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 1,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 3,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 2,  # 'ู'
+        11: 0,  # 'เ'
+        28: 1,  # 'แ'
+        41: 0,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    46: {  # 'ษ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 2,  # 'ฎ'
+        57: 1,  # 'ฏ'
+        49: 2,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 2,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    18: {  # 'ส'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 2,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 3,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 2,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 1,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 3,  # 'ำ'
+        23: 3,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 2,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 3,  # 'ู'
+        11: 2,  # 'เ'
+        28: 0,  # 'แ'
+        41: 1,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 1,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    21: {  # 'ห'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 1,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 3,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 0,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 0,  # 'ำ'
+        23: 1,  # 'ิ'
+        13: 1,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 1,  # 'ุ'
+        35: 1,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 3,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    4: {  # 'อ'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 2,  # 'ะ'
+        10: 3,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 2,  # 'ิ'
+        13: 3,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 3,  # 'ื'
+        32: 3,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 1,  # '็'
+        6: 2,  # '่'
+        7: 2,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    63: {  # 'ฯ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    22: {  # 'ะ'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 1,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 2,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    10: {  # 'ั'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 3,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 3,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 2,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 3,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    1: {  # 'า'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 3,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 2,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 3,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 3,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    36: {  # 'ำ'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 3,  # 'ค'
+        8: 2,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 3,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    23: {  # 'ิ'
+        5: 3,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 3,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 2,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 3,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 3,  # 'ห'
+        4: 1,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 2,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    13: {  # 'ี'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 1,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 3,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 2,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    40: {  # 'ึ'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 3,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    27: {  # 'ื'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 3,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    32: {  # 'ุ'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 3,  # 'ค'
+        8: 3,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 1,  # 'ฒ'
+        43: 3,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 2,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 1,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 1,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 2,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 1,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 2,  # '้'
+        38: 1,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    35: {  # 'ู'
+        5: 3,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 2,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 2,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 2,  # 'น'
+        17: 0,  # 'บ'
+        25: 3,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 1,  # 'แ'
+        41: 1,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 3,  # '่'
+        7: 3,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    11: {  # 'เ'
+        5: 3,  # 'ก'
+        30: 3,  # 'ข'
+        24: 3,  # 'ค'
+        8: 2,  # 'ง'
+        26: 3,  # 'จ'
+        52: 3,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 2,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 1,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 3,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 3,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 3,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 3,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 3,  # 'ว'
+        42: 2,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    28: {  # 'แ'
+        5: 3,  # 'ก'
+        30: 2,  # 'ข'
+        24: 2,  # 'ค'
+        8: 1,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 3,  # 'ต'
+        44: 2,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 2,  # 'ป'
+        39: 3,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 2,  # 'พ'
+        54: 2,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    41: {  # 'โ'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 1,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 3,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 1,  # 'ภ'
+        9: 1,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 3,  # 'ล'
+        12: 0,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 0,  # 'ห'
+        4: 2,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    29: {  # 'ใ'
+        5: 2,  # 'ก'
+        30: 0,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 3,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 1,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 3,  # 'ส'
+        21: 3,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    33: {  # 'ไ'
+        5: 1,  # 'ก'
+        30: 2,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 3,  # 'ด'
+        19: 1,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 3,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 1,  # 'บ'
+        25: 3,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 2,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 0,  # 'ย'
+        2: 3,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 2,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    50: {  # 'ๆ'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    37: {  # '็'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 2,  # 'ง'
+        26: 3,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 1,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 3,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 1,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 2,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 0,  # 'ห'
+        4: 1,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 1,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    6: {  # '่'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 1,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 1,  # 'ธ'
+        3: 3,  # 'น'
+        17: 1,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 1,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 3,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 2,  # 'ล'
+        12: 3,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 1,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 1,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 3,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 1,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    7: {  # '้'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 2,  # 'ค'
+        8: 3,  # 'ง'
+        26: 2,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 1,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 1,  # 'ด'
+        19: 2,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 2,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 3,  # 'น'
+        17: 2,  # 'บ'
+        25: 2,  # 'ป'
+        39: 2,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 3,  # 'ม'
+        16: 2,  # 'ย'
+        2: 2,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 3,  # 'ว'
+        42: 1,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 2,  # 'ส'
+        21: 2,  # 'ห'
+        4: 3,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 3,  # 'า'
+        36: 2,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 2,  # 'ใ'
+        33: 2,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    38: {  # '์'
+        5: 2,  # 'ก'
+        30: 1,  # 'ข'
+        24: 1,  # 'ค'
+        8: 0,  # 'ง'
+        26: 1,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 1,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 2,  # 'ด'
+        19: 1,  # 'ต'
+        44: 1,  # 'ถ'
+        14: 1,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 1,  # 'น'
+        17: 1,  # 'บ'
+        25: 1,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 1,  # 'พ'
+        54: 1,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 2,  # 'ม'
+        16: 0,  # 'ย'
+        2: 1,  # 'ร'
+        61: 1,  # 'ฤ'
+        15: 1,  # 'ล'
+        12: 1,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 1,  # 'ส'
+        21: 1,  # 'ห'
+        4: 2,  # 'อ'
+        63: 1,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 2,  # 'เ'
+        28: 2,  # 'แ'
+        41: 1,  # 'โ'
+        29: 1,  # 'ใ'
+        33: 1,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 0,  # '๑'
+        59: 0,  # '๒'
+        60: 0,  # '๕'
+    },
+    56: {  # '๑'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 2,  # '๑'
+        59: 1,  # '๒'
+        60: 1,  # '๕'
+    },
+    59: {  # '๒'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 1,  # '๑'
+        59: 1,  # '๒'
+        60: 3,  # '๕'
+    },
+    60: {  # '๕'
+        5: 0,  # 'ก'
+        30: 0,  # 'ข'
+        24: 0,  # 'ค'
+        8: 0,  # 'ง'
+        26: 0,  # 'จ'
+        52: 0,  # 'ฉ'
+        34: 0,  # 'ช'
+        51: 0,  # 'ซ'
+        47: 0,  # 'ญ'
+        58: 0,  # 'ฎ'
+        57: 0,  # 'ฏ'
+        49: 0,  # 'ฐ'
+        53: 0,  # 'ฑ'
+        55: 0,  # 'ฒ'
+        43: 0,  # 'ณ'
+        20: 0,  # 'ด'
+        19: 0,  # 'ต'
+        44: 0,  # 'ถ'
+        14: 0,  # 'ท'
+        48: 0,  # 'ธ'
+        3: 0,  # 'น'
+        17: 0,  # 'บ'
+        25: 0,  # 'ป'
+        39: 0,  # 'ผ'
+        62: 0,  # 'ฝ'
+        31: 0,  # 'พ'
+        54: 0,  # 'ฟ'
+        45: 0,  # 'ภ'
+        9: 0,  # 'ม'
+        16: 0,  # 'ย'
+        2: 0,  # 'ร'
+        61: 0,  # 'ฤ'
+        15: 0,  # 'ล'
+        12: 0,  # 'ว'
+        42: 0,  # 'ศ'
+        46: 0,  # 'ษ'
+        18: 0,  # 'ส'
+        21: 0,  # 'ห'
+        4: 0,  # 'อ'
+        63: 0,  # 'ฯ'
+        22: 0,  # 'ะ'
+        10: 0,  # 'ั'
+        1: 0,  # 'า'
+        36: 0,  # 'ำ'
+        23: 0,  # 'ิ'
+        13: 0,  # 'ี'
+        40: 0,  # 'ึ'
+        27: 0,  # 'ื'
+        32: 0,  # 'ุ'
+        35: 0,  # 'ู'
+        11: 0,  # 'เ'
+        28: 0,  # 'แ'
+        41: 0,  # 'โ'
+        29: 0,  # 'ใ'
+        33: 0,  # 'ไ'
+        50: 0,  # 'ๆ'
+        37: 0,  # '็'
+        6: 0,  # '่'
+        7: 0,  # '้'
+        38: 0,  # '์'
+        56: 2,  # '๑'
+        59: 1,  # '๒'
+        60: 0,  # '๕'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
+# 251: Control characters
 
-# The following result for thai was collected from a limited sample (1M).
-
-# Character Mapping Table:
-TIS620CharToOrderMap = (
-255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
-253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
-252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253,  # 30
-253,182,106,107,100,183,184,185,101, 94,186,187,108,109,110,111,  # 40
-188,189,190, 89, 95,112,113,191,192,193,194,253,253,253,253,253,  # 50
-253, 64, 72, 73,114, 74,115,116,102, 81,201,117, 90,103, 78, 82,  # 60
- 96,202, 91, 79, 84,104,105, 97, 98, 92,203,253,253,253,253,253,  # 70
-209,210,211,212,213, 88,214,215,216,217,218,219,220,118,221,222,
-223,224, 99, 85, 83,225,226,227,228,229,230,231,232,233,234,235,
-236,  5, 30,237, 24,238, 75,  8, 26, 52, 34, 51,119, 47, 58, 57,
- 49, 53, 55, 43, 20, 19, 44, 14, 48,  3, 17, 25, 39, 62, 31, 54,
- 45,  9, 16,  2, 61, 15,239, 12, 42, 46, 18, 21, 76,  4, 66, 63,
- 22, 10,  1, 36, 23, 13, 40, 27, 32, 35, 86,240,241,242,243,244,
- 11, 28, 41, 29, 33,245, 50, 37,  6,  7, 67, 77, 38, 93,246,247,
- 68, 56, 59, 65, 69, 60, 70, 80, 71, 87,248,249,250,251,252,253,
-)
+# Character Mapping Table(s):
+TIS_620_THAI_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 254,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 254,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 253,  # ' '
+     33: 253,  # '!'
+     34: 253,  # '"'
+     35: 253,  # '#'
+     36: 253,  # '$'
+     37: 253,  # '%'
+     38: 253,  # '&'
+     39: 253,  # "'"
+     40: 253,  # '('
+     41: 253,  # ')'
+     42: 253,  # '*'
+     43: 253,  # '+'
+     44: 253,  # ','
+     45: 253,  # '-'
+     46: 253,  # '.'
+     47: 253,  # '/'
+     48: 252,  # '0'
+     49: 252,  # '1'
+     50: 252,  # '2'
+     51: 252,  # '3'
+     52: 252,  # '4'
+     53: 252,  # '5'
+     54: 252,  # '6'
+     55: 252,  # '7'
+     56: 252,  # '8'
+     57: 252,  # '9'
+     58: 253,  # ':'
+     59: 253,  # ';'
+     60: 253,  # '<'
+     61: 253,  # '='
+     62: 253,  # '>'
+     63: 253,  # '?'
+     64: 253,  # '@'
+     65: 182,  # 'A'
+     66: 106,  # 'B'
+     67: 107,  # 'C'
+     68: 100,  # 'D'
+     69: 183,  # 'E'
+     70: 184,  # 'F'
+     71: 185,  # 'G'
+     72: 101,  # 'H'
+     73: 94,  # 'I'
+     74: 186,  # 'J'
+     75: 187,  # 'K'
+     76: 108,  # 'L'
+     77: 109,  # 'M'
+     78: 110,  # 'N'
+     79: 111,  # 'O'
+     80: 188,  # 'P'
+     81: 189,  # 'Q'
+     82: 190,  # 'R'
+     83: 89,  # 'S'
+     84: 95,  # 'T'
+     85: 112,  # 'U'
+     86: 113,  # 'V'
+     87: 191,  # 'W'
+     88: 192,  # 'X'
+     89: 193,  # 'Y'
+     90: 194,  # 'Z'
+     91: 253,  # '['
+     92: 253,  # '\\'
+     93: 253,  # ']'
+     94: 253,  # '^'
+     95: 253,  # '_'
+     96: 253,  # '`'
+     97: 64,  # 'a'
+     98: 72,  # 'b'
+     99: 73,  # 'c'
+     100: 114,  # 'd'
+     101: 74,  # 'e'
+     102: 115,  # 'f'
+     103: 116,  # 'g'
+     104: 102,  # 'h'
+     105: 81,  # 'i'
+     106: 201,  # 'j'
+     107: 117,  # 'k'
+     108: 90,  # 'l'
+     109: 103,  # 'm'
+     110: 78,  # 'n'
+     111: 82,  # 'o'
+     112: 96,  # 'p'
+     113: 202,  # 'q'
+     114: 91,  # 'r'
+     115: 79,  # 's'
+     116: 84,  # 't'
+     117: 104,  # 'u'
+     118: 105,  # 'v'
+     119: 97,  # 'w'
+     120: 98,  # 'x'
+     121: 92,  # 'y'
+     122: 203,  # 'z'
+     123: 253,  # '{'
+     124: 253,  # '|'
+     125: 253,  # '}'
+     126: 253,  # '~'
+     127: 253,  # '\x7f'
+     128: 209,  # '\x80'
+     129: 210,  # '\x81'
+     130: 211,  # '\x82'
+     131: 212,  # '\x83'
+     132: 213,  # '\x84'
+     133: 88,  # '\x85'
+     134: 214,  # '\x86'
+     135: 215,  # '\x87'
+     136: 216,  # '\x88'
+     137: 217,  # '\x89'
+     138: 218,  # '\x8a'
+     139: 219,  # '\x8b'
+     140: 220,  # '\x8c'
+     141: 118,  # '\x8d'
+     142: 221,  # '\x8e'
+     143: 222,  # '\x8f'
+     144: 223,  # '\x90'
+     145: 224,  # '\x91'
+     146: 99,  # '\x92'
+     147: 85,  # '\x93'
+     148: 83,  # '\x94'
+     149: 225,  # '\x95'
+     150: 226,  # '\x96'
+     151: 227,  # '\x97'
+     152: 228,  # '\x98'
+     153: 229,  # '\x99'
+     154: 230,  # '\x9a'
+     155: 231,  # '\x9b'
+     156: 232,  # '\x9c'
+     157: 233,  # '\x9d'
+     158: 234,  # '\x9e'
+     159: 235,  # '\x9f'
+     160: 236,  # None
+     161: 5,  # 'ก'
+     162: 30,  # 'ข'
+     163: 237,  # 'ฃ'
+     164: 24,  # 'ค'
+     165: 238,  # 'ฅ'
+     166: 75,  # 'ฆ'
+     167: 8,  # 'ง'
+     168: 26,  # 'จ'
+     169: 52,  # 'ฉ'
+     170: 34,  # 'ช'
+     171: 51,  # 'ซ'
+     172: 119,  # 'ฌ'
+     173: 47,  # 'ญ'
+     174: 58,  # 'ฎ'
+     175: 57,  # 'ฏ'
+     176: 49,  # 'ฐ'
+     177: 53,  # 'ฑ'
+     178: 55,  # 'ฒ'
+     179: 43,  # 'ณ'
+     180: 20,  # 'ด'
+     181: 19,  # 'ต'
+     182: 44,  # 'ถ'
+     183: 14,  # 'ท'
+     184: 48,  # 'ธ'
+     185: 3,  # 'น'
+     186: 17,  # 'บ'
+     187: 25,  # 'ป'
+     188: 39,  # 'ผ'
+     189: 62,  # 'ฝ'
+     190: 31,  # 'พ'
+     191: 54,  # 'ฟ'
+     192: 45,  # 'ภ'
+     193: 9,  # 'ม'
+     194: 16,  # 'ย'
+     195: 2,  # 'ร'
+     196: 61,  # 'ฤ'
+     197: 15,  # 'ล'
+     198: 239,  # 'ฦ'
+     199: 12,  # 'ว'
+     200: 42,  # 'ศ'
+     201: 46,  # 'ษ'
+     202: 18,  # 'ส'
+     203: 21,  # 'ห'
+     204: 76,  # 'ฬ'
+     205: 4,  # 'อ'
+     206: 66,  # 'ฮ'
+     207: 63,  # 'ฯ'
+     208: 22,  # 'ะ'
+     209: 10,  # 'ั'
+     210: 1,  # 'า'
+     211: 36,  # 'ำ'
+     212: 23,  # 'ิ'
+     213: 13,  # 'ี'
+     214: 40,  # 'ึ'
+     215: 27,  # 'ื'
+     216: 32,  # 'ุ'
+     217: 35,  # 'ู'
+     218: 86,  # 'ฺ'
+     219: 240,  # None
+     220: 241,  # None
+     221: 242,  # None
+     222: 243,  # None
+     223: 244,  # '฿'
+     224: 11,  # 'เ'
+     225: 28,  # 'แ'
+     226: 41,  # 'โ'
+     227: 29,  # 'ใ'
+     228: 33,  # 'ไ'
+     229: 245,  # 'ๅ'
+     230: 50,  # 'ๆ'
+     231: 37,  # '็'
+     232: 6,  # '่'
+     233: 7,  # '้'
+     234: 67,  # '๊'
+     235: 77,  # '๋'
+     236: 38,  # '์'
+     237: 93,  # 'ํ'
+     238: 246,  # '๎'
+     239: 247,  # '๏'
+     240: 68,  # '๐'
+     241: 56,  # '๑'
+     242: 59,  # '๒'
+     243: 65,  # '๓'
+     244: 69,  # '๔'
+     245: 60,  # '๕'
+     246: 70,  # '๖'
+     247: 80,  # '๗'
+     248: 71,  # '๘'
+     249: 87,  # '๙'
+     250: 248,  # '๚'
+     251: 249,  # '๛'
+     252: 250,  # None
+     253: 251,  # None
+     254: 252,  # None
+     255: 253,  # None
+}
 
-# Model Table:
-# total sequences: 100%
-# first 512 sequences: 92.6386%
-# first 1024 sequences:7.3177%
-# rest  sequences:     1.0230%
-# negative sequences:  0.0436%
-ThaiLangModel = (
-0,1,3,3,3,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,0,0,3,3,3,0,3,3,3,3,
-0,3,3,0,0,0,1,3,0,3,3,2,3,3,0,1,2,3,3,3,3,0,2,0,2,0,0,3,2,1,2,2,
-3,0,3,3,2,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,3,0,3,2,3,0,2,2,2,3,
-0,2,3,0,0,0,0,1,0,1,2,3,1,1,3,2,2,0,1,1,0,0,1,0,0,0,0,0,0,0,1,1,
-3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,3,3,2,3,2,3,3,2,2,2,
-3,1,2,3,0,3,3,2,2,1,2,3,3,1,2,0,1,3,0,1,0,0,1,0,0,0,0,0,0,0,1,1,
-3,3,2,2,3,3,3,3,1,2,3,3,3,3,3,2,2,2,2,3,3,2,2,3,3,2,2,3,2,3,2,2,
-3,3,1,2,3,1,2,2,3,3,1,0,2,1,0,0,3,1,2,1,0,0,1,0,0,0,0,0,0,1,0,1,
-3,3,3,3,3,3,2,2,3,3,3,3,2,3,2,2,3,3,2,2,3,2,2,2,2,1,1,3,1,2,1,1,
-3,2,1,0,2,1,0,1,0,1,1,0,1,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,
-3,3,3,2,3,2,3,3,2,2,3,2,3,3,2,3,1,1,2,3,2,2,2,3,2,2,2,2,2,1,2,1,
-2,2,1,1,3,3,2,1,0,1,2,2,0,1,3,0,0,0,1,1,0,0,0,0,0,2,3,0,0,2,1,1,
-3,3,2,3,3,2,0,0,3,3,0,3,3,0,2,2,3,1,2,2,1,1,1,0,2,2,2,0,2,2,1,1,
-0,2,1,0,2,0,0,2,0,1,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,
-3,3,2,3,3,2,0,0,3,3,0,2,3,0,2,1,2,2,2,2,1,2,0,0,2,2,2,0,2,2,1,1,
-0,2,1,0,2,0,0,2,0,1,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,
-3,3,2,3,2,3,2,0,2,2,1,3,2,1,3,2,1,2,3,2,2,3,0,2,3,2,2,1,2,2,2,2,
-1,2,2,0,0,0,0,2,0,1,2,0,1,1,1,0,1,0,3,1,1,0,0,0,0,0,0,0,0,0,1,0,
-3,3,2,3,3,2,3,2,2,2,3,2,2,3,2,2,1,2,3,2,2,3,1,3,2,2,2,3,2,2,2,3,
-3,2,1,3,0,1,1,1,0,2,1,1,1,1,1,0,1,0,1,1,0,0,0,0,0,0,0,0,0,2,0,0,
-1,0,0,3,0,3,3,3,3,3,0,0,3,0,2,2,3,3,3,3,3,0,0,0,1,1,3,0,0,0,0,2,
-0,0,1,0,0,0,0,0,0,0,2,3,0,0,0,3,0,2,0,0,0,0,0,3,0,0,0,0,0,0,0,0,
-2,0,3,3,3,3,0,0,2,3,0,0,3,0,3,3,2,3,3,3,3,3,0,0,3,3,3,0,0,0,3,3,
-0,0,3,0,0,0,0,2,0,0,2,1,1,3,0,0,1,0,0,2,3,0,1,0,0,0,0,0,0,0,1,0,
-3,3,3,3,2,3,3,3,3,3,3,3,1,2,1,3,3,2,2,1,2,2,2,3,1,1,2,0,2,1,2,1,
-2,2,1,0,0,0,1,1,0,1,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,
-3,0,2,1,2,3,3,3,0,2,0,2,2,0,2,1,3,2,2,1,2,1,0,0,2,2,1,0,2,1,2,2,
-0,1,1,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,2,1,3,3,1,1,3,0,2,3,1,1,3,2,1,1,2,0,2,2,3,2,1,1,1,1,1,2,
-3,0,0,1,3,1,2,1,2,0,3,0,0,0,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,
-3,3,1,1,3,2,3,3,3,1,3,2,1,3,2,1,3,2,2,2,2,1,3,3,1,2,1,3,1,2,3,0,
-2,1,1,3,2,2,2,1,2,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,
-3,3,2,3,2,3,3,2,3,2,3,2,3,3,2,1,0,3,2,2,2,1,2,2,2,1,2,2,1,2,1,1,
-2,2,2,3,0,1,3,1,1,1,1,0,1,1,0,2,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,2,3,2,2,1,1,3,2,3,2,3,2,0,3,2,2,1,2,0,2,2,2,1,2,2,2,2,1,
-3,2,1,2,2,1,0,2,0,1,0,0,1,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,1,
-3,3,3,3,3,2,3,1,2,3,3,2,2,3,0,1,1,2,0,3,3,2,2,3,0,1,1,3,0,0,0,0,
-3,1,0,3,3,0,2,0,2,1,0,0,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,2,3,2,3,3,0,1,3,1,1,2,1,2,1,1,3,1,1,0,2,3,1,1,1,1,1,1,1,1,
-3,1,1,2,2,2,2,1,1,1,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-3,2,2,1,1,2,1,3,3,2,3,2,2,3,2,2,3,1,2,2,1,2,0,3,2,1,2,2,2,2,2,1,
-3,2,1,2,2,2,1,1,1,1,0,0,1,1,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,1,3,3,0,2,1,0,3,2,0,0,3,1,0,1,1,0,1,0,0,0,0,0,1,
-1,0,0,1,0,3,2,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,2,2,2,3,0,0,1,3,0,3,2,0,3,2,2,3,3,3,3,3,1,0,2,2,2,0,2,2,1,2,
-0,2,3,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-3,0,2,3,1,3,3,2,3,3,0,3,3,0,3,2,2,3,2,3,3,3,0,0,2,2,3,0,1,1,1,3,
-0,0,3,0,0,0,2,2,0,1,3,0,1,2,2,2,3,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,
-3,2,3,3,2,0,3,3,2,2,3,1,3,2,1,3,2,0,1,2,2,0,2,3,2,1,0,3,0,0,0,0,
-3,0,0,2,3,1,3,0,0,3,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,1,3,2,2,2,1,2,0,1,3,1,1,3,1,3,0,0,2,1,1,1,1,2,1,1,1,0,2,1,0,1,
-1,2,0,0,0,3,1,1,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,3,1,0,0,0,1,0,
-3,3,3,3,2,2,2,2,2,1,3,1,1,1,2,0,1,1,2,1,2,1,3,2,0,0,3,1,1,1,1,1,
-3,1,0,2,3,0,0,0,3,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,2,3,0,3,3,0,2,0,0,0,0,0,0,0,3,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,2,3,1,3,0,0,1,2,0,0,2,0,3,3,2,3,3,3,2,3,0,0,2,2,2,0,0,0,2,2,
-0,0,1,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-0,0,0,3,0,2,0,0,0,0,0,0,0,0,0,0,1,2,3,1,3,3,0,0,1,0,3,0,0,0,0,0,
-0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,1,2,3,1,2,3,1,0,3,0,2,2,1,0,2,1,1,2,0,1,0,0,1,1,1,1,0,1,0,0,
-1,0,0,0,0,1,1,0,3,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,2,1,0,1,1,1,3,1,2,2,2,2,2,2,1,1,1,1,0,3,1,0,1,3,1,1,1,1,
-1,1,0,2,0,1,3,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,1,
-3,0,2,2,1,3,3,2,3,3,0,1,1,0,2,2,1,2,1,3,3,1,0,0,3,2,0,0,0,0,2,1,
-0,1,0,0,0,0,1,2,0,1,1,3,1,1,2,2,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,
-0,0,3,0,0,1,0,0,0,3,0,0,3,0,3,1,0,1,1,1,3,2,0,0,0,3,0,0,0,0,2,0,
-0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,
-3,3,1,3,2,1,3,3,1,2,2,0,1,2,1,0,1,2,0,0,0,0,0,3,0,0,0,3,0,0,0,0,
-3,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,1,2,0,3,3,3,2,2,0,1,1,0,1,3,0,0,0,2,2,0,0,0,0,3,1,0,1,0,0,0,
-0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,2,3,1,2,0,0,2,1,0,3,1,0,1,2,0,1,1,1,1,3,0,0,3,1,1,0,2,2,1,1,
-0,2,0,0,0,0,0,1,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,0,3,1,2,0,0,2,2,0,1,2,0,1,0,1,3,1,2,1,0,0,0,2,0,3,0,0,0,1,0,
-0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,1,1,2,2,0,0,0,2,0,2,1,0,1,1,0,1,1,1,2,1,0,0,1,1,1,0,2,1,1,1,
-0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,
-0,0,0,2,0,1,3,1,1,1,1,0,0,0,0,3,2,0,1,0,0,0,1,2,0,0,0,1,0,0,0,0,
-0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,3,3,3,3,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,0,2,3,2,2,0,0,0,1,0,0,0,0,2,3,2,1,2,2,3,0,0,0,2,3,1,0,0,0,1,1,
-0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,
-3,3,2,2,0,1,0,0,0,0,2,0,2,0,1,0,0,0,1,1,0,0,0,2,1,0,1,0,1,1,0,0,
-0,1,0,2,0,0,1,0,3,0,1,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,1,0,0,1,0,0,0,0,0,1,1,2,0,0,0,0,1,0,0,1,3,1,0,0,0,0,1,1,0,0,
-0,1,0,0,0,0,3,0,0,0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,
-3,3,1,1,1,1,2,3,0,0,2,1,1,1,1,1,0,2,1,1,0,0,0,2,1,0,1,2,1,1,0,1,
-2,1,0,3,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,3,1,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,
-0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,2,0,0,0,0,0,0,1,2,1,0,1,1,0,2,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,2,0,0,0,1,3,0,1,0,0,0,2,0,0,0,0,0,0,0,1,2,0,0,0,0,0,
-3,3,0,0,1,1,2,0,0,1,2,1,0,1,1,1,0,1,1,0,0,2,1,1,0,1,0,0,1,1,1,0,
-0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,1,0,0,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,
-2,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,3,0,0,1,1,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-1,1,0,1,2,0,1,2,0,0,1,1,0,2,0,1,0,0,1,0,0,0,0,1,0,0,0,2,0,0,0,0,
-1,0,0,1,0,1,1,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,1,0,0,0,0,0,0,0,1,1,0,1,1,0,2,1,3,0,0,0,0,1,1,0,0,0,0,0,0,0,3,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,1,0,1,0,0,2,0,0,2,0,0,1,1,2,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,
-1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
-1,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,1,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,3,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,0,0,0,
-1,0,0,0,0,0,0,0,0,1,0,0,0,0,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,1,1,0,0,2,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-)
+TIS_620_THAI_MODEL = SingleByteCharSetModel(charset_name='TIS-620',
+                                            language='Thai',
+                                            char_to_order_map=TIS_620_THAI_CHAR_TO_ORDER,
+                                            language_model=THAI_LANG_MODEL,
+                                            typical_positive_ratio=0.926386,
+                                            keep_ascii_letters=False,
+                                            alphabet='กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛')
 
-TIS620ThaiModel = {
-  'char_to_order_map': TIS620CharToOrderMap,
-  'precedence_matrix': ThaiLangModel,
-  'typical_positive_ratio': 0.926386,
-  'keep_english_letter': False,
-  'charset_name': "TIS-620",
-  'language': 'Thai',
-}
diff --git a/src/pip/_vendor/chardet/langturkishmodel.py b/src/pip/_vendor/chardet/langturkishmodel.py
index a427a457398..43f4230aead 100644
--- a/src/pip/_vendor/chardet/langturkishmodel.py
+++ b/src/pip/_vendor/chardet/langturkishmodel.py
@@ -1,193 +1,4383 @@
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#   Özgür Baskın - Turkish Language Model
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
 
-# 255: Control characters that usually does not exist in any text
+from pip._vendor.chardet.sbcharsetprober import SingleByteCharSetModel
+
+
+# 3: Positive
+# 2: Likely
+# 1: Unlikely
+# 0: Negative
+
+TURKISH_LANG_MODEL = {
+    23: {  # 'A'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 1,  # 'i'
+        24: 0,  # 'j'
+        10: 2,  # 'k'
+        5: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    37: {  # 'B'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 2,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    47: {  # 'C'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 2,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    39: {  # 'D'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    29: {  # 'E'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 1,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    52: {  # 'F'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 1,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 1,  # 'c'
+        12: 1,  # 'd'
+        2: 0,  # 'e'
+        18: 1,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 2,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    36: {  # 'G'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 2,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 2,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 1,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 1,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    45: {  # 'H'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 2,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 2,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    53: {  # 'I'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    60: {  # 'J'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 0,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    16: {  # 'K'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 1,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 0,  # 'u'
+        32: 3,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    49: {  # 'L'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 2,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 2,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 1,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    20: {  # 'M'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 0,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    46: {  # 'N'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    42: {  # 'O'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 2,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 2,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    48: {  # 'P'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 2,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    44: {  # 'R'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 1,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 1,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    35: {  # 'S'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 1,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 2,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 2,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    31: {  # 'T'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 2,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 2,  # 'r'
+        8: 0,  # 's'
+        9: 2,  # 't'
+        14: 2,  # 'u'
+        32: 1,  # 'v'
+        57: 1,  # 'w'
+        58: 1,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    51: {  # 'U'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 1,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    38: {  # 'V'
+        23: 1,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 2,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 1,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    62: {  # 'W'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    43: {  # 'Y'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 2,  # 'N'
+        42: 0,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 1,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 1,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    56: {  # 'Z'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 2,  # 'Z'
+        1: 2,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 1,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    1: {  # 'a'
+        23: 3,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 2,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 1,  # 'î'
+        34: 1,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    21: {  # 'b'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 3,  # 'g'
+        25: 1,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 1,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 2,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    28: {  # 'c'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 2,  # 'E'
+        52: 0,  # 'F'
+        36: 2,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 2,  # 'T'
+        51: 2,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 3,  # 'Y'
+        56: 0,  # 'Z'
+        1: 1,  # 'a'
+        21: 1,  # 'b'
+        28: 2,  # 'c'
+        12: 2,  # 'd'
+        2: 1,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 1,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 1,  # 'î'
+        34: 2,  # 'ö'
+        17: 2,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    12: {  # 'd'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    2: {  # 'e'
+        23: 2,  # 'A'
+        37: 0,  # 'B'
+        47: 2,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    18: {  # 'f'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 1,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 1,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    27: {  # 'g'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 2,  # 'r'
+        8: 2,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    25: {  # 'h'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    3: {  # 'i'
+        23: 2,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 1,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 3,  # 'g'
+        25: 1,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 1,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 1,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    24: {  # 'j'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 2,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 2,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    10: {  # 'k'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 2,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 3,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    5: {  # 'l'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 1,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 2,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    13: {  # 'm'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 2,  # 'u'
+        32: 2,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    4: {  # 'n'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 3,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 3,  # 'p'
+        7: 2,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    15: {  # 'o'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 2,  # 'L'
+        20: 0,  # 'M'
+        46: 2,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 2,  # 'İ'
+        6: 3,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    26: {  # 'p'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 2,  # 'r'
+        8: 1,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 1,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    7: {  # 'r'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 1,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 1,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 3,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    8: {  # 's'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 2,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    9: {  # 't'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 2,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 0,  # 'w'
+        58: 2,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    14: {  # 'u'
+        23: 3,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 2,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 3,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 2,  # 'Z'
+        1: 2,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 2,  # 'e'
+        18: 2,  # 'f'
+        27: 3,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 2,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    32: {  # 'v'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 1,  # 'k'
+        5: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 1,  # 'r'
+        8: 2,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    57: {  # 'w'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 1,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 1,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 1,  # 's'
+        9: 0,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 2,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    58: {  # 'x'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 1,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 2,  # 's'
+        9: 1,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    11: {  # 'y'
+        23: 1,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 2,  # 'r'
+        8: 1,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 3,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    22: {  # 'z'
+        23: 2,  # 'A'
+        37: 2,  # 'B'
+        47: 1,  # 'C'
+        39: 2,  # 'D'
+        29: 3,  # 'E'
+        52: 1,  # 'F'
+        36: 2,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 2,  # 'N'
+        42: 2,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 3,  # 'T'
+        51: 2,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 1,  # 'Z'
+        1: 1,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 2,  # 'd'
+        2: 2,  # 'e'
+        18: 3,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 0,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 3,  # 'y'
+        22: 2,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 1,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 2,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 3,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    63: {  # '·'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 1,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    54: {  # 'Ç'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 1,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 0,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 3,  # 'i'
+        24: 0,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 2,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 2,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 0,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 2,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    50: {  # 'Ö'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 2,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 2,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 1,  # 'N'
+        42: 2,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 1,  # 'f'
+        27: 1,  # 'g'
+        25: 1,  # 'h'
+        3: 2,  # 'i'
+        24: 0,  # 'j'
+        10: 2,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 2,  # 'p'
+        7: 3,  # 'r'
+        8: 1,  # 's'
+        9: 2,  # 't'
+        14: 0,  # 'u'
+        32: 1,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 2,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    55: {  # 'Ü'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 1,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 1,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 1,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 1,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 1,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 0,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    59: {  # 'â'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 0,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 2,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 2,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    33: {  # 'ç'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 3,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 0,  # 'Z'
+        1: 0,  # 'a'
+        21: 3,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 2,  # 'f'
+        27: 1,  # 'g'
+        25: 3,  # 'h'
+        3: 3,  # 'i'
+        24: 0,  # 'j'
+        10: 3,  # 'k'
+        5: 0,  # 'l'
+        13: 0,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 2,  # 's'
+        9: 3,  # 't'
+        14: 0,  # 'u'
+        32: 2,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 1,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    61: {  # 'î'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 0,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 0,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 2,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 1,  # 'j'
+        10: 0,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 1,  # 'n'
+        15: 0,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 1,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 1,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 1,  # 'î'
+        34: 0,  # 'ö'
+        17: 0,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 1,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    34: {  # 'ö'
+        23: 0,  # 'A'
+        37: 1,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 1,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 1,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 2,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 2,  # 'h'
+        3: 1,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 2,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 0,  # 'r'
+        8: 3,  # 's'
+        9: 1,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 1,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 1,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    17: {  # 'ü'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 0,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 1,  # 'J'
+        16: 1,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 0,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 0,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 0,  # 'c'
+        12: 1,  # 'd'
+        2: 3,  # 'e'
+        18: 1,  # 'f'
+        27: 2,  # 'g'
+        25: 0,  # 'h'
+        3: 1,  # 'i'
+        24: 1,  # 'j'
+        10: 2,  # 'k'
+        5: 3,  # 'l'
+        13: 2,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 2,  # 'p'
+        7: 2,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 3,  # 'u'
+        32: 1,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 2,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    30: {  # 'ğ'
+        23: 0,  # 'A'
+        37: 2,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 1,  # 'M'
+        46: 2,  # 'N'
+        42: 2,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 0,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 2,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 0,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 2,  # 'e'
+        18: 0,  # 'f'
+        27: 0,  # 'g'
+        25: 0,  # 'h'
+        3: 0,  # 'i'
+        24: 3,  # 'j'
+        10: 1,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 1,  # 'o'
+        26: 0,  # 'p'
+        7: 1,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 2,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 2,  # 'İ'
+        6: 2,  # 'ı'
+        40: 2,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    41: {  # 'İ'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 2,  # 'G'
+        45: 2,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 0,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 0,  # 'Z'
+        1: 1,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 2,  # 'd'
+        2: 1,  # 'e'
+        18: 0,  # 'f'
+        27: 3,  # 'g'
+        25: 2,  # 'h'
+        3: 2,  # 'i'
+        24: 2,  # 'j'
+        10: 2,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        15: 1,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 2,  # 't'
+        14: 0,  # 'u'
+        32: 0,  # 'v'
+        57: 1,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 1,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 1,  # 'ö'
+        17: 1,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+    6: {  # 'ı'
+        23: 2,  # 'A'
+        37: 0,  # 'B'
+        47: 0,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 2,  # 'J'
+        16: 3,  # 'K'
+        49: 0,  # 'L'
+        20: 3,  # 'M'
+        46: 1,  # 'N'
+        42: 0,  # 'O'
+        48: 0,  # 'P'
+        44: 0,  # 'R'
+        35: 0,  # 'S'
+        31: 2,  # 'T'
+        51: 0,  # 'U'
+        38: 0,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 1,  # 'Z'
+        1: 3,  # 'a'
+        21: 2,  # 'b'
+        28: 1,  # 'c'
+        12: 3,  # 'd'
+        2: 3,  # 'e'
+        18: 3,  # 'f'
+        27: 3,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 3,  # 'j'
+        10: 3,  # 'k'
+        5: 3,  # 'l'
+        13: 3,  # 'm'
+        4: 3,  # 'n'
+        15: 0,  # 'o'
+        26: 3,  # 'p'
+        7: 3,  # 'r'
+        8: 3,  # 's'
+        9: 3,  # 't'
+        14: 3,  # 'u'
+        32: 3,  # 'v'
+        57: 1,  # 'w'
+        58: 1,  # 'x'
+        11: 3,  # 'y'
+        22: 0,  # 'z'
+        63: 1,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 2,  # 'ç'
+        61: 0,  # 'î'
+        34: 0,  # 'ö'
+        17: 3,  # 'ü'
+        30: 0,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 3,  # 'ı'
+        40: 0,  # 'Ş'
+        19: 0,  # 'ş'
+    },
+    40: {  # 'Ş'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 1,  # 'D'
+        29: 1,  # 'E'
+        52: 0,  # 'F'
+        36: 1,  # 'G'
+        45: 2,  # 'H'
+        53: 1,  # 'I'
+        60: 0,  # 'J'
+        16: 0,  # 'K'
+        49: 0,  # 'L'
+        20: 2,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 2,  # 'P'
+        44: 2,  # 'R'
+        35: 1,  # 'S'
+        31: 1,  # 'T'
+        51: 0,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 2,  # 'Y'
+        56: 1,  # 'Z'
+        1: 0,  # 'a'
+        21: 2,  # 'b'
+        28: 0,  # 'c'
+        12: 2,  # 'd'
+        2: 0,  # 'e'
+        18: 3,  # 'f'
+        27: 0,  # 'g'
+        25: 2,  # 'h'
+        3: 3,  # 'i'
+        24: 2,  # 'j'
+        10: 1,  # 'k'
+        5: 0,  # 'l'
+        13: 1,  # 'm'
+        4: 3,  # 'n'
+        15: 2,  # 'o'
+        26: 0,  # 'p'
+        7: 3,  # 'r'
+        8: 2,  # 's'
+        9: 2,  # 't'
+        14: 1,  # 'u'
+        32: 3,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 2,  # 'y'
+        22: 0,  # 'z'
+        63: 0,  # '·'
+        54: 0,  # 'Ç'
+        50: 0,  # 'Ö'
+        55: 1,  # 'Ü'
+        59: 0,  # 'â'
+        33: 0,  # 'ç'
+        61: 0,  # 'î'
+        34: 2,  # 'ö'
+        17: 1,  # 'ü'
+        30: 2,  # 'ğ'
+        41: 0,  # 'İ'
+        6: 2,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 2,  # 'ş'
+    },
+    19: {  # 'ş'
+        23: 0,  # 'A'
+        37: 0,  # 'B'
+        47: 1,  # 'C'
+        39: 0,  # 'D'
+        29: 0,  # 'E'
+        52: 2,  # 'F'
+        36: 1,  # 'G'
+        45: 0,  # 'H'
+        53: 0,  # 'I'
+        60: 0,  # 'J'
+        16: 3,  # 'K'
+        49: 2,  # 'L'
+        20: 0,  # 'M'
+        46: 1,  # 'N'
+        42: 1,  # 'O'
+        48: 1,  # 'P'
+        44: 1,  # 'R'
+        35: 1,  # 'S'
+        31: 0,  # 'T'
+        51: 1,  # 'U'
+        38: 1,  # 'V'
+        62: 0,  # 'W'
+        43: 1,  # 'Y'
+        56: 0,  # 'Z'
+        1: 3,  # 'a'
+        21: 1,  # 'b'
+        28: 2,  # 'c'
+        12: 0,  # 'd'
+        2: 3,  # 'e'
+        18: 0,  # 'f'
+        27: 2,  # 'g'
+        25: 1,  # 'h'
+        3: 1,  # 'i'
+        24: 0,  # 'j'
+        10: 2,  # 'k'
+        5: 2,  # 'l'
+        13: 3,  # 'm'
+        4: 0,  # 'n'
+        15: 0,  # 'o'
+        26: 1,  # 'p'
+        7: 3,  # 'r'
+        8: 0,  # 's'
+        9: 0,  # 't'
+        14: 3,  # 'u'
+        32: 0,  # 'v'
+        57: 0,  # 'w'
+        58: 0,  # 'x'
+        11: 0,  # 'y'
+        22: 2,  # 'z'
+        63: 0,  # '·'
+        54: 1,  # 'Ç'
+        50: 2,  # 'Ö'
+        55: 0,  # 'Ü'
+        59: 0,  # 'â'
+        33: 1,  # 'ç'
+        61: 1,  # 'î'
+        34: 2,  # 'ö'
+        17: 0,  # 'ü'
+        30: 1,  # 'ğ'
+        41: 1,  # 'İ'
+        6: 1,  # 'ı'
+        40: 1,  # 'Ş'
+        19: 1,  # 'ş'
+    },
+}
+
+# 255: Undefined characters that did not exist in training text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
+# 251: Control characters
 
-# Character Mapping Table:
-Latin5_TurkishCharToOrderMap = (
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
-255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,
-255, 23, 37, 47, 39, 29, 52, 36, 45, 53, 60, 16, 49, 20, 46, 42,
- 48, 69, 44, 35, 31, 51, 38, 62, 65, 43, 56,255,255,255,255,255,
-255,  1, 21, 28, 12,  2, 18, 27, 25,  3, 24, 10,  5, 13,  4, 15,
- 26, 64,  7,  8,  9, 14, 32, 57, 58, 11, 22,255,255,255,255,255,
-180,179,178,177,176,175,174,173,172,171,170,169,168,167,166,165,
-164,163,162,161,160,159,101,158,157,156,155,154,153,152,151,106,
-150,149,148,147,146,145,144,100,143,142,141,140,139,138,137,136,
- 94, 80, 93,135,105,134,133, 63,132,131,130,129,128,127,126,125,
-124,104, 73, 99, 79, 85,123, 54,122, 98, 92,121,120, 91,103,119,
- 68,118,117, 97,116,115, 50, 90,114,113,112,111, 55, 41, 40, 86,
- 89, 70, 59, 78, 71, 82, 88, 33, 77, 66, 84, 83,110, 75, 61, 96,
- 30, 67,109, 74, 87,102, 34, 95, 81,108, 76, 72, 17,  6, 19,107,
-)
+# Character Mapping Table(s):
+ISO_8859_9_TURKISH_CHAR_TO_ORDER = {
+     0: 255,  # '\x00'
+     1: 255,  # '\x01'
+     2: 255,  # '\x02'
+     3: 255,  # '\x03'
+     4: 255,  # '\x04'
+     5: 255,  # '\x05'
+     6: 255,  # '\x06'
+     7: 255,  # '\x07'
+     8: 255,  # '\x08'
+     9: 255,  # '\t'
+     10: 255,  # '\n'
+     11: 255,  # '\x0b'
+     12: 255,  # '\x0c'
+     13: 255,  # '\r'
+     14: 255,  # '\x0e'
+     15: 255,  # '\x0f'
+     16: 255,  # '\x10'
+     17: 255,  # '\x11'
+     18: 255,  # '\x12'
+     19: 255,  # '\x13'
+     20: 255,  # '\x14'
+     21: 255,  # '\x15'
+     22: 255,  # '\x16'
+     23: 255,  # '\x17'
+     24: 255,  # '\x18'
+     25: 255,  # '\x19'
+     26: 255,  # '\x1a'
+     27: 255,  # '\x1b'
+     28: 255,  # '\x1c'
+     29: 255,  # '\x1d'
+     30: 255,  # '\x1e'
+     31: 255,  # '\x1f'
+     32: 255,  # ' '
+     33: 255,  # '!'
+     34: 255,  # '"'
+     35: 255,  # '#'
+     36: 255,  # '$'
+     37: 255,  # '%'
+     38: 255,  # '&'
+     39: 255,  # "'"
+     40: 255,  # '('
+     41: 255,  # ')'
+     42: 255,  # '*'
+     43: 255,  # '+'
+     44: 255,  # ','
+     45: 255,  # '-'
+     46: 255,  # '.'
+     47: 255,  # '/'
+     48: 255,  # '0'
+     49: 255,  # '1'
+     50: 255,  # '2'
+     51: 255,  # '3'
+     52: 255,  # '4'
+     53: 255,  # '5'
+     54: 255,  # '6'
+     55: 255,  # '7'
+     56: 255,  # '8'
+     57: 255,  # '9'
+     58: 255,  # ':'
+     59: 255,  # ';'
+     60: 255,  # '<'
+     61: 255,  # '='
+     62: 255,  # '>'
+     63: 255,  # '?'
+     64: 255,  # '@'
+     65: 23,  # 'A'
+     66: 37,  # 'B'
+     67: 47,  # 'C'
+     68: 39,  # 'D'
+     69: 29,  # 'E'
+     70: 52,  # 'F'
+     71: 36,  # 'G'
+     72: 45,  # 'H'
+     73: 53,  # 'I'
+     74: 60,  # 'J'
+     75: 16,  # 'K'
+     76: 49,  # 'L'
+     77: 20,  # 'M'
+     78: 46,  # 'N'
+     79: 42,  # 'O'
+     80: 48,  # 'P'
+     81: 69,  # 'Q'
+     82: 44,  # 'R'
+     83: 35,  # 'S'
+     84: 31,  # 'T'
+     85: 51,  # 'U'
+     86: 38,  # 'V'
+     87: 62,  # 'W'
+     88: 65,  # 'X'
+     89: 43,  # 'Y'
+     90: 56,  # 'Z'
+     91: 255,  # '['
+     92: 255,  # '\\'
+     93: 255,  # ']'
+     94: 255,  # '^'
+     95: 255,  # '_'
+     96: 255,  # '`'
+     97: 1,  # 'a'
+     98: 21,  # 'b'
+     99: 28,  # 'c'
+     100: 12,  # 'd'
+     101: 2,  # 'e'
+     102: 18,  # 'f'
+     103: 27,  # 'g'
+     104: 25,  # 'h'
+     105: 3,  # 'i'
+     106: 24,  # 'j'
+     107: 10,  # 'k'
+     108: 5,  # 'l'
+     109: 13,  # 'm'
+     110: 4,  # 'n'
+     111: 15,  # 'o'
+     112: 26,  # 'p'
+     113: 64,  # 'q'
+     114: 7,  # 'r'
+     115: 8,  # 's'
+     116: 9,  # 't'
+     117: 14,  # 'u'
+     118: 32,  # 'v'
+     119: 57,  # 'w'
+     120: 58,  # 'x'
+     121: 11,  # 'y'
+     122: 22,  # 'z'
+     123: 255,  # '{'
+     124: 255,  # '|'
+     125: 255,  # '}'
+     126: 255,  # '~'
+     127: 255,  # '\x7f'
+     128: 180,  # '\x80'
+     129: 179,  # '\x81'
+     130: 178,  # '\x82'
+     131: 177,  # '\x83'
+     132: 176,  # '\x84'
+     133: 175,  # '\x85'
+     134: 174,  # '\x86'
+     135: 173,  # '\x87'
+     136: 172,  # '\x88'
+     137: 171,  # '\x89'
+     138: 170,  # '\x8a'
+     139: 169,  # '\x8b'
+     140: 168,  # '\x8c'
+     141: 167,  # '\x8d'
+     142: 166,  # '\x8e'
+     143: 165,  # '\x8f'
+     144: 164,  # '\x90'
+     145: 163,  # '\x91'
+     146: 162,  # '\x92'
+     147: 161,  # '\x93'
+     148: 160,  # '\x94'
+     149: 159,  # '\x95'
+     150: 101,  # '\x96'
+     151: 158,  # '\x97'
+     152: 157,  # '\x98'
+     153: 156,  # '\x99'
+     154: 155,  # '\x9a'
+     155: 154,  # '\x9b'
+     156: 153,  # '\x9c'
+     157: 152,  # '\x9d'
+     158: 151,  # '\x9e'
+     159: 106,  # '\x9f'
+     160: 150,  # '\xa0'
+     161: 149,  # '¡'
+     162: 148,  # '¢'
+     163: 147,  # '£'
+     164: 146,  # '¤'
+     165: 145,  # '¥'
+     166: 144,  # '¦'
+     167: 100,  # '§'
+     168: 143,  # '¨'
+     169: 142,  # '©'
+     170: 141,  # 'ª'
+     171: 140,  # '«'
+     172: 139,  # '¬'
+     173: 138,  # '\xad'
+     174: 137,  # '®'
+     175: 136,  # '¯'
+     176: 94,  # '°'
+     177: 80,  # '±'
+     178: 93,  # '²'
+     179: 135,  # '³'
+     180: 105,  # '´'
+     181: 134,  # 'µ'
+     182: 133,  # '¶'
+     183: 63,  # '·'
+     184: 132,  # '¸'
+     185: 131,  # '¹'
+     186: 130,  # 'º'
+     187: 129,  # '»'
+     188: 128,  # '¼'
+     189: 127,  # '½'
+     190: 126,  # '¾'
+     191: 125,  # '¿'
+     192: 124,  # 'À'
+     193: 104,  # 'Á'
+     194: 73,  # 'Â'
+     195: 99,  # 'Ã'
+     196: 79,  # 'Ä'
+     197: 85,  # 'Å'
+     198: 123,  # 'Æ'
+     199: 54,  # 'Ç'
+     200: 122,  # 'È'
+     201: 98,  # 'É'
+     202: 92,  # 'Ê'
+     203: 121,  # 'Ë'
+     204: 120,  # 'Ì'
+     205: 91,  # 'Í'
+     206: 103,  # 'Î'
+     207: 119,  # 'Ï'
+     208: 68,  # 'Ğ'
+     209: 118,  # 'Ñ'
+     210: 117,  # 'Ò'
+     211: 97,  # 'Ó'
+     212: 116,  # 'Ô'
+     213: 115,  # 'Õ'
+     214: 50,  # 'Ö'
+     215: 90,  # '×'
+     216: 114,  # 'Ø'
+     217: 113,  # 'Ù'
+     218: 112,  # 'Ú'
+     219: 111,  # 'Û'
+     220: 55,  # 'Ü'
+     221: 41,  # 'İ'
+     222: 40,  # 'Ş'
+     223: 86,  # 'ß'
+     224: 89,  # 'à'
+     225: 70,  # 'á'
+     226: 59,  # 'â'
+     227: 78,  # 'ã'
+     228: 71,  # 'ä'
+     229: 82,  # 'å'
+     230: 88,  # 'æ'
+     231: 33,  # 'ç'
+     232: 77,  # 'è'
+     233: 66,  # 'é'
+     234: 84,  # 'ê'
+     235: 83,  # 'ë'
+     236: 110,  # 'ì'
+     237: 75,  # 'í'
+     238: 61,  # 'î'
+     239: 96,  # 'ï'
+     240: 30,  # 'ğ'
+     241: 67,  # 'ñ'
+     242: 109,  # 'ò'
+     243: 74,  # 'ó'
+     244: 87,  # 'ô'
+     245: 102,  # 'õ'
+     246: 34,  # 'ö'
+     247: 95,  # '÷'
+     248: 81,  # 'ø'
+     249: 108,  # 'ù'
+     250: 76,  # 'ú'
+     251: 72,  # 'û'
+     252: 17,  # 'ü'
+     253: 6,  # 'ı'
+     254: 19,  # 'ş'
+     255: 107,  # 'ÿ'
+}
 
-TurkishLangModel = (
-3,2,3,3,3,1,3,3,3,3,3,3,3,3,2,1,1,3,3,1,3,3,0,3,3,3,3,3,0,3,1,3,
-3,2,1,0,0,1,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,2,2,0,0,1,0,0,1,
-3,2,2,3,3,0,3,3,3,3,3,3,3,2,3,1,0,3,3,1,3,3,0,3,3,3,3,3,0,3,0,3,
-3,1,1,0,1,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,2,2,0,0,0,1,0,1,
-3,3,2,3,3,0,3,3,3,3,3,3,3,2,3,1,1,3,3,0,3,3,1,2,3,3,3,3,0,3,0,3,
-3,1,1,0,0,0,1,0,0,0,0,1,1,0,1,2,1,0,0,0,1,0,0,0,0,2,0,0,0,0,0,1,
-3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,1,3,3,2,0,3,2,1,2,2,1,3,3,0,0,0,2,
-2,2,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,1,0,0,1,
-3,3,3,2,3,3,1,2,3,3,3,3,3,3,3,1,3,2,1,0,3,2,0,1,2,3,3,2,1,0,0,2,
-2,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,2,0,0,0,
-1,0,1,3,3,1,3,3,3,3,3,3,3,1,2,0,0,2,3,0,2,3,0,0,2,2,2,3,0,3,0,1,
-2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,0,3,2,0,2,3,2,3,3,1,0,0,2,
-3,2,0,0,1,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,1,1,1,0,2,0,0,1,
-3,3,3,2,3,3,2,3,3,3,3,2,3,3,3,0,3,3,0,0,2,1,0,0,2,3,2,2,0,0,0,2,
-2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,1,0,2,0,0,1,
-3,3,3,2,3,3,3,3,3,3,3,2,3,3,3,0,3,2,0,1,3,2,1,1,3,2,3,2,1,0,0,2,
-2,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,
-3,3,3,2,3,3,3,3,3,3,3,2,3,3,3,0,3,2,2,0,2,3,0,0,2,2,2,2,0,0,0,2,
-3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,1,0,0,0,
-3,3,3,3,3,3,3,2,2,2,2,3,2,3,3,0,3,3,1,1,2,2,0,0,2,2,3,2,0,0,1,3,
-0,3,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,
-3,3,3,2,3,3,3,2,1,2,2,3,2,3,3,0,3,2,0,0,1,1,0,1,1,2,1,2,0,0,0,1,
-0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,
-3,3,3,2,3,3,2,3,2,2,2,3,3,3,3,1,3,1,1,0,3,2,1,1,3,3,2,3,1,0,0,1,
-1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,1,
-3,2,2,3,3,0,3,3,3,3,3,3,3,2,2,1,0,3,3,1,3,3,0,1,3,3,2,3,0,3,0,3,
-2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,
-2,2,2,3,3,0,3,3,3,3,3,3,3,3,3,0,0,3,2,0,3,3,0,3,2,3,3,3,0,3,1,3,
-2,0,0,0,0,0,0,0,0,0,0,1,0,1,2,0,1,0,0,0,0,0,0,0,2,2,0,0,1,0,0,1,
-3,3,3,1,2,3,3,1,0,0,1,0,0,3,3,2,3,0,0,2,0,0,2,0,2,0,0,0,2,0,2,0,
-0,3,1,0,1,0,0,0,2,2,1,0,1,1,2,1,2,2,2,0,2,1,1,0,0,0,2,0,0,0,0,0,
-1,2,1,3,3,0,3,3,3,3,3,2,3,0,0,0,0,2,3,0,2,3,1,0,2,3,1,3,0,3,0,2,
-3,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,1,3,3,2,2,3,2,2,0,1,2,3,0,1,2,1,0,1,0,0,0,1,0,2,2,0,0,0,1,
-1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,
-3,3,3,1,3,3,1,1,3,3,1,1,3,3,1,0,2,1,2,0,2,1,0,0,1,1,2,1,0,0,0,2,
-2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,1,0,2,1,3,0,0,2,0,0,3,3,0,3,0,0,1,0,1,2,0,0,1,1,2,2,0,1,0,
-0,1,2,1,1,0,1,0,1,1,1,1,1,0,1,1,1,2,2,1,2,0,1,0,0,0,0,0,0,1,0,0,
-3,3,3,2,3,2,3,3,0,2,2,2,3,3,3,0,3,0,0,0,2,2,0,1,2,1,1,1,0,0,0,1,
-0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
-3,3,3,3,3,3,2,1,2,2,3,3,3,3,2,0,2,0,0,0,2,2,0,0,2,1,3,3,0,0,1,1,
-1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,
-1,1,2,3,3,0,3,3,3,3,3,3,2,2,0,2,0,2,3,2,3,2,2,2,2,2,2,2,1,3,2,3,
-2,0,2,1,2,2,2,2,1,1,2,2,1,2,2,1,2,0,0,2,1,1,0,2,1,0,0,1,0,0,0,1,
-2,3,3,1,1,1,0,1,1,1,2,3,2,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,
-0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,2,2,2,3,2,3,2,2,1,3,3,3,0,2,1,2,0,2,1,0,0,1,1,1,1,1,0,0,1,
-2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,2,0,1,0,0,0,
-3,3,3,2,3,3,3,3,3,2,3,1,2,3,3,1,2,0,0,0,0,0,0,0,3,2,1,1,0,0,0,0,
-2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,
-3,3,3,2,2,3,3,2,1,1,1,1,1,3,3,0,3,1,0,0,1,1,0,0,3,1,2,1,0,0,0,0,
-0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,
-3,3,3,2,2,3,2,2,2,3,2,1,1,3,3,0,3,0,0,0,0,1,0,0,3,1,1,2,0,0,0,1,
-1,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
-1,1,1,3,3,0,3,3,3,3,3,2,2,2,1,2,0,2,1,2,2,1,1,0,1,2,2,2,2,2,2,2,
-0,0,2,1,2,1,2,1,0,1,1,3,1,2,1,1,2,0,0,2,0,1,0,1,0,1,0,0,0,1,0,1,
-3,3,3,1,3,3,3,0,1,1,0,2,2,3,1,0,3,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,2,0,0,2,2,1,0,0,1,0,0,3,3,1,3,0,0,1,1,0,2,0,3,0,0,0,2,0,1,1,
-0,1,2,0,1,2,2,0,2,2,2,2,1,0,2,1,1,0,2,0,2,1,2,0,0,0,0,0,0,0,0,0,
-3,3,3,1,3,2,3,2,0,2,2,2,1,3,2,0,2,1,2,0,1,2,0,0,1,0,2,2,0,0,0,2,
-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,
-3,3,3,0,3,3,1,1,2,3,1,0,3,2,3,0,3,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,
-1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,3,3,0,3,3,2,3,3,2,2,0,0,0,0,1,2,0,1,3,0,0,0,3,1,1,0,3,0,2,
-2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,1,2,2,1,0,3,1,1,1,1,3,3,2,3,0,0,1,0,1,2,0,2,2,0,2,2,0,2,1,
-0,2,2,1,1,1,1,0,2,1,1,0,1,1,1,1,2,1,2,1,2,0,1,0,1,0,0,0,0,0,0,0,
-3,3,3,0,1,1,3,0,0,1,1,0,0,2,2,0,3,0,0,1,1,0,1,0,0,0,0,0,2,0,0,0,
-0,3,1,0,1,0,1,0,2,0,0,1,0,1,0,1,1,1,2,1,1,0,2,0,0,0,0,0,0,0,0,0,
-3,3,3,0,2,0,2,0,1,1,1,0,0,3,3,0,2,0,0,1,0,0,2,1,1,0,1,0,1,0,1,0,
-0,2,0,1,2,0,2,0,2,1,1,0,1,0,2,1,1,0,2,1,1,0,1,0,0,0,1,1,0,0,0,0,
-3,2,3,0,1,0,0,0,0,0,0,0,0,1,2,0,1,0,0,1,0,0,1,0,0,0,0,0,2,0,0,0,
-0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,2,1,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,0,0,2,3,0,0,1,0,1,0,2,3,2,3,0,0,1,3,0,2,1,0,0,0,0,2,0,1,0,
-0,2,1,0,0,1,1,0,2,1,0,0,1,0,0,1,1,0,1,1,2,0,1,0,0,0,0,1,0,0,0,0,
-3,2,2,0,0,1,1,0,0,0,0,0,0,3,1,1,1,0,0,0,0,0,1,0,0,0,0,0,2,0,1,0,
-0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,
-0,0,0,3,3,0,2,3,2,2,1,2,2,1,1,2,0,1,3,2,2,2,0,0,2,2,0,0,0,1,2,1,
-3,0,2,1,1,0,1,1,1,0,1,2,2,2,1,1,2,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,
-0,1,1,2,3,0,3,3,3,2,2,2,2,1,0,1,0,1,0,1,2,2,0,0,2,2,1,3,1,1,2,1,
-0,0,1,1,2,0,1,1,0,0,1,2,0,2,1,1,2,0,0,1,0,0,0,1,0,1,0,1,0,0,0,0,
-3,3,2,0,0,3,1,0,0,0,0,0,0,3,2,1,2,0,0,1,0,0,2,0,0,0,0,0,2,0,1,0,
-0,2,1,1,0,0,1,0,1,2,0,0,1,1,0,0,2,1,1,1,1,0,2,0,0,0,0,0,0,0,0,0,
-3,3,2,0,0,1,0,0,0,0,1,0,0,3,3,2,2,0,0,1,0,0,2,0,1,0,0,0,2,0,1,0,
-0,0,1,1,0,0,2,0,2,1,0,0,1,1,2,1,2,0,2,1,2,1,1,1,0,0,1,1,0,0,0,0,
-3,3,2,0,0,2,2,0,0,0,1,1,0,2,2,1,3,1,0,1,0,1,2,0,0,0,0,0,1,0,1,0,
-0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,2,0,0,0,1,0,0,1,0,0,2,3,1,2,0,0,1,0,0,2,0,0,0,1,0,2,0,2,0,
-0,1,1,2,2,1,2,0,2,1,1,0,0,1,1,0,1,1,1,1,2,1,1,0,0,0,0,0,0,0,0,0,
-3,3,3,0,2,1,2,1,0,0,1,1,0,3,3,1,2,0,0,1,0,0,2,0,2,0,1,1,2,0,0,0,
-0,0,1,1,1,1,2,0,1,1,0,1,1,1,1,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0,0,
-3,3,3,0,2,2,3,2,0,0,1,0,0,2,3,1,0,0,0,0,0,0,2,0,2,0,0,0,2,0,0,0,
-0,1,1,0,0,0,1,0,0,1,0,1,1,0,1,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,
-3,2,3,0,0,0,0,0,0,0,1,0,0,2,2,2,2,0,0,1,0,0,2,0,0,0,0,0,2,0,1,0,
-0,0,2,1,1,0,1,0,2,1,1,0,0,1,1,2,1,0,2,0,2,0,1,0,0,0,2,0,0,0,0,0,
-0,0,0,2,2,0,2,1,1,1,1,2,2,0,0,1,0,1,0,0,1,3,0,0,0,0,1,0,0,2,1,0,
-0,0,1,0,1,0,0,0,0,0,2,1,0,1,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,
-2,0,0,2,3,0,2,3,1,2,2,0,2,0,0,2,0,2,1,1,1,2,1,0,0,1,2,1,1,2,1,0,
-1,0,2,0,1,0,1,1,0,0,2,2,1,2,1,1,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,
-3,3,3,0,2,1,2,0,0,0,1,0,0,3,2,0,1,0,0,1,0,0,2,0,0,0,1,2,1,0,1,0,
-0,0,0,0,1,0,1,0,0,1,0,0,0,0,1,0,1,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,
-0,0,0,2,2,0,2,2,1,1,0,1,1,1,1,1,0,0,1,2,1,1,1,0,1,0,0,0,1,1,1,1,
-0,0,2,1,0,1,1,1,0,1,1,2,1,2,1,1,2,0,1,1,2,1,0,2,0,0,0,0,0,0,0,0,
-3,2,2,0,0,2,0,0,0,0,0,0,0,2,2,0,2,0,0,1,0,0,2,0,0,0,0,0,2,0,0,0,
-0,2,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,
-0,0,0,3,2,0,2,2,0,1,1,0,1,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,
-2,0,1,0,1,0,1,1,0,0,1,2,0,1,0,1,1,0,0,1,0,1,0,2,0,0,0,0,0,0,0,0,
-2,2,2,0,1,1,0,0,0,1,0,0,0,1,2,0,1,0,0,1,0,0,1,0,0,0,0,1,2,0,1,0,
-0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,2,1,0,1,1,1,0,0,0,0,1,2,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,
-1,1,2,0,1,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,0,1,
-0,0,1,2,2,0,2,1,2,1,1,2,2,0,0,0,0,1,0,0,1,1,0,0,2,0,0,0,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
-2,2,2,0,0,0,1,0,0,0,0,0,0,2,2,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
-0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-2,2,2,0,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,1,0,0,0,0,0,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-)
+ISO_8859_9_TURKISH_MODEL = SingleByteCharSetModel(charset_name='ISO-8859-9',
+                                                  language='Turkish',
+                                                  char_to_order_map=ISO_8859_9_TURKISH_CHAR_TO_ORDER,
+                                                  language_model=TURKISH_LANG_MODEL,
+                                                  typical_positive_ratio=0.97029,
+                                                  keep_ascii_letters=True,
+                                                  alphabet='ABCDEFGHIJKLMNOPRSTUVYZabcdefghijklmnoprstuvyzÂÇÎÖÛÜâçîöûüĞğİıŞş')
 
-Latin5TurkishModel = {
-  'char_to_order_map': Latin5_TurkishCharToOrderMap,
-  'precedence_matrix': TurkishLangModel,
-  'typical_positive_ratio': 0.970290,
-  'keep_english_letter': True,
-  'charset_name': "ISO-8859-9",
-  'language': 'Turkish',
-}
diff --git a/news/0dd6ac6d-b127-4551-a404-404b0ee8dcd3.trivial.rst b/src/pip/_vendor/chardet/metadata/__init__.py
similarity index 100%
rename from news/0dd6ac6d-b127-4551-a404-404b0ee8dcd3.trivial.rst
rename to src/pip/_vendor/chardet/metadata/__init__.py
diff --git a/src/pip/_vendor/chardet/metadata/languages.py b/src/pip/_vendor/chardet/metadata/languages.py
new file mode 100644
index 00000000000..3237d5abf60
--- /dev/null
+++ b/src/pip/_vendor/chardet/metadata/languages.py
@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Metadata about languages used by our model training code for our
+SingleByteCharSetProbers.  Could be used for other things in the future.
+
+This code is based on the language metadata from the uchardet project.
+"""
+from __future__ import absolute_import, print_function
+
+from string import ascii_letters
+
+
+# TODO: Add Ukranian (KOI8-U)
+
+class Language(object):
+    """Metadata about a language useful for training models
+
+    :ivar name: The human name for the language, in English.
+    :type name: str
+    :ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
+                    or use another catalog as a last resort.
+    :type iso_code: str
+    :ivar use_ascii: Whether or not ASCII letters should be included in trained
+                     models.
+    :type use_ascii: bool
+    :ivar charsets: The charsets we want to support and create data for.
+    :type charsets: list of str
+    :ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
+                    `True`, you only need to add those not in the ASCII set.
+    :type alphabet: str
+    :ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling
+                            Wikipedia for training data.
+    :type wiki_start_pages: list of str
+    """
+    def __init__(self, name=None, iso_code=None, use_ascii=True, charsets=None,
+                 alphabet=None, wiki_start_pages=None):
+        super(Language, self).__init__()
+        self.name = name
+        self.iso_code = iso_code
+        self.use_ascii = use_ascii
+        self.charsets = charsets
+        if self.use_ascii:
+            if alphabet:
+                alphabet += ascii_letters
+            else:
+                alphabet = ascii_letters
+        elif not alphabet:
+            raise ValueError('Must supply alphabet if use_ascii is False')
+        self.alphabet = ''.join(sorted(set(alphabet))) if alphabet else None
+        self.wiki_start_pages = wiki_start_pages
+
+    def __repr__(self):
+        return '{}({})'.format(self.__class__.__name__,
+                               ', '.join('{}={!r}'.format(k, v)
+                                         for k, v in self.__dict__.items()
+                                         if not k.startswith('_')))
+
+
+LANGUAGES = {'Arabic': Language(name='Arabic',
+                                iso_code='ar',
+                                use_ascii=False,
+                                # We only support encodings that use isolated
+                                # forms, because the current recommendation is
+                                # that the rendering system handles presentation
+                                # forms. This means we purposefully skip IBM864.
+                                charsets=['ISO-8859-6', 'WINDOWS-1256',
+                                          'CP720', 'CP864'],
+                                alphabet=u'ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ',
+                                wiki_start_pages=[u'الصفحة_الرئيسية']),
+             'Belarusian': Language(name='Belarusian',
+                                    iso_code='be',
+                                    use_ascii=False,
+                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                              'IBM866', 'MacCyrillic'],
+                                    alphabet=(u'АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯ'
+                                              u'абвгдеёжзійклмнопрстуўфхцчшыьэюяʼ'),
+                                    wiki_start_pages=[u'Галоўная_старонка']),
+             'Bulgarian': Language(name='Bulgarian',
+                                   iso_code='bg',
+                                   use_ascii=False,
+                                   charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                             'IBM855'],
+                                   alphabet=(u'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ'
+                                             u'абвгдежзийклмнопрстуфхцчшщъьюя'),
+                                   wiki_start_pages=[u'Начална_страница']),
+             'Czech': Language(name='Czech',
+                               iso_code='cz',
+                               use_ascii=True,
+                               charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                               alphabet=u'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ',
+                               wiki_start_pages=[u'Hlavní_strana']),
+             'Danish': Language(name='Danish',
+                                iso_code='da',
+                                use_ascii=True,
+                                charsets=['ISO-8859-1', 'ISO-8859-15',
+                                          'WINDOWS-1252'],
+                                alphabet=u'æøåÆØÅ',
+                                wiki_start_pages=[u'Forside']),
+             'German': Language(name='German',
+                                iso_code='de',
+                                use_ascii=True,
+                                charsets=['ISO-8859-1', 'WINDOWS-1252'],
+                                alphabet=u'äöüßÄÖÜ',
+                                wiki_start_pages=[u'Wikipedia:Hauptseite']),
+             'Greek': Language(name='Greek',
+                               iso_code='el',
+                               use_ascii=False,
+                               charsets=['ISO-8859-7', 'WINDOWS-1253'],
+                               alphabet=(u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώ'
+                                         u'ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ'),
+                               wiki_start_pages=[u'Πύλη:Κύρια']),
+             'English': Language(name='English',
+                                 iso_code='en',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'WINDOWS-1252'],
+                                 wiki_start_pages=[u'Main_Page']),
+             'Esperanto': Language(name='Esperanto',
+                                   iso_code='eo',
+                                   # Q, W, X, and Y not used at all
+                                   use_ascii=False,
+                                   charsets=['ISO-8859-3'],
+                                   alphabet=(u'abcĉdefgĝhĥijĵklmnoprsŝtuŭvz'
+                                             u'ABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ'),
+                                   wiki_start_pages=[u'Vikipedio:Ĉefpaĝo']),
+             'Spanish': Language(name='Spanish',
+                                 iso_code='es',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ñáéíóúüÑÁÉÍÓÚÜ',
+                                 wiki_start_pages=[u'Wikipedia:Portada']),
+             'Estonian': Language(name='Estonian',
+                                  iso_code='et',
+                                  use_ascii=False,
+                                  charsets=['ISO-8859-4', 'ISO-8859-13',
+                                            'WINDOWS-1257'],
+                                  # C, F, Š, Q, W, X, Y, Z, Ž are only for
+                                  # loanwords
+                                  alphabet=(u'ABDEGHIJKLMNOPRSTUVÕÄÖÜ'
+                                            u'abdeghijklmnoprstuvõäöü'),
+                                  wiki_start_pages=[u'Esileht']),
+             'Finnish': Language(name='Finnish',
+                                 iso_code='fi',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ÅÄÖŠŽåäöšž',
+                                 wiki_start_pages=[u'Wikipedia:Etusivu']),
+             'French': Language(name='French',
+                                iso_code='fr',
+                                use_ascii=True,
+                                charsets=['ISO-8859-1', 'ISO-8859-15',
+                                          'WINDOWS-1252'],
+                                alphabet=u'œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ',
+                                wiki_start_pages=[u'Wikipédia:Accueil_principal',
+                                                  u'Bœuf (animal)']),
+             'Hebrew': Language(name='Hebrew',
+                                iso_code='he',
+                                use_ascii=False,
+                                charsets=['ISO-8859-8', 'WINDOWS-1255'],
+                                alphabet=u'אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ',
+                                wiki_start_pages=[u'עמוד_ראשי']),
+             'Croatian': Language(name='Croatian',
+                                  iso_code='hr',
+                                  # Q, W, X, Y are only used for foreign words.
+                                  use_ascii=False,
+                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                  alphabet=(u'abcčćdđefghijklmnoprsštuvzž'
+                                            u'ABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ'),
+                                  wiki_start_pages=[u'Glavna_stranica']),
+             'Hungarian': Language(name='Hungarian',
+                                   iso_code='hu',
+                                   # Q, W, X, Y are only used for foreign words.
+                                   use_ascii=False,
+                                   charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                   alphabet=(u'abcdefghijklmnoprstuvzáéíóöőúüű'
+                                             u'ABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ'),
+                                   wiki_start_pages=[u'Kezdőlap']),
+             'Italian': Language(name='Italian',
+                                 iso_code='it',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ÀÈÉÌÒÓÙàèéìòóù',
+                                 wiki_start_pages=[u'Pagina_principale']),
+             'Lithuanian': Language(name='Lithuanian',
+                                    iso_code='lt',
+                                    use_ascii=False,
+                                    charsets=['ISO-8859-13', 'WINDOWS-1257',
+                                              'ISO-8859-4'],
+                                    # Q, W, and X not used at all
+                                    alphabet=(u'AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽ'
+                                              u'aąbcčdeęėfghiįyjklmnoprsštuųūvzž'),
+                                    wiki_start_pages=[u'Pagrindinis_puslapis']),
+             'Latvian': Language(name='Latvian',
+                                 iso_code='lv',
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-13', 'WINDOWS-1257',
+                                           'ISO-8859-4'],
+                                 # Q, W, X, Y are only for loanwords
+                                 alphabet=(u'AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ'
+                                           u'aābcčdeēfgģhiījkķlļmnņoprsštuūvzž'),
+                                 wiki_start_pages=[u'Sākumlapa']),
+             'Macedonian': Language(name='Macedonian',
+                                    iso_code='mk',
+                                    use_ascii=False,
+                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                              'MacCyrillic', 'IBM855'],
+                                    alphabet=(u'АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ'
+                                              u'абвгдѓежзѕијклљмнњопрстќуфхцчџш'),
+                                    wiki_start_pages=[u'Главна_страница']),
+             'Dutch': Language(name='Dutch',
+                               iso_code='nl',
+                               use_ascii=True,
+                               charsets=['ISO-8859-1', 'WINDOWS-1252'],
+                               wiki_start_pages=[u'Hoofdpagina']),
+             'Polish': Language(name='Polish',
+                                iso_code='pl',
+                                # Q and X are only used for foreign words.
+                                use_ascii=False,
+                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                alphabet=(u'AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻ'
+                                          u'aąbcćdeęfghijklłmnńoóprsśtuwyzźż'),
+                                wiki_start_pages=[u'Wikipedia:Strona_główna']),
+             'Portuguese': Language(name='Portuguese',
+                                 iso_code='pt',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú',
+                                 wiki_start_pages=[u'Wikipédia:Página_principal']),
+             'Romanian': Language(name='Romanian',
+                                  iso_code='ro',
+                                  use_ascii=True,
+                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                  alphabet=u'ăâîșțĂÂÎȘȚ',
+                                  wiki_start_pages=[u'Pagina_principală']),
+             'Russian': Language(name='Russian',
+                                 iso_code='ru',
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                           'KOI8-R', 'MacCyrillic', 'IBM866',
+                                           'IBM855'],
+                                 alphabet=(u'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
+                                           u'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'),
+                                 wiki_start_pages=[u'Заглавная_страница']),
+             'Slovak': Language(name='Slovak',
+                                iso_code='sk',
+                                use_ascii=True,
+                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                alphabet=u'áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ',
+                                wiki_start_pages=[u'Hlavná_stránka']),
+             'Slovene': Language(name='Slovene',
+                                 iso_code='sl',
+                                 # Q, W, X, Y are only used for foreign words.
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                 alphabet=(u'abcčdefghijklmnoprsštuvzž'
+                                           u'ABCČDEFGHIJKLMNOPRSŠTUVZŽ'),
+                                 wiki_start_pages=[u'Glavna_stran']),
+             # Serbian can be written in both Latin and Cyrillic, but there's no
+             # simple way to get the Latin alphabet pages from Wikipedia through
+             # the API, so for now we just support Cyrillic.
+             'Serbian': Language(name='Serbian',
+                                 iso_code='sr',
+                                 alphabet=(u'АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШ'
+                                           u'абвгдђежзијклљмнњопрстћуфхцчџш'),
+                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                           'MacCyrillic', 'IBM855'],
+                                 wiki_start_pages=[u'Главна_страна']),
+             'Thai': Language(name='Thai',
+                              iso_code='th',
+                              use_ascii=False,
+                              charsets=['ISO-8859-11', 'TIS-620', 'CP874'],
+                              alphabet=u'กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛',
+                              wiki_start_pages=[u'หน้าหลัก']),
+             'Turkish': Language(name='Turkish',
+                                 iso_code='tr',
+                                 # Q, W, and X are not used by Turkish
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-3', 'ISO-8859-9',
+                                           'WINDOWS-1254'],
+                                 alphabet=(u'abcçdefgğhıijklmnoöprsştuüvyzâîû'
+                                           u'ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ'),
+                                 wiki_start_pages=[u'Ana_Sayfa']),
+             'Vietnamese': Language(name='Vietnamese',
+                                    iso_code='vi',
+                                    use_ascii=False,
+                                    # Windows-1258 is the only common 8-bit
+                                    # Vietnamese encoding supported by Python.
+                                    # From Wikipedia:
+                                    # For systems that lack support for Unicode,
+                                    # dozens of 8-bit Vietnamese code pages are
+                                    # available.[1] The most common are VISCII
+                                    # (TCVN 5712:1993), VPS, and Windows-1258.[3]
+                                    # Where ASCII is required, such as when
+                                    # ensuring readability in plain text e-mail,
+                                    # Vietnamese letters are often encoded
+                                    # according to Vietnamese Quoted-Readable
+                                    # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
+                                    # though usage of either variable-width
+                                    # scheme has declined dramatically following
+                                    # the adoption of Unicode on the World Wide
+                                    # Web.
+                                    charsets=['WINDOWS-1258'],
+                                    alphabet=(u'aăâbcdđeêghiklmnoôơpqrstuưvxy'
+                                              u'AĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY'),
+                                    wiki_start_pages=[u'Chữ_Quốc_ngữ']),
+            }
diff --git a/src/pip/_vendor/chardet/sbcharsetprober.py b/src/pip/_vendor/chardet/sbcharsetprober.py
index 0adb51de5a2..46ba835c66c 100644
--- a/src/pip/_vendor/chardet/sbcharsetprober.py
+++ b/src/pip/_vendor/chardet/sbcharsetprober.py
@@ -26,10 +26,22 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
+from collections import namedtuple
+
 from .charsetprober import CharSetProber
 from .enums import CharacterCategory, ProbingState, SequenceLikelihood
 
 
+SingleByteCharSetModel = namedtuple('SingleByteCharSetModel',
+                                    ['charset_name',
+                                     'language',
+                                     'char_to_order_map',
+                                     'language_model',
+                                     'typical_positive_ratio',
+                                     'keep_ascii_letters',
+                                     'alphabet'])
+
+
 class SingleByteCharSetProber(CharSetProber):
     SAMPLE_SIZE = 64
     SB_ENOUGH_REL_THRESHOLD = 1024  #  0.25 * SAMPLE_SIZE^2
@@ -65,25 +77,25 @@ def charset_name(self):
         if self._name_prober:
             return self._name_prober.charset_name
         else:
-            return self._model['charset_name']
+            return self._model.charset_name
 
     @property
     def language(self):
         if self._name_prober:
             return self._name_prober.language
         else:
-            return self._model.get('language')
+            return self._model.language
 
     def feed(self, byte_str):
-        if not self._model['keep_english_letter']:
+        # TODO: Make filter_international_words keep things in self.alphabet
+        if not self._model.keep_ascii_letters:
             byte_str = self.filter_international_words(byte_str)
         if not byte_str:
             return self.state
-        char_to_order_map = self._model['char_to_order_map']
-        for i, c in enumerate(byte_str):
-            # XXX: Order is in range 1-64, so one would think we want 0-63 here,
-            #      but that leads to 27 more test failures than before.
-            order = char_to_order_map[c]
+        char_to_order_map = self._model.char_to_order_map
+        language_model = self._model.language_model
+        for char in byte_str:
+            order = char_to_order_map.get(char, CharacterCategory.UNDEFINED)
             # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
             #      CharacterCategory.SYMBOL is actually 253, so we use CONTROL
             #      to make it closer to the original intent. The only difference
@@ -91,20 +103,21 @@ def feed(self, byte_str):
             #      _total_char purposes.
             if order < CharacterCategory.CONTROL:
                 self._total_char += 1
+            # TODO: Follow uchardet's lead and discount confidence for frequent
+            #       control characters.
+            #       See https://github.com/BYVoid/uchardet/commit/55b4f23971db61
             if order < self.SAMPLE_SIZE:
                 self._freq_char += 1
                 if self._last_order < self.SAMPLE_SIZE:
                     self._total_seqs += 1
                     if not self._reversed:
-                        i = (self._last_order * self.SAMPLE_SIZE) + order
-                        model = self._model['precedence_matrix'][i]
-                    else:  # reverse the order of the letters in the lookup
-                        i = (order * self.SAMPLE_SIZE) + self._last_order
-                        model = self._model['precedence_matrix'][i]
-                    self._seq_counters[model] += 1
+                        lm_cat = language_model[self._last_order][order]
+                    else:
+                        lm_cat = language_model[order][self._last_order]
+                    self._seq_counters[lm_cat] += 1
             self._last_order = order
 
-        charset_name = self._model['charset_name']
+        charset_name = self._model.charset_name
         if self.state == ProbingState.DETECTING:
             if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
                 confidence = self.get_confidence()
@@ -125,7 +138,7 @@ def get_confidence(self):
         r = 0.01
         if self._total_seqs > 0:
             r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
-                 self._total_seqs / self._model['typical_positive_ratio'])
+                 self._total_seqs / self._model.typical_positive_ratio)
             r = r * self._freq_char / self._total_char
             if r >= 1.0:
                 r = 0.99
diff --git a/src/pip/_vendor/chardet/sbcsgroupprober.py b/src/pip/_vendor/chardet/sbcsgroupprober.py
index 98e95dc1a3c..bdeef4e15b0 100644
--- a/src/pip/_vendor/chardet/sbcsgroupprober.py
+++ b/src/pip/_vendor/chardet/sbcsgroupprober.py
@@ -27,47 +27,57 @@
 ######################### END LICENSE BLOCK #########################
 
 from .charsetgroupprober import CharSetGroupProber
-from .sbcharsetprober import SingleByteCharSetProber
-from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel,
-                                Latin5CyrillicModel, MacCyrillicModel,
-                                Ibm866Model, Ibm855Model)
-from .langgreekmodel import Latin7GreekModel, Win1253GreekModel
-from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
-# from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
-from .langthaimodel import TIS620ThaiModel
-from .langhebrewmodel import Win1255HebrewModel
 from .hebrewprober import HebrewProber
-from .langturkishmodel import Latin5TurkishModel
+from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
+                                 WINDOWS_1251_BULGARIAN_MODEL)
+from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
+from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
+# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
+#                                  WINDOWS_1250_HUNGARIAN_MODEL)
+from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
+                               ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
+                               MACCYRILLIC_RUSSIAN_MODEL,
+                               WINDOWS_1251_RUSSIAN_MODEL)
+from .langthaimodel import TIS_620_THAI_MODEL
+from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
+from .sbcharsetprober import SingleByteCharSetProber
 
 
 class SBCSGroupProber(CharSetGroupProber):
     def __init__(self):
         super(SBCSGroupProber, self).__init__()
+        hebrew_prober = HebrewProber()
+        logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
+                                                        False, hebrew_prober)
+        # TODO: See if using ISO-8859-8 Hebrew model works better here, since
+        #       it's actually the visual one
+        visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
+                                                       True, hebrew_prober)
+        hebrew_prober.set_model_probers(logical_hebrew_prober,
+                                        visual_hebrew_prober)
+        # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
+        #       and several tests failed that did not before. Some thought
+        #       should be put into the ordering, and we should consider making
+        #       order not matter here, because that is very counter-intuitive.
         self.probers = [
-            SingleByteCharSetProber(Win1251CyrillicModel),
-            SingleByteCharSetProber(Koi8rModel),
-            SingleByteCharSetProber(Latin5CyrillicModel),
-            SingleByteCharSetProber(MacCyrillicModel),
-            SingleByteCharSetProber(Ibm866Model),
-            SingleByteCharSetProber(Ibm855Model),
-            SingleByteCharSetProber(Latin7GreekModel),
-            SingleByteCharSetProber(Win1253GreekModel),
-            SingleByteCharSetProber(Latin5BulgarianModel),
-            SingleByteCharSetProber(Win1251BulgarianModel),
+            SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
+            SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
+            SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
+            SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
+            SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
+            SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
+            SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
+            SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
+            SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
+            SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
             # TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
             #       after we retrain model.
-            # SingleByteCharSetProber(Latin2HungarianModel),
-            # SingleByteCharSetProber(Win1250HungarianModel),
-            SingleByteCharSetProber(TIS620ThaiModel),
-            SingleByteCharSetProber(Latin5TurkishModel),
+            # SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
+            # SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
+            SingleByteCharSetProber(TIS_620_THAI_MODEL),
+            SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
+            hebrew_prober,
+            logical_hebrew_prober,
+            visual_hebrew_prober,
         ]
-        hebrew_prober = HebrewProber()
-        logical_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel,
-                                                        False, hebrew_prober)
-        visual_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel, True,
-                                                       hebrew_prober)
-        hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
-        self.probers.extend([hebrew_prober, logical_hebrew_prober,
-                             visual_hebrew_prober])
-
         self.reset()
diff --git a/src/pip/_vendor/chardet/universaldetector.py b/src/pip/_vendor/chardet/universaldetector.py
index 7b4e92d6158..055a8ac1b1d 100644
--- a/src/pip/_vendor/chardet/universaldetector.py
+++ b/src/pip/_vendor/chardet/universaldetector.py
@@ -266,7 +266,7 @@ def close(self):
                                'language': max_prober.language}
 
         # Log all prober confidences if none met MINIMUM_THRESHOLD
-        if self.logger.getEffectiveLevel() == logging.DEBUG:
+        if self.logger.getEffectiveLevel() <= logging.DEBUG:
             if self.result['encoding'] is None:
                 self.logger.debug('no probers hit minimum threshold')
                 for group_prober in self._charset_probers:
@@ -280,7 +280,7 @@ def close(self):
                                               prober.get_confidence())
                     else:
                         self.logger.debug('%s %s confidence = %s',
-                                          prober.charset_name,
-                                          prober.language,
-                                          prober.get_confidence())
+                                          group_prober.charset_name,
+                                          group_prober.language,
+                                          group_prober.get_confidence())
         return self.result
diff --git a/src/pip/_vendor/chardet/version.py b/src/pip/_vendor/chardet/version.py
index bb2a34a70ea..70369b9d663 100644
--- a/src/pip/_vendor/chardet/version.py
+++ b/src/pip/_vendor/chardet/version.py
@@ -5,5 +5,5 @@
 :author: Dan Blanchard (dan.blanchard@gmail.com)
 """
 
-__version__ = "3.0.4"
+__version__ = "4.0.0"
 VERSION = __version__.split('.')
diff --git a/src/pip/_vendor/contextlib2.LICENSE.txt b/src/pip/_vendor/contextlib2.LICENSE.txt
deleted file mode 100644
index 5de20277df9..00000000000
--- a/src/pip/_vendor/contextlib2.LICENSE.txt
+++ /dev/null
@@ -1,122 +0,0 @@
-
-
-A. HISTORY OF THE SOFTWARE
-==========================
-
-contextlib2 is a derivative of the contextlib module distributed by the PSF
-as part of the Python standard library. According, it is itself redistributed
-under the PSF license (reproduced in full below). As the contextlib module
-was added only in Python 2.5, the licenses for earlier Python versions are
-not applicable and have not been included.
-
-Python was created in the early 1990s by Guido van Rossum at Stichting
-Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
-as a successor of a language called ABC.  Guido remains Python's
-principal author, although it includes many contributions from others.
-
-In 1995, Guido continued his work on Python at the Corporation for
-National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
-in Reston, Virginia where he released several versions of the
-software.
-
-In May 2000, Guido and the Python core development team moved to
-BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
-year, the PythonLabs team moved to Digital Creations (now Zope
-Corporation, see http://www.zope.com).  In 2001, the Python Software
-Foundation (PSF, see http://www.python.org/psf/) was formed, a
-non-profit organization created specifically to own Python-related
-Intellectual Property.  Zope Corporation is a sponsoring member of
-the PSF.
-
-All Python releases are Open Source (see http://www.opensource.org for
-the Open Source Definition).  Historically, most, but not all, Python
-releases have also been GPL-compatible; the table below summarizes
-the various releases that included the contextlib module.
-
-    Release         Derived     Year        Owner       GPL-
-                    from                                compatible? (1)
-
-    2.5             2.4         2006        PSF         yes
-    2.5.1           2.5         2007        PSF         yes
-    2.5.2           2.5.1       2008        PSF         yes
-    2.5.3           2.5.2       2008        PSF         yes
-    2.6             2.5         2008        PSF         yes
-    2.6.1           2.6         2008        PSF         yes
-    2.6.2           2.6.1       2009        PSF         yes
-    2.6.3           2.6.2       2009        PSF         yes
-    2.6.4           2.6.3       2009        PSF         yes
-    2.6.5           2.6.4       2010        PSF         yes
-    3.0             2.6         2008        PSF         yes
-    3.0.1           3.0         2009        PSF         yes
-    3.1             3.0.1       2009        PSF         yes
-    3.1.1           3.1         2009        PSF         yes
-    3.1.2           3.1.1       2010        PSF         yes
-    3.1.3           3.1.2       2010        PSF         yes
-    3.1.4           3.1.3       2011        PSF         yes
-    3.2             3.1         2011        PSF         yes
-    3.2.1           3.2         2011        PSF         yes
-    3.2.2           3.2.1       2011        PSF         yes
-    3.3             3.2         2012        PSF         yes
-
-Footnotes:
-
-(1) GPL-compatible doesn't mean that we're distributing Python under
-    the GPL.  All Python licenses, unlike the GPL, let you distribute
-    a modified version without making your changes open source.  The
-    GPL-compatible licenses make it possible to combine Python with
-    other software that is released under the GPL; the others don't.
-
-Thanks to the many outside volunteers who have worked under Guido's
-direction to make these releases possible.
-
-
-B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
-===============================================================
-
-PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
---------------------------------------------
-
-1. This LICENSE AGREEMENT is between the Python Software Foundation
-("PSF"), and the Individual or Organization ("Licensee") accessing and
-otherwise using this software ("Python") in source or binary form and
-its associated documentation.
-
-2. Subject to the terms and conditions of this License Agreement, PSF hereby
-grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
-analyze, test, perform and/or display publicly, prepare derivative works,
-distribute, and otherwise use Python alone or in any derivative version,
-provided, however, that PSF's License Agreement and PSF's notice of copyright,
-i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011 Python Software Foundation; All Rights Reserved" are retained in Python
-alone or in any derivative version prepared by Licensee.
-
-3. In the event Licensee prepares a derivative work that is based on
-or incorporates Python or any part thereof, and wants to make
-the derivative work available to others as provided herein, then
-Licensee hereby agrees to include in any such work a brief summary of
-the changes made to Python.
-
-4. PSF is making Python available to Licensee on an "AS IS"
-basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
-IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
-DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
-FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
-INFRINGE ANY THIRD PARTY RIGHTS.
-
-5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
-FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
-A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
-OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
-
-6. This License Agreement will automatically terminate upon a material
-breach of its terms and conditions.
-
-7. Nothing in this License Agreement shall be deemed to create any
-relationship of agency, partnership, or joint venture between PSF and
-Licensee.  This License Agreement does not grant permission to use PSF
-trademarks or trade name in a trademark sense to endorse or promote
-products or services of Licensee, or any third party.
-
-8. By copying, installing or otherwise using Python, Licensee
-agrees to be bound by the terms and conditions of this License
-Agreement.
diff --git a/src/pip/_vendor/contextlib2.py b/src/pip/_vendor/contextlib2.py
deleted file mode 100644
index 3aae8f4117c..00000000000
--- a/src/pip/_vendor/contextlib2.py
+++ /dev/null
@@ -1,518 +0,0 @@
-"""contextlib2 - backports and enhancements to the contextlib module"""
-
-import abc
-import sys
-import warnings
-from collections import deque
-from functools import wraps
-
-__all__ = ["contextmanager", "closing", "nullcontext",
-           "AbstractContextManager",
-           "ContextDecorator", "ExitStack",
-           "redirect_stdout", "redirect_stderr", "suppress"]
-
-# Backwards compatibility
-__all__ += ["ContextStack"]
-
-
-# Backport abc.ABC
-if sys.version_info[:2] >= (3, 4):
-    _abc_ABC = abc.ABC
-else:
-    _abc_ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()})
-
-
-# Backport classic class MRO
-def _classic_mro(C, result):
-    if C in result:
-        return
-    result.append(C)
-    for B in C.__bases__:
-        _classic_mro(B, result)
-    return result
-
-
-# Backport _collections_abc._check_methods
-def _check_methods(C, *methods):
-    try:
-        mro = C.__mro__
-    except AttributeError:
-        mro = tuple(_classic_mro(C, []))
-
-    for method in methods:
-        for B in mro:
-            if method in B.__dict__:
-                if B.__dict__[method] is None:
-                    return NotImplemented
-                break
-        else:
-            return NotImplemented
-    return True
-
-
-class AbstractContextManager(_abc_ABC):
-    """An abstract base class for context managers."""
-
-    def __enter__(self):
-        """Return `self` upon entering the runtime context."""
-        return self
-
-    @abc.abstractmethod
-    def __exit__(self, exc_type, exc_value, traceback):
-        """Raise any exception triggered within the runtime context."""
-        return None
-
-    @classmethod
-    def __subclasshook__(cls, C):
-        """Check whether subclass is considered a subclass of this ABC."""
-        if cls is AbstractContextManager:
-            return _check_methods(C, "__enter__", "__exit__")
-        return NotImplemented
-
-
-class ContextDecorator(object):
-    """A base class or mixin that enables context managers to work as decorators."""
-
-    def refresh_cm(self):
-        """Returns the context manager used to actually wrap the call to the
-        decorated function.
-
-        The default implementation just returns *self*.
-
-        Overriding this method allows otherwise one-shot context managers
-        like _GeneratorContextManager to support use as decorators via
-        implicit recreation.
-
-        DEPRECATED: refresh_cm was never added to the standard library's
-                    ContextDecorator API
-        """
-        warnings.warn("refresh_cm was never added to the standard library",
-                      DeprecationWarning)
-        return self._recreate_cm()
-
-    def _recreate_cm(self):
-        """Return a recreated instance of self.
-
-        Allows an otherwise one-shot context manager like
-        _GeneratorContextManager to support use as
-        a decorator via implicit recreation.
-
-        This is a private interface just for _GeneratorContextManager.
-        See issue #11647 for details.
-        """
-        return self
-
-    def __call__(self, func):
-        @wraps(func)
-        def inner(*args, **kwds):
-            with self._recreate_cm():
-                return func(*args, **kwds)
-        return inner
-
-
-class _GeneratorContextManager(ContextDecorator):
-    """Helper for @contextmanager decorator."""
-
-    def __init__(self, func, args, kwds):
-        self.gen = func(*args, **kwds)
-        self.func, self.args, self.kwds = func, args, kwds
-        # Issue 19330: ensure context manager instances have good docstrings
-        doc = getattr(func, "__doc__", None)
-        if doc is None:
-            doc = type(self).__doc__
-        self.__doc__ = doc
-        # Unfortunately, this still doesn't provide good help output when
-        # inspecting the created context manager instances, since pydoc
-        # currently bypasses the instance docstring and shows the docstring
-        # for the class instead.
-        # See http://bugs.python.org/issue19404 for more details.
-
-    def _recreate_cm(self):
-        # _GCM instances are one-shot context managers, so the
-        # CM must be recreated each time a decorated function is
-        # called
-        return self.__class__(self.func, self.args, self.kwds)
-
-    def __enter__(self):
-        try:
-            return next(self.gen)
-        except StopIteration:
-            raise RuntimeError("generator didn't yield")
-
-    def __exit__(self, type, value, traceback):
-        if type is None:
-            try:
-                next(self.gen)
-            except StopIteration:
-                return
-            else:
-                raise RuntimeError("generator didn't stop")
-        else:
-            if value is None:
-                # Need to force instantiation so we can reliably
-                # tell if we get the same exception back
-                value = type()
-            try:
-                self.gen.throw(type, value, traceback)
-                raise RuntimeError("generator didn't stop after throw()")
-            except StopIteration as exc:
-                # Suppress StopIteration *unless* it's the same exception that
-                # was passed to throw().  This prevents a StopIteration
-                # raised inside the "with" statement from being suppressed.
-                return exc is not value
-            except RuntimeError as exc:
-                # Don't re-raise the passed in exception
-                if exc is value:
-                    return False
-                # Likewise, avoid suppressing if a StopIteration exception
-                # was passed to throw() and later wrapped into a RuntimeError
-                # (see PEP 479).
-                if _HAVE_EXCEPTION_CHAINING and exc.__cause__ is value:
-                    return False
-                raise
-            except:
-                # only re-raise if it's *not* the exception that was
-                # passed to throw(), because __exit__() must not raise
-                # an exception unless __exit__() itself failed.  But throw()
-                # has to raise the exception to signal propagation, so this
-                # fixes the impedance mismatch between the throw() protocol
-                # and the __exit__() protocol.
-                #
-                if sys.exc_info()[1] is not value:
-                    raise
-
-
-def contextmanager(func):
-    """@contextmanager decorator.
-
-    Typical usage:
-
-        @contextmanager
-        def some_generator():
-            
-            try:
-                yield 
-            finally:
-                
-
-    This makes this:
-
-        with some_generator() as :
-            
-
-    equivalent to this:
-
-        
-        try:
-             = 
-            
-        finally:
-            
-
-    """
-    @wraps(func)
-    def helper(*args, **kwds):
-        return _GeneratorContextManager(func, args, kwds)
-    return helper
-
-
-class closing(object):
-    """Context to automatically close something at the end of a block.
-
-    Code like this:
-
-        with closing(.open()) as f:
-            
-
-    is equivalent to this:
-
-        f = .open()
-        try:
-            
-        finally:
-            f.close()
-
-    """
-    def __init__(self, thing):
-        self.thing = thing
-
-    def __enter__(self):
-        return self.thing
-
-    def __exit__(self, *exc_info):
-        self.thing.close()
-
-
-class _RedirectStream(object):
-
-    _stream = None
-
-    def __init__(self, new_target):
-        self._new_target = new_target
-        # We use a list of old targets to make this CM re-entrant
-        self._old_targets = []
-
-    def __enter__(self):
-        self._old_targets.append(getattr(sys, self._stream))
-        setattr(sys, self._stream, self._new_target)
-        return self._new_target
-
-    def __exit__(self, exctype, excinst, exctb):
-        setattr(sys, self._stream, self._old_targets.pop())
-
-
-class redirect_stdout(_RedirectStream):
-    """Context manager for temporarily redirecting stdout to another file.
-
-        # How to send help() to stderr
-        with redirect_stdout(sys.stderr):
-            help(dir)
-
-        # How to write help() to a file
-        with open('help.txt', 'w') as f:
-            with redirect_stdout(f):
-                help(pow)
-    """
-
-    _stream = "stdout"
-
-
-class redirect_stderr(_RedirectStream):
-    """Context manager for temporarily redirecting stderr to another file."""
-
-    _stream = "stderr"
-
-
-class suppress(object):
-    """Context manager to suppress specified exceptions
-
-    After the exception is suppressed, execution proceeds with the next
-    statement following the with statement.
-
-         with suppress(FileNotFoundError):
-             os.remove(somefile)
-         # Execution still resumes here if the file was already removed
-    """
-
-    def __init__(self, *exceptions):
-        self._exceptions = exceptions
-
-    def __enter__(self):
-        pass
-
-    def __exit__(self, exctype, excinst, exctb):
-        # Unlike isinstance and issubclass, CPython exception handling
-        # currently only looks at the concrete type hierarchy (ignoring
-        # the instance and subclass checking hooks). While Guido considers
-        # that a bug rather than a feature, it's a fairly hard one to fix
-        # due to various internal implementation details. suppress provides
-        # the simpler issubclass based semantics, rather than trying to
-        # exactly reproduce the limitations of the CPython interpreter.
-        #
-        # See http://bugs.python.org/issue12029 for more details
-        return exctype is not None and issubclass(exctype, self._exceptions)
-
-
-# Context manipulation is Python 3 only
-_HAVE_EXCEPTION_CHAINING = sys.version_info[0] >= 3
-if _HAVE_EXCEPTION_CHAINING:
-    def _make_context_fixer(frame_exc):
-        def _fix_exception_context(new_exc, old_exc):
-            # Context may not be correct, so find the end of the chain
-            while 1:
-                exc_context = new_exc.__context__
-                if exc_context is old_exc:
-                    # Context is already set correctly (see issue 20317)
-                    return
-                if exc_context is None or exc_context is frame_exc:
-                    break
-                new_exc = exc_context
-            # Change the end of the chain to point to the exception
-            # we expect it to reference
-            new_exc.__context__ = old_exc
-        return _fix_exception_context
-
-    def _reraise_with_existing_context(exc_details):
-        try:
-            # bare "raise exc_details[1]" replaces our carefully
-            # set-up context
-            fixed_ctx = exc_details[1].__context__
-            raise exc_details[1]
-        except BaseException:
-            exc_details[1].__context__ = fixed_ctx
-            raise
-else:
-    # No exception context in Python 2
-    def _make_context_fixer(frame_exc):
-        return lambda new_exc, old_exc: None
-
-    # Use 3 argument raise in Python 2,
-    # but use exec to avoid SyntaxError in Python 3
-    def _reraise_with_existing_context(exc_details):
-        exc_type, exc_value, exc_tb = exc_details
-        exec("raise exc_type, exc_value, exc_tb")
-
-# Handle old-style classes if they exist
-try:
-    from types import InstanceType
-except ImportError:
-    # Python 3 doesn't have old-style classes
-    _get_type = type
-else:
-    # Need to handle old-style context managers on Python 2
-    def _get_type(obj):
-        obj_type = type(obj)
-        if obj_type is InstanceType:
-            return obj.__class__  # Old-style class
-        return obj_type  # New-style class
-
-
-# Inspired by discussions on http://bugs.python.org/issue13585
-class ExitStack(object):
-    """Context manager for dynamic management of a stack of exit callbacks
-
-    For example:
-
-        with ExitStack() as stack:
-            files = [stack.enter_context(open(fname)) for fname in filenames]
-            # All opened files will automatically be closed at the end of
-            # the with statement, even if attempts to open files later
-            # in the list raise an exception
-
-    """
-    def __init__(self):
-        self._exit_callbacks = deque()
-
-    def pop_all(self):
-        """Preserve the context stack by transferring it to a new instance"""
-        new_stack = type(self)()
-        new_stack._exit_callbacks = self._exit_callbacks
-        self._exit_callbacks = deque()
-        return new_stack
-
-    def _push_cm_exit(self, cm, cm_exit):
-        """Helper to correctly register callbacks to __exit__ methods"""
-        def _exit_wrapper(*exc_details):
-            return cm_exit(cm, *exc_details)
-        _exit_wrapper.__self__ = cm
-        self.push(_exit_wrapper)
-
-    def push(self, exit):
-        """Registers a callback with the standard __exit__ method signature
-
-        Can suppress exceptions the same way __exit__ methods can.
-
-        Also accepts any object with an __exit__ method (registering a call
-        to the method instead of the object itself)
-        """
-        # We use an unbound method rather than a bound method to follow
-        # the standard lookup behaviour for special methods
-        _cb_type = _get_type(exit)
-        try:
-            exit_method = _cb_type.__exit__
-        except AttributeError:
-            # Not a context manager, so assume its a callable
-            self._exit_callbacks.append(exit)
-        else:
-            self._push_cm_exit(exit, exit_method)
-        return exit # Allow use as a decorator
-
-    def callback(self, callback, *args, **kwds):
-        """Registers an arbitrary callback and arguments.
-
-        Cannot suppress exceptions.
-        """
-        def _exit_wrapper(exc_type, exc, tb):
-            callback(*args, **kwds)
-        # We changed the signature, so using @wraps is not appropriate, but
-        # setting __wrapped__ may still help with introspection
-        _exit_wrapper.__wrapped__ = callback
-        self.push(_exit_wrapper)
-        return callback # Allow use as a decorator
-
-    def enter_context(self, cm):
-        """Enters the supplied context manager
-
-        If successful, also pushes its __exit__ method as a callback and
-        returns the result of the __enter__ method.
-        """
-        # We look up the special methods on the type to match the with statement
-        _cm_type = _get_type(cm)
-        _exit = _cm_type.__exit__
-        result = _cm_type.__enter__(cm)
-        self._push_cm_exit(cm, _exit)
-        return result
-
-    def close(self):
-        """Immediately unwind the context stack"""
-        self.__exit__(None, None, None)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *exc_details):
-        received_exc = exc_details[0] is not None
-
-        # We manipulate the exception state so it behaves as though
-        # we were actually nesting multiple with statements
-        frame_exc = sys.exc_info()[1]
-        _fix_exception_context = _make_context_fixer(frame_exc)
-
-        # Callbacks are invoked in LIFO order to match the behaviour of
-        # nested context managers
-        suppressed_exc = False
-        pending_raise = False
-        while self._exit_callbacks:
-            cb = self._exit_callbacks.pop()
-            try:
-                if cb(*exc_details):
-                    suppressed_exc = True
-                    pending_raise = False
-                    exc_details = (None, None, None)
-            except:
-                new_exc_details = sys.exc_info()
-                # simulate the stack of exceptions by setting the context
-                _fix_exception_context(new_exc_details[1], exc_details[1])
-                pending_raise = True
-                exc_details = new_exc_details
-        if pending_raise:
-            _reraise_with_existing_context(exc_details)
-        return received_exc and suppressed_exc
-
-
-# Preserve backwards compatibility
-class ContextStack(ExitStack):
-    """Backwards compatibility alias for ExitStack"""
-
-    def __init__(self):
-        warnings.warn("ContextStack has been renamed to ExitStack",
-                      DeprecationWarning)
-        super(ContextStack, self).__init__()
-
-    def register_exit(self, callback):
-        return self.push(callback)
-
-    def register(self, callback, *args, **kwds):
-        return self.callback(callback, *args, **kwds)
-
-    def preserve(self):
-        return self.pop_all()
-
-
-class nullcontext(AbstractContextManager):
-    """Context manager that does no additional processing.
-    Used as a stand-in for a normal context manager, when a particular
-    block of code is only sometimes used with a normal context manager:
-    cm = optional_cm if condition else nullcontext()
-    with cm:
-        # Perform operation, using optional_cm if condition is True
-    """
-
-    def __init__(self, enter_result=None):
-        self.enter_result = enter_result
-
-    def __enter__(self):
-        return self.enter_result
-
-    def __exit__(self, *excinfo):
-        pass
diff --git a/src/pip/_vendor/distlib/__init__.py b/src/pip/_vendor/distlib/__init__.py
index 63d916e345b..11549481074 100644
--- a/src/pip/_vendor/distlib/__init__.py
+++ b/src/pip/_vendor/distlib/__init__.py
@@ -6,7 +6,7 @@
 #
 import logging
 
-__version__ = '0.3.1'
+__version__ = '0.3.3'
 
 class DistlibException(Exception):
     pass
diff --git a/src/pip/_vendor/distlib/compat.py b/src/pip/_vendor/distlib/compat.py
index c316fd973ad..e594106956f 100644
--- a/src/pip/_vendor/distlib/compat.py
+++ b/src/pip/_vendor/distlib/compat.py
@@ -48,17 +48,18 @@ def quote(s):
     from itertools import ifilter as filter
     from itertools import ifilterfalse as filterfalse
 
-    _userprog = None
-    def splituser(host):
-        """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
-        global _userprog
-        if _userprog is None:
-            import re
-            _userprog = re.compile('^(.*)@(.*)$')
-
-        match = _userprog.match(host)
-        if match: return match.group(1, 2)
-        return None, host
+    # Leaving this around for now, in case it needs resurrecting in some way
+    # _userprog = None
+    # def splituser(host):
+        # """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
+        # global _userprog
+        # if _userprog is None:
+            # import re
+            # _userprog = re.compile('^(.*)@(.*)$')
+
+        # match = _userprog.match(host)
+        # if match: return match.group(1, 2)
+        # return None, host
 
 else:  # pragma: no cover
     from io import StringIO
@@ -68,7 +69,7 @@ def splituser(host):
     import builtins
     import configparser
     import shutil
-    from urllib.parse import (urlparse, urlunparse, urljoin, splituser, quote,
+    from urllib.parse import (urlparse, urlunparse, urljoin, quote,
                               unquote, urlsplit, urlunsplit, splittype)
     from urllib.request import (urlopen, urlretrieve, Request, url2pathname,
                                 pathname2url,
@@ -88,6 +89,7 @@ def splituser(host):
     from itertools import filterfalse
     filter = filter
 
+
 try:
     from ssl import match_hostname, CertificateError
 except ImportError: # pragma: no cover
diff --git a/src/pip/_vendor/distlib/index.py b/src/pip/_vendor/distlib/index.py
index 7a87cdcf7a1..b1fbbf8e8d2 100644
--- a/src/pip/_vendor/distlib/index.py
+++ b/src/pip/_vendor/distlib/index.py
@@ -18,7 +18,7 @@
 from . import DistlibException
 from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr,
                      urlparse, build_opener, string_types)
-from .util import cached_property, zip_dir, ServerProxy
+from .util import zip_dir, ServerProxy
 
 logger = logging.getLogger(__name__)
 
@@ -67,21 +67,17 @@ def _get_pypirc_command(self):
         Get the distutils command for interacting with PyPI configurations.
         :return: the command.
         """
-        from distutils.core import Distribution
-        from distutils.config import PyPIRCCommand
-        d = Distribution()
-        return PyPIRCCommand(d)
+        from .util import _get_pypirc_command as cmd
+        return cmd()
 
     def read_configuration(self):
         """
-        Read the PyPI access configuration as supported by distutils, getting
-        PyPI to do the actual work. This populates ``username``, ``password``,
-        ``realm`` and ``url`` attributes from the configuration.
+        Read the PyPI access configuration as supported by distutils. This populates
+        ``username``, ``password``, ``realm`` and ``url`` attributes from the
+        configuration.
         """
-        # get distutils to do the work
-        c = self._get_pypirc_command()
-        c.repository = self.url
-        cfg = c._read_pypirc()
+        from .util import _load_pypirc
+        cfg = _load_pypirc(self)
         self.username = cfg.get('username')
         self.password = cfg.get('password')
         self.realm = cfg.get('realm', 'pypi')
@@ -91,13 +87,10 @@ def save_configuration(self):
         """
         Save the PyPI access configuration. You must have set ``username`` and
         ``password`` attributes before calling this method.
-
-        Again, distutils is used to do the actual work.
         """
         self.check_credentials()
-        # get distutils to do the work
-        c = self._get_pypirc_command()
-        c._store_pypirc(self.username, self.password)
+        from .util import _store_pypirc
+        _store_pypirc(self)
 
     def check_credentials(self):
         """
diff --git a/src/pip/_vendor/distlib/locators.py b/src/pip/_vendor/distlib/locators.py
index 12a1d06351e..0c7d6391438 100644
--- a/src/pip/_vendor/distlib/locators.py
+++ b/src/pip/_vendor/distlib/locators.py
@@ -20,14 +20,14 @@
 
 from . import DistlibException
 from .compat import (urljoin, urlparse, urlunparse, url2pathname, pathname2url,
-                     queue, quote, unescape, string_types, build_opener,
+                     queue, quote, unescape, build_opener,
                      HTTPRedirectHandler as BaseRedirectHandler, text_type,
                      Request, HTTPError, URLError)
 from .database import Distribution, DistributionPath, make_dist
 from .metadata import Metadata, MetadataInvalidError
-from .util import (cached_property, parse_credentials, ensure_slash,
-                   split_filename, get_project_data, parse_requirement,
-                   parse_name_and_version, ServerProxy, normalize_name)
+from .util import (cached_property, ensure_slash, split_filename, get_project_data,
+                   parse_requirement, parse_name_and_version, ServerProxy,
+                   normalize_name)
 from .version import get_scheme, UnsupportedVersionError
 from .wheel import Wheel, is_compatible
 
@@ -378,13 +378,13 @@ def locate(self, requirement, prereleases=False):
                     continue
                 try:
                     if not matcher.match(k):
-                        logger.debug('%s did not match %r', matcher, k)
+                        pass  # logger.debug('%s did not match %r', matcher, k)
                     else:
                         if prereleases or not vcls(k).is_prerelease:
                             slist.append(k)
-                        else:
-                            logger.debug('skipping pre-release '
-                                         'version %s of %s', k, matcher.name)
+                        # else:
+                            # logger.debug('skipping pre-release '
+                                         # 'version %s of %s', k, matcher.name)
                 except Exception:  # pragma: no cover
                     logger.warning('error matching %s with %r', matcher, k)
                     pass # slist.append(k)
@@ -593,7 +593,7 @@ class SimpleScrapingLocator(Locator):
     # These are used to deal with various Content-Encoding schemes.
     decoders = {
         'deflate': zlib.decompress,
-        'gzip': lambda b: gzip.GzipFile(fileobj=BytesIO(d)).read(),
+        'gzip': lambda b: gzip.GzipFile(fileobj=BytesIO(b)).read(),
         'none': lambda b: b,
     }
 
@@ -1062,8 +1062,6 @@ def get_distribution_names(self):
 
 locate = default_locator.locate
 
-NAME_VERSION_RE = re.compile(r'(?P[\w-]+)\s*'
-                             r'\(\s*(==\s*)?(?P[^)]+)\)$')
 
 class DependencyFinder(object):
     """
diff --git a/src/pip/_vendor/distlib/markers.py b/src/pip/_vendor/distlib/markers.py
index ee1f3e23655..b43136fa11e 100644
--- a/src/pip/_vendor/distlib/markers.py
+++ b/src/pip/_vendor/distlib/markers.py
@@ -13,20 +13,29 @@
 # as ~= and === which aren't in Python, necessitating a different approach.
 
 import os
+import re
 import sys
 import platform
-import re
 
-from .compat import python_implementation, urlparse, string_types
+from .compat import string_types
 from .util import in_venv, parse_marker
+from .version import NormalizedVersion as NV
 
 __all__ = ['interpret']
 
+_VERSION_PATTERN = re.compile(r'((\d+(\.\d+)*\w*)|\'(\d+(\.\d+)*\w*)\'|\"(\d+(\.\d+)*\w*)\")')
+
 def _is_literal(o):
     if not isinstance(o, string_types) or not o:
         return False
     return o[0] in '\'"'
 
+def _get_versions(s):
+    result = []
+    for m in _VERSION_PATTERN.finditer(s):
+        result.append(NV(m.groups()[0]))
+    return set(result)
+
 class Evaluator(object):
     """
     This class is used to evaluate marker expessions.
@@ -71,6 +80,13 @@ def evaluate(self, expr, context):
 
             lhs = self.evaluate(elhs, context)
             rhs = self.evaluate(erhs, context)
+            if ((elhs == 'python_version' or erhs == 'python_version') and
+                op in ('<', '<=', '>', '>=', '===', '==', '!=', '~=')):
+                lhs = NV(lhs)
+                rhs = NV(rhs)
+            elif elhs == 'python_version' and op in ('in', 'not in'):
+                lhs = NV(lhs)
+                rhs = _get_versions(rhs)
             result = self.operations[op](lhs, rhs)
         return result
 
diff --git a/src/pip/_vendor/distlib/metadata.py b/src/pip/_vendor/distlib/metadata.py
index 6d5e236090d..6a26b0ab232 100644
--- a/src/pip/_vendor/distlib/metadata.py
+++ b/src/pip/_vendor/distlib/metadata.py
@@ -94,8 +94,9 @@ class MetadataInvalidError(DistlibException):
 # See issue #106: Sometimes 'Requires' and 'Provides' occur wrongly in
 # the metadata. Include them in the tuple literal below to allow them
 # (for now).
+# Ditto for Obsoletes - see issue #140.
 _566_FIELDS = _426_FIELDS + ('Description-Content-Type',
-                             'Requires', 'Provides')
+                             'Requires', 'Provides', 'Obsoletes')
 
 _566_MARKERS = ('Description-Content-Type',)
 
@@ -117,7 +118,8 @@ def _version2fieldlist(version):
     elif version == '1.2':
         return _345_FIELDS
     elif version in ('1.3', '2.1'):
-        return _345_FIELDS + _566_FIELDS
+        # avoid adding field names if already there
+        return _345_FIELDS + tuple(f for f in _566_FIELDS if f not in _345_FIELDS)
     elif version == '2.0':
         return _426_FIELDS
     raise MetadataUnrecognizedVersionError(version)
diff --git a/src/pip/_vendor/distlib/resources.py b/src/pip/_vendor/distlib/resources.py
index 18840167a9e..fef52aa103e 100644
--- a/src/pip/_vendor/distlib/resources.py
+++ b/src/pip/_vendor/distlib/resources.py
@@ -11,13 +11,12 @@
 import logging
 import os
 import pkgutil
-import shutil
 import sys
 import types
 import zipimport
 
 from . import DistlibException
-from .util import cached_property, get_cache_base, path_to_cache_dir, Cache
+from .util import cached_property, get_cache_base, Cache
 
 logger = logging.getLogger(__name__)
 
@@ -283,6 +282,7 @@ def _is_directory(self, path):
             result = False
         return result
 
+
 _finder_registry = {
     type(None): ResourceFinder,
     zipimport.zipimporter: ZipResourceFinder
@@ -296,6 +296,8 @@ def _is_directory(self, path):
         import _frozen_importlib as _fi
     _finder_registry[_fi.SourceFileLoader] = ResourceFinder
     _finder_registry[_fi.FileFinder] = ResourceFinder
+    # See issue #146
+    _finder_registry[_fi.SourcelessFileLoader] = ResourceFinder
     del _fi
 except (ImportError, AttributeError):
     pass
@@ -304,6 +306,7 @@ def _is_directory(self, path):
 def register_finder(loader, finder_maker):
     _finder_registry[type(loader)] = finder_maker
 
+
 _finder_cache = {}
 
 
diff --git a/src/pip/_vendor/distlib/scripts.py b/src/pip/_vendor/distlib/scripts.py
index 03f8f21e0ff..913912c7b8e 100644
--- a/src/pip/_vendor/distlib/scripts.py
+++ b/src/pip/_vendor/distlib/scripts.py
@@ -14,7 +14,7 @@
 from .compat import sysconfig, detect_encoding, ZipFile
 from .resources import finder
 from .util import (FileOperator, get_export_entry, convert_path,
-                   get_executable, in_venv)
+                   get_executable, get_platform, in_venv)
 
 logger = logging.getLogger(__name__)
 
@@ -170,6 +170,11 @@ def _get_shebang(self, encoding, post_interp=b'', options=None):
                 sysconfig.get_config_var('BINDIR'),
                'python%s%s' % (sysconfig.get_config_var('VERSION'),
                                sysconfig.get_config_var('EXE')))
+            if not os.path.isfile(executable):
+                # for Python builds from source on Windows, no Python executables with
+                # a version suffix are created, so we use python.exe
+                executable = os.path.join(sysconfig.get_config_var('BINDIR'),
+                                'python%s' % (sysconfig.get_config_var('EXE')))
         if options:
             executable = self._get_alternate_executable(executable, options)
 
@@ -282,6 +287,19 @@ def _write_script(self, names, shebang, script_bytes, filenames, ext):
                     self._fileop.set_executable_mode([outname])
             filenames.append(outname)
 
+    variant_separator = '-'
+
+    def get_script_filenames(self, name):
+        result = set()
+        if '' in self.variants:
+            result.add(name)
+        if 'X' in self.variants:
+            result.add('%s%s' % (name, self.version_info[0]))
+        if 'X.Y' in self.variants:
+            result.add('%s%s%s.%s' % (name, self.variant_separator,
+                                      self.version_info[0], self.version_info[1]))
+        return result
+
     def _make_script(self, entry, filenames, options=None):
         post_interp = b''
         if options:
@@ -291,15 +309,7 @@ def _make_script(self, entry, filenames, options=None):
                 post_interp = args.encode('utf-8')
         shebang = self._get_shebang('utf-8', post_interp, options=options)
         script = self._get_script_text(entry).encode('utf-8')
-        name = entry.name
-        scriptnames = set()
-        if '' in self.variants:
-            scriptnames.add(name)
-        if 'X' in self.variants:
-            scriptnames.add('%s%s' % (name, self.version_info[0]))
-        if 'X.Y' in self.variants:
-            scriptnames.add('%s-%s.%s' % (name, self.version_info[0],
-                                          self.version_info[1]))
+        scriptnames = self.get_script_filenames(entry.name)
         if options and options.get('gui', False):
             ext = 'pyw'
         else:
@@ -326,8 +336,7 @@ def _copy_script(self, script, filenames):
         else:
             first_line = f.readline()
             if not first_line:  # pragma: no cover
-                logger.warning('%s: %s is an empty file (skipping)',
-                               self.get_command_name(),  script)
+                logger.warning('%s is an empty file (skipping)', script)
                 return
 
             match = FIRST_LINE_RE.match(first_line.replace(b'\r\n', b'\n'))
@@ -375,7 +384,8 @@ def _get_launcher(self, kind):
                 bits = '64'
             else:
                 bits = '32'
-            name = '%s%s.exe' % (kind, bits)
+            platform_suffix = '-arm' if get_platform() == 'win-arm64' else ''
+            name = '%s%s%s.exe' % (kind, bits, platform_suffix)
             # Issue 31: don't hardcode an absolute package name, but
             # determine it relative to the current package
             distlib_package = __name__.rsplit('.', 1)[0]
diff --git a/src/pip/_vendor/distlib/t64-arm.exe b/src/pip/_vendor/distlib/t64-arm.exe
new file mode 100644
index 00000000000..c5df4869da5
Binary files /dev/null and b/src/pip/_vendor/distlib/t64-arm.exe differ
diff --git a/src/pip/_vendor/distlib/util.py b/src/pip/_vendor/distlib/util.py
index 01324eae462..80bfc864bcb 100644
--- a/src/pip/_vendor/distlib/util.py
+++ b/src/pip/_vendor/distlib/util.py
@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2012-2017 The Python Software Foundation.
+# Copyright (C) 2012-2021 The Python Software Foundation.
 # See LICENSE.txt and CONTRIBUTORS.txt.
 #
 import codecs
@@ -215,6 +215,10 @@ def get_versions(ver_remaining):
                         if not ver_remaining or ver_remaining[0] != ',':
                             break
                         ver_remaining = ver_remaining[1:].lstrip()
+                        # Some packages have a trailing comma which would break things
+                        # See issue #148
+                        if not ver_remaining:
+                            break
                         m = COMPARE_OP.match(ver_remaining)
                         if not m:
                             raise SyntaxError('invalid constraint: %s' % ver_remaining)
@@ -309,7 +313,9 @@ def get_executable():
 #    else:
 #        result = sys.executable
 #    return result
-    result = os.path.normcase(sys.executable)
+    # Avoid normcasing: see issue #143
+    # result = os.path.normcase(sys.executable)
+    result = sys.executable
     if not isinstance(result, text_type):
         result = fsdecode(result)
     return result
@@ -1570,7 +1576,8 @@ def __init__(self, uri, **kwargs):
         # The above classes only come into play if a timeout
         # is specified
         if timeout is not None:
-            scheme, _ = splittype(uri)
+            # scheme = splittype(uri)  # deprecated as of Python 3.8
+            scheme = urlparse(uri)[0]
             use_datetime = kwargs.get('use_datetime', 0)
             if scheme == 'https':
                 tcls = SafeTransport
@@ -1759,3 +1766,204 @@ def normalize_name(name):
     """Normalize a python package name a la PEP 503"""
     # https://www.python.org/dev/peps/pep-0503/#normalized-names
     return re.sub('[-_.]+', '-', name).lower()
+
+# def _get_pypirc_command():
+    # """
+    # Get the distutils command for interacting with PyPI configurations.
+    # :return: the command.
+    # """
+    # from distutils.core import Distribution
+    # from distutils.config import PyPIRCCommand
+    # d = Distribution()
+    # return PyPIRCCommand(d)
+
+class PyPIRCFile(object):
+
+    DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/'
+    DEFAULT_REALM = 'pypi'
+
+    def __init__(self, fn=None, url=None):
+        if fn is None:
+            fn = os.path.join(os.path.expanduser('~'), '.pypirc')
+        self.filename = fn
+        self.url = url
+
+    def read(self):
+        result = {}
+
+        if os.path.exists(self.filename):
+            repository = self.url or self.DEFAULT_REPOSITORY
+
+            config = configparser.RawConfigParser()
+            config.read(self.filename)
+            sections = config.sections()
+            if 'distutils' in sections:
+                # let's get the list of servers
+                index_servers = config.get('distutils', 'index-servers')
+                _servers = [server.strip() for server in
+                            index_servers.split('\n')
+                            if server.strip() != '']
+                if _servers == []:
+                    # nothing set, let's try to get the default pypi
+                    if 'pypi' in sections:
+                        _servers = ['pypi']
+                else:
+                    for server in _servers:
+                        result = {'server': server}
+                        result['username'] = config.get(server, 'username')
+
+                        # optional params
+                        for key, default in (('repository', self.DEFAULT_REPOSITORY),
+                                             ('realm', self.DEFAULT_REALM),
+                                             ('password', None)):
+                            if config.has_option(server, key):
+                                result[key] = config.get(server, key)
+                            else:
+                                result[key] = default
+
+                        # work around people having "repository" for the "pypi"
+                        # section of their config set to the HTTP (rather than
+                        # HTTPS) URL
+                        if (server == 'pypi' and
+                            repository in (self.DEFAULT_REPOSITORY, 'pypi')):
+                            result['repository'] = self.DEFAULT_REPOSITORY
+                        elif (result['server'] != repository and
+                              result['repository'] != repository):
+                            result = {}
+            elif 'server-login' in sections:
+                # old format
+                server = 'server-login'
+                if config.has_option(server, 'repository'):
+                    repository = config.get(server, 'repository')
+                else:
+                    repository = self.DEFAULT_REPOSITORY
+                result = {
+                    'username': config.get(server, 'username'),
+                    'password': config.get(server, 'password'),
+                    'repository': repository,
+                    'server': server,
+                    'realm': self.DEFAULT_REALM
+                }
+        return result
+
+    def update(self, username, password):
+        # import pdb; pdb.set_trace()
+        config = configparser.RawConfigParser()
+        fn = self.filename
+        config.read(fn)
+        if not config.has_section('pypi'):
+            config.add_section('pypi')
+        config.set('pypi', 'username', username)
+        config.set('pypi', 'password', password)
+        with open(fn, 'w') as f:
+            config.write(f)
+
+def _load_pypirc(index):
+    """
+    Read the PyPI access configuration as supported by distutils.
+    """
+    return PyPIRCFile(url=index.url).read()
+
+def _store_pypirc(index):
+    PyPIRCFile().update(index.username, index.password)
+
+#
+# get_platform()/get_host_platform() copied from Python 3.10.a0 source, with some minor
+# tweaks
+#
+
+def get_host_platform():
+    """Return a string that identifies the current platform.  This is used mainly to
+    distinguish platform-specific build directories and platform-specific built
+    distributions.  Typically includes the OS name and version and the
+    architecture (as supplied by 'os.uname()'), although the exact information
+    included depends on the OS; eg. on Linux, the kernel version isn't
+    particularly important.
+
+    Examples of returned values:
+       linux-i586
+       linux-alpha (?)
+       solaris-2.6-sun4u
+
+    Windows will return one of:
+       win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
+       win32 (all others - specifically, sys.platform is returned)
+
+    For other non-POSIX platforms, currently just returns 'sys.platform'.
+
+    """
+    if os.name == 'nt':
+        if 'amd64' in sys.version.lower():
+            return 'win-amd64'
+        if '(arm)' in sys.version.lower():
+            return 'win-arm32'
+        if '(arm64)' in sys.version.lower():
+            return 'win-arm64'
+        return sys.platform
+
+    # Set for cross builds explicitly
+    if "_PYTHON_HOST_PLATFORM" in os.environ:
+        return os.environ["_PYTHON_HOST_PLATFORM"]
+
+    if os.name != 'posix' or not hasattr(os, 'uname'):
+        # XXX what about the architecture? NT is Intel or Alpha,
+        # Mac OS is M68k or PPC, etc.
+        return sys.platform
+
+    # Try to distinguish various flavours of Unix
+
+    (osname, host, release, version, machine) = os.uname()
+
+    # Convert the OS name to lowercase, remove '/' characters, and translate
+    # spaces (for "Power Macintosh")
+    osname = osname.lower().replace('/', '')
+    machine = machine.replace(' ', '_').replace('/', '-')
+
+    if osname[:5] == 'linux':
+        # At least on Linux/Intel, 'machine' is the processor --
+        # i386, etc.
+        # XXX what about Alpha, SPARC, etc?
+        return  "%s-%s" % (osname, machine)
+
+    elif osname[:5] == 'sunos':
+        if release[0] >= '5':           # SunOS 5 == Solaris 2
+            osname = 'solaris'
+            release = '%d.%s' % (int(release[0]) - 3, release[2:])
+            # We can't use 'platform.architecture()[0]' because a
+            # bootstrap problem. We use a dict to get an error
+            # if some suspicious happens.
+            bitness = {2147483647:'32bit', 9223372036854775807:'64bit'}
+            machine += '.%s' % bitness[sys.maxsize]
+        # fall through to standard osname-release-machine representation
+    elif osname[:3] == 'aix':
+        from _aix_support import aix_platform
+        return aix_platform()
+    elif osname[:6] == 'cygwin':
+        osname = 'cygwin'
+        rel_re = re.compile (r'[\d.]+', re.ASCII)
+        m = rel_re.match(release)
+        if m:
+            release = m.group()
+    elif osname[:6] == 'darwin':
+        import _osx_support, distutils.sysconfig
+        osname, release, machine = _osx_support.get_platform_osx(
+                                        distutils.sysconfig.get_config_vars(),
+                                        osname, release, machine)
+
+    return '%s-%s-%s' % (osname, release, machine)
+
+
+_TARGET_TO_PLAT = {
+    'x86' : 'win32',
+    'x64' : 'win-amd64',
+    'arm' : 'win-arm32',
+}
+
+
+def get_platform():
+    if os.name != 'nt':
+        return get_host_platform()
+    cross_compilation_target = os.environ.get('VSCMD_ARG_TGT_ARCH')
+    if cross_compilation_target not in _TARGET_TO_PLAT:
+        return get_host_platform()
+    return _TARGET_TO_PLAT[cross_compilation_target]
diff --git a/src/pip/_vendor/distlib/version.py b/src/pip/_vendor/distlib/version.py
index 3eebe18ee84..c7c8bb6ff4f 100644
--- a/src/pip/_vendor/distlib/version.py
+++ b/src/pip/_vendor/distlib/version.py
@@ -194,7 +194,7 @@ def _pep_440_key(s):
     if not groups[0]:
         epoch = 0
     else:
-        epoch = int(groups[0])
+        epoch = int(groups[0][:-1])
     pre = groups[4:6]
     post = groups[7:9]
     dev = groups[10:12]
@@ -710,6 +710,9 @@ def is_valid_constraint_list(self, s):
         """
         Used for processing some metadata fields
         """
+        # See issue #140. Be tolerant of a single trailing comma.
+        if s.endswith(','):
+            s = s[:-1]
         return self.is_valid_matcher('dummy_name (%s)' % s)
 
     def suggest(self, s):
diff --git a/src/pip/_vendor/distlib/w64-arm.exe b/src/pip/_vendor/distlib/w64-arm.exe
new file mode 100644
index 00000000000..70a2ec26852
Binary files /dev/null and b/src/pip/_vendor/distlib/w64-arm.exe differ
diff --git a/src/pip/_vendor/distlib/wheel.py b/src/pip/_vendor/distlib/wheel.py
index 1e2c7a020c9..48abfde5b52 100644
--- a/src/pip/_vendor/distlib/wheel.py
+++ b/src/pip/_vendor/distlib/wheel.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013-2017 Vinay Sajip.
+# Copyright (C) 2013-2020 Vinay Sajip.
 # Licensed to the Python Software Foundation under a contributor agreement.
 # See LICENSE.txt and CONTRIBUTORS.txt.
 #
@@ -9,7 +9,6 @@
 import base64
 import codecs
 import datetime
-import distutils.util
 from email import message_from_file
 import hashlib
 import imp
@@ -29,7 +28,8 @@
 from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME,
                        LEGACY_METADATA_FILENAME)
 from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache,
-                   cached_property, get_cache_base, read_exports, tempdir)
+                   cached_property, get_cache_base, read_exports, tempdir,
+                   get_platform)
 from .version import NormalizedVersion, UnsupportedVersionError
 
 logger = logging.getLogger(__name__)
@@ -51,11 +51,11 @@
 PYVER = 'py' + VER_SUFFIX
 IMPVER = IMP_PREFIX + VER_SUFFIX
 
-ARCH = distutils.util.get_platform().replace('-', '_').replace('.', '_')
+ARCH = get_platform().replace('-', '_').replace('.', '_')
 
 ABI = sysconfig.get_config_var('SOABI')
 if ABI and ABI.startswith('cpython-'):
-    ABI = ABI.replace('cpython-', 'cp')
+    ABI = ABI.replace('cpython-', 'cp').split('-')[0]
 else:
     def _derive_abi():
         parts = ['cp', VER_SUFFIX]
@@ -576,6 +576,13 @@ def install(self, paths, maker, **kwargs):
                     if not is_script:
                         with zf.open(arcname) as bf:
                             fileop.copy_stream(bf, outfile)
+                        # Issue #147: permission bits aren't preserved. Using
+                        # zf.extract(zinfo, libdir) should have worked, but didn't,
+                        # see https://www.thetopsites.net/article/53834422.shtml
+                        # So ... manually preserve permission bits as given in zinfo
+                        if os.name == 'posix':
+                            # just set the normal permission bits
+                            os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF)
                         outfiles.append(outfile)
                         # Double check the digest of the written file
                         if not dry_run and row[1]:
@@ -938,6 +945,16 @@ def update_version(version, path):
                     shutil.copyfile(newpath, pathname)
         return modified
 
+def _get_glibc_version():
+    import platform
+    ver = platform.libc_ver()
+    result = []
+    if ver[0] == 'glibc':
+        for s in ver[1].split('.'):
+            result.append(int(s) if s.isdigit() else 0)
+        result = tuple(result)
+    return result
+
 def compatible_tags():
     """
     Return (pyver, abi, arch) tuples compatible with this Python.
@@ -985,6 +1002,23 @@ def compatible_tags():
     for abi in abis:
         for arch in arches:
             result.append((''.join((IMP_PREFIX, versions[0])), abi, arch))
+            # manylinux
+            if abi != 'none' and sys.platform.startswith('linux'):
+                arch = arch.replace('linux_', '')
+                parts = _get_glibc_version()
+                if len(parts) == 2:
+                    if parts >= (2, 5):
+                        result.append((''.join((IMP_PREFIX, versions[0])), abi,
+                                       'manylinux1_%s' % arch))
+                    if parts >= (2, 12):
+                        result.append((''.join((IMP_PREFIX, versions[0])), abi,
+                                       'manylinux2010_%s' % arch))
+                    if parts >= (2, 17):
+                        result.append((''.join((IMP_PREFIX, versions[0])), abi,
+                                       'manylinux2014_%s' % arch))
+                    result.append((''.join((IMP_PREFIX, versions[0])), abi,
+                                   'manylinux_%s_%s_%s' % (parts[0], parts[1],
+                                                           arch)))
 
     # where no ABI / arch dependency, but IMP_PREFIX dependency
     for i, version in enumerate(versions):
@@ -997,6 +1031,7 @@ def compatible_tags():
         result.append((''.join(('py', version)), 'none', 'any'))
         if i == 0:
             result.append((''.join(('py', version[0])), 'none', 'any'))
+
     return set(result)
 
 
diff --git a/src/pip/_vendor/distro.py b/src/pip/_vendor/distro.py
index 0611b62a3a8..7892741347d 100644
--- a/src/pip/_vendor/distro.py
+++ b/src/pip/_vendor/distro.py
@@ -20,26 +20,61 @@
 It is the recommended replacement for Python's original
 :py:func:`platform.linux_distribution` function, but it provides much more
 functionality. An alternative implementation became necessary because Python
-3.5 deprecated this function, and Python 3.8 will remove it altogether.
-Its predecessor function :py:func:`platform.dist` was already
-deprecated since Python 2.6 and will also be removed in Python 3.8.
-Still, there are many cases in which access to OS distribution information
-is needed. See `Python issue 1322 `_ for
-more information.
+3.5 deprecated this function, and Python 3.8 removed it altogether. Its
+predecessor function :py:func:`platform.dist` was already deprecated since
+Python 2.6 and removed in Python 3.8. Still, there are many cases in which
+access to OS distribution information is needed. See `Python issue 1322
+`_ for more information.
 """
 
+import argparse
+import json
+import logging
 import os
 import re
-import sys
-import json
 import shlex
-import logging
-import argparse
 import subprocess
+import sys
+import warnings
+
+__version__ = "1.6.0"
+
+# Use `if False` to avoid an ImportError on Python 2. After dropping Python 2
+# support, can use typing.TYPE_CHECKING instead. See:
+# https://docs.python.org/3/library/typing.html#typing.TYPE_CHECKING
+if False:  # pragma: nocover
+    from typing import (
+        Any,
+        Callable,
+        Dict,
+        Iterable,
+        Optional,
+        Sequence,
+        TextIO,
+        Tuple,
+        Type,
+        TypedDict,
+        Union,
+    )
+
+    VersionDict = TypedDict(
+        "VersionDict", {"major": str, "minor": str, "build_number": str}
+    )
+    InfoDict = TypedDict(
+        "InfoDict",
+        {
+            "id": str,
+            "version": str,
+            "version_parts": VersionDict,
+            "like": str,
+            "codename": str,
+        },
+    )
 
 
-_UNIXCONFDIR = os.environ.get('UNIXCONFDIR', '/etc')
-_OS_RELEASE_BASENAME = 'os-release'
+_UNIXCONFDIR = os.environ.get("UNIXCONFDIR", "/etc")
+_UNIXUSRLIBDIR = os.environ.get("UNIXUSRLIBDIR", "/usr/lib")
+_OS_RELEASE_BASENAME = "os-release"
 
 #: Translation table for normalizing the "ID" attribute defined in os-release
 #: files, for use by the :func:`distro.id` method.
@@ -49,7 +84,7 @@
 #:
 #: * Value: Normalized value.
 NORMALIZED_OS_ID = {
-    'ol': 'oracle',  # Oracle Linux
+    "ol": "oracle",  # Oracle Linux
 }
 
 #: Translation table for normalizing the "Distributor ID" attribute returned by
@@ -60,11 +95,11 @@
 #:
 #: * Value: Normalized value.
 NORMALIZED_LSB_ID = {
-    'enterpriseenterpriseas': 'oracle',  # Oracle Enterprise Linux 4
-    'enterpriseenterpriseserver': 'oracle',  # Oracle Linux 5
-    'redhatenterpriseworkstation': 'rhel',  # RHEL 6, 7 Workstation
-    'redhatenterpriseserver': 'rhel',  # RHEL 6, 7 Server
-    'redhatenterprisecomputenode': 'rhel',  # RHEL 6 ComputeNode
+    "enterpriseenterpriseas": "oracle",  # Oracle Enterprise Linux 4
+    "enterpriseenterpriseserver": "oracle",  # Oracle Linux 5
+    "redhatenterpriseworkstation": "rhel",  # RHEL 6, 7 Workstation
+    "redhatenterpriseserver": "rhel",  # RHEL 6, 7 Server
+    "redhatenterprisecomputenode": "rhel",  # RHEL 6 ComputeNode
 }
 
 #: Translation table for normalizing the distro ID derived from the file name
@@ -75,30 +110,39 @@
 #:
 #: * Value: Normalized value.
 NORMALIZED_DISTRO_ID = {
-    'redhat': 'rhel',  # RHEL 6.x, 7.x
+    "redhat": "rhel",  # RHEL 6.x, 7.x
 }
 
 # Pattern for content of distro release file (reversed)
 _DISTRO_RELEASE_CONTENT_REVERSED_PATTERN = re.compile(
-    r'(?:[^)]*\)(.*)\()? *(?:STL )?([\d.+\-a-z]*\d) *(?:esaeler *)?(.+)')
+    r"(?:[^)]*\)(.*)\()? *(?:STL )?([\d.+\-a-z]*\d) *(?:esaeler *)?(.+)"
+)
 
 # Pattern for base file name of distro release file
-_DISTRO_RELEASE_BASENAME_PATTERN = re.compile(
-    r'(\w+)[-_](release|version)$')
+_DISTRO_RELEASE_BASENAME_PATTERN = re.compile(r"(\w+)[-_](release|version)$")
 
 # Base file names to be ignored when searching for distro release file
 _DISTRO_RELEASE_IGNORE_BASENAMES = (
-    'debian_version',
-    'lsb-release',
-    'oem-release',
+    "debian_version",
+    "lsb-release",
+    "oem-release",
     _OS_RELEASE_BASENAME,
-    'system-release',
-    'plesk-release',
+    "system-release",
+    "plesk-release",
+    "iredmail-release",
 )
 
 
 def linux_distribution(full_distribution_name=True):
+    # type: (bool) -> Tuple[str, str, str]
     """
+    .. deprecated:: 1.6.0
+
+        :func:`distro.linux_distribution()` is deprecated. It should only be
+        used as a compatibility shim with Python's
+        :py:func:`platform.linux_distribution()`. Please use :func:`distro.id`,
+        :func:`distro.version` and :func:`distro.name` instead.
+
     Return information about the current OS distribution as a tuple
     ``(id_name, version, codename)`` with items as follows:
 
@@ -122,10 +166,18 @@ def linux_distribution(full_distribution_name=True):
     method normalizes the distro ID string to a reliable machine-readable value
     for a number of popular OS distributions.
     """
+    warnings.warn(
+        "distro.linux_distribution() is deprecated. It should only be used as a "
+        "compatibility shim with Python's platform.linux_distribution(). Please use "
+        "distro.id(), distro.version() and distro.name() instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     return _distro.linux_distribution(full_distribution_name)
 
 
 def id():
+    # type: () -> str
     """
     Return the distro ID of the current distribution, as a
     machine-readable string.
@@ -205,6 +257,7 @@ def id():
 
 
 def name(pretty=False):
+    # type: (bool) -> str
     """
     Return the name of the current OS distribution, as a human-readable
     string.
@@ -244,6 +297,7 @@ def name(pretty=False):
 
 
 def version(pretty=False, best=False):
+    # type: (bool, bool) -> str
     """
     Return the version of the current OS distribution, as a human-readable
     string.
@@ -288,6 +342,7 @@ def version(pretty=False, best=False):
 
 
 def version_parts(best=False):
+    # type: (bool) -> Tuple[str, str, str]
     """
     Return the version of the current OS distribution as a tuple
     ``(major, minor, build_number)`` with items as follows:
@@ -305,6 +360,7 @@ def version_parts(best=False):
 
 
 def major_version(best=False):
+    # type: (bool) -> str
     """
     Return the major version of the current OS distribution, as a string,
     if provided.
@@ -318,6 +374,7 @@ def major_version(best=False):
 
 
 def minor_version(best=False):
+    # type: (bool) -> str
     """
     Return the minor version of the current OS distribution, as a string,
     if provided.
@@ -331,6 +388,7 @@ def minor_version(best=False):
 
 
 def build_number(best=False):
+    # type: (bool) -> str
     """
     Return the build number of the current OS distribution, as a string,
     if provided.
@@ -344,6 +402,7 @@ def build_number(best=False):
 
 
 def like():
+    # type: () -> str
     """
     Return a space-separated list of distro IDs of distributions that are
     closely related to the current OS distribution in regards to packaging
@@ -361,6 +420,7 @@ def like():
 
 
 def codename():
+    # type: () -> str
     """
     Return the codename for the release of the current OS distribution,
     as a string.
@@ -385,6 +445,7 @@ def codename():
 
 
 def info(pretty=False, best=False):
+    # type: (bool, bool) -> InfoDict
     """
     Return certain machine-readable information items about the current OS
     distribution in a dictionary, as shown in the following example:
@@ -429,6 +490,7 @@ def info(pretty=False, best=False):
 
 
 def os_release_info():
+    # type: () -> Dict[str, str]
     """
     Return a dictionary containing key-value pairs for the information items
     from the os-release file data source of the current OS distribution.
@@ -439,6 +501,7 @@ def os_release_info():
 
 
 def lsb_release_info():
+    # type: () -> Dict[str, str]
     """
     Return a dictionary containing key-value pairs for the information items
     from the lsb_release command data source of the current OS distribution.
@@ -450,6 +513,7 @@ def lsb_release_info():
 
 
 def distro_release_info():
+    # type: () -> Dict[str, str]
     """
     Return a dictionary containing key-value pairs for the information items
     from the distro release file data source of the current OS distribution.
@@ -460,6 +524,7 @@ def distro_release_info():
 
 
 def uname_info():
+    # type: () -> Dict[str, str]
     """
     Return a dictionary containing key-value pairs for the information items
     from the distro release file data source of the current OS distribution.
@@ -468,6 +533,7 @@ def uname_info():
 
 
 def os_release_attr(attribute):
+    # type: (str) -> str
     """
     Return a single named information item from the os-release file data source
     of the current OS distribution.
@@ -487,6 +553,7 @@ def os_release_attr(attribute):
 
 
 def lsb_release_attr(attribute):
+    # type: (str) -> str
     """
     Return a single named information item from the lsb_release command output
     data source of the current OS distribution.
@@ -507,6 +574,7 @@ def lsb_release_attr(attribute):
 
 
 def distro_release_attr(attribute):
+    # type: (str) -> str
     """
     Return a single named information item from the distro release file
     data source of the current OS distribution.
@@ -526,6 +594,7 @@ def distro_release_attr(attribute):
 
 
 def uname_attr(attribute):
+    # type: (str) -> str
     """
     Return a single named information item from the distro release file
     data source of the current OS distribution.
@@ -542,19 +611,26 @@ def uname_attr(attribute):
     return _distro.uname_attr(attribute)
 
 
-class cached_property(object):
-    """A version of @property which caches the value.  On access, it calls the
-    underlying function and sets the value in `__dict__` so future accesses
-    will not re-call the property.
-    """
-    def __init__(self, f):
-        self._fname = f.__name__
-        self._f = f
+try:
+    from functools import cached_property
+except ImportError:
+    # Python < 3.8
+    class cached_property(object):  # type: ignore
+        """A version of @property which caches the value.  On access, it calls the
+        underlying function and sets the value in `__dict__` so future accesses
+        will not re-call the property.
+        """
+
+        def __init__(self, f):
+            # type: (Callable[[Any], Any]) -> None
+            self._fname = f.__name__
+            self._f = f
 
-    def __get__(self, obj, owner):
-        assert obj is not None, 'call {} on an instance'.format(self._fname)
-        ret = obj.__dict__[self._fname] = self._f(obj)
-        return ret
+        def __get__(self, obj, owner):
+            # type: (Any, Type[Any]) -> Any
+            assert obj is not None, "call {} on an instance".format(self._fname)
+            ret = obj.__dict__[self._fname] = self._f(obj)
+            return ret
 
 
 class LinuxDistribution(object):
@@ -575,11 +651,15 @@ class LinuxDistribution(object):
     lsb_release command.
     """
 
-    def __init__(self,
-                 include_lsb=True,
-                 os_release_file='',
-                 distro_release_file='',
-                 include_uname=True):
+    def __init__(
+        self,
+        include_lsb=True,
+        os_release_file="",
+        distro_release_file="",
+        include_uname=True,
+        root_dir=None,
+    ):
+        # type: (bool, str, str, bool, Optional[str]) -> None
         """
         The initialization method of this class gathers information from the
         available data sources, and stores that in private instance attributes.
@@ -618,6 +698,9 @@ def __init__(self,
           the program execution path the data source for the uname command will
           be empty.
 
+        * ``root_dir`` (string): The absolute path to the root directory to use
+          to find distro-related information files.
+
         Public instance attributes:
 
         * ``os_release_file`` (string): The path name of the
@@ -647,28 +730,50 @@ def __init__(self,
         * :py:exc:`UnicodeError`: A data source has unexpected characters or
           uses an unexpected encoding.
         """
-        self.os_release_file = os_release_file or \
-            os.path.join(_UNIXCONFDIR, _OS_RELEASE_BASENAME)
-        self.distro_release_file = distro_release_file or ''  # updated later
+        self.root_dir = root_dir
+        self.etc_dir = os.path.join(root_dir, "etc") if root_dir else _UNIXCONFDIR
+        self.usr_lib_dir = (
+            os.path.join(root_dir, "usr/lib") if root_dir else _UNIXUSRLIBDIR
+        )
+
+        if os_release_file:
+            self.os_release_file = os_release_file
+        else:
+            etc_dir_os_release_file = os.path.join(self.etc_dir, _OS_RELEASE_BASENAME)
+            usr_lib_os_release_file = os.path.join(
+                self.usr_lib_dir, _OS_RELEASE_BASENAME
+            )
+
+            # NOTE: The idea is to respect order **and** have it set
+            #       at all times for API backwards compatibility.
+            if os.path.isfile(etc_dir_os_release_file) or not os.path.isfile(
+                usr_lib_os_release_file
+            ):
+                self.os_release_file = etc_dir_os_release_file
+            else:
+                self.os_release_file = usr_lib_os_release_file
+
+        self.distro_release_file = distro_release_file or ""  # updated later
         self.include_lsb = include_lsb
         self.include_uname = include_uname
 
     def __repr__(self):
-        """Return repr of all info
-        """
-        return \
-            "LinuxDistribution(" \
-            "os_release_file={self.os_release_file!r}, " \
-            "distro_release_file={self.distro_release_file!r}, " \
-            "include_lsb={self.include_lsb!r}, " \
-            "include_uname={self.include_uname!r}, " \
-            "_os_release_info={self._os_release_info!r}, " \
-            "_lsb_release_info={self._lsb_release_info!r}, " \
-            "_distro_release_info={self._distro_release_info!r}, " \
-            "_uname_info={self._uname_info!r})".format(
-                self=self)
+        # type: () -> str
+        """Return repr of all info"""
+        return (
+            "LinuxDistribution("
+            "os_release_file={self.os_release_file!r}, "
+            "distro_release_file={self.distro_release_file!r}, "
+            "include_lsb={self.include_lsb!r}, "
+            "include_uname={self.include_uname!r}, "
+            "_os_release_info={self._os_release_info!r}, "
+            "_lsb_release_info={self._lsb_release_info!r}, "
+            "_distro_release_info={self._distro_release_info!r}, "
+            "_uname_info={self._uname_info!r})".format(self=self)
+        )
 
     def linux_distribution(self, full_distribution_name=True):
+        # type: (bool) -> Tuple[str, str, str]
         """
         Return information about the OS distribution that is compatible
         with Python's :func:`platform.linux_distribution`, supporting a subset
@@ -679,92 +784,102 @@ def linux_distribution(self, full_distribution_name=True):
         return (
             self.name() if full_distribution_name else self.id(),
             self.version(),
-            self.codename()
+            self.codename(),
         )
 
     def id(self):
+        # type: () -> str
         """Return the distro ID of the OS distribution, as a string.
 
         For details, see :func:`distro.id`.
         """
+
         def normalize(distro_id, table):
-            distro_id = distro_id.lower().replace(' ', '_')
+            # type: (str, Dict[str, str]) -> str
+            distro_id = distro_id.lower().replace(" ", "_")
             return table.get(distro_id, distro_id)
 
-        distro_id = self.os_release_attr('id')
+        distro_id = self.os_release_attr("id")
         if distro_id:
             return normalize(distro_id, NORMALIZED_OS_ID)
 
-        distro_id = self.lsb_release_attr('distributor_id')
+        distro_id = self.lsb_release_attr("distributor_id")
         if distro_id:
             return normalize(distro_id, NORMALIZED_LSB_ID)
 
-        distro_id = self.distro_release_attr('id')
+        distro_id = self.distro_release_attr("id")
         if distro_id:
             return normalize(distro_id, NORMALIZED_DISTRO_ID)
 
-        distro_id = self.uname_attr('id')
+        distro_id = self.uname_attr("id")
         if distro_id:
             return normalize(distro_id, NORMALIZED_DISTRO_ID)
 
-        return ''
+        return ""
 
     def name(self, pretty=False):
+        # type: (bool) -> str
         """
         Return the name of the OS distribution, as a string.
 
         For details, see :func:`distro.name`.
         """
-        name = self.os_release_attr('name') \
-            or self.lsb_release_attr('distributor_id') \
-            or self.distro_release_attr('name') \
-            or self.uname_attr('name')
+        name = (
+            self.os_release_attr("name")
+            or self.lsb_release_attr("distributor_id")
+            or self.distro_release_attr("name")
+            or self.uname_attr("name")
+        )
         if pretty:
-            name = self.os_release_attr('pretty_name') \
-                or self.lsb_release_attr('description')
+            name = self.os_release_attr("pretty_name") or self.lsb_release_attr(
+                "description"
+            )
             if not name:
-                name = self.distro_release_attr('name') \
-                       or self.uname_attr('name')
+                name = self.distro_release_attr("name") or self.uname_attr("name")
                 version = self.version(pretty=True)
                 if version:
-                    name = name + ' ' + version
-        return name or ''
+                    name = name + " " + version
+        return name or ""
 
     def version(self, pretty=False, best=False):
+        # type: (bool, bool) -> str
         """
         Return the version of the OS distribution, as a string.
 
         For details, see :func:`distro.version`.
         """
         versions = [
-            self.os_release_attr('version_id'),
-            self.lsb_release_attr('release'),
-            self.distro_release_attr('version_id'),
-            self._parse_distro_release_content(
-                self.os_release_attr('pretty_name')).get('version_id', ''),
+            self.os_release_attr("version_id"),
+            self.lsb_release_attr("release"),
+            self.distro_release_attr("version_id"),
+            self._parse_distro_release_content(self.os_release_attr("pretty_name")).get(
+                "version_id", ""
+            ),
             self._parse_distro_release_content(
-                self.lsb_release_attr('description')).get('version_id', ''),
-            self.uname_attr('release')
+                self.lsb_release_attr("description")
+            ).get("version_id", ""),
+            self.uname_attr("release"),
         ]
-        version = ''
+        version = ""
         if best:
             # This algorithm uses the last version in priority order that has
             # the best precision. If the versions are not in conflict, that
             # does not matter; otherwise, using the last one instead of the
             # first one might be considered a surprise.
             for v in versions:
-                if v.count(".") > version.count(".") or version == '':
+                if v.count(".") > version.count(".") or version == "":
                     version = v
         else:
             for v in versions:
-                if v != '':
+                if v != "":
                     version = v
                     break
         if pretty and version and self.codename():
-            version = '{0} ({1})'.format(version, self.codename())
+            version = "{0} ({1})".format(version, self.codename())
         return version
 
     def version_parts(self, best=False):
+        # type: (bool) -> Tuple[str, str, str]
         """
         Return the version of the OS distribution, as a tuple of version
         numbers.
@@ -773,14 +888,15 @@ def version_parts(self, best=False):
         """
         version_str = self.version(best=best)
         if version_str:
-            version_regex = re.compile(r'(\d+)\.?(\d+)?\.?(\d+)?')
+            version_regex = re.compile(r"(\d+)\.?(\d+)?\.?(\d+)?")
             matches = version_regex.match(version_str)
             if matches:
                 major, minor, build_number = matches.groups()
-                return major, minor or '', build_number or ''
-        return '', '', ''
+                return major, minor or "", build_number or ""
+        return "", "", ""
 
     def major_version(self, best=False):
+        # type: (bool) -> str
         """
         Return the major version number of the current distribution.
 
@@ -789,6 +905,7 @@ def major_version(self, best=False):
         return self.version_parts(best)[0]
 
     def minor_version(self, best=False):
+        # type: (bool) -> str
         """
         Return the minor version number of the current distribution.
 
@@ -797,6 +914,7 @@ def minor_version(self, best=False):
         return self.version_parts(best)[1]
 
     def build_number(self, best=False):
+        # type: (bool) -> str
         """
         Return the build number of the current distribution.
 
@@ -805,14 +923,16 @@ def build_number(self, best=False):
         return self.version_parts(best)[2]
 
     def like(self):
+        # type: () -> str
         """
         Return the IDs of distributions that are like the OS distribution.
 
         For details, see :func:`distro.like`.
         """
-        return self.os_release_attr('id_like') or ''
+        return self.os_release_attr("id_like") or ""
 
     def codename(self):
+        # type: () -> str
         """
         Return the codename of the OS distribution.
 
@@ -821,13 +941,16 @@ def codename(self):
         try:
             # Handle os_release specially since distros might purposefully set
             # this to empty string to have no codename
-            return self._os_release_info['codename']
+            return self._os_release_info["codename"]
         except KeyError:
-            return self.lsb_release_attr('codename') \
-                or self.distro_release_attr('codename') \
-                or ''
+            return (
+                self.lsb_release_attr("codename")
+                or self.distro_release_attr("codename")
+                or ""
+            )
 
     def info(self, pretty=False, best=False):
+        # type: (bool, bool) -> InfoDict
         """
         Return certain machine-readable information about the OS
         distribution.
@@ -840,13 +963,14 @@ def info(self, pretty=False, best=False):
             version_parts=dict(
                 major=self.major_version(best),
                 minor=self.minor_version(best),
-                build_number=self.build_number(best)
+                build_number=self.build_number(best),
             ),
             like=self.like(),
             codename=self.codename(),
         )
 
     def os_release_info(self):
+        # type: () -> Dict[str, str]
         """
         Return a dictionary containing key-value pairs for the information
         items from the os-release file data source of the OS distribution.
@@ -856,6 +980,7 @@ def os_release_info(self):
         return self._os_release_info
 
     def lsb_release_info(self):
+        # type: () -> Dict[str, str]
         """
         Return a dictionary containing key-value pairs for the information
         items from the lsb_release command data source of the OS
@@ -866,6 +991,7 @@ def lsb_release_info(self):
         return self._lsb_release_info
 
     def distro_release_info(self):
+        # type: () -> Dict[str, str]
         """
         Return a dictionary containing key-value pairs for the information
         items from the distro release file data source of the OS
@@ -876,6 +1002,7 @@ def distro_release_info(self):
         return self._distro_release_info
 
     def uname_info(self):
+        # type: () -> Dict[str, str]
         """
         Return a dictionary containing key-value pairs for the information
         items from the uname command data source of the OS distribution.
@@ -885,43 +1012,48 @@ def uname_info(self):
         return self._uname_info
 
     def os_release_attr(self, attribute):
+        # type: (str) -> str
         """
         Return a single named information item from the os-release file data
         source of the OS distribution.
 
         For details, see :func:`distro.os_release_attr`.
         """
-        return self._os_release_info.get(attribute, '')
+        return self._os_release_info.get(attribute, "")
 
     def lsb_release_attr(self, attribute):
+        # type: (str) -> str
         """
         Return a single named information item from the lsb_release command
         output data source of the OS distribution.
 
         For details, see :func:`distro.lsb_release_attr`.
         """
-        return self._lsb_release_info.get(attribute, '')
+        return self._lsb_release_info.get(attribute, "")
 
     def distro_release_attr(self, attribute):
+        # type: (str) -> str
         """
         Return a single named information item from the distro release file
         data source of the OS distribution.
 
         For details, see :func:`distro.distro_release_attr`.
         """
-        return self._distro_release_info.get(attribute, '')
+        return self._distro_release_info.get(attribute, "")
 
     def uname_attr(self, attribute):
+        # type: (str) -> str
         """
         Return a single named information item from the uname command
         output data source of the OS distribution.
 
-        For details, see :func:`distro.uname_release_attr`.
+        For details, see :func:`distro.uname_attr`.
         """
-        return self._uname_info.get(attribute, '')
+        return self._uname_info.get(attribute, "")
 
     @cached_property
     def _os_release_info(self):
+        # type: () -> Dict[str, str]
         """
         Get the information items from the specified os-release file.
 
@@ -935,6 +1067,7 @@ def _os_release_info(self):
 
     @staticmethod
     def _parse_os_release_content(lines):
+        # type: (TextIO) -> Dict[str, str]
         """
         Parse the lines of an os-release file.
 
@@ -959,7 +1092,7 @@ def _parse_os_release_content(lines):
         # parsed content is a unicode object. The following fix resolves that
         # (... but it should be fixed in shlex...):
         if sys.version_info[0] == 2 and isinstance(lexer.wordchars, bytes):
-            lexer.wordchars = lexer.wordchars.decode('iso-8859-1')
+            lexer.wordchars = lexer.wordchars.decode("iso-8859-1")
 
         tokens = list(lexer)
         for token in tokens:
@@ -969,37 +1102,38 @@ def _parse_os_release_content(lines):
             # stripped, etc.), so the tokens are now either:
             # * variable assignments: var=value
             # * commands or their arguments (not allowed in os-release)
-            if '=' in token:
-                k, v = token.split('=', 1)
+            if "=" in token:
+                k, v = token.split("=", 1)
                 props[k.lower()] = v
             else:
                 # Ignore any tokens that are not variable assignments
                 pass
 
-        if 'version_codename' in props:
+        if "version_codename" in props:
             # os-release added a version_codename field.  Use that in
             # preference to anything else Note that some distros purposefully
             # do not have code names.  They should be setting
             # version_codename=""
-            props['codename'] = props['version_codename']
-        elif 'ubuntu_codename' in props:
+            props["codename"] = props["version_codename"]
+        elif "ubuntu_codename" in props:
             # Same as above but a non-standard field name used on older Ubuntus
-            props['codename'] = props['ubuntu_codename']
-        elif 'version' in props:
+            props["codename"] = props["ubuntu_codename"]
+        elif "version" in props:
             # If there is no version_codename, parse it from the version
-            codename = re.search(r'(\(\D+\))|,(\s+)?\D+', props['version'])
-            if codename:
-                codename = codename.group()
-                codename = codename.strip('()')
-                codename = codename.strip(',')
+            match = re.search(r"(\(\D+\))|,(\s+)?\D+", props["version"])
+            if match:
+                codename = match.group()
+                codename = codename.strip("()")
+                codename = codename.strip(",")
                 codename = codename.strip()
                 # codename appears within paranthese.
-                props['codename'] = codename
+                props["codename"] = codename
 
         return props
 
     @cached_property
     def _lsb_release_info(self):
+        # type: () -> Dict[str, str]
         """
         Get the information items from the lsb_release command output.
 
@@ -1008,17 +1142,19 @@ def _lsb_release_info(self):
         """
         if not self.include_lsb:
             return {}
-        with open(os.devnull, 'w') as devnull:
+        with open(os.devnull, "wb") as devnull:
             try:
-                cmd = ('lsb_release', '-a')
+                cmd = ("lsb_release", "-a")
                 stdout = subprocess.check_output(cmd, stderr=devnull)
-            except OSError:  # Command not found
+            # Command not found or lsb_release returned error
+            except (OSError, subprocess.CalledProcessError):
                 return {}
         content = self._to_str(stdout).splitlines()
         return self._parse_lsb_release_content(content)
 
     @staticmethod
     def _parse_lsb_release_content(lines):
+        # type: (Iterable[str]) -> Dict[str, str]
         """
         Parse the output of the lsb_release command.
 
@@ -1033,19 +1169,20 @@ def _parse_lsb_release_content(lines):
         """
         props = {}
         for line in lines:
-            kv = line.strip('\n').split(':', 1)
+            kv = line.strip("\n").split(":", 1)
             if len(kv) != 2:
                 # Ignore lines without colon.
                 continue
             k, v = kv
-            props.update({k.replace(' ', '_').lower(): v.strip()})
+            props.update({k.replace(" ", "_").lower(): v.strip()})
         return props
 
     @cached_property
     def _uname_info(self):
-        with open(os.devnull, 'w') as devnull:
+        # type: () -> Dict[str, str]
+        with open(os.devnull, "wb") as devnull:
             try:
-                cmd = ('uname', '-rs')
+                cmd = ("uname", "-rs")
                 stdout = subprocess.check_output(cmd, stderr=devnull)
             except OSError:
                 return {}
@@ -1054,25 +1191,27 @@ def _uname_info(self):
 
     @staticmethod
     def _parse_uname_content(lines):
+        # type: (Sequence[str]) -> Dict[str, str]
         props = {}
-        match = re.search(r'^([^\s]+)\s+([\d\.]+)', lines[0].strip())
+        match = re.search(r"^([^\s]+)\s+([\d\.]+)", lines[0].strip())
         if match:
             name, version = match.groups()
 
             # This is to prevent the Linux kernel version from
             # appearing as the 'best' version on otherwise
             # identifiable distributions.
-            if name == 'Linux':
+            if name == "Linux":
                 return {}
-            props['id'] = name.lower()
-            props['name'] = name
-            props['release'] = version
+            props["id"] = name.lower()
+            props["name"] = name
+            props["release"] = version
         return props
 
     @staticmethod
     def _to_str(text):
+        # type: (Union[bytes, str]) -> str
         encoding = sys.getfilesystemencoding()
-        encoding = 'utf-8' if encoding == 'ascii' else encoding
+        encoding = "utf-8" if encoding == "ascii" else encoding
 
         if sys.version_info[0] >= 3:
             if isinstance(text, bytes):
@@ -1085,6 +1224,7 @@ def _to_str(text):
 
     @cached_property
     def _distro_release_info(self):
+        # type: () -> Dict[str, str]
         """
         Get the information items from the specified distro release file.
 
@@ -1094,23 +1234,21 @@ def _distro_release_info(self):
         if self.distro_release_file:
             # If it was specified, we use it and parse what we can, even if
             # its file name or content does not match the expected pattern.
-            distro_info = self._parse_distro_release_file(
-                self.distro_release_file)
+            distro_info = self._parse_distro_release_file(self.distro_release_file)
             basename = os.path.basename(self.distro_release_file)
             # The file name pattern for user-specified distro release files
             # is somewhat more tolerant (compared to when searching for the
             # file), because we want to use what was specified as best as
             # possible.
             match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
-            if 'name' in distro_info \
-               and 'cloudlinux' in distro_info['name'].lower():
-                distro_info['id'] = 'cloudlinux'
+            if "name" in distro_info and "cloudlinux" in distro_info["name"].lower():
+                distro_info["id"] = "cloudlinux"
             elif match:
-                distro_info['id'] = match.group(1)
+                distro_info["id"] = match.group(1)
             return distro_info
         else:
             try:
-                basenames = os.listdir(_UNIXCONFDIR)
+                basenames = os.listdir(self.etc_dir)
                 # We sort for repeatability in cases where there are multiple
                 # distro specific files; e.g. CentOS, Oracle, Enterprise all
                 # containing `redhat-release` on top of their own.
@@ -1120,38 +1258,41 @@ def _distro_release_info(self):
                 # sure about the *-release files. Check common entries of
                 # /etc for information. If they turn out to not be there the
                 # error is handled in `_parse_distro_release_file()`.
-                basenames = ['SuSE-release',
-                             'arch-release',
-                             'base-release',
-                             'centos-release',
-                             'fedora-release',
-                             'gentoo-release',
-                             'mageia-release',
-                             'mandrake-release',
-                             'mandriva-release',
-                             'mandrivalinux-release',
-                             'manjaro-release',
-                             'oracle-release',
-                             'redhat-release',
-                             'sl-release',
-                             'slackware-version']
+                basenames = [
+                    "SuSE-release",
+                    "arch-release",
+                    "base-release",
+                    "centos-release",
+                    "fedora-release",
+                    "gentoo-release",
+                    "mageia-release",
+                    "mandrake-release",
+                    "mandriva-release",
+                    "mandrivalinux-release",
+                    "manjaro-release",
+                    "oracle-release",
+                    "redhat-release",
+                    "sl-release",
+                    "slackware-version",
+                ]
             for basename in basenames:
                 if basename in _DISTRO_RELEASE_IGNORE_BASENAMES:
                     continue
                 match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
                 if match:
-                    filepath = os.path.join(_UNIXCONFDIR, basename)
+                    filepath = os.path.join(self.etc_dir, basename)
                     distro_info = self._parse_distro_release_file(filepath)
-                    if 'name' in distro_info:
+                    if "name" in distro_info:
                         # The name is always present if the pattern matches
                         self.distro_release_file = filepath
-                        distro_info['id'] = match.group(1)
-                        if 'cloudlinux' in distro_info['name'].lower():
-                            distro_info['id'] = 'cloudlinux'
+                        distro_info["id"] = match.group(1)
+                        if "cloudlinux" in distro_info["name"].lower():
+                            distro_info["id"] = "cloudlinux"
                         return distro_info
             return {}
 
     def _parse_distro_release_file(self, filepath):
+        # type: (str) -> Dict[str, str]
         """
         Parse a distro release file.
 
@@ -1170,11 +1311,12 @@ def _parse_distro_release_file(self, filepath):
         except (OSError, IOError):
             # Ignore not being able to read a specific, seemingly version
             # related file.
-            # See https://github.com/nir0s/distro/issues/162
+            # See https://github.com/python-distro/distro/issues/162
             return {}
 
     @staticmethod
     def _parse_distro_release_content(line):
+        # type: (str) -> Dict[str, str]
         """
         Parse a line from a distro release file.
 
@@ -1185,18 +1327,17 @@ def _parse_distro_release_content(line):
         Returns:
             A dictionary containing all information items.
         """
-        matches = _DISTRO_RELEASE_CONTENT_REVERSED_PATTERN.match(
-            line.strip()[::-1])
+        matches = _DISTRO_RELEASE_CONTENT_REVERSED_PATTERN.match(line.strip()[::-1])
         distro_info = {}
         if matches:
             # regexp ensures non-None
-            distro_info['name'] = matches.group(3)[::-1]
+            distro_info["name"] = matches.group(3)[::-1]
             if matches.group(2):
-                distro_info['version_id'] = matches.group(2)[::-1]
+                distro_info["version_id"] = matches.group(2)[::-1]
             if matches.group(1):
-                distro_info['codename'] = matches.group(1)[::-1]
+                distro_info["codename"] = matches.group(1)[::-1]
         elif line:
-            distro_info['name'] = line.strip()
+            distro_info["name"] = line.strip()
         return distro_info
 
 
@@ -1204,27 +1345,42 @@ def _parse_distro_release_content(line):
 
 
 def main():
+    # type: () -> None
     logger = logging.getLogger(__name__)
     logger.setLevel(logging.DEBUG)
     logger.addHandler(logging.StreamHandler(sys.stdout))
 
     parser = argparse.ArgumentParser(description="OS distro info tool")
     parser.add_argument(
-        '--json',
-        '-j',
-        help="Output in machine readable format",
-        action="store_true")
+        "--json", "-j", help="Output in machine readable format", action="store_true"
+    )
+
+    parser.add_argument(
+        "--root-dir",
+        "-r",
+        type=str,
+        dest="root_dir",
+        help="Path to the root filesystem directory (defaults to /)",
+    )
+
     args = parser.parse_args()
 
+    if args.root_dir:
+        dist = LinuxDistribution(
+            include_lsb=False, include_uname=False, root_dir=args.root_dir
+        )
+    else:
+        dist = _distro
+
     if args.json:
-        logger.info(json.dumps(info(), indent=4, sort_keys=True))
+        logger.info(json.dumps(dist.info(), indent=4, sort_keys=True))
     else:
-        logger.info('Name: %s', name(pretty=True))
-        distribution_version = version(pretty=True)
-        logger.info('Version: %s', distribution_version)
-        distribution_codename = codename()
-        logger.info('Codename: %s', distribution_codename)
+        logger.info("Name: %s", dist.name(pretty=True))
+        distribution_version = dist.version(pretty=True)
+        logger.info("Version: %s", distribution_version)
+        distribution_codename = dist.codename()
+        logger.info("Codename: %s", distribution_codename)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/src/pip/_vendor/distro.pyi b/src/pip/_vendor/distro.pyi
deleted file mode 100644
index c7ea94b37ba..00000000000
--- a/src/pip/_vendor/distro.pyi
+++ /dev/null
@@ -1 +0,0 @@
-from distro import *
\ No newline at end of file
diff --git a/src/pip/_vendor/idna.pyi b/src/pip/_vendor/idna.pyi
deleted file mode 100644
index 7410d72fe7d..00000000000
--- a/src/pip/_vendor/idna.pyi
+++ /dev/null
@@ -1 +0,0 @@
-from idna import *
\ No newline at end of file
diff --git a/src/pip/_vendor/idna/LICENSE.md b/src/pip/_vendor/idna/LICENSE.md
new file mode 100644
index 00000000000..b6f87326ffb
--- /dev/null
+++ b/src/pip/_vendor/idna/LICENSE.md
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2013-2021, Kim Davies
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/pip/_vendor/idna/LICENSE.rst b/src/pip/_vendor/idna/LICENSE.rst
deleted file mode 100644
index 63664b82e7a..00000000000
--- a/src/pip/_vendor/idna/LICENSE.rst
+++ /dev/null
@@ -1,34 +0,0 @@
-License
--------
-
-License: bsd-3-clause
-
-Copyright (c) 2013-2020, Kim Davies. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-#. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-#. Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following
-   disclaimer in the documentation and/or other materials provided with
-   the distribution.
-
-#. Neither the name of the copyright holder nor the names of the 
-   contributors may be used to endorse or promote products derived 
-   from this software without specific prior written permission.
-
-#. THIS SOFTWARE IS PROVIDED BY THE CONTRIBUTORS "AS IS" AND ANY
-   EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR 
-   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-   USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
-   DAMAGE.
diff --git a/src/pip/_vendor/idna/__init__.py b/src/pip/_vendor/idna/__init__.py
index 847bf935478..a40eeafcc91 100644
--- a/src/pip/_vendor/idna/__init__.py
+++ b/src/pip/_vendor/idna/__init__.py
@@ -1,2 +1,44 @@
 from .package_data import __version__
-from .core import *
+from .core import (
+    IDNABidiError,
+    IDNAError,
+    InvalidCodepoint,
+    InvalidCodepointContext,
+    alabel,
+    check_bidi,
+    check_hyphen_ok,
+    check_initial_combiner,
+    check_label,
+    check_nfc,
+    decode,
+    encode,
+    ulabel,
+    uts46_remap,
+    valid_contextj,
+    valid_contexto,
+    valid_label_length,
+    valid_string_length,
+)
+from .intranges import intranges_contain
+
+__all__ = [
+    "IDNABidiError",
+    "IDNAError",
+    "InvalidCodepoint",
+    "InvalidCodepointContext",
+    "alabel",
+    "check_bidi",
+    "check_hyphen_ok",
+    "check_initial_combiner",
+    "check_label",
+    "check_nfc",
+    "decode",
+    "encode",
+    "intranges_contain",
+    "ulabel",
+    "uts46_remap",
+    "valid_contextj",
+    "valid_contexto",
+    "valid_label_length",
+    "valid_string_length",
+]
diff --git a/src/pip/_vendor/idna/codec.py b/src/pip/_vendor/idna/codec.py
index 98c65ead146..1ca9ba62c20 100644
--- a/src/pip/_vendor/idna/codec.py
+++ b/src/pip/_vendor/idna/codec.py
@@ -1,41 +1,40 @@
 from .core import encode, decode, alabel, ulabel, IDNAError
 import codecs
 import re
+from typing import Tuple, Optional
 
-_unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]')
+_unicode_dots_re = re.compile('[\u002e\u3002\uff0e\uff61]')
 
 class Codec(codecs.Codec):
 
-    def encode(self, data, errors='strict'):
-
+    def encode(self, data: str, errors: str = 'strict') -> Tuple[bytes, int]:
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return "", 0
+            return b"", 0
 
         return encode(data), len(data)
 
-    def decode(self, data, errors='strict'):
-
+    def decode(self, data: bytes, errors: str = 'strict') -> Tuple[str, int]:
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return u"", 0
+            return '', 0
 
         return decode(data), len(data)
 
 class IncrementalEncoder(codecs.BufferedIncrementalEncoder):
-    def _buffer_encode(self, data, errors, final):
+    def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[str, int]:  # type: ignore
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return ("", 0)
+            return "", 0
 
         labels = _unicode_dots_re.split(data)
-        trailing_dot = u''
+        trailing_dot = ''
         if labels:
             if not labels[-1]:
                 trailing_dot = '.'
@@ -55,37 +54,29 @@ def _buffer_encode(self, data, errors, final):
             size += len(label)
 
         # Join with U+002E
-        result = ".".join(result) + trailing_dot
+        result_str = '.'.join(result) + trailing_dot  # type: ignore
         size += len(trailing_dot)
-        return (result, size)
+        return result_str, size
 
 class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
-    def _buffer_decode(self, data, errors, final):
+    def _buffer_decode(self, data: str, errors: str, final: bool) -> Tuple[str, int]:  # type: ignore
         if errors != 'strict':
-            raise IDNAError("Unsupported error handling \"{0}\"".format(errors))
+            raise IDNAError('Unsupported error handling \"{}\"'.format(errors))
 
         if not data:
-            return (u"", 0)
-
-        # IDNA allows decoding to operate on Unicode strings, too.
-        if isinstance(data, unicode):
-            labels = _unicode_dots_re.split(data)
-        else:
-            # Must be ASCII string
-            data = str(data)
-            unicode(data, "ascii")
-            labels = data.split(".")
-
-        trailing_dot = u''
+            return ('', 0)
+
+        labels = _unicode_dots_re.split(data)
+        trailing_dot = ''
         if labels:
             if not labels[-1]:
-                trailing_dot = u'.'
+                trailing_dot = '.'
                 del labels[-1]
             elif not final:
                 # Keep potentially unfinished label until the next call
                 del labels[-1]
                 if labels:
-                    trailing_dot = u'.'
+                    trailing_dot = '.'
 
         result = []
         size = 0
@@ -95,22 +86,25 @@ def _buffer_decode(self, data, errors, final):
                 size += 1
             size += len(label)
 
-        result = u".".join(result) + trailing_dot
+        result_str = '.'.join(result) + trailing_dot
         size += len(trailing_dot)
-        return (result, size)
+        return (result_str, size)
 
 
 class StreamWriter(Codec, codecs.StreamWriter):
     pass
 
+
 class StreamReader(Codec, codecs.StreamReader):
     pass
 
-def getregentry():
+
+def getregentry() -> codecs.CodecInfo:
+    # Compatibility as a search_function for codecs.register()
     return codecs.CodecInfo(
         name='idna',
-        encode=Codec().encode,
-        decode=Codec().decode,
+        encode=Codec().encode,  # type: ignore
+        decode=Codec().decode,  # type: ignore
         incrementalencoder=IncrementalEncoder,
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
diff --git a/src/pip/_vendor/idna/compat.py b/src/pip/_vendor/idna/compat.py
index 4d47f336dbc..786e6bda636 100644
--- a/src/pip/_vendor/idna/compat.py
+++ b/src/pip/_vendor/idna/compat.py
@@ -1,12 +1,13 @@
 from .core import *
 from .codec import *
+from typing import Any, Union
 
-def ToASCII(label):
+def ToASCII(label: str) -> bytes:
     return encode(label)
 
-def ToUnicode(label):
+def ToUnicode(label: Union[bytes, bytearray]) -> str:
     return decode(label)
 
-def nameprep(s):
-    raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol")
+def nameprep(s: Any) -> None:
+    raise NotImplementedError('IDNA 2008 does not utilise nameprep protocol')
 
diff --git a/src/pip/_vendor/idna/core.py b/src/pip/_vendor/idna/core.py
index 41ec5c711d1..55ab9678850 100644
--- a/src/pip/_vendor/idna/core.py
+++ b/src/pip/_vendor/idna/core.py
@@ -2,16 +2,12 @@
 import bisect
 import unicodedata
 import re
-import sys
+from typing import Union, Optional
 from .intranges import intranges_contain
 
 _virama_combining_class = 9
 _alabel_prefix = b'xn--'
-_unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]')
-
-if sys.version_info[0] >= 3:
-    unicode = str
-    unichr = chr
+_unicode_dots_re = re.compile('[\u002e\u3002\uff0e\uff61]')
 
 class IDNAError(UnicodeError):
     """ Base exception for all IDNA-encoding related problems """
@@ -33,46 +29,43 @@ class InvalidCodepointContext(IDNAError):
     pass
 
 
-def _combining_class(cp):
-    v = unicodedata.combining(unichr(cp))
+def _combining_class(cp: int) -> int:
+    v = unicodedata.combining(chr(cp))
     if v == 0:
-        if not unicodedata.name(unichr(cp)):
-            raise ValueError("Unknown character in unicodedata")
+        if not unicodedata.name(chr(cp)):
+            raise ValueError('Unknown character in unicodedata')
     return v
 
-def _is_script(cp, script):
+def _is_script(cp: str, script: str) -> bool:
     return intranges_contain(ord(cp), idnadata.scripts[script])
 
-def _punycode(s):
+def _punycode(s: str) -> bytes:
     return s.encode('punycode')
 
-def _unot(s):
-    return 'U+{0:04X}'.format(s)
-
+def _unot(s: int) -> str:
+    return 'U+{:04X}'.format(s)
 
-def valid_label_length(label):
 
+def valid_label_length(label: Union[bytes, str]) -> bool:
     if len(label) > 63:
         return False
     return True
 
 
-def valid_string_length(label, trailing_dot):
-
+def valid_string_length(label: Union[bytes, str], trailing_dot: bool) -> bool:
     if len(label) > (254 if trailing_dot else 253):
         return False
     return True
 
 
-def check_bidi(label, check_ltr=False):
-
+def check_bidi(label: str, check_ltr: bool = False) -> bool:
     # Bidi rules should only be applied if string contains RTL characters
     bidi_label = False
     for (idx, cp) in enumerate(label, 1):
         direction = unicodedata.bidirectional(cp)
         if direction == '':
             # String likely comes from a newer version of Unicode
-            raise IDNABidiError('Unknown directionality in label {0} at position {1}'.format(repr(label), idx))
+            raise IDNABidiError('Unknown directionality in label {} at position {}'.format(repr(label), idx))
         if direction in ['R', 'AL', 'AN']:
             bidi_label = True
     if not bidi_label and not check_ltr:
@@ -85,17 +78,17 @@ def check_bidi(label, check_ltr=False):
     elif direction == 'L':
         rtl = False
     else:
-        raise IDNABidiError('First codepoint in label {0} must be directionality L, R or AL'.format(repr(label)))
+        raise IDNABidiError('First codepoint in label {} must be directionality L, R or AL'.format(repr(label)))
 
     valid_ending = False
-    number_type = False
+    number_type = None  # type: Optional[str]
     for (idx, cp) in enumerate(label, 1):
         direction = unicodedata.bidirectional(cp)
 
         if rtl:
             # Bidi rule 2
             if not direction in ['R', 'AL', 'AN', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']:
-                raise IDNABidiError('Invalid direction for codepoint at position {0} in a right-to-left label'.format(idx))
+                raise IDNABidiError('Invalid direction for codepoint at position {} in a right-to-left label'.format(idx))
             # Bidi rule 3
             if direction in ['R', 'AL', 'EN', 'AN']:
                 valid_ending = True
@@ -111,7 +104,7 @@ def check_bidi(label, check_ltr=False):
         else:
             # Bidi rule 5
             if not direction in ['L', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']:
-                raise IDNABidiError('Invalid direction for codepoint at position {0} in a left-to-right label'.format(idx))
+                raise IDNABidiError('Invalid direction for codepoint at position {} in a left-to-right label'.format(idx))
             # Bidi rule 6
             if direction in ['L', 'EN']:
                 valid_ending = True
@@ -124,15 +117,13 @@ def check_bidi(label, check_ltr=False):
     return True
 
 
-def check_initial_combiner(label):
-
+def check_initial_combiner(label: str) -> bool:
     if unicodedata.category(label[0])[0] == 'M':
         raise IDNAError('Label begins with an illegal combining character')
     return True
 
 
-def check_hyphen_ok(label):
-
+def check_hyphen_ok(label: str) -> bool:
     if label[2:4] == '--':
         raise IDNAError('Label has disallowed hyphens in 3rd and 4th position')
     if label[0] == '-' or label[-1] == '-':
@@ -140,14 +131,12 @@ def check_hyphen_ok(label):
     return True
 
 
-def check_nfc(label):
-
+def check_nfc(label: str) -> None:
     if unicodedata.normalize('NFC', label) != label:
         raise IDNAError('Label must be in Normalization Form C')
 
 
-def valid_contextj(label, pos):
-
+def valid_contextj(label: str, pos: int) -> bool:
     cp_value = ord(label[pos])
 
     if cp_value == 0x200c:
@@ -190,8 +179,7 @@ def valid_contextj(label, pos):
         return False
 
 
-def valid_contexto(label, pos, exception=False):
-
+def valid_contexto(label: str, pos: int, exception: bool = False) -> bool:
     cp_value = ord(label[pos])
 
     if cp_value == 0x00b7:
@@ -212,7 +200,7 @@ def valid_contexto(label, pos, exception=False):
 
     elif cp_value == 0x30fb:
         for cp in label:
-            if cp == u'\u30fb':
+            if cp == '\u30fb':
                 continue
             if _is_script(cp, 'Hiragana') or _is_script(cp, 'Katakana') or _is_script(cp, 'Han'):
                 return True
@@ -230,9 +218,10 @@ def valid_contexto(label, pos, exception=False):
                 return False
         return True
 
+    return False
 
-def check_label(label):
 
+def check_label(label: Union[str, bytes, bytearray]) -> None:
     if isinstance(label, (bytes, bytearray)):
         label = label.decode('utf-8')
     if len(label) == 0:
@@ -249,102 +238,108 @@ def check_label(label):
         elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']):
             try:
                 if not valid_contextj(label, pos):
-                    raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format(
+                    raise InvalidCodepointContext('Joiner {} not allowed at position {} in {}'.format(
                         _unot(cp_value), pos+1, repr(label)))
             except ValueError:
-                raise IDNAError('Unknown codepoint adjacent to joiner {0} at position {1} in {2}'.format(
+                raise IDNAError('Unknown codepoint adjacent to joiner {} at position {} in {}'.format(
                     _unot(cp_value), pos+1, repr(label)))
         elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']):
             if not valid_contexto(label, pos):
-                raise InvalidCodepointContext('Codepoint {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label)))
+                raise InvalidCodepointContext('Codepoint {} not allowed at position {} in {}'.format(_unot(cp_value), pos+1, repr(label)))
         else:
-            raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
+            raise InvalidCodepoint('Codepoint {} at position {} of {} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
 
     check_bidi(label)
 
 
-def alabel(label):
-
+def alabel(label: str) -> bytes:
     try:
-        label = label.encode('ascii')
-        ulabel(label)
-        if not valid_label_length(label):
+        label_bytes = label.encode('ascii')
+        ulabel(label_bytes)
+        if not valid_label_length(label_bytes):
             raise IDNAError('Label too long')
-        return label
+        return label_bytes
     except UnicodeEncodeError:
         pass
 
     if not label:
         raise IDNAError('No Input')
 
-    label = unicode(label)
+    label = str(label)
     check_label(label)
-    label = _punycode(label)
-    label = _alabel_prefix + label
+    label_bytes = _punycode(label)
+    label_bytes = _alabel_prefix + label_bytes
 
-    if not valid_label_length(label):
+    if not valid_label_length(label_bytes):
         raise IDNAError('Label too long')
 
-    return label
+    return label_bytes
 
 
-def ulabel(label):
-
+def ulabel(label: Union[str, bytes, bytearray]) -> str:
     if not isinstance(label, (bytes, bytearray)):
         try:
-            label = label.encode('ascii')
+            label_bytes = label.encode('ascii')
         except UnicodeEncodeError:
             check_label(label)
             return label
+    else:
+        label_bytes = label
 
-    label = label.lower()
-    if label.startswith(_alabel_prefix):
-        label = label[len(_alabel_prefix):]
-        if not label:
+    label_bytes = label_bytes.lower()
+    if label_bytes.startswith(_alabel_prefix):
+        label_bytes = label_bytes[len(_alabel_prefix):]
+        if not label_bytes:
             raise IDNAError('Malformed A-label, no Punycode eligible content found')
-        if label.decode('ascii')[-1] == '-':
+        if label_bytes.decode('ascii')[-1] == '-':
             raise IDNAError('A-label must not end with a hyphen')
     else:
-        check_label(label)
-        return label.decode('ascii')
+        check_label(label_bytes)
+        return label_bytes.decode('ascii')
 
-    label = label.decode('punycode')
+    try:
+        label = label_bytes.decode('punycode')
+    except UnicodeError:
+        raise IDNAError('Invalid A-label')
     check_label(label)
     return label
 
 
-def uts46_remap(domain, std3_rules=True, transitional=False):
+def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str:
     """Re-map the characters in the string according to UTS46 processing."""
     from .uts46data import uts46data
-    output = u""
-    try:
-        for pos, char in enumerate(domain):
-            code_point = ord(char)
+    output = ''
+
+    for pos, char in enumerate(domain):
+        code_point = ord(char)
+        try:
             uts46row = uts46data[code_point if code_point < 256 else
-                bisect.bisect_left(uts46data, (code_point, "Z")) - 1]
+                bisect.bisect_left(uts46data, (code_point, 'Z')) - 1]
             status = uts46row[1]
-            replacement = uts46row[2] if len(uts46row) == 3 else None
-            if (status == "V" or
-                    (status == "D" and not transitional) or
-                    (status == "3" and not std3_rules and replacement is None)):
+            replacement = None  # type: Optional[str]
+            if len(uts46row) == 3:
+                replacement = uts46row[2]  # type: ignore
+            if (status == 'V' or
+                    (status == 'D' and not transitional) or
+                    (status == '3' and not std3_rules and replacement is None)):
                 output += char
-            elif replacement is not None and (status == "M" or
-                    (status == "3" and not std3_rules) or
-                    (status == "D" and transitional)):
+            elif replacement is not None and (status == 'M' or
+                    (status == '3' and not std3_rules) or
+                    (status == 'D' and transitional)):
                 output += replacement
-            elif status != "I":
+            elif status != 'I':
                 raise IndexError()
-        return unicodedata.normalize("NFC", output)
-    except IndexError:
-        raise InvalidCodepoint(
-            "Codepoint {0} not allowed at position {1} in {2}".format(
-            _unot(code_point), pos + 1, repr(domain)))
+        except IndexError:
+            raise InvalidCodepoint(
+                'Codepoint {} not allowed at position {} in {}'.format(
+                _unot(code_point), pos + 1, repr(domain)))
 
+    return unicodedata.normalize('NFC', output)
 
-def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False):
 
+def encode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False, transitional: bool = False) -> bytes:
     if isinstance(s, (bytes, bytearray)):
-        s = s.decode("ascii")
+        s = s.decode('ascii')
     if uts46:
         s = uts46_remap(s, std3_rules, transitional)
     trailing_dot = False
@@ -372,10 +367,12 @@ def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False):
     return s
 
 
-def decode(s, strict=False, uts46=False, std3_rules=False):
-
-    if isinstance(s, (bytes, bytearray)):
-        s = s.decode("ascii")
+def decode(s: Union[str, bytes, bytearray], strict: bool = False, uts46: bool = False, std3_rules: bool = False) -> str:
+    try:
+        if isinstance(s, (bytes, bytearray)):
+            s = s.decode('ascii')
+    except UnicodeDecodeError:
+        raise IDNAError('Invalid ASCII in A-label')
     if uts46:
         s = uts46_remap(s, std3_rules, False)
     trailing_dot = False
@@ -383,7 +380,7 @@ def decode(s, strict=False, uts46=False, std3_rules=False):
     if not strict:
         labels = _unicode_dots_re.split(s)
     else:
-        labels = s.split(u'.')
+        labels = s.split('.')
     if not labels or labels == ['']:
         raise IDNAError('Empty domain')
     if not labels[-1]:
@@ -396,5 +393,5 @@ def decode(s, strict=False, uts46=False, std3_rules=False):
         else:
             raise IDNAError('Empty label')
     if trailing_dot:
-        result.append(u'')
-    return u'.'.join(result)
+        result.append('')
+    return '.'.join(result)
diff --git a/src/pip/_vendor/idna/idnadata.py b/src/pip/_vendor/idna/idnadata.py
index a284e4c84ac..1b5805d15e5 100644
--- a/src/pip/_vendor/idna/idnadata.py
+++ b/src/pip/_vendor/idna/idnadata.py
@@ -1,6 +1,6 @@
 # This file is automatically generated by tools/idna-data
 
-__version__ = "13.0.0"
+__version__ = '14.0.0'
 scripts = {
     'Greek': (
         0x37000000374,
@@ -49,12 +49,13 @@
         0x30210000302a,
         0x30380000303c,
         0x340000004dc0,
-        0x4e0000009ffd,
+        0x4e000000a000,
         0xf9000000fa6e,
         0xfa700000fada,
+        0x16fe200016fe4,
         0x16ff000016ff2,
-        0x200000002a6de,
-        0x2a7000002b735,
+        0x200000002a6e0,
+        0x2a7000002b739,
         0x2b7400002b81e,
         0x2b8200002cea2,
         0x2ceb00002ebe1,
@@ -75,7 +76,7 @@
     'Hiragana': (
         0x304100003097,
         0x309d000030a0,
-        0x1b0010001b11f,
+        0x1b0010001b120,
         0x1b1500001b153,
         0x1f2000001f201,
     ),
@@ -87,7 +88,11 @@
         0x330000003358,
         0xff660000ff70,
         0xff710000ff9e,
+        0x1aff00001aff4,
+        0x1aff50001affc,
+        0x1affd0001afff,
         0x1b0000001b001,
+        0x1b1200001b123,
         0x1b1640001b168,
     ),
 }
@@ -405,6 +410,39 @@
     0x868: 68,
     0x869: 82,
     0x86a: 82,
+    0x870: 82,
+    0x871: 82,
+    0x872: 82,
+    0x873: 82,
+    0x874: 82,
+    0x875: 82,
+    0x876: 82,
+    0x877: 82,
+    0x878: 82,
+    0x879: 82,
+    0x87a: 82,
+    0x87b: 82,
+    0x87c: 82,
+    0x87d: 82,
+    0x87e: 82,
+    0x87f: 82,
+    0x880: 82,
+    0x881: 82,
+    0x882: 82,
+    0x883: 67,
+    0x884: 67,
+    0x885: 67,
+    0x886: 68,
+    0x887: 85,
+    0x888: 85,
+    0x889: 68,
+    0x88a: 68,
+    0x88b: 68,
+    0x88c: 68,
+    0x88d: 68,
+    0x88e: 82,
+    0x890: 85,
+    0x891: 85,
     0x8a0: 68,
     0x8a1: 68,
     0x8a2: 68,
@@ -426,6 +464,7 @@
     0x8b2: 82,
     0x8b3: 68,
     0x8b4: 68,
+    0x8b5: 68,
     0x8b6: 68,
     0x8b7: 68,
     0x8b8: 68,
@@ -444,6 +483,7 @@
     0x8c5: 68,
     0x8c6: 68,
     0x8c7: 68,
+    0x8c8: 68,
     0x8e2: 85,
     0x1806: 85,
     0x1807: 68,
@@ -768,6 +808,24 @@
     0x10f52: 68,
     0x10f53: 68,
     0x10f54: 82,
+    0x10f70: 68,
+    0x10f71: 68,
+    0x10f72: 68,
+    0x10f73: 68,
+    0x10f74: 82,
+    0x10f75: 82,
+    0x10f76: 68,
+    0x10f77: 68,
+    0x10f78: 68,
+    0x10f79: 68,
+    0x10f7a: 68,
+    0x10f7b: 68,
+    0x10f7c: 68,
+    0x10f7d: 68,
+    0x10f7e: 68,
+    0x10f7f: 68,
+    0x10f80: 68,
+    0x10f81: 68,
     0x10fb0: 68,
     0x10fb1: 85,
     0x10fb2: 68,
@@ -1168,9 +1226,9 @@
         0x8000000082e,
         0x8400000085c,
         0x8600000086b,
-        0x8a0000008b5,
-        0x8b6000008c8,
-        0x8d3000008e2,
+        0x87000000888,
+        0x8890000088f,
+        0x898000008e2,
         0x8e300000958,
         0x96000000964,
         0x96600000970,
@@ -1252,11 +1310,12 @@
         0xc0e00000c11,
         0xc1200000c29,
         0xc2a00000c3a,
-        0xc3d00000c45,
+        0xc3c00000c45,
         0xc4600000c49,
         0xc4a00000c4e,
         0xc5500000c57,
         0xc5800000c5b,
+        0xc5d00000c5e,
         0xc6000000c64,
         0xc6600000c70,
         0xc8000000c84,
@@ -1269,7 +1328,7 @@
         0xcc600000cc9,
         0xcca00000cce,
         0xcd500000cd7,
-        0xcde00000cdf,
+        0xcdd00000cdf,
         0xce000000ce4,
         0xce600000cf0,
         0xcf100000cf3,
@@ -1366,9 +1425,8 @@
         0x16810000169b,
         0x16a0000016eb,
         0x16f1000016f9,
-        0x17000000170d,
-        0x170e00001715,
-        0x172000001735,
+        0x170000001716,
+        0x171f00001735,
         0x174000001754,
         0x17600000176d,
         0x176e00001771,
@@ -1397,8 +1455,8 @@
         0x1a9000001a9a,
         0x1aa700001aa8,
         0x1ab000001abe,
-        0x1abf00001ac1,
-        0x1b0000001b4c,
+        0x1abf00001acf,
+        0x1b0000001b4d,
         0x1b5000001b5a,
         0x1b6b00001b74,
         0x1b8000001bf4,
@@ -1413,8 +1471,7 @@
         0x1d4e00001d4f,
         0x1d6b00001d78,
         0x1d7900001d9b,
-        0x1dc000001dfa,
-        0x1dfb00001e00,
+        0x1dc000001e00,
         0x1e0100001e02,
         0x1e0300001e04,
         0x1e0500001e06,
@@ -1563,7 +1620,7 @@
         0x1ff600001ff7,
         0x214e0000214f,
         0x218400002185,
-        0x2c3000002c5f,
+        0x2c3000002c60,
         0x2c6100002c62,
         0x2c6500002c67,
         0x2c6800002c69,
@@ -1652,8 +1709,7 @@
         0x31a0000031c0,
         0x31f000003200,
         0x340000004dc0,
-        0x4e0000009ffd,
-        0xa0000000a48d,
+        0x4e000000a48d,
         0xa4d00000a4fe,
         0xa5000000a60d,
         0xa6100000a62c,
@@ -1766,9 +1822,16 @@
         0xa7bb0000a7bc,
         0xa7bd0000a7be,
         0xa7bf0000a7c0,
+        0xa7c10000a7c2,
         0xa7c30000a7c4,
         0xa7c80000a7c9,
         0xa7ca0000a7cb,
+        0xa7d10000a7d2,
+        0xa7d30000a7d4,
+        0xa7d50000a7d6,
+        0xa7d70000a7d8,
+        0xa7d90000a7da,
+        0xa7f20000a7f5,
         0xa7f60000a7f8,
         0xa7fa0000a828,
         0xa82c0000a82d,
@@ -1834,9 +1897,16 @@
         0x104d8000104fc,
         0x1050000010528,
         0x1053000010564,
+        0x10597000105a2,
+        0x105a3000105b2,
+        0x105b3000105ba,
+        0x105bb000105bd,
         0x1060000010737,
         0x1074000010756,
         0x1076000010768,
+        0x1078000010786,
+        0x10787000107b1,
+        0x107b2000107bb,
         0x1080000010806,
         0x1080800010809,
         0x1080a00010836,
@@ -1876,11 +1946,13 @@
         0x10f0000010f1d,
         0x10f2700010f28,
         0x10f3000010f51,
+        0x10f7000010f86,
         0x10fb000010fc5,
         0x10fe000010ff7,
         0x1100000011047,
-        0x1106600011070,
+        0x1106600011076,
         0x1107f000110bb,
+        0x110c2000110c3,
         0x110d0000110e9,
         0x110f0000110fa,
         0x1110000011135,
@@ -1934,6 +2006,7 @@
         0x117000001171b,
         0x1171d0001172c,
         0x117300001173a,
+        0x1174000011747,
         0x118000001183b,
         0x118c0000118ea,
         0x118ff00011907,
@@ -1952,7 +2025,7 @@
         0x11a4700011a48,
         0x11a5000011a9a,
         0x11a9d00011a9e,
-        0x11ac000011af9,
+        0x11ab000011af9,
         0x11c0000011c09,
         0x11c0a00011c37,
         0x11c3800011c41,
@@ -1977,11 +2050,14 @@
         0x11fb000011fb1,
         0x120000001239a,
         0x1248000012544,
+        0x12f9000012ff1,
         0x130000001342f,
         0x1440000014647,
         0x1680000016a39,
         0x16a4000016a5f,
         0x16a6000016a6a,
+        0x16a7000016abf,
+        0x16ac000016aca,
         0x16ad000016aee,
         0x16af000016af5,
         0x16b0000016b37,
@@ -1999,7 +2075,10 @@
         0x17000000187f8,
         0x1880000018cd6,
         0x18d0000018d09,
-        0x1b0000001b11f,
+        0x1aff00001aff4,
+        0x1aff50001affc,
+        0x1affd0001afff,
+        0x1b0000001b123,
         0x1b1500001b153,
         0x1b1640001b168,
         0x1b1700001b2fc,
@@ -2008,12 +2087,15 @@
         0x1bc800001bc89,
         0x1bc900001bc9a,
         0x1bc9d0001bc9f,
+        0x1cf000001cf2e,
+        0x1cf300001cf47,
         0x1da000001da37,
         0x1da3b0001da6d,
         0x1da750001da76,
         0x1da840001da85,
         0x1da9b0001daa0,
         0x1daa10001dab0,
+        0x1df000001df1f,
         0x1e0000001e007,
         0x1e0080001e019,
         0x1e01b0001e022,
@@ -2023,14 +2105,19 @@
         0x1e1300001e13e,
         0x1e1400001e14a,
         0x1e14e0001e14f,
+        0x1e2900001e2af,
         0x1e2c00001e2fa,
+        0x1e7e00001e7e7,
+        0x1e7e80001e7ec,
+        0x1e7ed0001e7ef,
+        0x1e7f00001e7ff,
         0x1e8000001e8c5,
         0x1e8d00001e8d7,
         0x1e9220001e94c,
         0x1e9500001e95a,
         0x1fbf00001fbfa,
-        0x200000002a6de,
-        0x2a7000002b735,
+        0x200000002a6e0,
+        0x2a7000002b739,
         0x2b7400002b81e,
         0x2b8200002cea2,
         0x2ceb00002ebe1,
diff --git a/src/pip/_vendor/idna/intranges.py b/src/pip/_vendor/idna/intranges.py
index fa8a735662d..6a43b047534 100644
--- a/src/pip/_vendor/idna/intranges.py
+++ b/src/pip/_vendor/idna/intranges.py
@@ -6,8 +6,9 @@
 """
 
 import bisect
+from typing import List, Tuple
 
-def intranges_from_list(list_):
+def intranges_from_list(list_: List[int]) -> Tuple[int, ...]:
     """Represent a list of integers as a sequence of ranges:
     ((start_0, end_0), (start_1, end_1), ...), such that the original
     integers are exactly those x such that start_i <= x < end_i for some i.
@@ -28,14 +29,14 @@ def intranges_from_list(list_):
 
     return tuple(ranges)
 
-def _encode_range(start, end):
+def _encode_range(start: int, end: int) -> int:
     return (start << 32) | end
 
-def _decode_range(r):
+def _decode_range(r: int) -> Tuple[int, int]:
     return (r >> 32), (r & ((1 << 32) - 1))
 
 
-def intranges_contain(int_, ranges):
+def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool:
     """Determine if `int_` falls into one of the ranges in `ranges`."""
     tuple_ = _encode_range(int_, 0)
     pos = bisect.bisect_left(ranges, tuple_)
diff --git a/src/pip/_vendor/idna/package_data.py b/src/pip/_vendor/idna/package_data.py
index ce1c521d23a..f5ea87c12bd 100644
--- a/src/pip/_vendor/idna/package_data.py
+++ b/src/pip/_vendor/idna/package_data.py
@@ -1,2 +1,2 @@
-__version__ = '2.10'
+__version__ = '3.3'
 
diff --git a/news/1170af15-1373-4226-a1ec-efe54b7ad480.trivial.rst b/src/pip/_vendor/idna/py.typed
similarity index 100%
rename from news/1170af15-1373-4226-a1ec-efe54b7ad480.trivial.rst
rename to src/pip/_vendor/idna/py.typed
diff --git a/src/pip/_vendor/idna/uts46data.py b/src/pip/_vendor/idna/uts46data.py
index 3766dd49f6d..8f65705ee91 100644
--- a/src/pip/_vendor/idna/uts46data.py
+++ b/src/pip/_vendor/idna/uts46data.py
@@ -1,11 +1,14 @@
 # This file is automatically generated by tools/idna-data
 # vim: set fileencoding=utf-8 :
 
+from typing import List, Tuple, Union
+
+
 """IDNA Mapping Table from UTS46."""
 
 
-__version__ = "13.0.0"
-def _seg_0():
+__version__ = '14.0.0'
+def _seg_0() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0x0, '3'),
     (0x1, '3'),
@@ -72,32 +75,32 @@ def _seg_0():
     (0x3E, '3'),
     (0x3F, '3'),
     (0x40, '3'),
-    (0x41, 'M', u'a'),
-    (0x42, 'M', u'b'),
-    (0x43, 'M', u'c'),
-    (0x44, 'M', u'd'),
-    (0x45, 'M', u'e'),
-    (0x46, 'M', u'f'),
-    (0x47, 'M', u'g'),
-    (0x48, 'M', u'h'),
-    (0x49, 'M', u'i'),
-    (0x4A, 'M', u'j'),
-    (0x4B, 'M', u'k'),
-    (0x4C, 'M', u'l'),
-    (0x4D, 'M', u'm'),
-    (0x4E, 'M', u'n'),
-    (0x4F, 'M', u'o'),
-    (0x50, 'M', u'p'),
-    (0x51, 'M', u'q'),
-    (0x52, 'M', u'r'),
-    (0x53, 'M', u's'),
-    (0x54, 'M', u't'),
-    (0x55, 'M', u'u'),
-    (0x56, 'M', u'v'),
-    (0x57, 'M', u'w'),
-    (0x58, 'M', u'x'),
-    (0x59, 'M', u'y'),
-    (0x5A, 'M', u'z'),
+    (0x41, 'M', 'a'),
+    (0x42, 'M', 'b'),
+    (0x43, 'M', 'c'),
+    (0x44, 'M', 'd'),
+    (0x45, 'M', 'e'),
+    (0x46, 'M', 'f'),
+    (0x47, 'M', 'g'),
+    (0x48, 'M', 'h'),
+    (0x49, 'M', 'i'),
+    (0x4A, 'M', 'j'),
+    (0x4B, 'M', 'k'),
+    (0x4C, 'M', 'l'),
+    (0x4D, 'M', 'm'),
+    (0x4E, 'M', 'n'),
+    (0x4F, 'M', 'o'),
+    (0x50, 'M', 'p'),
+    (0x51, 'M', 'q'),
+    (0x52, 'M', 'r'),
+    (0x53, 'M', 's'),
+    (0x54, 'M', 't'),
+    (0x55, 'M', 'u'),
+    (0x56, 'M', 'v'),
+    (0x57, 'M', 'w'),
+    (0x58, 'M', 'x'),
+    (0x59, 'M', 'y'),
+    (0x5A, 'M', 'z'),
     (0x5B, '3'),
     (0x5C, '3'),
     (0x5D, '3'),
@@ -109,7 +112,7 @@ def _seg_0():
     (0x63, 'V'),
     ]
 
-def _seg_1():
+def _seg_1() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0x64, 'V'),
     (0x65, 'V'),
@@ -171,7 +174,7 @@ def _seg_1():
     (0x9D, 'X'),
     (0x9E, 'X'),
     (0x9F, 'X'),
-    (0xA0, '3', u' '),
+    (0xA0, '3', ' '),
     (0xA1, 'V'),
     (0xA2, 'V'),
     (0xA3, 'V'),
@@ -179,66 +182,66 @@ def _seg_1():
     (0xA5, 'V'),
     (0xA6, 'V'),
     (0xA7, 'V'),
-    (0xA8, '3', u' ̈'),
+    (0xA8, '3', ' ̈'),
     (0xA9, 'V'),
-    (0xAA, 'M', u'a'),
+    (0xAA, 'M', 'a'),
     (0xAB, 'V'),
     (0xAC, 'V'),
     (0xAD, 'I'),
     (0xAE, 'V'),
-    (0xAF, '3', u' ̄'),
+    (0xAF, '3', ' ̄'),
     (0xB0, 'V'),
     (0xB1, 'V'),
-    (0xB2, 'M', u'2'),
-    (0xB3, 'M', u'3'),
-    (0xB4, '3', u' ́'),
-    (0xB5, 'M', u'μ'),
+    (0xB2, 'M', '2'),
+    (0xB3, 'M', '3'),
+    (0xB4, '3', ' ́'),
+    (0xB5, 'M', 'μ'),
     (0xB6, 'V'),
     (0xB7, 'V'),
-    (0xB8, '3', u' ̧'),
-    (0xB9, 'M', u'1'),
-    (0xBA, 'M', u'o'),
+    (0xB8, '3', ' ̧'),
+    (0xB9, 'M', '1'),
+    (0xBA, 'M', 'o'),
     (0xBB, 'V'),
-    (0xBC, 'M', u'1⁄4'),
-    (0xBD, 'M', u'1⁄2'),
-    (0xBE, 'M', u'3⁄4'),
+    (0xBC, 'M', '1⁄4'),
+    (0xBD, 'M', '1⁄2'),
+    (0xBE, 'M', '3⁄4'),
     (0xBF, 'V'),
-    (0xC0, 'M', u'à'),
-    (0xC1, 'M', u'á'),
-    (0xC2, 'M', u'â'),
-    (0xC3, 'M', u'ã'),
-    (0xC4, 'M', u'ä'),
-    (0xC5, 'M', u'å'),
-    (0xC6, 'M', u'æ'),
-    (0xC7, 'M', u'ç'),
+    (0xC0, 'M', 'à'),
+    (0xC1, 'M', 'á'),
+    (0xC2, 'M', 'â'),
+    (0xC3, 'M', 'ã'),
+    (0xC4, 'M', 'ä'),
+    (0xC5, 'M', 'å'),
+    (0xC6, 'M', 'æ'),
+    (0xC7, 'M', 'ç'),
     ]
 
-def _seg_2():
+def _seg_2() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xC8, 'M', u'è'),
-    (0xC9, 'M', u'é'),
-    (0xCA, 'M', u'ê'),
-    (0xCB, 'M', u'ë'),
-    (0xCC, 'M', u'ì'),
-    (0xCD, 'M', u'í'),
-    (0xCE, 'M', u'î'),
-    (0xCF, 'M', u'ï'),
-    (0xD0, 'M', u'ð'),
-    (0xD1, 'M', u'ñ'),
-    (0xD2, 'M', u'ò'),
-    (0xD3, 'M', u'ó'),
-    (0xD4, 'M', u'ô'),
-    (0xD5, 'M', u'õ'),
-    (0xD6, 'M', u'ö'),
+    (0xC8, 'M', 'è'),
+    (0xC9, 'M', 'é'),
+    (0xCA, 'M', 'ê'),
+    (0xCB, 'M', 'ë'),
+    (0xCC, 'M', 'ì'),
+    (0xCD, 'M', 'í'),
+    (0xCE, 'M', 'î'),
+    (0xCF, 'M', 'ï'),
+    (0xD0, 'M', 'ð'),
+    (0xD1, 'M', 'ñ'),
+    (0xD2, 'M', 'ò'),
+    (0xD3, 'M', 'ó'),
+    (0xD4, 'M', 'ô'),
+    (0xD5, 'M', 'õ'),
+    (0xD6, 'M', 'ö'),
     (0xD7, 'V'),
-    (0xD8, 'M', u'ø'),
-    (0xD9, 'M', u'ù'),
-    (0xDA, 'M', u'ú'),
-    (0xDB, 'M', u'û'),
-    (0xDC, 'M', u'ü'),
-    (0xDD, 'M', u'ý'),
-    (0xDE, 'M', u'þ'),
-    (0xDF, 'D', u'ss'),
+    (0xD8, 'M', 'ø'),
+    (0xD9, 'M', 'ù'),
+    (0xDA, 'M', 'ú'),
+    (0xDB, 'M', 'û'),
+    (0xDC, 'M', 'ü'),
+    (0xDD, 'M', 'ý'),
+    (0xDE, 'M', 'þ'),
+    (0xDF, 'D', 'ss'),
     (0xE0, 'V'),
     (0xE1, 'V'),
     (0xE2, 'V'),
@@ -271,765 +274,765 @@ def _seg_2():
     (0xFD, 'V'),
     (0xFE, 'V'),
     (0xFF, 'V'),
-    (0x100, 'M', u'ā'),
+    (0x100, 'M', 'ā'),
     (0x101, 'V'),
-    (0x102, 'M', u'ă'),
+    (0x102, 'M', 'ă'),
     (0x103, 'V'),
-    (0x104, 'M', u'ą'),
+    (0x104, 'M', 'ą'),
     (0x105, 'V'),
-    (0x106, 'M', u'ć'),
+    (0x106, 'M', 'ć'),
     (0x107, 'V'),
-    (0x108, 'M', u'ĉ'),
+    (0x108, 'M', 'ĉ'),
     (0x109, 'V'),
-    (0x10A, 'M', u'ċ'),
+    (0x10A, 'M', 'ċ'),
     (0x10B, 'V'),
-    (0x10C, 'M', u'č'),
+    (0x10C, 'M', 'č'),
     (0x10D, 'V'),
-    (0x10E, 'M', u'ď'),
+    (0x10E, 'M', 'ď'),
     (0x10F, 'V'),
-    (0x110, 'M', u'đ'),
+    (0x110, 'M', 'đ'),
     (0x111, 'V'),
-    (0x112, 'M', u'ē'),
+    (0x112, 'M', 'ē'),
     (0x113, 'V'),
-    (0x114, 'M', u'ĕ'),
+    (0x114, 'M', 'ĕ'),
     (0x115, 'V'),
-    (0x116, 'M', u'ė'),
+    (0x116, 'M', 'ė'),
     (0x117, 'V'),
-    (0x118, 'M', u'ę'),
+    (0x118, 'M', 'ę'),
     (0x119, 'V'),
-    (0x11A, 'M', u'ě'),
+    (0x11A, 'M', 'ě'),
     (0x11B, 'V'),
-    (0x11C, 'M', u'ĝ'),
+    (0x11C, 'M', 'ĝ'),
     (0x11D, 'V'),
-    (0x11E, 'M', u'ğ'),
+    (0x11E, 'M', 'ğ'),
     (0x11F, 'V'),
-    (0x120, 'M', u'ġ'),
+    (0x120, 'M', 'ġ'),
     (0x121, 'V'),
-    (0x122, 'M', u'ģ'),
+    (0x122, 'M', 'ģ'),
     (0x123, 'V'),
-    (0x124, 'M', u'ĥ'),
+    (0x124, 'M', 'ĥ'),
     (0x125, 'V'),
-    (0x126, 'M', u'ħ'),
+    (0x126, 'M', 'ħ'),
     (0x127, 'V'),
-    (0x128, 'M', u'ĩ'),
+    (0x128, 'M', 'ĩ'),
     (0x129, 'V'),
-    (0x12A, 'M', u'ī'),
+    (0x12A, 'M', 'ī'),
     (0x12B, 'V'),
     ]
 
-def _seg_3():
+def _seg_3() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x12C, 'M', u'ĭ'),
+    (0x12C, 'M', 'ĭ'),
     (0x12D, 'V'),
-    (0x12E, 'M', u'į'),
+    (0x12E, 'M', 'į'),
     (0x12F, 'V'),
-    (0x130, 'M', u'i̇'),
+    (0x130, 'M', 'i̇'),
     (0x131, 'V'),
-    (0x132, 'M', u'ij'),
-    (0x134, 'M', u'ĵ'),
+    (0x132, 'M', 'ij'),
+    (0x134, 'M', 'ĵ'),
     (0x135, 'V'),
-    (0x136, 'M', u'ķ'),
+    (0x136, 'M', 'ķ'),
     (0x137, 'V'),
-    (0x139, 'M', u'ĺ'),
+    (0x139, 'M', 'ĺ'),
     (0x13A, 'V'),
-    (0x13B, 'M', u'ļ'),
+    (0x13B, 'M', 'ļ'),
     (0x13C, 'V'),
-    (0x13D, 'M', u'ľ'),
+    (0x13D, 'M', 'ľ'),
     (0x13E, 'V'),
-    (0x13F, 'M', u'l·'),
-    (0x141, 'M', u'ł'),
+    (0x13F, 'M', 'l·'),
+    (0x141, 'M', 'ł'),
     (0x142, 'V'),
-    (0x143, 'M', u'ń'),
+    (0x143, 'M', 'ń'),
     (0x144, 'V'),
-    (0x145, 'M', u'ņ'),
+    (0x145, 'M', 'ņ'),
     (0x146, 'V'),
-    (0x147, 'M', u'ň'),
+    (0x147, 'M', 'ň'),
     (0x148, 'V'),
-    (0x149, 'M', u'ʼn'),
-    (0x14A, 'M', u'ŋ'),
+    (0x149, 'M', 'ʼn'),
+    (0x14A, 'M', 'ŋ'),
     (0x14B, 'V'),
-    (0x14C, 'M', u'ō'),
+    (0x14C, 'M', 'ō'),
     (0x14D, 'V'),
-    (0x14E, 'M', u'ŏ'),
+    (0x14E, 'M', 'ŏ'),
     (0x14F, 'V'),
-    (0x150, 'M', u'ő'),
+    (0x150, 'M', 'ő'),
     (0x151, 'V'),
-    (0x152, 'M', u'œ'),
+    (0x152, 'M', 'œ'),
     (0x153, 'V'),
-    (0x154, 'M', u'ŕ'),
+    (0x154, 'M', 'ŕ'),
     (0x155, 'V'),
-    (0x156, 'M', u'ŗ'),
+    (0x156, 'M', 'ŗ'),
     (0x157, 'V'),
-    (0x158, 'M', u'ř'),
+    (0x158, 'M', 'ř'),
     (0x159, 'V'),
-    (0x15A, 'M', u'ś'),
+    (0x15A, 'M', 'ś'),
     (0x15B, 'V'),
-    (0x15C, 'M', u'ŝ'),
+    (0x15C, 'M', 'ŝ'),
     (0x15D, 'V'),
-    (0x15E, 'M', u'ş'),
+    (0x15E, 'M', 'ş'),
     (0x15F, 'V'),
-    (0x160, 'M', u'š'),
+    (0x160, 'M', 'š'),
     (0x161, 'V'),
-    (0x162, 'M', u'ţ'),
+    (0x162, 'M', 'ţ'),
     (0x163, 'V'),
-    (0x164, 'M', u'ť'),
+    (0x164, 'M', 'ť'),
     (0x165, 'V'),
-    (0x166, 'M', u'ŧ'),
+    (0x166, 'M', 'ŧ'),
     (0x167, 'V'),
-    (0x168, 'M', u'ũ'),
+    (0x168, 'M', 'ũ'),
     (0x169, 'V'),
-    (0x16A, 'M', u'ū'),
+    (0x16A, 'M', 'ū'),
     (0x16B, 'V'),
-    (0x16C, 'M', u'ŭ'),
+    (0x16C, 'M', 'ŭ'),
     (0x16D, 'V'),
-    (0x16E, 'M', u'ů'),
+    (0x16E, 'M', 'ů'),
     (0x16F, 'V'),
-    (0x170, 'M', u'ű'),
+    (0x170, 'M', 'ű'),
     (0x171, 'V'),
-    (0x172, 'M', u'ų'),
+    (0x172, 'M', 'ų'),
     (0x173, 'V'),
-    (0x174, 'M', u'ŵ'),
+    (0x174, 'M', 'ŵ'),
     (0x175, 'V'),
-    (0x176, 'M', u'ŷ'),
+    (0x176, 'M', 'ŷ'),
     (0x177, 'V'),
-    (0x178, 'M', u'ÿ'),
-    (0x179, 'M', u'ź'),
+    (0x178, 'M', 'ÿ'),
+    (0x179, 'M', 'ź'),
     (0x17A, 'V'),
-    (0x17B, 'M', u'ż'),
+    (0x17B, 'M', 'ż'),
     (0x17C, 'V'),
-    (0x17D, 'M', u'ž'),
+    (0x17D, 'M', 'ž'),
     (0x17E, 'V'),
-    (0x17F, 'M', u's'),
+    (0x17F, 'M', 's'),
     (0x180, 'V'),
-    (0x181, 'M', u'ɓ'),
-    (0x182, 'M', u'ƃ'),
+    (0x181, 'M', 'ɓ'),
+    (0x182, 'M', 'ƃ'),
     (0x183, 'V'),
-    (0x184, 'M', u'ƅ'),
+    (0x184, 'M', 'ƅ'),
     (0x185, 'V'),
-    (0x186, 'M', u'ɔ'),
-    (0x187, 'M', u'ƈ'),
+    (0x186, 'M', 'ɔ'),
+    (0x187, 'M', 'ƈ'),
     (0x188, 'V'),
-    (0x189, 'M', u'ɖ'),
-    (0x18A, 'M', u'ɗ'),
-    (0x18B, 'M', u'ƌ'),
+    (0x189, 'M', 'ɖ'),
+    (0x18A, 'M', 'ɗ'),
+    (0x18B, 'M', 'ƌ'),
     (0x18C, 'V'),
-    (0x18E, 'M', u'ǝ'),
-    (0x18F, 'M', u'ə'),
-    (0x190, 'M', u'ɛ'),
-    (0x191, 'M', u'ƒ'),
+    (0x18E, 'M', 'ǝ'),
+    (0x18F, 'M', 'ə'),
+    (0x190, 'M', 'ɛ'),
+    (0x191, 'M', 'ƒ'),
     (0x192, 'V'),
-    (0x193, 'M', u'ɠ'),
+    (0x193, 'M', 'ɠ'),
     ]
 
-def _seg_4():
+def _seg_4() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x194, 'M', u'ɣ'),
+    (0x194, 'M', 'ɣ'),
     (0x195, 'V'),
-    (0x196, 'M', u'ɩ'),
-    (0x197, 'M', u'ɨ'),
-    (0x198, 'M', u'ƙ'),
+    (0x196, 'M', 'ɩ'),
+    (0x197, 'M', 'ɨ'),
+    (0x198, 'M', 'ƙ'),
     (0x199, 'V'),
-    (0x19C, 'M', u'ɯ'),
-    (0x19D, 'M', u'ɲ'),
+    (0x19C, 'M', 'ɯ'),
+    (0x19D, 'M', 'ɲ'),
     (0x19E, 'V'),
-    (0x19F, 'M', u'ɵ'),
-    (0x1A0, 'M', u'ơ'),
+    (0x19F, 'M', 'ɵ'),
+    (0x1A0, 'M', 'ơ'),
     (0x1A1, 'V'),
-    (0x1A2, 'M', u'ƣ'),
+    (0x1A2, 'M', 'ƣ'),
     (0x1A3, 'V'),
-    (0x1A4, 'M', u'ƥ'),
+    (0x1A4, 'M', 'ƥ'),
     (0x1A5, 'V'),
-    (0x1A6, 'M', u'ʀ'),
-    (0x1A7, 'M', u'ƨ'),
+    (0x1A6, 'M', 'ʀ'),
+    (0x1A7, 'M', 'ƨ'),
     (0x1A8, 'V'),
-    (0x1A9, 'M', u'ʃ'),
+    (0x1A9, 'M', 'ʃ'),
     (0x1AA, 'V'),
-    (0x1AC, 'M', u'ƭ'),
+    (0x1AC, 'M', 'ƭ'),
     (0x1AD, 'V'),
-    (0x1AE, 'M', u'ʈ'),
-    (0x1AF, 'M', u'ư'),
+    (0x1AE, 'M', 'ʈ'),
+    (0x1AF, 'M', 'ư'),
     (0x1B0, 'V'),
-    (0x1B1, 'M', u'ʊ'),
-    (0x1B2, 'M', u'ʋ'),
-    (0x1B3, 'M', u'ƴ'),
+    (0x1B1, 'M', 'ʊ'),
+    (0x1B2, 'M', 'ʋ'),
+    (0x1B3, 'M', 'ƴ'),
     (0x1B4, 'V'),
-    (0x1B5, 'M', u'ƶ'),
+    (0x1B5, 'M', 'ƶ'),
     (0x1B6, 'V'),
-    (0x1B7, 'M', u'ʒ'),
-    (0x1B8, 'M', u'ƹ'),
+    (0x1B7, 'M', 'ʒ'),
+    (0x1B8, 'M', 'ƹ'),
     (0x1B9, 'V'),
-    (0x1BC, 'M', u'ƽ'),
+    (0x1BC, 'M', 'ƽ'),
     (0x1BD, 'V'),
-    (0x1C4, 'M', u'dž'),
-    (0x1C7, 'M', u'lj'),
-    (0x1CA, 'M', u'nj'),
-    (0x1CD, 'M', u'ǎ'),
+    (0x1C4, 'M', 'dž'),
+    (0x1C7, 'M', 'lj'),
+    (0x1CA, 'M', 'nj'),
+    (0x1CD, 'M', 'ǎ'),
     (0x1CE, 'V'),
-    (0x1CF, 'M', u'ǐ'),
+    (0x1CF, 'M', 'ǐ'),
     (0x1D0, 'V'),
-    (0x1D1, 'M', u'ǒ'),
+    (0x1D1, 'M', 'ǒ'),
     (0x1D2, 'V'),
-    (0x1D3, 'M', u'ǔ'),
+    (0x1D3, 'M', 'ǔ'),
     (0x1D4, 'V'),
-    (0x1D5, 'M', u'ǖ'),
+    (0x1D5, 'M', 'ǖ'),
     (0x1D6, 'V'),
-    (0x1D7, 'M', u'ǘ'),
+    (0x1D7, 'M', 'ǘ'),
     (0x1D8, 'V'),
-    (0x1D9, 'M', u'ǚ'),
+    (0x1D9, 'M', 'ǚ'),
     (0x1DA, 'V'),
-    (0x1DB, 'M', u'ǜ'),
+    (0x1DB, 'M', 'ǜ'),
     (0x1DC, 'V'),
-    (0x1DE, 'M', u'ǟ'),
+    (0x1DE, 'M', 'ǟ'),
     (0x1DF, 'V'),
-    (0x1E0, 'M', u'ǡ'),
+    (0x1E0, 'M', 'ǡ'),
     (0x1E1, 'V'),
-    (0x1E2, 'M', u'ǣ'),
+    (0x1E2, 'M', 'ǣ'),
     (0x1E3, 'V'),
-    (0x1E4, 'M', u'ǥ'),
+    (0x1E4, 'M', 'ǥ'),
     (0x1E5, 'V'),
-    (0x1E6, 'M', u'ǧ'),
+    (0x1E6, 'M', 'ǧ'),
     (0x1E7, 'V'),
-    (0x1E8, 'M', u'ǩ'),
+    (0x1E8, 'M', 'ǩ'),
     (0x1E9, 'V'),
-    (0x1EA, 'M', u'ǫ'),
+    (0x1EA, 'M', 'ǫ'),
     (0x1EB, 'V'),
-    (0x1EC, 'M', u'ǭ'),
+    (0x1EC, 'M', 'ǭ'),
     (0x1ED, 'V'),
-    (0x1EE, 'M', u'ǯ'),
+    (0x1EE, 'M', 'ǯ'),
     (0x1EF, 'V'),
-    (0x1F1, 'M', u'dz'),
-    (0x1F4, 'M', u'ǵ'),
+    (0x1F1, 'M', 'dz'),
+    (0x1F4, 'M', 'ǵ'),
     (0x1F5, 'V'),
-    (0x1F6, 'M', u'ƕ'),
-    (0x1F7, 'M', u'ƿ'),
-    (0x1F8, 'M', u'ǹ'),
+    (0x1F6, 'M', 'ƕ'),
+    (0x1F7, 'M', 'ƿ'),
+    (0x1F8, 'M', 'ǹ'),
     (0x1F9, 'V'),
-    (0x1FA, 'M', u'ǻ'),
+    (0x1FA, 'M', 'ǻ'),
     (0x1FB, 'V'),
-    (0x1FC, 'M', u'ǽ'),
+    (0x1FC, 'M', 'ǽ'),
     (0x1FD, 'V'),
-    (0x1FE, 'M', u'ǿ'),
+    (0x1FE, 'M', 'ǿ'),
     (0x1FF, 'V'),
-    (0x200, 'M', u'ȁ'),
+    (0x200, 'M', 'ȁ'),
     (0x201, 'V'),
-    (0x202, 'M', u'ȃ'),
+    (0x202, 'M', 'ȃ'),
     (0x203, 'V'),
-    (0x204, 'M', u'ȅ'),
+    (0x204, 'M', 'ȅ'),
     (0x205, 'V'),
-    (0x206, 'M', u'ȇ'),
+    (0x206, 'M', 'ȇ'),
     (0x207, 'V'),
-    (0x208, 'M', u'ȉ'),
+    (0x208, 'M', 'ȉ'),
     (0x209, 'V'),
-    (0x20A, 'M', u'ȋ'),
+    (0x20A, 'M', 'ȋ'),
     (0x20B, 'V'),
-    (0x20C, 'M', u'ȍ'),
+    (0x20C, 'M', 'ȍ'),
     ]
 
-def _seg_5():
+def _seg_5() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0x20D, 'V'),
-    (0x20E, 'M', u'ȏ'),
+    (0x20E, 'M', 'ȏ'),
     (0x20F, 'V'),
-    (0x210, 'M', u'ȑ'),
+    (0x210, 'M', 'ȑ'),
     (0x211, 'V'),
-    (0x212, 'M', u'ȓ'),
+    (0x212, 'M', 'ȓ'),
     (0x213, 'V'),
-    (0x214, 'M', u'ȕ'),
+    (0x214, 'M', 'ȕ'),
     (0x215, 'V'),
-    (0x216, 'M', u'ȗ'),
+    (0x216, 'M', 'ȗ'),
     (0x217, 'V'),
-    (0x218, 'M', u'ș'),
+    (0x218, 'M', 'ș'),
     (0x219, 'V'),
-    (0x21A, 'M', u'ț'),
+    (0x21A, 'M', 'ț'),
     (0x21B, 'V'),
-    (0x21C, 'M', u'ȝ'),
+    (0x21C, 'M', 'ȝ'),
     (0x21D, 'V'),
-    (0x21E, 'M', u'ȟ'),
+    (0x21E, 'M', 'ȟ'),
     (0x21F, 'V'),
-    (0x220, 'M', u'ƞ'),
+    (0x220, 'M', 'ƞ'),
     (0x221, 'V'),
-    (0x222, 'M', u'ȣ'),
+    (0x222, 'M', 'ȣ'),
     (0x223, 'V'),
-    (0x224, 'M', u'ȥ'),
+    (0x224, 'M', 'ȥ'),
     (0x225, 'V'),
-    (0x226, 'M', u'ȧ'),
+    (0x226, 'M', 'ȧ'),
     (0x227, 'V'),
-    (0x228, 'M', u'ȩ'),
+    (0x228, 'M', 'ȩ'),
     (0x229, 'V'),
-    (0x22A, 'M', u'ȫ'),
+    (0x22A, 'M', 'ȫ'),
     (0x22B, 'V'),
-    (0x22C, 'M', u'ȭ'),
+    (0x22C, 'M', 'ȭ'),
     (0x22D, 'V'),
-    (0x22E, 'M', u'ȯ'),
+    (0x22E, 'M', 'ȯ'),
     (0x22F, 'V'),
-    (0x230, 'M', u'ȱ'),
+    (0x230, 'M', 'ȱ'),
     (0x231, 'V'),
-    (0x232, 'M', u'ȳ'),
+    (0x232, 'M', 'ȳ'),
     (0x233, 'V'),
-    (0x23A, 'M', u'ⱥ'),
-    (0x23B, 'M', u'ȼ'),
+    (0x23A, 'M', 'ⱥ'),
+    (0x23B, 'M', 'ȼ'),
     (0x23C, 'V'),
-    (0x23D, 'M', u'ƚ'),
-    (0x23E, 'M', u'ⱦ'),
+    (0x23D, 'M', 'ƚ'),
+    (0x23E, 'M', 'ⱦ'),
     (0x23F, 'V'),
-    (0x241, 'M', u'ɂ'),
+    (0x241, 'M', 'ɂ'),
     (0x242, 'V'),
-    (0x243, 'M', u'ƀ'),
-    (0x244, 'M', u'ʉ'),
-    (0x245, 'M', u'ʌ'),
-    (0x246, 'M', u'ɇ'),
+    (0x243, 'M', 'ƀ'),
+    (0x244, 'M', 'ʉ'),
+    (0x245, 'M', 'ʌ'),
+    (0x246, 'M', 'ɇ'),
     (0x247, 'V'),
-    (0x248, 'M', u'ɉ'),
+    (0x248, 'M', 'ɉ'),
     (0x249, 'V'),
-    (0x24A, 'M', u'ɋ'),
+    (0x24A, 'M', 'ɋ'),
     (0x24B, 'V'),
-    (0x24C, 'M', u'ɍ'),
+    (0x24C, 'M', 'ɍ'),
     (0x24D, 'V'),
-    (0x24E, 'M', u'ɏ'),
+    (0x24E, 'M', 'ɏ'),
     (0x24F, 'V'),
-    (0x2B0, 'M', u'h'),
-    (0x2B1, 'M', u'ɦ'),
-    (0x2B2, 'M', u'j'),
-    (0x2B3, 'M', u'r'),
-    (0x2B4, 'M', u'ɹ'),
-    (0x2B5, 'M', u'ɻ'),
-    (0x2B6, 'M', u'ʁ'),
-    (0x2B7, 'M', u'w'),
-    (0x2B8, 'M', u'y'),
+    (0x2B0, 'M', 'h'),
+    (0x2B1, 'M', 'ɦ'),
+    (0x2B2, 'M', 'j'),
+    (0x2B3, 'M', 'r'),
+    (0x2B4, 'M', 'ɹ'),
+    (0x2B5, 'M', 'ɻ'),
+    (0x2B6, 'M', 'ʁ'),
+    (0x2B7, 'M', 'w'),
+    (0x2B8, 'M', 'y'),
     (0x2B9, 'V'),
-    (0x2D8, '3', u' ̆'),
-    (0x2D9, '3', u' ̇'),
-    (0x2DA, '3', u' ̊'),
-    (0x2DB, '3', u' ̨'),
-    (0x2DC, '3', u' ̃'),
-    (0x2DD, '3', u' ̋'),
+    (0x2D8, '3', ' ̆'),
+    (0x2D9, '3', ' ̇'),
+    (0x2DA, '3', ' ̊'),
+    (0x2DB, '3', ' ̨'),
+    (0x2DC, '3', ' ̃'),
+    (0x2DD, '3', ' ̋'),
     (0x2DE, 'V'),
-    (0x2E0, 'M', u'ɣ'),
-    (0x2E1, 'M', u'l'),
-    (0x2E2, 'M', u's'),
-    (0x2E3, 'M', u'x'),
-    (0x2E4, 'M', u'ʕ'),
+    (0x2E0, 'M', 'ɣ'),
+    (0x2E1, 'M', 'l'),
+    (0x2E2, 'M', 's'),
+    (0x2E3, 'M', 'x'),
+    (0x2E4, 'M', 'ʕ'),
     (0x2E5, 'V'),
-    (0x340, 'M', u'̀'),
-    (0x341, 'M', u'́'),
+    (0x340, 'M', '̀'),
+    (0x341, 'M', '́'),
     (0x342, 'V'),
-    (0x343, 'M', u'̓'),
-    (0x344, 'M', u'̈́'),
-    (0x345, 'M', u'ι'),
+    (0x343, 'M', '̓'),
+    (0x344, 'M', '̈́'),
+    (0x345, 'M', 'ι'),
     (0x346, 'V'),
     (0x34F, 'I'),
     (0x350, 'V'),
-    (0x370, 'M', u'ͱ'),
+    (0x370, 'M', 'ͱ'),
     (0x371, 'V'),
-    (0x372, 'M', u'ͳ'),
+    (0x372, 'M', 'ͳ'),
     (0x373, 'V'),
-    (0x374, 'M', u'ʹ'),
+    (0x374, 'M', 'ʹ'),
     (0x375, 'V'),
-    (0x376, 'M', u'ͷ'),
+    (0x376, 'M', 'ͷ'),
     (0x377, 'V'),
     ]
 
-def _seg_6():
+def _seg_6() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0x378, 'X'),
-    (0x37A, '3', u' ι'),
+    (0x37A, '3', ' ι'),
     (0x37B, 'V'),
-    (0x37E, '3', u';'),
-    (0x37F, 'M', u'ϳ'),
+    (0x37E, '3', ';'),
+    (0x37F, 'M', 'ϳ'),
     (0x380, 'X'),
-    (0x384, '3', u' ́'),
-    (0x385, '3', u' ̈́'),
-    (0x386, 'M', u'ά'),
-    (0x387, 'M', u'·'),
-    (0x388, 'M', u'έ'),
-    (0x389, 'M', u'ή'),
-    (0x38A, 'M', u'ί'),
+    (0x384, '3', ' ́'),
+    (0x385, '3', ' ̈́'),
+    (0x386, 'M', 'ά'),
+    (0x387, 'M', '·'),
+    (0x388, 'M', 'έ'),
+    (0x389, 'M', 'ή'),
+    (0x38A, 'M', 'ί'),
     (0x38B, 'X'),
-    (0x38C, 'M', u'ό'),
+    (0x38C, 'M', 'ό'),
     (0x38D, 'X'),
-    (0x38E, 'M', u'ύ'),
-    (0x38F, 'M', u'ώ'),
+    (0x38E, 'M', 'ύ'),
+    (0x38F, 'M', 'ώ'),
     (0x390, 'V'),
-    (0x391, 'M', u'α'),
-    (0x392, 'M', u'β'),
-    (0x393, 'M', u'γ'),
-    (0x394, 'M', u'δ'),
-    (0x395, 'M', u'ε'),
-    (0x396, 'M', u'ζ'),
-    (0x397, 'M', u'η'),
-    (0x398, 'M', u'θ'),
-    (0x399, 'M', u'ι'),
-    (0x39A, 'M', u'κ'),
-    (0x39B, 'M', u'λ'),
-    (0x39C, 'M', u'μ'),
-    (0x39D, 'M', u'ν'),
-    (0x39E, 'M', u'ξ'),
-    (0x39F, 'M', u'ο'),
-    (0x3A0, 'M', u'π'),
-    (0x3A1, 'M', u'ρ'),
+    (0x391, 'M', 'α'),
+    (0x392, 'M', 'β'),
+    (0x393, 'M', 'γ'),
+    (0x394, 'M', 'δ'),
+    (0x395, 'M', 'ε'),
+    (0x396, 'M', 'ζ'),
+    (0x397, 'M', 'η'),
+    (0x398, 'M', 'θ'),
+    (0x399, 'M', 'ι'),
+    (0x39A, 'M', 'κ'),
+    (0x39B, 'M', 'λ'),
+    (0x39C, 'M', 'μ'),
+    (0x39D, 'M', 'ν'),
+    (0x39E, 'M', 'ξ'),
+    (0x39F, 'M', 'ο'),
+    (0x3A0, 'M', 'π'),
+    (0x3A1, 'M', 'ρ'),
     (0x3A2, 'X'),
-    (0x3A3, 'M', u'σ'),
-    (0x3A4, 'M', u'τ'),
-    (0x3A5, 'M', u'υ'),
-    (0x3A6, 'M', u'φ'),
-    (0x3A7, 'M', u'χ'),
-    (0x3A8, 'M', u'ψ'),
-    (0x3A9, 'M', u'ω'),
-    (0x3AA, 'M', u'ϊ'),
-    (0x3AB, 'M', u'ϋ'),
+    (0x3A3, 'M', 'σ'),
+    (0x3A4, 'M', 'τ'),
+    (0x3A5, 'M', 'υ'),
+    (0x3A6, 'M', 'φ'),
+    (0x3A7, 'M', 'χ'),
+    (0x3A8, 'M', 'ψ'),
+    (0x3A9, 'M', 'ω'),
+    (0x3AA, 'M', 'ϊ'),
+    (0x3AB, 'M', 'ϋ'),
     (0x3AC, 'V'),
-    (0x3C2, 'D', u'σ'),
+    (0x3C2, 'D', 'σ'),
     (0x3C3, 'V'),
-    (0x3CF, 'M', u'ϗ'),
-    (0x3D0, 'M', u'β'),
-    (0x3D1, 'M', u'θ'),
-    (0x3D2, 'M', u'υ'),
-    (0x3D3, 'M', u'ύ'),
-    (0x3D4, 'M', u'ϋ'),
-    (0x3D5, 'M', u'φ'),
-    (0x3D6, 'M', u'π'),
+    (0x3CF, 'M', 'ϗ'),
+    (0x3D0, 'M', 'β'),
+    (0x3D1, 'M', 'θ'),
+    (0x3D2, 'M', 'υ'),
+    (0x3D3, 'M', 'ύ'),
+    (0x3D4, 'M', 'ϋ'),
+    (0x3D5, 'M', 'φ'),
+    (0x3D6, 'M', 'π'),
     (0x3D7, 'V'),
-    (0x3D8, 'M', u'ϙ'),
+    (0x3D8, 'M', 'ϙ'),
     (0x3D9, 'V'),
-    (0x3DA, 'M', u'ϛ'),
+    (0x3DA, 'M', 'ϛ'),
     (0x3DB, 'V'),
-    (0x3DC, 'M', u'ϝ'),
+    (0x3DC, 'M', 'ϝ'),
     (0x3DD, 'V'),
-    (0x3DE, 'M', u'ϟ'),
+    (0x3DE, 'M', 'ϟ'),
     (0x3DF, 'V'),
-    (0x3E0, 'M', u'ϡ'),
+    (0x3E0, 'M', 'ϡ'),
     (0x3E1, 'V'),
-    (0x3E2, 'M', u'ϣ'),
+    (0x3E2, 'M', 'ϣ'),
     (0x3E3, 'V'),
-    (0x3E4, 'M', u'ϥ'),
+    (0x3E4, 'M', 'ϥ'),
     (0x3E5, 'V'),
-    (0x3E6, 'M', u'ϧ'),
+    (0x3E6, 'M', 'ϧ'),
     (0x3E7, 'V'),
-    (0x3E8, 'M', u'ϩ'),
+    (0x3E8, 'M', 'ϩ'),
     (0x3E9, 'V'),
-    (0x3EA, 'M', u'ϫ'),
+    (0x3EA, 'M', 'ϫ'),
     (0x3EB, 'V'),
-    (0x3EC, 'M', u'ϭ'),
+    (0x3EC, 'M', 'ϭ'),
     (0x3ED, 'V'),
-    (0x3EE, 'M', u'ϯ'),
+    (0x3EE, 'M', 'ϯ'),
     (0x3EF, 'V'),
-    (0x3F0, 'M', u'κ'),
-    (0x3F1, 'M', u'ρ'),
-    (0x3F2, 'M', u'σ'),
+    (0x3F0, 'M', 'κ'),
+    (0x3F1, 'M', 'ρ'),
+    (0x3F2, 'M', 'σ'),
     (0x3F3, 'V'),
-    (0x3F4, 'M', u'θ'),
-    (0x3F5, 'M', u'ε'),
+    (0x3F4, 'M', 'θ'),
+    (0x3F5, 'M', 'ε'),
     (0x3F6, 'V'),
-    (0x3F7, 'M', u'ϸ'),
+    (0x3F7, 'M', 'ϸ'),
     (0x3F8, 'V'),
-    (0x3F9, 'M', u'σ'),
-    (0x3FA, 'M', u'ϻ'),
+    (0x3F9, 'M', 'σ'),
+    (0x3FA, 'M', 'ϻ'),
     (0x3FB, 'V'),
-    (0x3FD, 'M', u'ͻ'),
-    (0x3FE, 'M', u'ͼ'),
-    (0x3FF, 'M', u'ͽ'),
-    (0x400, 'M', u'ѐ'),
-    (0x401, 'M', u'ё'),
-    (0x402, 'M', u'ђ'),
+    (0x3FD, 'M', 'ͻ'),
+    (0x3FE, 'M', 'ͼ'),
+    (0x3FF, 'M', 'ͽ'),
+    (0x400, 'M', 'ѐ'),
+    (0x401, 'M', 'ё'),
+    (0x402, 'M', 'ђ'),
     ]
 
-def _seg_7():
+def _seg_7() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x403, 'M', u'ѓ'),
-    (0x404, 'M', u'є'),
-    (0x405, 'M', u'ѕ'),
-    (0x406, 'M', u'і'),
-    (0x407, 'M', u'ї'),
-    (0x408, 'M', u'ј'),
-    (0x409, 'M', u'љ'),
-    (0x40A, 'M', u'њ'),
-    (0x40B, 'M', u'ћ'),
-    (0x40C, 'M', u'ќ'),
-    (0x40D, 'M', u'ѝ'),
-    (0x40E, 'M', u'ў'),
-    (0x40F, 'M', u'џ'),
-    (0x410, 'M', u'а'),
-    (0x411, 'M', u'б'),
-    (0x412, 'M', u'в'),
-    (0x413, 'M', u'г'),
-    (0x414, 'M', u'д'),
-    (0x415, 'M', u'е'),
-    (0x416, 'M', u'ж'),
-    (0x417, 'M', u'з'),
-    (0x418, 'M', u'и'),
-    (0x419, 'M', u'й'),
-    (0x41A, 'M', u'к'),
-    (0x41B, 'M', u'л'),
-    (0x41C, 'M', u'м'),
-    (0x41D, 'M', u'н'),
-    (0x41E, 'M', u'о'),
-    (0x41F, 'M', u'п'),
-    (0x420, 'M', u'р'),
-    (0x421, 'M', u'с'),
-    (0x422, 'M', u'т'),
-    (0x423, 'M', u'у'),
-    (0x424, 'M', u'ф'),
-    (0x425, 'M', u'х'),
-    (0x426, 'M', u'ц'),
-    (0x427, 'M', u'ч'),
-    (0x428, 'M', u'ш'),
-    (0x429, 'M', u'щ'),
-    (0x42A, 'M', u'ъ'),
-    (0x42B, 'M', u'ы'),
-    (0x42C, 'M', u'ь'),
-    (0x42D, 'M', u'э'),
-    (0x42E, 'M', u'ю'),
-    (0x42F, 'M', u'я'),
+    (0x403, 'M', 'ѓ'),
+    (0x404, 'M', 'є'),
+    (0x405, 'M', 'ѕ'),
+    (0x406, 'M', 'і'),
+    (0x407, 'M', 'ї'),
+    (0x408, 'M', 'ј'),
+    (0x409, 'M', 'љ'),
+    (0x40A, 'M', 'њ'),
+    (0x40B, 'M', 'ћ'),
+    (0x40C, 'M', 'ќ'),
+    (0x40D, 'M', 'ѝ'),
+    (0x40E, 'M', 'ў'),
+    (0x40F, 'M', 'џ'),
+    (0x410, 'M', 'а'),
+    (0x411, 'M', 'б'),
+    (0x412, 'M', 'в'),
+    (0x413, 'M', 'г'),
+    (0x414, 'M', 'д'),
+    (0x415, 'M', 'е'),
+    (0x416, 'M', 'ж'),
+    (0x417, 'M', 'з'),
+    (0x418, 'M', 'и'),
+    (0x419, 'M', 'й'),
+    (0x41A, 'M', 'к'),
+    (0x41B, 'M', 'л'),
+    (0x41C, 'M', 'м'),
+    (0x41D, 'M', 'н'),
+    (0x41E, 'M', 'о'),
+    (0x41F, 'M', 'п'),
+    (0x420, 'M', 'р'),
+    (0x421, 'M', 'с'),
+    (0x422, 'M', 'т'),
+    (0x423, 'M', 'у'),
+    (0x424, 'M', 'ф'),
+    (0x425, 'M', 'х'),
+    (0x426, 'M', 'ц'),
+    (0x427, 'M', 'ч'),
+    (0x428, 'M', 'ш'),
+    (0x429, 'M', 'щ'),
+    (0x42A, 'M', 'ъ'),
+    (0x42B, 'M', 'ы'),
+    (0x42C, 'M', 'ь'),
+    (0x42D, 'M', 'э'),
+    (0x42E, 'M', 'ю'),
+    (0x42F, 'M', 'я'),
     (0x430, 'V'),
-    (0x460, 'M', u'ѡ'),
+    (0x460, 'M', 'ѡ'),
     (0x461, 'V'),
-    (0x462, 'M', u'ѣ'),
+    (0x462, 'M', 'ѣ'),
     (0x463, 'V'),
-    (0x464, 'M', u'ѥ'),
+    (0x464, 'M', 'ѥ'),
     (0x465, 'V'),
-    (0x466, 'M', u'ѧ'),
+    (0x466, 'M', 'ѧ'),
     (0x467, 'V'),
-    (0x468, 'M', u'ѩ'),
+    (0x468, 'M', 'ѩ'),
     (0x469, 'V'),
-    (0x46A, 'M', u'ѫ'),
+    (0x46A, 'M', 'ѫ'),
     (0x46B, 'V'),
-    (0x46C, 'M', u'ѭ'),
+    (0x46C, 'M', 'ѭ'),
     (0x46D, 'V'),
-    (0x46E, 'M', u'ѯ'),
+    (0x46E, 'M', 'ѯ'),
     (0x46F, 'V'),
-    (0x470, 'M', u'ѱ'),
+    (0x470, 'M', 'ѱ'),
     (0x471, 'V'),
-    (0x472, 'M', u'ѳ'),
+    (0x472, 'M', 'ѳ'),
     (0x473, 'V'),
-    (0x474, 'M', u'ѵ'),
+    (0x474, 'M', 'ѵ'),
     (0x475, 'V'),
-    (0x476, 'M', u'ѷ'),
+    (0x476, 'M', 'ѷ'),
     (0x477, 'V'),
-    (0x478, 'M', u'ѹ'),
+    (0x478, 'M', 'ѹ'),
     (0x479, 'V'),
-    (0x47A, 'M', u'ѻ'),
+    (0x47A, 'M', 'ѻ'),
     (0x47B, 'V'),
-    (0x47C, 'M', u'ѽ'),
+    (0x47C, 'M', 'ѽ'),
     (0x47D, 'V'),
-    (0x47E, 'M', u'ѿ'),
+    (0x47E, 'M', 'ѿ'),
     (0x47F, 'V'),
-    (0x480, 'M', u'ҁ'),
+    (0x480, 'M', 'ҁ'),
     (0x481, 'V'),
-    (0x48A, 'M', u'ҋ'),
+    (0x48A, 'M', 'ҋ'),
     (0x48B, 'V'),
-    (0x48C, 'M', u'ҍ'),
+    (0x48C, 'M', 'ҍ'),
     (0x48D, 'V'),
-    (0x48E, 'M', u'ҏ'),
+    (0x48E, 'M', 'ҏ'),
     (0x48F, 'V'),
-    (0x490, 'M', u'ґ'),
+    (0x490, 'M', 'ґ'),
     (0x491, 'V'),
-    (0x492, 'M', u'ғ'),
+    (0x492, 'M', 'ғ'),
     (0x493, 'V'),
-    (0x494, 'M', u'ҕ'),
+    (0x494, 'M', 'ҕ'),
     (0x495, 'V'),
-    (0x496, 'M', u'җ'),
+    (0x496, 'M', 'җ'),
     (0x497, 'V'),
-    (0x498, 'M', u'ҙ'),
+    (0x498, 'M', 'ҙ'),
     (0x499, 'V'),
-    (0x49A, 'M', u'қ'),
+    (0x49A, 'M', 'қ'),
     (0x49B, 'V'),
-    (0x49C, 'M', u'ҝ'),
+    (0x49C, 'M', 'ҝ'),
     (0x49D, 'V'),
     ]
 
-def _seg_8():
+def _seg_8() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x49E, 'M', u'ҟ'),
+    (0x49E, 'M', 'ҟ'),
     (0x49F, 'V'),
-    (0x4A0, 'M', u'ҡ'),
+    (0x4A0, 'M', 'ҡ'),
     (0x4A1, 'V'),
-    (0x4A2, 'M', u'ң'),
+    (0x4A2, 'M', 'ң'),
     (0x4A3, 'V'),
-    (0x4A4, 'M', u'ҥ'),
+    (0x4A4, 'M', 'ҥ'),
     (0x4A5, 'V'),
-    (0x4A6, 'M', u'ҧ'),
+    (0x4A6, 'M', 'ҧ'),
     (0x4A7, 'V'),
-    (0x4A8, 'M', u'ҩ'),
+    (0x4A8, 'M', 'ҩ'),
     (0x4A9, 'V'),
-    (0x4AA, 'M', u'ҫ'),
+    (0x4AA, 'M', 'ҫ'),
     (0x4AB, 'V'),
-    (0x4AC, 'M', u'ҭ'),
+    (0x4AC, 'M', 'ҭ'),
     (0x4AD, 'V'),
-    (0x4AE, 'M', u'ү'),
+    (0x4AE, 'M', 'ү'),
     (0x4AF, 'V'),
-    (0x4B0, 'M', u'ұ'),
+    (0x4B0, 'M', 'ұ'),
     (0x4B1, 'V'),
-    (0x4B2, 'M', u'ҳ'),
+    (0x4B2, 'M', 'ҳ'),
     (0x4B3, 'V'),
-    (0x4B4, 'M', u'ҵ'),
+    (0x4B4, 'M', 'ҵ'),
     (0x4B5, 'V'),
-    (0x4B6, 'M', u'ҷ'),
+    (0x4B6, 'M', 'ҷ'),
     (0x4B7, 'V'),
-    (0x4B8, 'M', u'ҹ'),
+    (0x4B8, 'M', 'ҹ'),
     (0x4B9, 'V'),
-    (0x4BA, 'M', u'һ'),
+    (0x4BA, 'M', 'һ'),
     (0x4BB, 'V'),
-    (0x4BC, 'M', u'ҽ'),
+    (0x4BC, 'M', 'ҽ'),
     (0x4BD, 'V'),
-    (0x4BE, 'M', u'ҿ'),
+    (0x4BE, 'M', 'ҿ'),
     (0x4BF, 'V'),
     (0x4C0, 'X'),
-    (0x4C1, 'M', u'ӂ'),
+    (0x4C1, 'M', 'ӂ'),
     (0x4C2, 'V'),
-    (0x4C3, 'M', u'ӄ'),
+    (0x4C3, 'M', 'ӄ'),
     (0x4C4, 'V'),
-    (0x4C5, 'M', u'ӆ'),
+    (0x4C5, 'M', 'ӆ'),
     (0x4C6, 'V'),
-    (0x4C7, 'M', u'ӈ'),
+    (0x4C7, 'M', 'ӈ'),
     (0x4C8, 'V'),
-    (0x4C9, 'M', u'ӊ'),
+    (0x4C9, 'M', 'ӊ'),
     (0x4CA, 'V'),
-    (0x4CB, 'M', u'ӌ'),
+    (0x4CB, 'M', 'ӌ'),
     (0x4CC, 'V'),
-    (0x4CD, 'M', u'ӎ'),
+    (0x4CD, 'M', 'ӎ'),
     (0x4CE, 'V'),
-    (0x4D0, 'M', u'ӑ'),
+    (0x4D0, 'M', 'ӑ'),
     (0x4D1, 'V'),
-    (0x4D2, 'M', u'ӓ'),
+    (0x4D2, 'M', 'ӓ'),
     (0x4D3, 'V'),
-    (0x4D4, 'M', u'ӕ'),
+    (0x4D4, 'M', 'ӕ'),
     (0x4D5, 'V'),
-    (0x4D6, 'M', u'ӗ'),
+    (0x4D6, 'M', 'ӗ'),
     (0x4D7, 'V'),
-    (0x4D8, 'M', u'ә'),
+    (0x4D8, 'M', 'ә'),
     (0x4D9, 'V'),
-    (0x4DA, 'M', u'ӛ'),
+    (0x4DA, 'M', 'ӛ'),
     (0x4DB, 'V'),
-    (0x4DC, 'M', u'ӝ'),
+    (0x4DC, 'M', 'ӝ'),
     (0x4DD, 'V'),
-    (0x4DE, 'M', u'ӟ'),
+    (0x4DE, 'M', 'ӟ'),
     (0x4DF, 'V'),
-    (0x4E0, 'M', u'ӡ'),
+    (0x4E0, 'M', 'ӡ'),
     (0x4E1, 'V'),
-    (0x4E2, 'M', u'ӣ'),
+    (0x4E2, 'M', 'ӣ'),
     (0x4E3, 'V'),
-    (0x4E4, 'M', u'ӥ'),
+    (0x4E4, 'M', 'ӥ'),
     (0x4E5, 'V'),
-    (0x4E6, 'M', u'ӧ'),
+    (0x4E6, 'M', 'ӧ'),
     (0x4E7, 'V'),
-    (0x4E8, 'M', u'ө'),
+    (0x4E8, 'M', 'ө'),
     (0x4E9, 'V'),
-    (0x4EA, 'M', u'ӫ'),
+    (0x4EA, 'M', 'ӫ'),
     (0x4EB, 'V'),
-    (0x4EC, 'M', u'ӭ'),
+    (0x4EC, 'M', 'ӭ'),
     (0x4ED, 'V'),
-    (0x4EE, 'M', u'ӯ'),
+    (0x4EE, 'M', 'ӯ'),
     (0x4EF, 'V'),
-    (0x4F0, 'M', u'ӱ'),
+    (0x4F0, 'M', 'ӱ'),
     (0x4F1, 'V'),
-    (0x4F2, 'M', u'ӳ'),
+    (0x4F2, 'M', 'ӳ'),
     (0x4F3, 'V'),
-    (0x4F4, 'M', u'ӵ'),
+    (0x4F4, 'M', 'ӵ'),
     (0x4F5, 'V'),
-    (0x4F6, 'M', u'ӷ'),
+    (0x4F6, 'M', 'ӷ'),
     (0x4F7, 'V'),
-    (0x4F8, 'M', u'ӹ'),
+    (0x4F8, 'M', 'ӹ'),
     (0x4F9, 'V'),
-    (0x4FA, 'M', u'ӻ'),
+    (0x4FA, 'M', 'ӻ'),
     (0x4FB, 'V'),
-    (0x4FC, 'M', u'ӽ'),
+    (0x4FC, 'M', 'ӽ'),
     (0x4FD, 'V'),
-    (0x4FE, 'M', u'ӿ'),
+    (0x4FE, 'M', 'ӿ'),
     (0x4FF, 'V'),
-    (0x500, 'M', u'ԁ'),
+    (0x500, 'M', 'ԁ'),
     (0x501, 'V'),
-    (0x502, 'M', u'ԃ'),
+    (0x502, 'M', 'ԃ'),
     ]
 
-def _seg_9():
+def _seg_9() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0x503, 'V'),
-    (0x504, 'M', u'ԅ'),
+    (0x504, 'M', 'ԅ'),
     (0x505, 'V'),
-    (0x506, 'M', u'ԇ'),
+    (0x506, 'M', 'ԇ'),
     (0x507, 'V'),
-    (0x508, 'M', u'ԉ'),
+    (0x508, 'M', 'ԉ'),
     (0x509, 'V'),
-    (0x50A, 'M', u'ԋ'),
+    (0x50A, 'M', 'ԋ'),
     (0x50B, 'V'),
-    (0x50C, 'M', u'ԍ'),
+    (0x50C, 'M', 'ԍ'),
     (0x50D, 'V'),
-    (0x50E, 'M', u'ԏ'),
+    (0x50E, 'M', 'ԏ'),
     (0x50F, 'V'),
-    (0x510, 'M', u'ԑ'),
+    (0x510, 'M', 'ԑ'),
     (0x511, 'V'),
-    (0x512, 'M', u'ԓ'),
+    (0x512, 'M', 'ԓ'),
     (0x513, 'V'),
-    (0x514, 'M', u'ԕ'),
+    (0x514, 'M', 'ԕ'),
     (0x515, 'V'),
-    (0x516, 'M', u'ԗ'),
+    (0x516, 'M', 'ԗ'),
     (0x517, 'V'),
-    (0x518, 'M', u'ԙ'),
+    (0x518, 'M', 'ԙ'),
     (0x519, 'V'),
-    (0x51A, 'M', u'ԛ'),
+    (0x51A, 'M', 'ԛ'),
     (0x51B, 'V'),
-    (0x51C, 'M', u'ԝ'),
+    (0x51C, 'M', 'ԝ'),
     (0x51D, 'V'),
-    (0x51E, 'M', u'ԟ'),
+    (0x51E, 'M', 'ԟ'),
     (0x51F, 'V'),
-    (0x520, 'M', u'ԡ'),
+    (0x520, 'M', 'ԡ'),
     (0x521, 'V'),
-    (0x522, 'M', u'ԣ'),
+    (0x522, 'M', 'ԣ'),
     (0x523, 'V'),
-    (0x524, 'M', u'ԥ'),
+    (0x524, 'M', 'ԥ'),
     (0x525, 'V'),
-    (0x526, 'M', u'ԧ'),
+    (0x526, 'M', 'ԧ'),
     (0x527, 'V'),
-    (0x528, 'M', u'ԩ'),
+    (0x528, 'M', 'ԩ'),
     (0x529, 'V'),
-    (0x52A, 'M', u'ԫ'),
+    (0x52A, 'M', 'ԫ'),
     (0x52B, 'V'),
-    (0x52C, 'M', u'ԭ'),
+    (0x52C, 'M', 'ԭ'),
     (0x52D, 'V'),
-    (0x52E, 'M', u'ԯ'),
+    (0x52E, 'M', 'ԯ'),
     (0x52F, 'V'),
     (0x530, 'X'),
-    (0x531, 'M', u'ա'),
-    (0x532, 'M', u'բ'),
-    (0x533, 'M', u'գ'),
-    (0x534, 'M', u'դ'),
-    (0x535, 'M', u'ե'),
-    (0x536, 'M', u'զ'),
-    (0x537, 'M', u'է'),
-    (0x538, 'M', u'ը'),
-    (0x539, 'M', u'թ'),
-    (0x53A, 'M', u'ժ'),
-    (0x53B, 'M', u'ի'),
-    (0x53C, 'M', u'լ'),
-    (0x53D, 'M', u'խ'),
-    (0x53E, 'M', u'ծ'),
-    (0x53F, 'M', u'կ'),
-    (0x540, 'M', u'հ'),
-    (0x541, 'M', u'ձ'),
-    (0x542, 'M', u'ղ'),
-    (0x543, 'M', u'ճ'),
-    (0x544, 'M', u'մ'),
-    (0x545, 'M', u'յ'),
-    (0x546, 'M', u'ն'),
-    (0x547, 'M', u'շ'),
-    (0x548, 'M', u'ո'),
-    (0x549, 'M', u'չ'),
-    (0x54A, 'M', u'պ'),
-    (0x54B, 'M', u'ջ'),
-    (0x54C, 'M', u'ռ'),
-    (0x54D, 'M', u'ս'),
-    (0x54E, 'M', u'վ'),
-    (0x54F, 'M', u'տ'),
-    (0x550, 'M', u'ր'),
-    (0x551, 'M', u'ց'),
-    (0x552, 'M', u'ւ'),
-    (0x553, 'M', u'փ'),
-    (0x554, 'M', u'ք'),
-    (0x555, 'M', u'օ'),
-    (0x556, 'M', u'ֆ'),
+    (0x531, 'M', 'ա'),
+    (0x532, 'M', 'բ'),
+    (0x533, 'M', 'գ'),
+    (0x534, 'M', 'դ'),
+    (0x535, 'M', 'ե'),
+    (0x536, 'M', 'զ'),
+    (0x537, 'M', 'է'),
+    (0x538, 'M', 'ը'),
+    (0x539, 'M', 'թ'),
+    (0x53A, 'M', 'ժ'),
+    (0x53B, 'M', 'ի'),
+    (0x53C, 'M', 'լ'),
+    (0x53D, 'M', 'խ'),
+    (0x53E, 'M', 'ծ'),
+    (0x53F, 'M', 'կ'),
+    (0x540, 'M', 'հ'),
+    (0x541, 'M', 'ձ'),
+    (0x542, 'M', 'ղ'),
+    (0x543, 'M', 'ճ'),
+    (0x544, 'M', 'մ'),
+    (0x545, 'M', 'յ'),
+    (0x546, 'M', 'ն'),
+    (0x547, 'M', 'շ'),
+    (0x548, 'M', 'ո'),
+    (0x549, 'M', 'չ'),
+    (0x54A, 'M', 'պ'),
+    (0x54B, 'M', 'ջ'),
+    (0x54C, 'M', 'ռ'),
+    (0x54D, 'M', 'ս'),
+    (0x54E, 'M', 'վ'),
+    (0x54F, 'M', 'տ'),
+    (0x550, 'M', 'ր'),
+    (0x551, 'M', 'ց'),
+    (0x552, 'M', 'ւ'),
+    (0x553, 'M', 'փ'),
+    (0x554, 'M', 'ք'),
+    (0x555, 'M', 'օ'),
+    (0x556, 'M', 'ֆ'),
     (0x557, 'X'),
     (0x559, 'V'),
-    (0x587, 'M', u'եւ'),
+    (0x587, 'M', 'եւ'),
     (0x588, 'V'),
     (0x58B, 'X'),
     (0x58D, 'V'),
@@ -1042,15 +1045,15 @@ def _seg_9():
     (0x5F5, 'X'),
     (0x606, 'V'),
     (0x61C, 'X'),
-    (0x61E, 'V'),
+    (0x61D, 'V'),
     ]
 
-def _seg_10():
+def _seg_10() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x675, 'M', u'اٴ'),
-    (0x676, 'M', u'وٴ'),
-    (0x677, 'M', u'ۇٴ'),
-    (0x678, 'M', u'يٴ'),
+    (0x675, 'M', 'اٴ'),
+    (0x676, 'M', 'وٴ'),
+    (0x677, 'M', 'ۇٴ'),
+    (0x678, 'M', 'يٴ'),
     (0x679, 'V'),
     (0x6DD, 'X'),
     (0x6DE, 'V'),
@@ -1071,21 +1074,19 @@ def _seg_10():
     (0x85F, 'X'),
     (0x860, 'V'),
     (0x86B, 'X'),
-    (0x8A0, 'V'),
-    (0x8B5, 'X'),
-    (0x8B6, 'V'),
-    (0x8C8, 'X'),
-    (0x8D3, 'V'),
+    (0x870, 'V'),
+    (0x88F, 'X'),
+    (0x898, 'V'),
     (0x8E2, 'X'),
     (0x8E3, 'V'),
-    (0x958, 'M', u'क़'),
-    (0x959, 'M', u'ख़'),
-    (0x95A, 'M', u'ग़'),
-    (0x95B, 'M', u'ज़'),
-    (0x95C, 'M', u'ड़'),
-    (0x95D, 'M', u'ढ़'),
-    (0x95E, 'M', u'फ़'),
-    (0x95F, 'M', u'य़'),
+    (0x958, 'M', 'क़'),
+    (0x959, 'M', 'ख़'),
+    (0x95A, 'M', 'ग़'),
+    (0x95B, 'M', 'ज़'),
+    (0x95C, 'M', 'ड़'),
+    (0x95D, 'M', 'ढ़'),
+    (0x95E, 'M', 'फ़'),
+    (0x95F, 'M', 'य़'),
     (0x960, 'V'),
     (0x984, 'X'),
     (0x985, 'V'),
@@ -1108,10 +1109,10 @@ def _seg_10():
     (0x9CF, 'X'),
     (0x9D7, 'V'),
     (0x9D8, 'X'),
-    (0x9DC, 'M', u'ড়'),
-    (0x9DD, 'M', u'ঢ়'),
+    (0x9DC, 'M', 'ড়'),
+    (0x9DD, 'M', 'ঢ়'),
     (0x9DE, 'X'),
-    (0x9DF, 'M', u'য়'),
+    (0x9DF, 'M', 'য়'),
     (0x9E0, 'V'),
     (0x9E4, 'X'),
     (0x9E6, 'V'),
@@ -1127,10 +1128,10 @@ def _seg_10():
     (0xA2A, 'V'),
     (0xA31, 'X'),
     (0xA32, 'V'),
-    (0xA33, 'M', u'ਲ਼'),
+    (0xA33, 'M', 'ਲ਼'),
     (0xA34, 'X'),
     (0xA35, 'V'),
-    (0xA36, 'M', u'ਸ਼'),
+    (0xA36, 'M', 'ਸ਼'),
     (0xA37, 'X'),
     (0xA38, 'V'),
     (0xA3A, 'X'),
@@ -1144,16 +1145,16 @@ def _seg_10():
     (0xA4E, 'X'),
     (0xA51, 'V'),
     (0xA52, 'X'),
-    (0xA59, 'M', u'ਖ਼'),
-    (0xA5A, 'M', u'ਗ਼'),
-    (0xA5B, 'M', u'ਜ਼'),
+    (0xA59, 'M', 'ਖ਼'),
+    (0xA5A, 'M', 'ਗ਼'),
+    (0xA5B, 'M', 'ਜ਼'),
+    (0xA5C, 'V'),
+    (0xA5D, 'X'),
     ]
 
-def _seg_11():
+def _seg_11() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xA5C, 'V'),
-    (0xA5D, 'X'),
-    (0xA5E, 'M', u'ਫ਼'),
+    (0xA5E, 'M', 'ਫ਼'),
     (0xA5F, 'X'),
     (0xA66, 'V'),
     (0xA77, 'X'),
@@ -1207,8 +1208,8 @@ def _seg_11():
     (0xB4E, 'X'),
     (0xB55, 'V'),
     (0xB58, 'X'),
-    (0xB5C, 'M', u'ଡ଼'),
-    (0xB5D, 'M', u'ଢ଼'),
+    (0xB5C, 'M', 'ଡ଼'),
+    (0xB5D, 'M', 'ଢ଼'),
     (0xB5E, 'X'),
     (0xB5F, 'V'),
     (0xB64, 'X'),
@@ -1251,14 +1252,14 @@ def _seg_11():
     (0xC0E, 'V'),
     (0xC11, 'X'),
     (0xC12, 'V'),
+    (0xC29, 'X'),
+    (0xC2A, 'V'),
     ]
 
-def _seg_12():
+def _seg_12() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xC29, 'X'),
-    (0xC2A, 'V'),
     (0xC3A, 'X'),
-    (0xC3D, 'V'),
+    (0xC3C, 'V'),
     (0xC45, 'X'),
     (0xC46, 'V'),
     (0xC49, 'X'),
@@ -1268,6 +1269,8 @@ def _seg_12():
     (0xC57, 'X'),
     (0xC58, 'V'),
     (0xC5B, 'X'),
+    (0xC5D, 'V'),
+    (0xC5E, 'X'),
     (0xC60, 'V'),
     (0xC64, 'X'),
     (0xC66, 'V'),
@@ -1290,7 +1293,7 @@ def _seg_12():
     (0xCCE, 'X'),
     (0xCD5, 'V'),
     (0xCD7, 'X'),
-    (0xCDE, 'V'),
+    (0xCDD, 'V'),
     (0xCDF, 'X'),
     (0xCE0, 'V'),
     (0xCE4, 'X'),
@@ -1337,7 +1340,7 @@ def _seg_12():
     (0xDF2, 'V'),
     (0xDF5, 'X'),
     (0xE01, 'V'),
-    (0xE33, 'M', u'ํา'),
+    (0xE33, 'M', 'ํา'),
     (0xE34, 'V'),
     (0xE3B, 'X'),
     (0xE3F, 'V'),
@@ -1353,11 +1356,11 @@ def _seg_12():
     (0xEA5, 'V'),
     (0xEA6, 'X'),
     (0xEA7, 'V'),
-    (0xEB3, 'M', u'ໍາ'),
+    (0xEB3, 'M', 'ໍາ'),
     (0xEB4, 'V'),
     ]
 
-def _seg_13():
+def _seg_13() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0xEBE, 'X'),
     (0xEC0, 'V'),
@@ -1368,52 +1371,52 @@ def _seg_13():
     (0xECE, 'X'),
     (0xED0, 'V'),
     (0xEDA, 'X'),
-    (0xEDC, 'M', u'ຫນ'),
-    (0xEDD, 'M', u'ຫມ'),
+    (0xEDC, 'M', 'ຫນ'),
+    (0xEDD, 'M', 'ຫມ'),
     (0xEDE, 'V'),
     (0xEE0, 'X'),
     (0xF00, 'V'),
-    (0xF0C, 'M', u'་'),
+    (0xF0C, 'M', '་'),
     (0xF0D, 'V'),
-    (0xF43, 'M', u'གྷ'),
+    (0xF43, 'M', 'གྷ'),
     (0xF44, 'V'),
     (0xF48, 'X'),
     (0xF49, 'V'),
-    (0xF4D, 'M', u'ཌྷ'),
+    (0xF4D, 'M', 'ཌྷ'),
     (0xF4E, 'V'),
-    (0xF52, 'M', u'དྷ'),
+    (0xF52, 'M', 'དྷ'),
     (0xF53, 'V'),
-    (0xF57, 'M', u'བྷ'),
+    (0xF57, 'M', 'བྷ'),
     (0xF58, 'V'),
-    (0xF5C, 'M', u'ཛྷ'),
+    (0xF5C, 'M', 'ཛྷ'),
     (0xF5D, 'V'),
-    (0xF69, 'M', u'ཀྵ'),
+    (0xF69, 'M', 'ཀྵ'),
     (0xF6A, 'V'),
     (0xF6D, 'X'),
     (0xF71, 'V'),
-    (0xF73, 'M', u'ཱི'),
+    (0xF73, 'M', 'ཱི'),
     (0xF74, 'V'),
-    (0xF75, 'M', u'ཱུ'),
-    (0xF76, 'M', u'ྲྀ'),
-    (0xF77, 'M', u'ྲཱྀ'),
-    (0xF78, 'M', u'ླྀ'),
-    (0xF79, 'M', u'ླཱྀ'),
+    (0xF75, 'M', 'ཱུ'),
+    (0xF76, 'M', 'ྲྀ'),
+    (0xF77, 'M', 'ྲཱྀ'),
+    (0xF78, 'M', 'ླྀ'),
+    (0xF79, 'M', 'ླཱྀ'),
     (0xF7A, 'V'),
-    (0xF81, 'M', u'ཱྀ'),
+    (0xF81, 'M', 'ཱྀ'),
     (0xF82, 'V'),
-    (0xF93, 'M', u'ྒྷ'),
+    (0xF93, 'M', 'ྒྷ'),
     (0xF94, 'V'),
     (0xF98, 'X'),
     (0xF99, 'V'),
-    (0xF9D, 'M', u'ྜྷ'),
+    (0xF9D, 'M', 'ྜྷ'),
     (0xF9E, 'V'),
-    (0xFA2, 'M', u'ྡྷ'),
+    (0xFA2, 'M', 'ྡྷ'),
     (0xFA3, 'V'),
-    (0xFA7, 'M', u'ྦྷ'),
+    (0xFA7, 'M', 'ྦྷ'),
     (0xFA8, 'V'),
-    (0xFAC, 'M', u'ྫྷ'),
+    (0xFAC, 'M', 'ྫྷ'),
     (0xFAD, 'V'),
-    (0xFB9, 'M', u'ྐྵ'),
+    (0xFB9, 'M', 'ྐྵ'),
     (0xFBA, 'V'),
     (0xFBD, 'X'),
     (0xFBE, 'V'),
@@ -1422,12 +1425,12 @@ def _seg_13():
     (0xFDB, 'X'),
     (0x1000, 'V'),
     (0x10A0, 'X'),
-    (0x10C7, 'M', u'ⴧ'),
+    (0x10C7, 'M', 'ⴧ'),
     (0x10C8, 'X'),
-    (0x10CD, 'M', u'ⴭ'),
+    (0x10CD, 'M', 'ⴭ'),
     (0x10CE, 'X'),
     (0x10D0, 'V'),
-    (0x10FC, 'M', u'ნ'),
+    (0x10FC, 'M', 'ნ'),
     (0x10FD, 'V'),
     (0x115F, 'X'),
     (0x1161, 'V'),
@@ -1461,7 +1464,7 @@ def _seg_13():
     (0x1312, 'V'),
     ]
 
-def _seg_14():
+def _seg_14() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
     (0x1316, 'X'),
     (0x1318, 'V'),
@@ -1472,12 +1475,12 @@ def _seg_14():
     (0x139A, 'X'),
     (0x13A0, 'V'),
     (0x13F6, 'X'),
-    (0x13F8, 'M', u'Ᏸ'),
-    (0x13F9, 'M', u'Ᏹ'),
-    (0x13FA, 'M', u'Ᏺ'),
-    (0x13FB, 'M', u'Ᏻ'),
-    (0x13FC, 'M', u'Ᏼ'),
-    (0x13FD, 'M', u'Ᏽ'),
+    (0x13F8, 'M', 'Ᏸ'),
+    (0x13F9, 'M', 'Ᏹ'),
+    (0x13FA, 'M', 'Ᏺ'),
+    (0x13FB, 'M', 'Ᏻ'),
+    (0x13FC, 'M', 'Ᏼ'),
+    (0x13FD, 'M', 'Ᏽ'),
     (0x13FE, 'X'),
     (0x1400, 'V'),
     (0x1680, 'X'),
@@ -1486,10 +1489,8 @@ def _seg_14():
     (0x16A0, 'V'),
     (0x16F9, 'X'),
     (0x1700, 'V'),
-    (0x170D, 'X'),
-    (0x170E, 'V'),
-    (0x1715, 'X'),
-    (0x1720, 'V'),
+    (0x1716, 'X'),
+    (0x171F, 'V'),
     (0x1737, 'X'),
     (0x1740, 'V'),
     (0x1754, 'X'),
@@ -1512,6 +1513,7 @@ def _seg_14():
     (0x1807, 'V'),
     (0x180B, 'I'),
     (0x180E, 'X'),
+    (0x180F, 'I'),
     (0x1810, 'V'),
     (0x181A, 'X'),
     (0x1820, 'V'),
@@ -1551,11 +1553,11 @@ def _seg_14():
     (0x1AA0, 'V'),
     (0x1AAE, 'X'),
     (0x1AB0, 'V'),
-    (0x1AC1, 'X'),
+    (0x1ACF, 'X'),
     (0x1B00, 'V'),
-    (0x1B4C, 'X'),
+    (0x1B4D, 'X'),
     (0x1B50, 'V'),
-    (0x1B7D, 'X'),
+    (0x1B7F, 'X'),
     (0x1B80, 'V'),
     (0x1BF4, 'X'),
     (0x1BFC, 'V'),
@@ -1563,1196 +1565,1193 @@ def _seg_14():
     (0x1C3B, 'V'),
     (0x1C4A, 'X'),
     (0x1C4D, 'V'),
+    (0x1C80, 'M', 'в'),
     ]
 
-def _seg_15():
+def _seg_15() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1C80, 'M', u'в'),
-    (0x1C81, 'M', u'д'),
-    (0x1C82, 'M', u'о'),
-    (0x1C83, 'M', u'с'),
-    (0x1C84, 'M', u'т'),
-    (0x1C86, 'M', u'ъ'),
-    (0x1C87, 'M', u'ѣ'),
-    (0x1C88, 'M', u'ꙋ'),
+    (0x1C81, 'M', 'д'),
+    (0x1C82, 'M', 'о'),
+    (0x1C83, 'M', 'с'),
+    (0x1C84, 'M', 'т'),
+    (0x1C86, 'M', 'ъ'),
+    (0x1C87, 'M', 'ѣ'),
+    (0x1C88, 'M', 'ꙋ'),
     (0x1C89, 'X'),
-    (0x1C90, 'M', u'ა'),
-    (0x1C91, 'M', u'ბ'),
-    (0x1C92, 'M', u'გ'),
-    (0x1C93, 'M', u'დ'),
-    (0x1C94, 'M', u'ე'),
-    (0x1C95, 'M', u'ვ'),
-    (0x1C96, 'M', u'ზ'),
-    (0x1C97, 'M', u'თ'),
-    (0x1C98, 'M', u'ი'),
-    (0x1C99, 'M', u'კ'),
-    (0x1C9A, 'M', u'ლ'),
-    (0x1C9B, 'M', u'მ'),
-    (0x1C9C, 'M', u'ნ'),
-    (0x1C9D, 'M', u'ო'),
-    (0x1C9E, 'M', u'პ'),
-    (0x1C9F, 'M', u'ჟ'),
-    (0x1CA0, 'M', u'რ'),
-    (0x1CA1, 'M', u'ს'),
-    (0x1CA2, 'M', u'ტ'),
-    (0x1CA3, 'M', u'უ'),
-    (0x1CA4, 'M', u'ფ'),
-    (0x1CA5, 'M', u'ქ'),
-    (0x1CA6, 'M', u'ღ'),
-    (0x1CA7, 'M', u'ყ'),
-    (0x1CA8, 'M', u'შ'),
-    (0x1CA9, 'M', u'ჩ'),
-    (0x1CAA, 'M', u'ც'),
-    (0x1CAB, 'M', u'ძ'),
-    (0x1CAC, 'M', u'წ'),
-    (0x1CAD, 'M', u'ჭ'),
-    (0x1CAE, 'M', u'ხ'),
-    (0x1CAF, 'M', u'ჯ'),
-    (0x1CB0, 'M', u'ჰ'),
-    (0x1CB1, 'M', u'ჱ'),
-    (0x1CB2, 'M', u'ჲ'),
-    (0x1CB3, 'M', u'ჳ'),
-    (0x1CB4, 'M', u'ჴ'),
-    (0x1CB5, 'M', u'ჵ'),
-    (0x1CB6, 'M', u'ჶ'),
-    (0x1CB7, 'M', u'ჷ'),
-    (0x1CB8, 'M', u'ჸ'),
-    (0x1CB9, 'M', u'ჹ'),
-    (0x1CBA, 'M', u'ჺ'),
+    (0x1C90, 'M', 'ა'),
+    (0x1C91, 'M', 'ბ'),
+    (0x1C92, 'M', 'გ'),
+    (0x1C93, 'M', 'დ'),
+    (0x1C94, 'M', 'ე'),
+    (0x1C95, 'M', 'ვ'),
+    (0x1C96, 'M', 'ზ'),
+    (0x1C97, 'M', 'თ'),
+    (0x1C98, 'M', 'ი'),
+    (0x1C99, 'M', 'კ'),
+    (0x1C9A, 'M', 'ლ'),
+    (0x1C9B, 'M', 'მ'),
+    (0x1C9C, 'M', 'ნ'),
+    (0x1C9D, 'M', 'ო'),
+    (0x1C9E, 'M', 'პ'),
+    (0x1C9F, 'M', 'ჟ'),
+    (0x1CA0, 'M', 'რ'),
+    (0x1CA1, 'M', 'ს'),
+    (0x1CA2, 'M', 'ტ'),
+    (0x1CA3, 'M', 'უ'),
+    (0x1CA4, 'M', 'ფ'),
+    (0x1CA5, 'M', 'ქ'),
+    (0x1CA6, 'M', 'ღ'),
+    (0x1CA7, 'M', 'ყ'),
+    (0x1CA8, 'M', 'შ'),
+    (0x1CA9, 'M', 'ჩ'),
+    (0x1CAA, 'M', 'ც'),
+    (0x1CAB, 'M', 'ძ'),
+    (0x1CAC, 'M', 'წ'),
+    (0x1CAD, 'M', 'ჭ'),
+    (0x1CAE, 'M', 'ხ'),
+    (0x1CAF, 'M', 'ჯ'),
+    (0x1CB0, 'M', 'ჰ'),
+    (0x1CB1, 'M', 'ჱ'),
+    (0x1CB2, 'M', 'ჲ'),
+    (0x1CB3, 'M', 'ჳ'),
+    (0x1CB4, 'M', 'ჴ'),
+    (0x1CB5, 'M', 'ჵ'),
+    (0x1CB6, 'M', 'ჶ'),
+    (0x1CB7, 'M', 'ჷ'),
+    (0x1CB8, 'M', 'ჸ'),
+    (0x1CB9, 'M', 'ჹ'),
+    (0x1CBA, 'M', 'ჺ'),
     (0x1CBB, 'X'),
-    (0x1CBD, 'M', u'ჽ'),
-    (0x1CBE, 'M', u'ჾ'),
-    (0x1CBF, 'M', u'ჿ'),
+    (0x1CBD, 'M', 'ჽ'),
+    (0x1CBE, 'M', 'ჾ'),
+    (0x1CBF, 'M', 'ჿ'),
     (0x1CC0, 'V'),
     (0x1CC8, 'X'),
     (0x1CD0, 'V'),
     (0x1CFB, 'X'),
     (0x1D00, 'V'),
-    (0x1D2C, 'M', u'a'),
-    (0x1D2D, 'M', u'æ'),
-    (0x1D2E, 'M', u'b'),
+    (0x1D2C, 'M', 'a'),
+    (0x1D2D, 'M', 'æ'),
+    (0x1D2E, 'M', 'b'),
     (0x1D2F, 'V'),
-    (0x1D30, 'M', u'd'),
-    (0x1D31, 'M', u'e'),
-    (0x1D32, 'M', u'ǝ'),
-    (0x1D33, 'M', u'g'),
-    (0x1D34, 'M', u'h'),
-    (0x1D35, 'M', u'i'),
-    (0x1D36, 'M', u'j'),
-    (0x1D37, 'M', u'k'),
-    (0x1D38, 'M', u'l'),
-    (0x1D39, 'M', u'm'),
-    (0x1D3A, 'M', u'n'),
+    (0x1D30, 'M', 'd'),
+    (0x1D31, 'M', 'e'),
+    (0x1D32, 'M', 'ǝ'),
+    (0x1D33, 'M', 'g'),
+    (0x1D34, 'M', 'h'),
+    (0x1D35, 'M', 'i'),
+    (0x1D36, 'M', 'j'),
+    (0x1D37, 'M', 'k'),
+    (0x1D38, 'M', 'l'),
+    (0x1D39, 'M', 'm'),
+    (0x1D3A, 'M', 'n'),
     (0x1D3B, 'V'),
-    (0x1D3C, 'M', u'o'),
-    (0x1D3D, 'M', u'ȣ'),
-    (0x1D3E, 'M', u'p'),
-    (0x1D3F, 'M', u'r'),
-    (0x1D40, 'M', u't'),
-    (0x1D41, 'M', u'u'),
-    (0x1D42, 'M', u'w'),
-    (0x1D43, 'M', u'a'),
-    (0x1D44, 'M', u'ɐ'),
-    (0x1D45, 'M', u'ɑ'),
-    (0x1D46, 'M', u'ᴂ'),
-    (0x1D47, 'M', u'b'),
-    (0x1D48, 'M', u'd'),
-    (0x1D49, 'M', u'e'),
-    (0x1D4A, 'M', u'ə'),
-    (0x1D4B, 'M', u'ɛ'),
-    (0x1D4C, 'M', u'ɜ'),
-    (0x1D4D, 'M', u'g'),
+    (0x1D3C, 'M', 'o'),
+    (0x1D3D, 'M', 'ȣ'),
+    (0x1D3E, 'M', 'p'),
+    (0x1D3F, 'M', 'r'),
+    (0x1D40, 'M', 't'),
+    (0x1D41, 'M', 'u'),
+    (0x1D42, 'M', 'w'),
+    (0x1D43, 'M', 'a'),
+    (0x1D44, 'M', 'ɐ'),
+    (0x1D45, 'M', 'ɑ'),
+    (0x1D46, 'M', 'ᴂ'),
+    (0x1D47, 'M', 'b'),
+    (0x1D48, 'M', 'd'),
+    (0x1D49, 'M', 'e'),
+    (0x1D4A, 'M', 'ə'),
+    (0x1D4B, 'M', 'ɛ'),
+    (0x1D4C, 'M', 'ɜ'),
+    (0x1D4D, 'M', 'g'),
     (0x1D4E, 'V'),
-    (0x1D4F, 'M', u'k'),
-    (0x1D50, 'M', u'm'),
-    (0x1D51, 'M', u'ŋ'),
-    (0x1D52, 'M', u'o'),
+    (0x1D4F, 'M', 'k'),
+    (0x1D50, 'M', 'm'),
+    (0x1D51, 'M', 'ŋ'),
+    (0x1D52, 'M', 'o'),
+    (0x1D53, 'M', 'ɔ'),
     ]
 
-def _seg_16():
+def _seg_16() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D53, 'M', u'ɔ'),
-    (0x1D54, 'M', u'ᴖ'),
-    (0x1D55, 'M', u'ᴗ'),
-    (0x1D56, 'M', u'p'),
-    (0x1D57, 'M', u't'),
-    (0x1D58, 'M', u'u'),
-    (0x1D59, 'M', u'ᴝ'),
-    (0x1D5A, 'M', u'ɯ'),
-    (0x1D5B, 'M', u'v'),
-    (0x1D5C, 'M', u'ᴥ'),
-    (0x1D5D, 'M', u'β'),
-    (0x1D5E, 'M', u'γ'),
-    (0x1D5F, 'M', u'δ'),
-    (0x1D60, 'M', u'φ'),
-    (0x1D61, 'M', u'χ'),
-    (0x1D62, 'M', u'i'),
-    (0x1D63, 'M', u'r'),
-    (0x1D64, 'M', u'u'),
-    (0x1D65, 'M', u'v'),
-    (0x1D66, 'M', u'β'),
-    (0x1D67, 'M', u'γ'),
-    (0x1D68, 'M', u'ρ'),
-    (0x1D69, 'M', u'φ'),
-    (0x1D6A, 'M', u'χ'),
+    (0x1D54, 'M', 'ᴖ'),
+    (0x1D55, 'M', 'ᴗ'),
+    (0x1D56, 'M', 'p'),
+    (0x1D57, 'M', 't'),
+    (0x1D58, 'M', 'u'),
+    (0x1D59, 'M', 'ᴝ'),
+    (0x1D5A, 'M', 'ɯ'),
+    (0x1D5B, 'M', 'v'),
+    (0x1D5C, 'M', 'ᴥ'),
+    (0x1D5D, 'M', 'β'),
+    (0x1D5E, 'M', 'γ'),
+    (0x1D5F, 'M', 'δ'),
+    (0x1D60, 'M', 'φ'),
+    (0x1D61, 'M', 'χ'),
+    (0x1D62, 'M', 'i'),
+    (0x1D63, 'M', 'r'),
+    (0x1D64, 'M', 'u'),
+    (0x1D65, 'M', 'v'),
+    (0x1D66, 'M', 'β'),
+    (0x1D67, 'M', 'γ'),
+    (0x1D68, 'M', 'ρ'),
+    (0x1D69, 'M', 'φ'),
+    (0x1D6A, 'M', 'χ'),
     (0x1D6B, 'V'),
-    (0x1D78, 'M', u'н'),
+    (0x1D78, 'M', 'н'),
     (0x1D79, 'V'),
-    (0x1D9B, 'M', u'ɒ'),
-    (0x1D9C, 'M', u'c'),
-    (0x1D9D, 'M', u'ɕ'),
-    (0x1D9E, 'M', u'ð'),
-    (0x1D9F, 'M', u'ɜ'),
-    (0x1DA0, 'M', u'f'),
-    (0x1DA1, 'M', u'ɟ'),
-    (0x1DA2, 'M', u'ɡ'),
-    (0x1DA3, 'M', u'ɥ'),
-    (0x1DA4, 'M', u'ɨ'),
-    (0x1DA5, 'M', u'ɩ'),
-    (0x1DA6, 'M', u'ɪ'),
-    (0x1DA7, 'M', u'ᵻ'),
-    (0x1DA8, 'M', u'ʝ'),
-    (0x1DA9, 'M', u'ɭ'),
-    (0x1DAA, 'M', u'ᶅ'),
-    (0x1DAB, 'M', u'ʟ'),
-    (0x1DAC, 'M', u'ɱ'),
-    (0x1DAD, 'M', u'ɰ'),
-    (0x1DAE, 'M', u'ɲ'),
-    (0x1DAF, 'M', u'ɳ'),
-    (0x1DB0, 'M', u'ɴ'),
-    (0x1DB1, 'M', u'ɵ'),
-    (0x1DB2, 'M', u'ɸ'),
-    (0x1DB3, 'M', u'ʂ'),
-    (0x1DB4, 'M', u'ʃ'),
-    (0x1DB5, 'M', u'ƫ'),
-    (0x1DB6, 'M', u'ʉ'),
-    (0x1DB7, 'M', u'ʊ'),
-    (0x1DB8, 'M', u'ᴜ'),
-    (0x1DB9, 'M', u'ʋ'),
-    (0x1DBA, 'M', u'ʌ'),
-    (0x1DBB, 'M', u'z'),
-    (0x1DBC, 'M', u'ʐ'),
-    (0x1DBD, 'M', u'ʑ'),
-    (0x1DBE, 'M', u'ʒ'),
-    (0x1DBF, 'M', u'θ'),
+    (0x1D9B, 'M', 'ɒ'),
+    (0x1D9C, 'M', 'c'),
+    (0x1D9D, 'M', 'ɕ'),
+    (0x1D9E, 'M', 'ð'),
+    (0x1D9F, 'M', 'ɜ'),
+    (0x1DA0, 'M', 'f'),
+    (0x1DA1, 'M', 'ɟ'),
+    (0x1DA2, 'M', 'ɡ'),
+    (0x1DA3, 'M', 'ɥ'),
+    (0x1DA4, 'M', 'ɨ'),
+    (0x1DA5, 'M', 'ɩ'),
+    (0x1DA6, 'M', 'ɪ'),
+    (0x1DA7, 'M', 'ᵻ'),
+    (0x1DA8, 'M', 'ʝ'),
+    (0x1DA9, 'M', 'ɭ'),
+    (0x1DAA, 'M', 'ᶅ'),
+    (0x1DAB, 'M', 'ʟ'),
+    (0x1DAC, 'M', 'ɱ'),
+    (0x1DAD, 'M', 'ɰ'),
+    (0x1DAE, 'M', 'ɲ'),
+    (0x1DAF, 'M', 'ɳ'),
+    (0x1DB0, 'M', 'ɴ'),
+    (0x1DB1, 'M', 'ɵ'),
+    (0x1DB2, 'M', 'ɸ'),
+    (0x1DB3, 'M', 'ʂ'),
+    (0x1DB4, 'M', 'ʃ'),
+    (0x1DB5, 'M', 'ƫ'),
+    (0x1DB6, 'M', 'ʉ'),
+    (0x1DB7, 'M', 'ʊ'),
+    (0x1DB8, 'M', 'ᴜ'),
+    (0x1DB9, 'M', 'ʋ'),
+    (0x1DBA, 'M', 'ʌ'),
+    (0x1DBB, 'M', 'z'),
+    (0x1DBC, 'M', 'ʐ'),
+    (0x1DBD, 'M', 'ʑ'),
+    (0x1DBE, 'M', 'ʒ'),
+    (0x1DBF, 'M', 'θ'),
     (0x1DC0, 'V'),
-    (0x1DFA, 'X'),
-    (0x1DFB, 'V'),
-    (0x1E00, 'M', u'ḁ'),
+    (0x1E00, 'M', 'ḁ'),
     (0x1E01, 'V'),
-    (0x1E02, 'M', u'ḃ'),
+    (0x1E02, 'M', 'ḃ'),
     (0x1E03, 'V'),
-    (0x1E04, 'M', u'ḅ'),
+    (0x1E04, 'M', 'ḅ'),
     (0x1E05, 'V'),
-    (0x1E06, 'M', u'ḇ'),
+    (0x1E06, 'M', 'ḇ'),
     (0x1E07, 'V'),
-    (0x1E08, 'M', u'ḉ'),
+    (0x1E08, 'M', 'ḉ'),
     (0x1E09, 'V'),
-    (0x1E0A, 'M', u'ḋ'),
+    (0x1E0A, 'M', 'ḋ'),
     (0x1E0B, 'V'),
-    (0x1E0C, 'M', u'ḍ'),
+    (0x1E0C, 'M', 'ḍ'),
     (0x1E0D, 'V'),
-    (0x1E0E, 'M', u'ḏ'),
+    (0x1E0E, 'M', 'ḏ'),
     (0x1E0F, 'V'),
-    (0x1E10, 'M', u'ḑ'),
+    (0x1E10, 'M', 'ḑ'),
     (0x1E11, 'V'),
-    (0x1E12, 'M', u'ḓ'),
+    (0x1E12, 'M', 'ḓ'),
     (0x1E13, 'V'),
-    (0x1E14, 'M', u'ḕ'),
+    (0x1E14, 'M', 'ḕ'),
     (0x1E15, 'V'),
-    (0x1E16, 'M', u'ḗ'),
+    (0x1E16, 'M', 'ḗ'),
     (0x1E17, 'V'),
-    (0x1E18, 'M', u'ḙ'),
+    (0x1E18, 'M', 'ḙ'),
     (0x1E19, 'V'),
-    (0x1E1A, 'M', u'ḛ'),
+    (0x1E1A, 'M', 'ḛ'),
     (0x1E1B, 'V'),
-    (0x1E1C, 'M', u'ḝ'),
+    (0x1E1C, 'M', 'ḝ'),
     (0x1E1D, 'V'),
-    (0x1E1E, 'M', u'ḟ'),
+    (0x1E1E, 'M', 'ḟ'),
     (0x1E1F, 'V'),
-    (0x1E20, 'M', u'ḡ'),
+    (0x1E20, 'M', 'ḡ'),
+    (0x1E21, 'V'),
+    (0x1E22, 'M', 'ḣ'),
+    (0x1E23, 'V'),
     ]
 
-def _seg_17():
+def _seg_17() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1E21, 'V'),
-    (0x1E22, 'M', u'ḣ'),
-    (0x1E23, 'V'),
-    (0x1E24, 'M', u'ḥ'),
+    (0x1E24, 'M', 'ḥ'),
     (0x1E25, 'V'),
-    (0x1E26, 'M', u'ḧ'),
+    (0x1E26, 'M', 'ḧ'),
     (0x1E27, 'V'),
-    (0x1E28, 'M', u'ḩ'),
+    (0x1E28, 'M', 'ḩ'),
     (0x1E29, 'V'),
-    (0x1E2A, 'M', u'ḫ'),
+    (0x1E2A, 'M', 'ḫ'),
     (0x1E2B, 'V'),
-    (0x1E2C, 'M', u'ḭ'),
+    (0x1E2C, 'M', 'ḭ'),
     (0x1E2D, 'V'),
-    (0x1E2E, 'M', u'ḯ'),
+    (0x1E2E, 'M', 'ḯ'),
     (0x1E2F, 'V'),
-    (0x1E30, 'M', u'ḱ'),
+    (0x1E30, 'M', 'ḱ'),
     (0x1E31, 'V'),
-    (0x1E32, 'M', u'ḳ'),
+    (0x1E32, 'M', 'ḳ'),
     (0x1E33, 'V'),
-    (0x1E34, 'M', u'ḵ'),
+    (0x1E34, 'M', 'ḵ'),
     (0x1E35, 'V'),
-    (0x1E36, 'M', u'ḷ'),
+    (0x1E36, 'M', 'ḷ'),
     (0x1E37, 'V'),
-    (0x1E38, 'M', u'ḹ'),
+    (0x1E38, 'M', 'ḹ'),
     (0x1E39, 'V'),
-    (0x1E3A, 'M', u'ḻ'),
+    (0x1E3A, 'M', 'ḻ'),
     (0x1E3B, 'V'),
-    (0x1E3C, 'M', u'ḽ'),
+    (0x1E3C, 'M', 'ḽ'),
     (0x1E3D, 'V'),
-    (0x1E3E, 'M', u'ḿ'),
+    (0x1E3E, 'M', 'ḿ'),
     (0x1E3F, 'V'),
-    (0x1E40, 'M', u'ṁ'),
+    (0x1E40, 'M', 'ṁ'),
     (0x1E41, 'V'),
-    (0x1E42, 'M', u'ṃ'),
+    (0x1E42, 'M', 'ṃ'),
     (0x1E43, 'V'),
-    (0x1E44, 'M', u'ṅ'),
+    (0x1E44, 'M', 'ṅ'),
     (0x1E45, 'V'),
-    (0x1E46, 'M', u'ṇ'),
+    (0x1E46, 'M', 'ṇ'),
     (0x1E47, 'V'),
-    (0x1E48, 'M', u'ṉ'),
+    (0x1E48, 'M', 'ṉ'),
     (0x1E49, 'V'),
-    (0x1E4A, 'M', u'ṋ'),
+    (0x1E4A, 'M', 'ṋ'),
     (0x1E4B, 'V'),
-    (0x1E4C, 'M', u'ṍ'),
+    (0x1E4C, 'M', 'ṍ'),
     (0x1E4D, 'V'),
-    (0x1E4E, 'M', u'ṏ'),
+    (0x1E4E, 'M', 'ṏ'),
     (0x1E4F, 'V'),
-    (0x1E50, 'M', u'ṑ'),
+    (0x1E50, 'M', 'ṑ'),
     (0x1E51, 'V'),
-    (0x1E52, 'M', u'ṓ'),
+    (0x1E52, 'M', 'ṓ'),
     (0x1E53, 'V'),
-    (0x1E54, 'M', u'ṕ'),
+    (0x1E54, 'M', 'ṕ'),
     (0x1E55, 'V'),
-    (0x1E56, 'M', u'ṗ'),
+    (0x1E56, 'M', 'ṗ'),
     (0x1E57, 'V'),
-    (0x1E58, 'M', u'ṙ'),
+    (0x1E58, 'M', 'ṙ'),
     (0x1E59, 'V'),
-    (0x1E5A, 'M', u'ṛ'),
+    (0x1E5A, 'M', 'ṛ'),
     (0x1E5B, 'V'),
-    (0x1E5C, 'M', u'ṝ'),
+    (0x1E5C, 'M', 'ṝ'),
     (0x1E5D, 'V'),
-    (0x1E5E, 'M', u'ṟ'),
+    (0x1E5E, 'M', 'ṟ'),
     (0x1E5F, 'V'),
-    (0x1E60, 'M', u'ṡ'),
+    (0x1E60, 'M', 'ṡ'),
     (0x1E61, 'V'),
-    (0x1E62, 'M', u'ṣ'),
+    (0x1E62, 'M', 'ṣ'),
     (0x1E63, 'V'),
-    (0x1E64, 'M', u'ṥ'),
+    (0x1E64, 'M', 'ṥ'),
     (0x1E65, 'V'),
-    (0x1E66, 'M', u'ṧ'),
+    (0x1E66, 'M', 'ṧ'),
     (0x1E67, 'V'),
-    (0x1E68, 'M', u'ṩ'),
+    (0x1E68, 'M', 'ṩ'),
     (0x1E69, 'V'),
-    (0x1E6A, 'M', u'ṫ'),
+    (0x1E6A, 'M', 'ṫ'),
     (0x1E6B, 'V'),
-    (0x1E6C, 'M', u'ṭ'),
+    (0x1E6C, 'M', 'ṭ'),
     (0x1E6D, 'V'),
-    (0x1E6E, 'M', u'ṯ'),
+    (0x1E6E, 'M', 'ṯ'),
     (0x1E6F, 'V'),
-    (0x1E70, 'M', u'ṱ'),
+    (0x1E70, 'M', 'ṱ'),
     (0x1E71, 'V'),
-    (0x1E72, 'M', u'ṳ'),
+    (0x1E72, 'M', 'ṳ'),
     (0x1E73, 'V'),
-    (0x1E74, 'M', u'ṵ'),
+    (0x1E74, 'M', 'ṵ'),
     (0x1E75, 'V'),
-    (0x1E76, 'M', u'ṷ'),
+    (0x1E76, 'M', 'ṷ'),
     (0x1E77, 'V'),
-    (0x1E78, 'M', u'ṹ'),
+    (0x1E78, 'M', 'ṹ'),
     (0x1E79, 'V'),
-    (0x1E7A, 'M', u'ṻ'),
+    (0x1E7A, 'M', 'ṻ'),
     (0x1E7B, 'V'),
-    (0x1E7C, 'M', u'ṽ'),
+    (0x1E7C, 'M', 'ṽ'),
     (0x1E7D, 'V'),
-    (0x1E7E, 'M', u'ṿ'),
+    (0x1E7E, 'M', 'ṿ'),
     (0x1E7F, 'V'),
-    (0x1E80, 'M', u'ẁ'),
+    (0x1E80, 'M', 'ẁ'),
     (0x1E81, 'V'),
-    (0x1E82, 'M', u'ẃ'),
+    (0x1E82, 'M', 'ẃ'),
     (0x1E83, 'V'),
-    (0x1E84, 'M', u'ẅ'),
+    (0x1E84, 'M', 'ẅ'),
+    (0x1E85, 'V'),
+    (0x1E86, 'M', 'ẇ'),
+    (0x1E87, 'V'),
     ]
 
-def _seg_18():
+def _seg_18() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1E85, 'V'),
-    (0x1E86, 'M', u'ẇ'),
-    (0x1E87, 'V'),
-    (0x1E88, 'M', u'ẉ'),
+    (0x1E88, 'M', 'ẉ'),
     (0x1E89, 'V'),
-    (0x1E8A, 'M', u'ẋ'),
+    (0x1E8A, 'M', 'ẋ'),
     (0x1E8B, 'V'),
-    (0x1E8C, 'M', u'ẍ'),
+    (0x1E8C, 'M', 'ẍ'),
     (0x1E8D, 'V'),
-    (0x1E8E, 'M', u'ẏ'),
+    (0x1E8E, 'M', 'ẏ'),
     (0x1E8F, 'V'),
-    (0x1E90, 'M', u'ẑ'),
+    (0x1E90, 'M', 'ẑ'),
     (0x1E91, 'V'),
-    (0x1E92, 'M', u'ẓ'),
+    (0x1E92, 'M', 'ẓ'),
     (0x1E93, 'V'),
-    (0x1E94, 'M', u'ẕ'),
+    (0x1E94, 'M', 'ẕ'),
     (0x1E95, 'V'),
-    (0x1E9A, 'M', u'aʾ'),
-    (0x1E9B, 'M', u'ṡ'),
+    (0x1E9A, 'M', 'aʾ'),
+    (0x1E9B, 'M', 'ṡ'),
     (0x1E9C, 'V'),
-    (0x1E9E, 'M', u'ss'),
+    (0x1E9E, 'M', 'ss'),
     (0x1E9F, 'V'),
-    (0x1EA0, 'M', u'ạ'),
+    (0x1EA0, 'M', 'ạ'),
     (0x1EA1, 'V'),
-    (0x1EA2, 'M', u'ả'),
+    (0x1EA2, 'M', 'ả'),
     (0x1EA3, 'V'),
-    (0x1EA4, 'M', u'ấ'),
+    (0x1EA4, 'M', 'ấ'),
     (0x1EA5, 'V'),
-    (0x1EA6, 'M', u'ầ'),
+    (0x1EA6, 'M', 'ầ'),
     (0x1EA7, 'V'),
-    (0x1EA8, 'M', u'ẩ'),
+    (0x1EA8, 'M', 'ẩ'),
     (0x1EA9, 'V'),
-    (0x1EAA, 'M', u'ẫ'),
+    (0x1EAA, 'M', 'ẫ'),
     (0x1EAB, 'V'),
-    (0x1EAC, 'M', u'ậ'),
+    (0x1EAC, 'M', 'ậ'),
     (0x1EAD, 'V'),
-    (0x1EAE, 'M', u'ắ'),
+    (0x1EAE, 'M', 'ắ'),
     (0x1EAF, 'V'),
-    (0x1EB0, 'M', u'ằ'),
+    (0x1EB0, 'M', 'ằ'),
     (0x1EB1, 'V'),
-    (0x1EB2, 'M', u'ẳ'),
+    (0x1EB2, 'M', 'ẳ'),
     (0x1EB3, 'V'),
-    (0x1EB4, 'M', u'ẵ'),
+    (0x1EB4, 'M', 'ẵ'),
     (0x1EB5, 'V'),
-    (0x1EB6, 'M', u'ặ'),
+    (0x1EB6, 'M', 'ặ'),
     (0x1EB7, 'V'),
-    (0x1EB8, 'M', u'ẹ'),
+    (0x1EB8, 'M', 'ẹ'),
     (0x1EB9, 'V'),
-    (0x1EBA, 'M', u'ẻ'),
+    (0x1EBA, 'M', 'ẻ'),
     (0x1EBB, 'V'),
-    (0x1EBC, 'M', u'ẽ'),
+    (0x1EBC, 'M', 'ẽ'),
     (0x1EBD, 'V'),
-    (0x1EBE, 'M', u'ế'),
+    (0x1EBE, 'M', 'ế'),
     (0x1EBF, 'V'),
-    (0x1EC0, 'M', u'ề'),
+    (0x1EC0, 'M', 'ề'),
     (0x1EC1, 'V'),
-    (0x1EC2, 'M', u'ể'),
+    (0x1EC2, 'M', 'ể'),
     (0x1EC3, 'V'),
-    (0x1EC4, 'M', u'ễ'),
+    (0x1EC4, 'M', 'ễ'),
     (0x1EC5, 'V'),
-    (0x1EC6, 'M', u'ệ'),
+    (0x1EC6, 'M', 'ệ'),
     (0x1EC7, 'V'),
-    (0x1EC8, 'M', u'ỉ'),
+    (0x1EC8, 'M', 'ỉ'),
     (0x1EC9, 'V'),
-    (0x1ECA, 'M', u'ị'),
+    (0x1ECA, 'M', 'ị'),
     (0x1ECB, 'V'),
-    (0x1ECC, 'M', u'ọ'),
+    (0x1ECC, 'M', 'ọ'),
     (0x1ECD, 'V'),
-    (0x1ECE, 'M', u'ỏ'),
+    (0x1ECE, 'M', 'ỏ'),
     (0x1ECF, 'V'),
-    (0x1ED0, 'M', u'ố'),
+    (0x1ED0, 'M', 'ố'),
     (0x1ED1, 'V'),
-    (0x1ED2, 'M', u'ồ'),
+    (0x1ED2, 'M', 'ồ'),
     (0x1ED3, 'V'),
-    (0x1ED4, 'M', u'ổ'),
+    (0x1ED4, 'M', 'ổ'),
     (0x1ED5, 'V'),
-    (0x1ED6, 'M', u'ỗ'),
+    (0x1ED6, 'M', 'ỗ'),
     (0x1ED7, 'V'),
-    (0x1ED8, 'M', u'ộ'),
+    (0x1ED8, 'M', 'ộ'),
     (0x1ED9, 'V'),
-    (0x1EDA, 'M', u'ớ'),
+    (0x1EDA, 'M', 'ớ'),
     (0x1EDB, 'V'),
-    (0x1EDC, 'M', u'ờ'),
+    (0x1EDC, 'M', 'ờ'),
     (0x1EDD, 'V'),
-    (0x1EDE, 'M', u'ở'),
+    (0x1EDE, 'M', 'ở'),
     (0x1EDF, 'V'),
-    (0x1EE0, 'M', u'ỡ'),
+    (0x1EE0, 'M', 'ỡ'),
     (0x1EE1, 'V'),
-    (0x1EE2, 'M', u'ợ'),
+    (0x1EE2, 'M', 'ợ'),
     (0x1EE3, 'V'),
-    (0x1EE4, 'M', u'ụ'),
+    (0x1EE4, 'M', 'ụ'),
     (0x1EE5, 'V'),
-    (0x1EE6, 'M', u'ủ'),
+    (0x1EE6, 'M', 'ủ'),
     (0x1EE7, 'V'),
-    (0x1EE8, 'M', u'ứ'),
+    (0x1EE8, 'M', 'ứ'),
     (0x1EE9, 'V'),
-    (0x1EEA, 'M', u'ừ'),
+    (0x1EEA, 'M', 'ừ'),
     (0x1EEB, 'V'),
-    (0x1EEC, 'M', u'ử'),
+    (0x1EEC, 'M', 'ử'),
     (0x1EED, 'V'),
+    (0x1EEE, 'M', 'ữ'),
+    (0x1EEF, 'V'),
+    (0x1EF0, 'M', 'ự'),
     ]
 
-def _seg_19():
+def _seg_19() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1EEE, 'M', u'ữ'),
-    (0x1EEF, 'V'),
-    (0x1EF0, 'M', u'ự'),
     (0x1EF1, 'V'),
-    (0x1EF2, 'M', u'ỳ'),
+    (0x1EF2, 'M', 'ỳ'),
     (0x1EF3, 'V'),
-    (0x1EF4, 'M', u'ỵ'),
+    (0x1EF4, 'M', 'ỵ'),
     (0x1EF5, 'V'),
-    (0x1EF6, 'M', u'ỷ'),
+    (0x1EF6, 'M', 'ỷ'),
     (0x1EF7, 'V'),
-    (0x1EF8, 'M', u'ỹ'),
+    (0x1EF8, 'M', 'ỹ'),
     (0x1EF9, 'V'),
-    (0x1EFA, 'M', u'ỻ'),
+    (0x1EFA, 'M', 'ỻ'),
     (0x1EFB, 'V'),
-    (0x1EFC, 'M', u'ỽ'),
+    (0x1EFC, 'M', 'ỽ'),
     (0x1EFD, 'V'),
-    (0x1EFE, 'M', u'ỿ'),
+    (0x1EFE, 'M', 'ỿ'),
     (0x1EFF, 'V'),
-    (0x1F08, 'M', u'ἀ'),
-    (0x1F09, 'M', u'ἁ'),
-    (0x1F0A, 'M', u'ἂ'),
-    (0x1F0B, 'M', u'ἃ'),
-    (0x1F0C, 'M', u'ἄ'),
-    (0x1F0D, 'M', u'ἅ'),
-    (0x1F0E, 'M', u'ἆ'),
-    (0x1F0F, 'M', u'ἇ'),
+    (0x1F08, 'M', 'ἀ'),
+    (0x1F09, 'M', 'ἁ'),
+    (0x1F0A, 'M', 'ἂ'),
+    (0x1F0B, 'M', 'ἃ'),
+    (0x1F0C, 'M', 'ἄ'),
+    (0x1F0D, 'M', 'ἅ'),
+    (0x1F0E, 'M', 'ἆ'),
+    (0x1F0F, 'M', 'ἇ'),
     (0x1F10, 'V'),
     (0x1F16, 'X'),
-    (0x1F18, 'M', u'ἐ'),
-    (0x1F19, 'M', u'ἑ'),
-    (0x1F1A, 'M', u'ἒ'),
-    (0x1F1B, 'M', u'ἓ'),
-    (0x1F1C, 'M', u'ἔ'),
-    (0x1F1D, 'M', u'ἕ'),
+    (0x1F18, 'M', 'ἐ'),
+    (0x1F19, 'M', 'ἑ'),
+    (0x1F1A, 'M', 'ἒ'),
+    (0x1F1B, 'M', 'ἓ'),
+    (0x1F1C, 'M', 'ἔ'),
+    (0x1F1D, 'M', 'ἕ'),
     (0x1F1E, 'X'),
     (0x1F20, 'V'),
-    (0x1F28, 'M', u'ἠ'),
-    (0x1F29, 'M', u'ἡ'),
-    (0x1F2A, 'M', u'ἢ'),
-    (0x1F2B, 'M', u'ἣ'),
-    (0x1F2C, 'M', u'ἤ'),
-    (0x1F2D, 'M', u'ἥ'),
-    (0x1F2E, 'M', u'ἦ'),
-    (0x1F2F, 'M', u'ἧ'),
+    (0x1F28, 'M', 'ἠ'),
+    (0x1F29, 'M', 'ἡ'),
+    (0x1F2A, 'M', 'ἢ'),
+    (0x1F2B, 'M', 'ἣ'),
+    (0x1F2C, 'M', 'ἤ'),
+    (0x1F2D, 'M', 'ἥ'),
+    (0x1F2E, 'M', 'ἦ'),
+    (0x1F2F, 'M', 'ἧ'),
     (0x1F30, 'V'),
-    (0x1F38, 'M', u'ἰ'),
-    (0x1F39, 'M', u'ἱ'),
-    (0x1F3A, 'M', u'ἲ'),
-    (0x1F3B, 'M', u'ἳ'),
-    (0x1F3C, 'M', u'ἴ'),
-    (0x1F3D, 'M', u'ἵ'),
-    (0x1F3E, 'M', u'ἶ'),
-    (0x1F3F, 'M', u'ἷ'),
+    (0x1F38, 'M', 'ἰ'),
+    (0x1F39, 'M', 'ἱ'),
+    (0x1F3A, 'M', 'ἲ'),
+    (0x1F3B, 'M', 'ἳ'),
+    (0x1F3C, 'M', 'ἴ'),
+    (0x1F3D, 'M', 'ἵ'),
+    (0x1F3E, 'M', 'ἶ'),
+    (0x1F3F, 'M', 'ἷ'),
     (0x1F40, 'V'),
     (0x1F46, 'X'),
-    (0x1F48, 'M', u'ὀ'),
-    (0x1F49, 'M', u'ὁ'),
-    (0x1F4A, 'M', u'ὂ'),
-    (0x1F4B, 'M', u'ὃ'),
-    (0x1F4C, 'M', u'ὄ'),
-    (0x1F4D, 'M', u'ὅ'),
+    (0x1F48, 'M', 'ὀ'),
+    (0x1F49, 'M', 'ὁ'),
+    (0x1F4A, 'M', 'ὂ'),
+    (0x1F4B, 'M', 'ὃ'),
+    (0x1F4C, 'M', 'ὄ'),
+    (0x1F4D, 'M', 'ὅ'),
     (0x1F4E, 'X'),
     (0x1F50, 'V'),
     (0x1F58, 'X'),
-    (0x1F59, 'M', u'ὑ'),
+    (0x1F59, 'M', 'ὑ'),
     (0x1F5A, 'X'),
-    (0x1F5B, 'M', u'ὓ'),
+    (0x1F5B, 'M', 'ὓ'),
     (0x1F5C, 'X'),
-    (0x1F5D, 'M', u'ὕ'),
+    (0x1F5D, 'M', 'ὕ'),
     (0x1F5E, 'X'),
-    (0x1F5F, 'M', u'ὗ'),
+    (0x1F5F, 'M', 'ὗ'),
     (0x1F60, 'V'),
-    (0x1F68, 'M', u'ὠ'),
-    (0x1F69, 'M', u'ὡ'),
-    (0x1F6A, 'M', u'ὢ'),
-    (0x1F6B, 'M', u'ὣ'),
-    (0x1F6C, 'M', u'ὤ'),
-    (0x1F6D, 'M', u'ὥ'),
-    (0x1F6E, 'M', u'ὦ'),
-    (0x1F6F, 'M', u'ὧ'),
+    (0x1F68, 'M', 'ὠ'),
+    (0x1F69, 'M', 'ὡ'),
+    (0x1F6A, 'M', 'ὢ'),
+    (0x1F6B, 'M', 'ὣ'),
+    (0x1F6C, 'M', 'ὤ'),
+    (0x1F6D, 'M', 'ὥ'),
+    (0x1F6E, 'M', 'ὦ'),
+    (0x1F6F, 'M', 'ὧ'),
     (0x1F70, 'V'),
-    (0x1F71, 'M', u'ά'),
+    (0x1F71, 'M', 'ά'),
     (0x1F72, 'V'),
-    (0x1F73, 'M', u'έ'),
+    (0x1F73, 'M', 'έ'),
     (0x1F74, 'V'),
-    (0x1F75, 'M', u'ή'),
+    (0x1F75, 'M', 'ή'),
     (0x1F76, 'V'),
-    (0x1F77, 'M', u'ί'),
+    (0x1F77, 'M', 'ί'),
     (0x1F78, 'V'),
-    (0x1F79, 'M', u'ό'),
+    (0x1F79, 'M', 'ό'),
     (0x1F7A, 'V'),
-    (0x1F7B, 'M', u'ύ'),
+    (0x1F7B, 'M', 'ύ'),
     (0x1F7C, 'V'),
-    (0x1F7D, 'M', u'ώ'),
+    (0x1F7D, 'M', 'ώ'),
     (0x1F7E, 'X'),
-    (0x1F80, 'M', u'ἀι'),
-    (0x1F81, 'M', u'ἁι'),
-    (0x1F82, 'M', u'ἂι'),
-    (0x1F83, 'M', u'ἃι'),
-    (0x1F84, 'M', u'ἄι'),
+    (0x1F80, 'M', 'ἀι'),
+    (0x1F81, 'M', 'ἁι'),
+    (0x1F82, 'M', 'ἂι'),
+    (0x1F83, 'M', 'ἃι'),
+    (0x1F84, 'M', 'ἄι'),
+    (0x1F85, 'M', 'ἅι'),
+    (0x1F86, 'M', 'ἆι'),
+    (0x1F87, 'M', 'ἇι'),
     ]
 
-def _seg_20():
+def _seg_20() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1F85, 'M', u'ἅι'),
-    (0x1F86, 'M', u'ἆι'),
-    (0x1F87, 'M', u'ἇι'),
-    (0x1F88, 'M', u'ἀι'),
-    (0x1F89, 'M', u'ἁι'),
-    (0x1F8A, 'M', u'ἂι'),
-    (0x1F8B, 'M', u'ἃι'),
-    (0x1F8C, 'M', u'ἄι'),
-    (0x1F8D, 'M', u'ἅι'),
-    (0x1F8E, 'M', u'ἆι'),
-    (0x1F8F, 'M', u'ἇι'),
-    (0x1F90, 'M', u'ἠι'),
-    (0x1F91, 'M', u'ἡι'),
-    (0x1F92, 'M', u'ἢι'),
-    (0x1F93, 'M', u'ἣι'),
-    (0x1F94, 'M', u'ἤι'),
-    (0x1F95, 'M', u'ἥι'),
-    (0x1F96, 'M', u'ἦι'),
-    (0x1F97, 'M', u'ἧι'),
-    (0x1F98, 'M', u'ἠι'),
-    (0x1F99, 'M', u'ἡι'),
-    (0x1F9A, 'M', u'ἢι'),
-    (0x1F9B, 'M', u'ἣι'),
-    (0x1F9C, 'M', u'ἤι'),
-    (0x1F9D, 'M', u'ἥι'),
-    (0x1F9E, 'M', u'ἦι'),
-    (0x1F9F, 'M', u'ἧι'),
-    (0x1FA0, 'M', u'ὠι'),
-    (0x1FA1, 'M', u'ὡι'),
-    (0x1FA2, 'M', u'ὢι'),
-    (0x1FA3, 'M', u'ὣι'),
-    (0x1FA4, 'M', u'ὤι'),
-    (0x1FA5, 'M', u'ὥι'),
-    (0x1FA6, 'M', u'ὦι'),
-    (0x1FA7, 'M', u'ὧι'),
-    (0x1FA8, 'M', u'ὠι'),
-    (0x1FA9, 'M', u'ὡι'),
-    (0x1FAA, 'M', u'ὢι'),
-    (0x1FAB, 'M', u'ὣι'),
-    (0x1FAC, 'M', u'ὤι'),
-    (0x1FAD, 'M', u'ὥι'),
-    (0x1FAE, 'M', u'ὦι'),
-    (0x1FAF, 'M', u'ὧι'),
+    (0x1F88, 'M', 'ἀι'),
+    (0x1F89, 'M', 'ἁι'),
+    (0x1F8A, 'M', 'ἂι'),
+    (0x1F8B, 'M', 'ἃι'),
+    (0x1F8C, 'M', 'ἄι'),
+    (0x1F8D, 'M', 'ἅι'),
+    (0x1F8E, 'M', 'ἆι'),
+    (0x1F8F, 'M', 'ἇι'),
+    (0x1F90, 'M', 'ἠι'),
+    (0x1F91, 'M', 'ἡι'),
+    (0x1F92, 'M', 'ἢι'),
+    (0x1F93, 'M', 'ἣι'),
+    (0x1F94, 'M', 'ἤι'),
+    (0x1F95, 'M', 'ἥι'),
+    (0x1F96, 'M', 'ἦι'),
+    (0x1F97, 'M', 'ἧι'),
+    (0x1F98, 'M', 'ἠι'),
+    (0x1F99, 'M', 'ἡι'),
+    (0x1F9A, 'M', 'ἢι'),
+    (0x1F9B, 'M', 'ἣι'),
+    (0x1F9C, 'M', 'ἤι'),
+    (0x1F9D, 'M', 'ἥι'),
+    (0x1F9E, 'M', 'ἦι'),
+    (0x1F9F, 'M', 'ἧι'),
+    (0x1FA0, 'M', 'ὠι'),
+    (0x1FA1, 'M', 'ὡι'),
+    (0x1FA2, 'M', 'ὢι'),
+    (0x1FA3, 'M', 'ὣι'),
+    (0x1FA4, 'M', 'ὤι'),
+    (0x1FA5, 'M', 'ὥι'),
+    (0x1FA6, 'M', 'ὦι'),
+    (0x1FA7, 'M', 'ὧι'),
+    (0x1FA8, 'M', 'ὠι'),
+    (0x1FA9, 'M', 'ὡι'),
+    (0x1FAA, 'M', 'ὢι'),
+    (0x1FAB, 'M', 'ὣι'),
+    (0x1FAC, 'M', 'ὤι'),
+    (0x1FAD, 'M', 'ὥι'),
+    (0x1FAE, 'M', 'ὦι'),
+    (0x1FAF, 'M', 'ὧι'),
     (0x1FB0, 'V'),
-    (0x1FB2, 'M', u'ὰι'),
-    (0x1FB3, 'M', u'αι'),
-    (0x1FB4, 'M', u'άι'),
+    (0x1FB2, 'M', 'ὰι'),
+    (0x1FB3, 'M', 'αι'),
+    (0x1FB4, 'M', 'άι'),
     (0x1FB5, 'X'),
     (0x1FB6, 'V'),
-    (0x1FB7, 'M', u'ᾶι'),
-    (0x1FB8, 'M', u'ᾰ'),
-    (0x1FB9, 'M', u'ᾱ'),
-    (0x1FBA, 'M', u'ὰ'),
-    (0x1FBB, 'M', u'ά'),
-    (0x1FBC, 'M', u'αι'),
-    (0x1FBD, '3', u' ̓'),
-    (0x1FBE, 'M', u'ι'),
-    (0x1FBF, '3', u' ̓'),
-    (0x1FC0, '3', u' ͂'),
-    (0x1FC1, '3', u' ̈͂'),
-    (0x1FC2, 'M', u'ὴι'),
-    (0x1FC3, 'M', u'ηι'),
-    (0x1FC4, 'M', u'ήι'),
+    (0x1FB7, 'M', 'ᾶι'),
+    (0x1FB8, 'M', 'ᾰ'),
+    (0x1FB9, 'M', 'ᾱ'),
+    (0x1FBA, 'M', 'ὰ'),
+    (0x1FBB, 'M', 'ά'),
+    (0x1FBC, 'M', 'αι'),
+    (0x1FBD, '3', ' ̓'),
+    (0x1FBE, 'M', 'ι'),
+    (0x1FBF, '3', ' ̓'),
+    (0x1FC0, '3', ' ͂'),
+    (0x1FC1, '3', ' ̈͂'),
+    (0x1FC2, 'M', 'ὴι'),
+    (0x1FC3, 'M', 'ηι'),
+    (0x1FC4, 'M', 'ήι'),
     (0x1FC5, 'X'),
     (0x1FC6, 'V'),
-    (0x1FC7, 'M', u'ῆι'),
-    (0x1FC8, 'M', u'ὲ'),
-    (0x1FC9, 'M', u'έ'),
-    (0x1FCA, 'M', u'ὴ'),
-    (0x1FCB, 'M', u'ή'),
-    (0x1FCC, 'M', u'ηι'),
-    (0x1FCD, '3', u' ̓̀'),
-    (0x1FCE, '3', u' ̓́'),
-    (0x1FCF, '3', u' ̓͂'),
+    (0x1FC7, 'M', 'ῆι'),
+    (0x1FC8, 'M', 'ὲ'),
+    (0x1FC9, 'M', 'έ'),
+    (0x1FCA, 'M', 'ὴ'),
+    (0x1FCB, 'M', 'ή'),
+    (0x1FCC, 'M', 'ηι'),
+    (0x1FCD, '3', ' ̓̀'),
+    (0x1FCE, '3', ' ̓́'),
+    (0x1FCF, '3', ' ̓͂'),
     (0x1FD0, 'V'),
-    (0x1FD3, 'M', u'ΐ'),
+    (0x1FD3, 'M', 'ΐ'),
     (0x1FD4, 'X'),
     (0x1FD6, 'V'),
-    (0x1FD8, 'M', u'ῐ'),
-    (0x1FD9, 'M', u'ῑ'),
-    (0x1FDA, 'M', u'ὶ'),
-    (0x1FDB, 'M', u'ί'),
+    (0x1FD8, 'M', 'ῐ'),
+    (0x1FD9, 'M', 'ῑ'),
+    (0x1FDA, 'M', 'ὶ'),
+    (0x1FDB, 'M', 'ί'),
     (0x1FDC, 'X'),
-    (0x1FDD, '3', u' ̔̀'),
-    (0x1FDE, '3', u' ̔́'),
-    (0x1FDF, '3', u' ̔͂'),
+    (0x1FDD, '3', ' ̔̀'),
+    (0x1FDE, '3', ' ̔́'),
+    (0x1FDF, '3', ' ̔͂'),
     (0x1FE0, 'V'),
-    (0x1FE3, 'M', u'ΰ'),
+    (0x1FE3, 'M', 'ΰ'),
     (0x1FE4, 'V'),
-    (0x1FE8, 'M', u'ῠ'),
-    (0x1FE9, 'M', u'ῡ'),
-    (0x1FEA, 'M', u'ὺ'),
-    (0x1FEB, 'M', u'ύ'),
-    (0x1FEC, 'M', u'ῥ'),
-    (0x1FED, '3', u' ̈̀'),
-    (0x1FEE, '3', u' ̈́'),
-    (0x1FEF, '3', u'`'),
+    (0x1FE8, 'M', 'ῠ'),
+    (0x1FE9, 'M', 'ῡ'),
+    (0x1FEA, 'M', 'ὺ'),
+    (0x1FEB, 'M', 'ύ'),
+    (0x1FEC, 'M', 'ῥ'),
+    (0x1FED, '3', ' ̈̀'),
+    (0x1FEE, '3', ' ̈́'),
+    (0x1FEF, '3', '`'),
     (0x1FF0, 'X'),
-    (0x1FF2, 'M', u'ὼι'),
-    (0x1FF3, 'M', u'ωι'),
+    (0x1FF2, 'M', 'ὼι'),
+    (0x1FF3, 'M', 'ωι'),
+    (0x1FF4, 'M', 'ώι'),
+    (0x1FF5, 'X'),
+    (0x1FF6, 'V'),
     ]
 
-def _seg_21():
+def _seg_21() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1FF4, 'M', u'ώι'),
-    (0x1FF5, 'X'),
-    (0x1FF6, 'V'),
-    (0x1FF7, 'M', u'ῶι'),
-    (0x1FF8, 'M', u'ὸ'),
-    (0x1FF9, 'M', u'ό'),
-    (0x1FFA, 'M', u'ὼ'),
-    (0x1FFB, 'M', u'ώ'),
-    (0x1FFC, 'M', u'ωι'),
-    (0x1FFD, '3', u' ́'),
-    (0x1FFE, '3', u' ̔'),
+    (0x1FF7, 'M', 'ῶι'),
+    (0x1FF8, 'M', 'ὸ'),
+    (0x1FF9, 'M', 'ό'),
+    (0x1FFA, 'M', 'ὼ'),
+    (0x1FFB, 'M', 'ώ'),
+    (0x1FFC, 'M', 'ωι'),
+    (0x1FFD, '3', ' ́'),
+    (0x1FFE, '3', ' ̔'),
     (0x1FFF, 'X'),
-    (0x2000, '3', u' '),
+    (0x2000, '3', ' '),
     (0x200B, 'I'),
-    (0x200C, 'D', u''),
+    (0x200C, 'D', ''),
     (0x200E, 'X'),
     (0x2010, 'V'),
-    (0x2011, 'M', u'‐'),
+    (0x2011, 'M', '‐'),
     (0x2012, 'V'),
-    (0x2017, '3', u' ̳'),
+    (0x2017, '3', ' ̳'),
     (0x2018, 'V'),
     (0x2024, 'X'),
     (0x2027, 'V'),
     (0x2028, 'X'),
-    (0x202F, '3', u' '),
+    (0x202F, '3', ' '),
     (0x2030, 'V'),
-    (0x2033, 'M', u'′′'),
-    (0x2034, 'M', u'′′′'),
+    (0x2033, 'M', '′′'),
+    (0x2034, 'M', '′′′'),
     (0x2035, 'V'),
-    (0x2036, 'M', u'‵‵'),
-    (0x2037, 'M', u'‵‵‵'),
+    (0x2036, 'M', '‵‵'),
+    (0x2037, 'M', '‵‵‵'),
     (0x2038, 'V'),
-    (0x203C, '3', u'!!'),
+    (0x203C, '3', '!!'),
     (0x203D, 'V'),
-    (0x203E, '3', u' ̅'),
+    (0x203E, '3', ' ̅'),
     (0x203F, 'V'),
-    (0x2047, '3', u'??'),
-    (0x2048, '3', u'?!'),
-    (0x2049, '3', u'!?'),
+    (0x2047, '3', '??'),
+    (0x2048, '3', '?!'),
+    (0x2049, '3', '!?'),
     (0x204A, 'V'),
-    (0x2057, 'M', u'′′′′'),
+    (0x2057, 'M', '′′′′'),
     (0x2058, 'V'),
-    (0x205F, '3', u' '),
+    (0x205F, '3', ' '),
     (0x2060, 'I'),
     (0x2061, 'X'),
     (0x2064, 'I'),
     (0x2065, 'X'),
-    (0x2070, 'M', u'0'),
-    (0x2071, 'M', u'i'),
+    (0x2070, 'M', '0'),
+    (0x2071, 'M', 'i'),
     (0x2072, 'X'),
-    (0x2074, 'M', u'4'),
-    (0x2075, 'M', u'5'),
-    (0x2076, 'M', u'6'),
-    (0x2077, 'M', u'7'),
-    (0x2078, 'M', u'8'),
-    (0x2079, 'M', u'9'),
-    (0x207A, '3', u'+'),
-    (0x207B, 'M', u'−'),
-    (0x207C, '3', u'='),
-    (0x207D, '3', u'('),
-    (0x207E, '3', u')'),
-    (0x207F, 'M', u'n'),
-    (0x2080, 'M', u'0'),
-    (0x2081, 'M', u'1'),
-    (0x2082, 'M', u'2'),
-    (0x2083, 'M', u'3'),
-    (0x2084, 'M', u'4'),
-    (0x2085, 'M', u'5'),
-    (0x2086, 'M', u'6'),
-    (0x2087, 'M', u'7'),
-    (0x2088, 'M', u'8'),
-    (0x2089, 'M', u'9'),
-    (0x208A, '3', u'+'),
-    (0x208B, 'M', u'−'),
-    (0x208C, '3', u'='),
-    (0x208D, '3', u'('),
-    (0x208E, '3', u')'),
+    (0x2074, 'M', '4'),
+    (0x2075, 'M', '5'),
+    (0x2076, 'M', '6'),
+    (0x2077, 'M', '7'),
+    (0x2078, 'M', '8'),
+    (0x2079, 'M', '9'),
+    (0x207A, '3', '+'),
+    (0x207B, 'M', '−'),
+    (0x207C, '3', '='),
+    (0x207D, '3', '('),
+    (0x207E, '3', ')'),
+    (0x207F, 'M', 'n'),
+    (0x2080, 'M', '0'),
+    (0x2081, 'M', '1'),
+    (0x2082, 'M', '2'),
+    (0x2083, 'M', '3'),
+    (0x2084, 'M', '4'),
+    (0x2085, 'M', '5'),
+    (0x2086, 'M', '6'),
+    (0x2087, 'M', '7'),
+    (0x2088, 'M', '8'),
+    (0x2089, 'M', '9'),
+    (0x208A, '3', '+'),
+    (0x208B, 'M', '−'),
+    (0x208C, '3', '='),
+    (0x208D, '3', '('),
+    (0x208E, '3', ')'),
     (0x208F, 'X'),
-    (0x2090, 'M', u'a'),
-    (0x2091, 'M', u'e'),
-    (0x2092, 'M', u'o'),
-    (0x2093, 'M', u'x'),
-    (0x2094, 'M', u'ə'),
-    (0x2095, 'M', u'h'),
-    (0x2096, 'M', u'k'),
-    (0x2097, 'M', u'l'),
-    (0x2098, 'M', u'm'),
-    (0x2099, 'M', u'n'),
-    (0x209A, 'M', u'p'),
-    (0x209B, 'M', u's'),
-    (0x209C, 'M', u't'),
+    (0x2090, 'M', 'a'),
+    (0x2091, 'M', 'e'),
+    (0x2092, 'M', 'o'),
+    (0x2093, 'M', 'x'),
+    (0x2094, 'M', 'ə'),
+    (0x2095, 'M', 'h'),
+    (0x2096, 'M', 'k'),
+    (0x2097, 'M', 'l'),
+    (0x2098, 'M', 'm'),
+    (0x2099, 'M', 'n'),
+    (0x209A, 'M', 'p'),
+    (0x209B, 'M', 's'),
+    (0x209C, 'M', 't'),
     (0x209D, 'X'),
     (0x20A0, 'V'),
-    (0x20A8, 'M', u'rs'),
+    (0x20A8, 'M', 'rs'),
     (0x20A9, 'V'),
-    (0x20C0, 'X'),
+    (0x20C1, 'X'),
     (0x20D0, 'V'),
     (0x20F1, 'X'),
-    (0x2100, '3', u'a/c'),
-    (0x2101, '3', u'a/s'),
+    (0x2100, '3', 'a/c'),
+    (0x2101, '3', 'a/s'),
+    (0x2102, 'M', 'c'),
+    (0x2103, 'M', '°c'),
+    (0x2104, 'V'),
     ]
 
-def _seg_22():
+def _seg_22() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2102, 'M', u'c'),
-    (0x2103, 'M', u'°c'),
-    (0x2104, 'V'),
-    (0x2105, '3', u'c/o'),
-    (0x2106, '3', u'c/u'),
-    (0x2107, 'M', u'ɛ'),
+    (0x2105, '3', 'c/o'),
+    (0x2106, '3', 'c/u'),
+    (0x2107, 'M', 'ɛ'),
     (0x2108, 'V'),
-    (0x2109, 'M', u'°f'),
-    (0x210A, 'M', u'g'),
-    (0x210B, 'M', u'h'),
-    (0x210F, 'M', u'ħ'),
-    (0x2110, 'M', u'i'),
-    (0x2112, 'M', u'l'),
+    (0x2109, 'M', '°f'),
+    (0x210A, 'M', 'g'),
+    (0x210B, 'M', 'h'),
+    (0x210F, 'M', 'ħ'),
+    (0x2110, 'M', 'i'),
+    (0x2112, 'M', 'l'),
     (0x2114, 'V'),
-    (0x2115, 'M', u'n'),
-    (0x2116, 'M', u'no'),
+    (0x2115, 'M', 'n'),
+    (0x2116, 'M', 'no'),
     (0x2117, 'V'),
-    (0x2119, 'M', u'p'),
-    (0x211A, 'M', u'q'),
-    (0x211B, 'M', u'r'),
+    (0x2119, 'M', 'p'),
+    (0x211A, 'M', 'q'),
+    (0x211B, 'M', 'r'),
     (0x211E, 'V'),
-    (0x2120, 'M', u'sm'),
-    (0x2121, 'M', u'tel'),
-    (0x2122, 'M', u'tm'),
+    (0x2120, 'M', 'sm'),
+    (0x2121, 'M', 'tel'),
+    (0x2122, 'M', 'tm'),
     (0x2123, 'V'),
-    (0x2124, 'M', u'z'),
+    (0x2124, 'M', 'z'),
     (0x2125, 'V'),
-    (0x2126, 'M', u'ω'),
+    (0x2126, 'M', 'ω'),
     (0x2127, 'V'),
-    (0x2128, 'M', u'z'),
+    (0x2128, 'M', 'z'),
     (0x2129, 'V'),
-    (0x212A, 'M', u'k'),
-    (0x212B, 'M', u'å'),
-    (0x212C, 'M', u'b'),
-    (0x212D, 'M', u'c'),
+    (0x212A, 'M', 'k'),
+    (0x212B, 'M', 'å'),
+    (0x212C, 'M', 'b'),
+    (0x212D, 'M', 'c'),
     (0x212E, 'V'),
-    (0x212F, 'M', u'e'),
-    (0x2131, 'M', u'f'),
+    (0x212F, 'M', 'e'),
+    (0x2131, 'M', 'f'),
     (0x2132, 'X'),
-    (0x2133, 'M', u'm'),
-    (0x2134, 'M', u'o'),
-    (0x2135, 'M', u'א'),
-    (0x2136, 'M', u'ב'),
-    (0x2137, 'M', u'ג'),
-    (0x2138, 'M', u'ד'),
-    (0x2139, 'M', u'i'),
+    (0x2133, 'M', 'm'),
+    (0x2134, 'M', 'o'),
+    (0x2135, 'M', 'א'),
+    (0x2136, 'M', 'ב'),
+    (0x2137, 'M', 'ג'),
+    (0x2138, 'M', 'ד'),
+    (0x2139, 'M', 'i'),
     (0x213A, 'V'),
-    (0x213B, 'M', u'fax'),
-    (0x213C, 'M', u'π'),
-    (0x213D, 'M', u'γ'),
-    (0x213F, 'M', u'π'),
-    (0x2140, 'M', u'∑'),
+    (0x213B, 'M', 'fax'),
+    (0x213C, 'M', 'π'),
+    (0x213D, 'M', 'γ'),
+    (0x213F, 'M', 'π'),
+    (0x2140, 'M', '∑'),
     (0x2141, 'V'),
-    (0x2145, 'M', u'd'),
-    (0x2147, 'M', u'e'),
-    (0x2148, 'M', u'i'),
-    (0x2149, 'M', u'j'),
+    (0x2145, 'M', 'd'),
+    (0x2147, 'M', 'e'),
+    (0x2148, 'M', 'i'),
+    (0x2149, 'M', 'j'),
     (0x214A, 'V'),
-    (0x2150, 'M', u'1⁄7'),
-    (0x2151, 'M', u'1⁄9'),
-    (0x2152, 'M', u'1⁄10'),
-    (0x2153, 'M', u'1⁄3'),
-    (0x2154, 'M', u'2⁄3'),
-    (0x2155, 'M', u'1⁄5'),
-    (0x2156, 'M', u'2⁄5'),
-    (0x2157, 'M', u'3⁄5'),
-    (0x2158, 'M', u'4⁄5'),
-    (0x2159, 'M', u'1⁄6'),
-    (0x215A, 'M', u'5⁄6'),
-    (0x215B, 'M', u'1⁄8'),
-    (0x215C, 'M', u'3⁄8'),
-    (0x215D, 'M', u'5⁄8'),
-    (0x215E, 'M', u'7⁄8'),
-    (0x215F, 'M', u'1⁄'),
-    (0x2160, 'M', u'i'),
-    (0x2161, 'M', u'ii'),
-    (0x2162, 'M', u'iii'),
-    (0x2163, 'M', u'iv'),
-    (0x2164, 'M', u'v'),
-    (0x2165, 'M', u'vi'),
-    (0x2166, 'M', u'vii'),
-    (0x2167, 'M', u'viii'),
-    (0x2168, 'M', u'ix'),
-    (0x2169, 'M', u'x'),
-    (0x216A, 'M', u'xi'),
-    (0x216B, 'M', u'xii'),
-    (0x216C, 'M', u'l'),
-    (0x216D, 'M', u'c'),
-    (0x216E, 'M', u'd'),
-    (0x216F, 'M', u'm'),
-    (0x2170, 'M', u'i'),
-    (0x2171, 'M', u'ii'),
-    (0x2172, 'M', u'iii'),
-    (0x2173, 'M', u'iv'),
-    (0x2174, 'M', u'v'),
-    (0x2175, 'M', u'vi'),
-    (0x2176, 'M', u'vii'),
-    (0x2177, 'M', u'viii'),
-    (0x2178, 'M', u'ix'),
-    (0x2179, 'M', u'x'),
+    (0x2150, 'M', '1⁄7'),
+    (0x2151, 'M', '1⁄9'),
+    (0x2152, 'M', '1⁄10'),
+    (0x2153, 'M', '1⁄3'),
+    (0x2154, 'M', '2⁄3'),
+    (0x2155, 'M', '1⁄5'),
+    (0x2156, 'M', '2⁄5'),
+    (0x2157, 'M', '3⁄5'),
+    (0x2158, 'M', '4⁄5'),
+    (0x2159, 'M', '1⁄6'),
+    (0x215A, 'M', '5⁄6'),
+    (0x215B, 'M', '1⁄8'),
+    (0x215C, 'M', '3⁄8'),
+    (0x215D, 'M', '5⁄8'),
+    (0x215E, 'M', '7⁄8'),
+    (0x215F, 'M', '1⁄'),
+    (0x2160, 'M', 'i'),
+    (0x2161, 'M', 'ii'),
+    (0x2162, 'M', 'iii'),
+    (0x2163, 'M', 'iv'),
+    (0x2164, 'M', 'v'),
+    (0x2165, 'M', 'vi'),
+    (0x2166, 'M', 'vii'),
+    (0x2167, 'M', 'viii'),
+    (0x2168, 'M', 'ix'),
+    (0x2169, 'M', 'x'),
+    (0x216A, 'M', 'xi'),
+    (0x216B, 'M', 'xii'),
+    (0x216C, 'M', 'l'),
+    (0x216D, 'M', 'c'),
+    (0x216E, 'M', 'd'),
+    (0x216F, 'M', 'm'),
+    (0x2170, 'M', 'i'),
+    (0x2171, 'M', 'ii'),
+    (0x2172, 'M', 'iii'),
+    (0x2173, 'M', 'iv'),
+    (0x2174, 'M', 'v'),
+    (0x2175, 'M', 'vi'),
+    (0x2176, 'M', 'vii'),
+    (0x2177, 'M', 'viii'),
+    (0x2178, 'M', 'ix'),
+    (0x2179, 'M', 'x'),
+    (0x217A, 'M', 'xi'),
+    (0x217B, 'M', 'xii'),
+    (0x217C, 'M', 'l'),
     ]
 
-def _seg_23():
+def _seg_23() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x217A, 'M', u'xi'),
-    (0x217B, 'M', u'xii'),
-    (0x217C, 'M', u'l'),
-    (0x217D, 'M', u'c'),
-    (0x217E, 'M', u'd'),
-    (0x217F, 'M', u'm'),
+    (0x217D, 'M', 'c'),
+    (0x217E, 'M', 'd'),
+    (0x217F, 'M', 'm'),
     (0x2180, 'V'),
     (0x2183, 'X'),
     (0x2184, 'V'),
-    (0x2189, 'M', u'0⁄3'),
+    (0x2189, 'M', '0⁄3'),
     (0x218A, 'V'),
     (0x218C, 'X'),
     (0x2190, 'V'),
-    (0x222C, 'M', u'∫∫'),
-    (0x222D, 'M', u'∫∫∫'),
+    (0x222C, 'M', '∫∫'),
+    (0x222D, 'M', '∫∫∫'),
     (0x222E, 'V'),
-    (0x222F, 'M', u'∮∮'),
-    (0x2230, 'M', u'∮∮∮'),
+    (0x222F, 'M', '∮∮'),
+    (0x2230, 'M', '∮∮∮'),
     (0x2231, 'V'),
     (0x2260, '3'),
     (0x2261, 'V'),
     (0x226E, '3'),
     (0x2270, 'V'),
-    (0x2329, 'M', u'〈'),
-    (0x232A, 'M', u'〉'),
+    (0x2329, 'M', '〈'),
+    (0x232A, 'M', '〉'),
     (0x232B, 'V'),
     (0x2427, 'X'),
     (0x2440, 'V'),
     (0x244B, 'X'),
-    (0x2460, 'M', u'1'),
-    (0x2461, 'M', u'2'),
-    (0x2462, 'M', u'3'),
-    (0x2463, 'M', u'4'),
-    (0x2464, 'M', u'5'),
-    (0x2465, 'M', u'6'),
-    (0x2466, 'M', u'7'),
-    (0x2467, 'M', u'8'),
-    (0x2468, 'M', u'9'),
-    (0x2469, 'M', u'10'),
-    (0x246A, 'M', u'11'),
-    (0x246B, 'M', u'12'),
-    (0x246C, 'M', u'13'),
-    (0x246D, 'M', u'14'),
-    (0x246E, 'M', u'15'),
-    (0x246F, 'M', u'16'),
-    (0x2470, 'M', u'17'),
-    (0x2471, 'M', u'18'),
-    (0x2472, 'M', u'19'),
-    (0x2473, 'M', u'20'),
-    (0x2474, '3', u'(1)'),
-    (0x2475, '3', u'(2)'),
-    (0x2476, '3', u'(3)'),
-    (0x2477, '3', u'(4)'),
-    (0x2478, '3', u'(5)'),
-    (0x2479, '3', u'(6)'),
-    (0x247A, '3', u'(7)'),
-    (0x247B, '3', u'(8)'),
-    (0x247C, '3', u'(9)'),
-    (0x247D, '3', u'(10)'),
-    (0x247E, '3', u'(11)'),
-    (0x247F, '3', u'(12)'),
-    (0x2480, '3', u'(13)'),
-    (0x2481, '3', u'(14)'),
-    (0x2482, '3', u'(15)'),
-    (0x2483, '3', u'(16)'),
-    (0x2484, '3', u'(17)'),
-    (0x2485, '3', u'(18)'),
-    (0x2486, '3', u'(19)'),
-    (0x2487, '3', u'(20)'),
+    (0x2460, 'M', '1'),
+    (0x2461, 'M', '2'),
+    (0x2462, 'M', '3'),
+    (0x2463, 'M', '4'),
+    (0x2464, 'M', '5'),
+    (0x2465, 'M', '6'),
+    (0x2466, 'M', '7'),
+    (0x2467, 'M', '8'),
+    (0x2468, 'M', '9'),
+    (0x2469, 'M', '10'),
+    (0x246A, 'M', '11'),
+    (0x246B, 'M', '12'),
+    (0x246C, 'M', '13'),
+    (0x246D, 'M', '14'),
+    (0x246E, 'M', '15'),
+    (0x246F, 'M', '16'),
+    (0x2470, 'M', '17'),
+    (0x2471, 'M', '18'),
+    (0x2472, 'M', '19'),
+    (0x2473, 'M', '20'),
+    (0x2474, '3', '(1)'),
+    (0x2475, '3', '(2)'),
+    (0x2476, '3', '(3)'),
+    (0x2477, '3', '(4)'),
+    (0x2478, '3', '(5)'),
+    (0x2479, '3', '(6)'),
+    (0x247A, '3', '(7)'),
+    (0x247B, '3', '(8)'),
+    (0x247C, '3', '(9)'),
+    (0x247D, '3', '(10)'),
+    (0x247E, '3', '(11)'),
+    (0x247F, '3', '(12)'),
+    (0x2480, '3', '(13)'),
+    (0x2481, '3', '(14)'),
+    (0x2482, '3', '(15)'),
+    (0x2483, '3', '(16)'),
+    (0x2484, '3', '(17)'),
+    (0x2485, '3', '(18)'),
+    (0x2486, '3', '(19)'),
+    (0x2487, '3', '(20)'),
     (0x2488, 'X'),
-    (0x249C, '3', u'(a)'),
-    (0x249D, '3', u'(b)'),
-    (0x249E, '3', u'(c)'),
-    (0x249F, '3', u'(d)'),
-    (0x24A0, '3', u'(e)'),
-    (0x24A1, '3', u'(f)'),
-    (0x24A2, '3', u'(g)'),
-    (0x24A3, '3', u'(h)'),
-    (0x24A4, '3', u'(i)'),
-    (0x24A5, '3', u'(j)'),
-    (0x24A6, '3', u'(k)'),
-    (0x24A7, '3', u'(l)'),
-    (0x24A8, '3', u'(m)'),
-    (0x24A9, '3', u'(n)'),
-    (0x24AA, '3', u'(o)'),
-    (0x24AB, '3', u'(p)'),
-    (0x24AC, '3', u'(q)'),
-    (0x24AD, '3', u'(r)'),
-    (0x24AE, '3', u'(s)'),
-    (0x24AF, '3', u'(t)'),
-    (0x24B0, '3', u'(u)'),
-    (0x24B1, '3', u'(v)'),
-    (0x24B2, '3', u'(w)'),
-    (0x24B3, '3', u'(x)'),
-    (0x24B4, '3', u'(y)'),
-    (0x24B5, '3', u'(z)'),
-    (0x24B6, 'M', u'a'),
-    (0x24B7, 'M', u'b'),
-    (0x24B8, 'M', u'c'),
-    (0x24B9, 'M', u'd'),
+    (0x249C, '3', '(a)'),
+    (0x249D, '3', '(b)'),
+    (0x249E, '3', '(c)'),
+    (0x249F, '3', '(d)'),
+    (0x24A0, '3', '(e)'),
+    (0x24A1, '3', '(f)'),
+    (0x24A2, '3', '(g)'),
+    (0x24A3, '3', '(h)'),
+    (0x24A4, '3', '(i)'),
+    (0x24A5, '3', '(j)'),
+    (0x24A6, '3', '(k)'),
+    (0x24A7, '3', '(l)'),
+    (0x24A8, '3', '(m)'),
+    (0x24A9, '3', '(n)'),
+    (0x24AA, '3', '(o)'),
+    (0x24AB, '3', '(p)'),
+    (0x24AC, '3', '(q)'),
+    (0x24AD, '3', '(r)'),
+    (0x24AE, '3', '(s)'),
+    (0x24AF, '3', '(t)'),
+    (0x24B0, '3', '(u)'),
+    (0x24B1, '3', '(v)'),
+    (0x24B2, '3', '(w)'),
+    (0x24B3, '3', '(x)'),
+    (0x24B4, '3', '(y)'),
+    (0x24B5, '3', '(z)'),
+    (0x24B6, 'M', 'a'),
+    (0x24B7, 'M', 'b'),
+    (0x24B8, 'M', 'c'),
+    (0x24B9, 'M', 'd'),
+    (0x24BA, 'M', 'e'),
+    (0x24BB, 'M', 'f'),
+    (0x24BC, 'M', 'g'),
     ]
 
-def _seg_24():
+def _seg_24() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x24BA, 'M', u'e'),
-    (0x24BB, 'M', u'f'),
-    (0x24BC, 'M', u'g'),
-    (0x24BD, 'M', u'h'),
-    (0x24BE, 'M', u'i'),
-    (0x24BF, 'M', u'j'),
-    (0x24C0, 'M', u'k'),
-    (0x24C1, 'M', u'l'),
-    (0x24C2, 'M', u'm'),
-    (0x24C3, 'M', u'n'),
-    (0x24C4, 'M', u'o'),
-    (0x24C5, 'M', u'p'),
-    (0x24C6, 'M', u'q'),
-    (0x24C7, 'M', u'r'),
-    (0x24C8, 'M', u's'),
-    (0x24C9, 'M', u't'),
-    (0x24CA, 'M', u'u'),
-    (0x24CB, 'M', u'v'),
-    (0x24CC, 'M', u'w'),
-    (0x24CD, 'M', u'x'),
-    (0x24CE, 'M', u'y'),
-    (0x24CF, 'M', u'z'),
-    (0x24D0, 'M', u'a'),
-    (0x24D1, 'M', u'b'),
-    (0x24D2, 'M', u'c'),
-    (0x24D3, 'M', u'd'),
-    (0x24D4, 'M', u'e'),
-    (0x24D5, 'M', u'f'),
-    (0x24D6, 'M', u'g'),
-    (0x24D7, 'M', u'h'),
-    (0x24D8, 'M', u'i'),
-    (0x24D9, 'M', u'j'),
-    (0x24DA, 'M', u'k'),
-    (0x24DB, 'M', u'l'),
-    (0x24DC, 'M', u'm'),
-    (0x24DD, 'M', u'n'),
-    (0x24DE, 'M', u'o'),
-    (0x24DF, 'M', u'p'),
-    (0x24E0, 'M', u'q'),
-    (0x24E1, 'M', u'r'),
-    (0x24E2, 'M', u's'),
-    (0x24E3, 'M', u't'),
-    (0x24E4, 'M', u'u'),
-    (0x24E5, 'M', u'v'),
-    (0x24E6, 'M', u'w'),
-    (0x24E7, 'M', u'x'),
-    (0x24E8, 'M', u'y'),
-    (0x24E9, 'M', u'z'),
-    (0x24EA, 'M', u'0'),
+    (0x24BD, 'M', 'h'),
+    (0x24BE, 'M', 'i'),
+    (0x24BF, 'M', 'j'),
+    (0x24C0, 'M', 'k'),
+    (0x24C1, 'M', 'l'),
+    (0x24C2, 'M', 'm'),
+    (0x24C3, 'M', 'n'),
+    (0x24C4, 'M', 'o'),
+    (0x24C5, 'M', 'p'),
+    (0x24C6, 'M', 'q'),
+    (0x24C7, 'M', 'r'),
+    (0x24C8, 'M', 's'),
+    (0x24C9, 'M', 't'),
+    (0x24CA, 'M', 'u'),
+    (0x24CB, 'M', 'v'),
+    (0x24CC, 'M', 'w'),
+    (0x24CD, 'M', 'x'),
+    (0x24CE, 'M', 'y'),
+    (0x24CF, 'M', 'z'),
+    (0x24D0, 'M', 'a'),
+    (0x24D1, 'M', 'b'),
+    (0x24D2, 'M', 'c'),
+    (0x24D3, 'M', 'd'),
+    (0x24D4, 'M', 'e'),
+    (0x24D5, 'M', 'f'),
+    (0x24D6, 'M', 'g'),
+    (0x24D7, 'M', 'h'),
+    (0x24D8, 'M', 'i'),
+    (0x24D9, 'M', 'j'),
+    (0x24DA, 'M', 'k'),
+    (0x24DB, 'M', 'l'),
+    (0x24DC, 'M', 'm'),
+    (0x24DD, 'M', 'n'),
+    (0x24DE, 'M', 'o'),
+    (0x24DF, 'M', 'p'),
+    (0x24E0, 'M', 'q'),
+    (0x24E1, 'M', 'r'),
+    (0x24E2, 'M', 's'),
+    (0x24E3, 'M', 't'),
+    (0x24E4, 'M', 'u'),
+    (0x24E5, 'M', 'v'),
+    (0x24E6, 'M', 'w'),
+    (0x24E7, 'M', 'x'),
+    (0x24E8, 'M', 'y'),
+    (0x24E9, 'M', 'z'),
+    (0x24EA, 'M', '0'),
     (0x24EB, 'V'),
-    (0x2A0C, 'M', u'∫∫∫∫'),
+    (0x2A0C, 'M', '∫∫∫∫'),
     (0x2A0D, 'V'),
-    (0x2A74, '3', u'::='),
-    (0x2A75, '3', u'=='),
-    (0x2A76, '3', u'==='),
+    (0x2A74, '3', '::='),
+    (0x2A75, '3', '=='),
+    (0x2A76, '3', '==='),
     (0x2A77, 'V'),
-    (0x2ADC, 'M', u'⫝̸'),
+    (0x2ADC, 'M', '⫝̸'),
     (0x2ADD, 'V'),
     (0x2B74, 'X'),
     (0x2B76, 'V'),
     (0x2B96, 'X'),
     (0x2B97, 'V'),
-    (0x2C00, 'M', u'ⰰ'),
-    (0x2C01, 'M', u'ⰱ'),
-    (0x2C02, 'M', u'ⰲ'),
-    (0x2C03, 'M', u'ⰳ'),
-    (0x2C04, 'M', u'ⰴ'),
-    (0x2C05, 'M', u'ⰵ'),
-    (0x2C06, 'M', u'ⰶ'),
-    (0x2C07, 'M', u'ⰷ'),
-    (0x2C08, 'M', u'ⰸ'),
-    (0x2C09, 'M', u'ⰹ'),
-    (0x2C0A, 'M', u'ⰺ'),
-    (0x2C0B, 'M', u'ⰻ'),
-    (0x2C0C, 'M', u'ⰼ'),
-    (0x2C0D, 'M', u'ⰽ'),
-    (0x2C0E, 'M', u'ⰾ'),
-    (0x2C0F, 'M', u'ⰿ'),
-    (0x2C10, 'M', u'ⱀ'),
-    (0x2C11, 'M', u'ⱁ'),
-    (0x2C12, 'M', u'ⱂ'),
-    (0x2C13, 'M', u'ⱃ'),
-    (0x2C14, 'M', u'ⱄ'),
-    (0x2C15, 'M', u'ⱅ'),
-    (0x2C16, 'M', u'ⱆ'),
-    (0x2C17, 'M', u'ⱇ'),
-    (0x2C18, 'M', u'ⱈ'),
-    (0x2C19, 'M', u'ⱉ'),
-    (0x2C1A, 'M', u'ⱊ'),
-    (0x2C1B, 'M', u'ⱋ'),
-    (0x2C1C, 'M', u'ⱌ'),
-    (0x2C1D, 'M', u'ⱍ'),
-    (0x2C1E, 'M', u'ⱎ'),
-    (0x2C1F, 'M', u'ⱏ'),
-    (0x2C20, 'M', u'ⱐ'),
-    (0x2C21, 'M', u'ⱑ'),
-    (0x2C22, 'M', u'ⱒ'),
-    (0x2C23, 'M', u'ⱓ'),
-    (0x2C24, 'M', u'ⱔ'),
-    (0x2C25, 'M', u'ⱕ'),
+    (0x2C00, 'M', 'ⰰ'),
+    (0x2C01, 'M', 'ⰱ'),
+    (0x2C02, 'M', 'ⰲ'),
+    (0x2C03, 'M', 'ⰳ'),
+    (0x2C04, 'M', 'ⰴ'),
+    (0x2C05, 'M', 'ⰵ'),
+    (0x2C06, 'M', 'ⰶ'),
+    (0x2C07, 'M', 'ⰷ'),
+    (0x2C08, 'M', 'ⰸ'),
+    (0x2C09, 'M', 'ⰹ'),
+    (0x2C0A, 'M', 'ⰺ'),
+    (0x2C0B, 'M', 'ⰻ'),
+    (0x2C0C, 'M', 'ⰼ'),
+    (0x2C0D, 'M', 'ⰽ'),
+    (0x2C0E, 'M', 'ⰾ'),
+    (0x2C0F, 'M', 'ⰿ'),
+    (0x2C10, 'M', 'ⱀ'),
+    (0x2C11, 'M', 'ⱁ'),
+    (0x2C12, 'M', 'ⱂ'),
+    (0x2C13, 'M', 'ⱃ'),
+    (0x2C14, 'M', 'ⱄ'),
+    (0x2C15, 'M', 'ⱅ'),
+    (0x2C16, 'M', 'ⱆ'),
+    (0x2C17, 'M', 'ⱇ'),
+    (0x2C18, 'M', 'ⱈ'),
+    (0x2C19, 'M', 'ⱉ'),
+    (0x2C1A, 'M', 'ⱊ'),
+    (0x2C1B, 'M', 'ⱋ'),
+    (0x2C1C, 'M', 'ⱌ'),
+    (0x2C1D, 'M', 'ⱍ'),
+    (0x2C1E, 'M', 'ⱎ'),
+    (0x2C1F, 'M', 'ⱏ'),
+    (0x2C20, 'M', 'ⱐ'),
+    (0x2C21, 'M', 'ⱑ'),
+    (0x2C22, 'M', 'ⱒ'),
+    (0x2C23, 'M', 'ⱓ'),
+    (0x2C24, 'M', 'ⱔ'),
+    (0x2C25, 'M', 'ⱕ'),
+    (0x2C26, 'M', 'ⱖ'),
+    (0x2C27, 'M', 'ⱗ'),
+    (0x2C28, 'M', 'ⱘ'),
     ]
 
-def _seg_25():
+def _seg_25() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2C26, 'M', u'ⱖ'),
-    (0x2C27, 'M', u'ⱗ'),
-    (0x2C28, 'M', u'ⱘ'),
-    (0x2C29, 'M', u'ⱙ'),
-    (0x2C2A, 'M', u'ⱚ'),
-    (0x2C2B, 'M', u'ⱛ'),
-    (0x2C2C, 'M', u'ⱜ'),
-    (0x2C2D, 'M', u'ⱝ'),
-    (0x2C2E, 'M', u'ⱞ'),
-    (0x2C2F, 'X'),
+    (0x2C29, 'M', 'ⱙ'),
+    (0x2C2A, 'M', 'ⱚ'),
+    (0x2C2B, 'M', 'ⱛ'),
+    (0x2C2C, 'M', 'ⱜ'),
+    (0x2C2D, 'M', 'ⱝ'),
+    (0x2C2E, 'M', 'ⱞ'),
+    (0x2C2F, 'M', 'ⱟ'),
     (0x2C30, 'V'),
-    (0x2C5F, 'X'),
-    (0x2C60, 'M', u'ⱡ'),
+    (0x2C60, 'M', 'ⱡ'),
     (0x2C61, 'V'),
-    (0x2C62, 'M', u'ɫ'),
-    (0x2C63, 'M', u'ᵽ'),
-    (0x2C64, 'M', u'ɽ'),
+    (0x2C62, 'M', 'ɫ'),
+    (0x2C63, 'M', 'ᵽ'),
+    (0x2C64, 'M', 'ɽ'),
     (0x2C65, 'V'),
-    (0x2C67, 'M', u'ⱨ'),
+    (0x2C67, 'M', 'ⱨ'),
     (0x2C68, 'V'),
-    (0x2C69, 'M', u'ⱪ'),
+    (0x2C69, 'M', 'ⱪ'),
     (0x2C6A, 'V'),
-    (0x2C6B, 'M', u'ⱬ'),
+    (0x2C6B, 'M', 'ⱬ'),
     (0x2C6C, 'V'),
-    (0x2C6D, 'M', u'ɑ'),
-    (0x2C6E, 'M', u'ɱ'),
-    (0x2C6F, 'M', u'ɐ'),
-    (0x2C70, 'M', u'ɒ'),
+    (0x2C6D, 'M', 'ɑ'),
+    (0x2C6E, 'M', 'ɱ'),
+    (0x2C6F, 'M', 'ɐ'),
+    (0x2C70, 'M', 'ɒ'),
     (0x2C71, 'V'),
-    (0x2C72, 'M', u'ⱳ'),
+    (0x2C72, 'M', 'ⱳ'),
     (0x2C73, 'V'),
-    (0x2C75, 'M', u'ⱶ'),
+    (0x2C75, 'M', 'ⱶ'),
     (0x2C76, 'V'),
-    (0x2C7C, 'M', u'j'),
-    (0x2C7D, 'M', u'v'),
-    (0x2C7E, 'M', u'ȿ'),
-    (0x2C7F, 'M', u'ɀ'),
-    (0x2C80, 'M', u'ⲁ'),
+    (0x2C7C, 'M', 'j'),
+    (0x2C7D, 'M', 'v'),
+    (0x2C7E, 'M', 'ȿ'),
+    (0x2C7F, 'M', 'ɀ'),
+    (0x2C80, 'M', 'ⲁ'),
     (0x2C81, 'V'),
-    (0x2C82, 'M', u'ⲃ'),
+    (0x2C82, 'M', 'ⲃ'),
     (0x2C83, 'V'),
-    (0x2C84, 'M', u'ⲅ'),
+    (0x2C84, 'M', 'ⲅ'),
     (0x2C85, 'V'),
-    (0x2C86, 'M', u'ⲇ'),
+    (0x2C86, 'M', 'ⲇ'),
     (0x2C87, 'V'),
-    (0x2C88, 'M', u'ⲉ'),
+    (0x2C88, 'M', 'ⲉ'),
     (0x2C89, 'V'),
-    (0x2C8A, 'M', u'ⲋ'),
+    (0x2C8A, 'M', 'ⲋ'),
     (0x2C8B, 'V'),
-    (0x2C8C, 'M', u'ⲍ'),
+    (0x2C8C, 'M', 'ⲍ'),
     (0x2C8D, 'V'),
-    (0x2C8E, 'M', u'ⲏ'),
+    (0x2C8E, 'M', 'ⲏ'),
     (0x2C8F, 'V'),
-    (0x2C90, 'M', u'ⲑ'),
+    (0x2C90, 'M', 'ⲑ'),
     (0x2C91, 'V'),
-    (0x2C92, 'M', u'ⲓ'),
+    (0x2C92, 'M', 'ⲓ'),
     (0x2C93, 'V'),
-    (0x2C94, 'M', u'ⲕ'),
+    (0x2C94, 'M', 'ⲕ'),
     (0x2C95, 'V'),
-    (0x2C96, 'M', u'ⲗ'),
+    (0x2C96, 'M', 'ⲗ'),
     (0x2C97, 'V'),
-    (0x2C98, 'M', u'ⲙ'),
+    (0x2C98, 'M', 'ⲙ'),
     (0x2C99, 'V'),
-    (0x2C9A, 'M', u'ⲛ'),
+    (0x2C9A, 'M', 'ⲛ'),
     (0x2C9B, 'V'),
-    (0x2C9C, 'M', u'ⲝ'),
+    (0x2C9C, 'M', 'ⲝ'),
     (0x2C9D, 'V'),
-    (0x2C9E, 'M', u'ⲟ'),
+    (0x2C9E, 'M', 'ⲟ'),
     (0x2C9F, 'V'),
-    (0x2CA0, 'M', u'ⲡ'),
+    (0x2CA0, 'M', 'ⲡ'),
     (0x2CA1, 'V'),
-    (0x2CA2, 'M', u'ⲣ'),
+    (0x2CA2, 'M', 'ⲣ'),
     (0x2CA3, 'V'),
-    (0x2CA4, 'M', u'ⲥ'),
+    (0x2CA4, 'M', 'ⲥ'),
     (0x2CA5, 'V'),
-    (0x2CA6, 'M', u'ⲧ'),
+    (0x2CA6, 'M', 'ⲧ'),
     (0x2CA7, 'V'),
-    (0x2CA8, 'M', u'ⲩ'),
+    (0x2CA8, 'M', 'ⲩ'),
     (0x2CA9, 'V'),
-    (0x2CAA, 'M', u'ⲫ'),
+    (0x2CAA, 'M', 'ⲫ'),
     (0x2CAB, 'V'),
-    (0x2CAC, 'M', u'ⲭ'),
+    (0x2CAC, 'M', 'ⲭ'),
     (0x2CAD, 'V'),
-    (0x2CAE, 'M', u'ⲯ'),
+    (0x2CAE, 'M', 'ⲯ'),
     (0x2CAF, 'V'),
-    (0x2CB0, 'M', u'ⲱ'),
+    (0x2CB0, 'M', 'ⲱ'),
     (0x2CB1, 'V'),
-    (0x2CB2, 'M', u'ⲳ'),
+    (0x2CB2, 'M', 'ⲳ'),
     (0x2CB3, 'V'),
-    (0x2CB4, 'M', u'ⲵ'),
+    (0x2CB4, 'M', 'ⲵ'),
     (0x2CB5, 'V'),
-    (0x2CB6, 'M', u'ⲷ'),
+    (0x2CB6, 'M', 'ⲷ'),
     (0x2CB7, 'V'),
-    (0x2CB8, 'M', u'ⲹ'),
+    (0x2CB8, 'M', 'ⲹ'),
     (0x2CB9, 'V'),
-    (0x2CBA, 'M', u'ⲻ'),
+    (0x2CBA, 'M', 'ⲻ'),
     (0x2CBB, 'V'),
-    (0x2CBC, 'M', u'ⲽ'),
+    (0x2CBC, 'M', 'ⲽ'),
     (0x2CBD, 'V'),
-    (0x2CBE, 'M', u'ⲿ'),
+    (0x2CBE, 'M', 'ⲿ'),
+    (0x2CBF, 'V'),
+    (0x2CC0, 'M', 'ⳁ'),
+    (0x2CC1, 'V'),
+    (0x2CC2, 'M', 'ⳃ'),
     ]
 
-def _seg_26():
+def _seg_26() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2CBF, 'V'),
-    (0x2CC0, 'M', u'ⳁ'),
-    (0x2CC1, 'V'),
-    (0x2CC2, 'M', u'ⳃ'),
     (0x2CC3, 'V'),
-    (0x2CC4, 'M', u'ⳅ'),
+    (0x2CC4, 'M', 'ⳅ'),
     (0x2CC5, 'V'),
-    (0x2CC6, 'M', u'ⳇ'),
+    (0x2CC6, 'M', 'ⳇ'),
     (0x2CC7, 'V'),
-    (0x2CC8, 'M', u'ⳉ'),
+    (0x2CC8, 'M', 'ⳉ'),
     (0x2CC9, 'V'),
-    (0x2CCA, 'M', u'ⳋ'),
+    (0x2CCA, 'M', 'ⳋ'),
     (0x2CCB, 'V'),
-    (0x2CCC, 'M', u'ⳍ'),
+    (0x2CCC, 'M', 'ⳍ'),
     (0x2CCD, 'V'),
-    (0x2CCE, 'M', u'ⳏ'),
+    (0x2CCE, 'M', 'ⳏ'),
     (0x2CCF, 'V'),
-    (0x2CD0, 'M', u'ⳑ'),
+    (0x2CD0, 'M', 'ⳑ'),
     (0x2CD1, 'V'),
-    (0x2CD2, 'M', u'ⳓ'),
+    (0x2CD2, 'M', 'ⳓ'),
     (0x2CD3, 'V'),
-    (0x2CD4, 'M', u'ⳕ'),
+    (0x2CD4, 'M', 'ⳕ'),
     (0x2CD5, 'V'),
-    (0x2CD6, 'M', u'ⳗ'),
+    (0x2CD6, 'M', 'ⳗ'),
     (0x2CD7, 'V'),
-    (0x2CD8, 'M', u'ⳙ'),
+    (0x2CD8, 'M', 'ⳙ'),
     (0x2CD9, 'V'),
-    (0x2CDA, 'M', u'ⳛ'),
+    (0x2CDA, 'M', 'ⳛ'),
     (0x2CDB, 'V'),
-    (0x2CDC, 'M', u'ⳝ'),
+    (0x2CDC, 'M', 'ⳝ'),
     (0x2CDD, 'V'),
-    (0x2CDE, 'M', u'ⳟ'),
+    (0x2CDE, 'M', 'ⳟ'),
     (0x2CDF, 'V'),
-    (0x2CE0, 'M', u'ⳡ'),
+    (0x2CE0, 'M', 'ⳡ'),
     (0x2CE1, 'V'),
-    (0x2CE2, 'M', u'ⳣ'),
+    (0x2CE2, 'M', 'ⳣ'),
     (0x2CE3, 'V'),
-    (0x2CEB, 'M', u'ⳬ'),
+    (0x2CEB, 'M', 'ⳬ'),
     (0x2CEC, 'V'),
-    (0x2CED, 'M', u'ⳮ'),
+    (0x2CED, 'M', 'ⳮ'),
     (0x2CEE, 'V'),
-    (0x2CF2, 'M', u'ⳳ'),
+    (0x2CF2, 'M', 'ⳳ'),
     (0x2CF3, 'V'),
     (0x2CF4, 'X'),
     (0x2CF9, 'V'),
@@ -2763,7 +2762,7 @@ def _seg_26():
     (0x2D2E, 'X'),
     (0x2D30, 'V'),
     (0x2D68, 'X'),
-    (0x2D6F, 'M', u'ⵡ'),
+    (0x2D6F, 'M', 'ⵡ'),
     (0x2D70, 'V'),
     (0x2D71, 'X'),
     (0x2D7F, 'V'),
@@ -2785,1159 +2784,1172 @@ def _seg_26():
     (0x2DD8, 'V'),
     (0x2DDF, 'X'),
     (0x2DE0, 'V'),
-    (0x2E53, 'X'),
+    (0x2E5E, 'X'),
     (0x2E80, 'V'),
     (0x2E9A, 'X'),
     (0x2E9B, 'V'),
-    (0x2E9F, 'M', u'母'),
+    (0x2E9F, 'M', '母'),
     (0x2EA0, 'V'),
-    (0x2EF3, 'M', u'龟'),
+    (0x2EF3, 'M', '龟'),
     (0x2EF4, 'X'),
-    (0x2F00, 'M', u'一'),
-    (0x2F01, 'M', u'丨'),
-    (0x2F02, 'M', u'丶'),
-    (0x2F03, 'M', u'丿'),
-    (0x2F04, 'M', u'乙'),
-    (0x2F05, 'M', u'亅'),
-    (0x2F06, 'M', u'二'),
-    (0x2F07, 'M', u'亠'),
-    (0x2F08, 'M', u'人'),
-    (0x2F09, 'M', u'儿'),
-    (0x2F0A, 'M', u'入'),
-    (0x2F0B, 'M', u'八'),
-    (0x2F0C, 'M', u'冂'),
-    (0x2F0D, 'M', u'冖'),
-    (0x2F0E, 'M', u'冫'),
-    (0x2F0F, 'M', u'几'),
-    (0x2F10, 'M', u'凵'),
-    (0x2F11, 'M', u'刀'),
+    (0x2F00, 'M', '一'),
+    (0x2F01, 'M', '丨'),
+    (0x2F02, 'M', '丶'),
+    (0x2F03, 'M', '丿'),
+    (0x2F04, 'M', '乙'),
+    (0x2F05, 'M', '亅'),
+    (0x2F06, 'M', '二'),
+    (0x2F07, 'M', '亠'),
+    (0x2F08, 'M', '人'),
+    (0x2F09, 'M', '儿'),
+    (0x2F0A, 'M', '入'),
+    (0x2F0B, 'M', '八'),
+    (0x2F0C, 'M', '冂'),
+    (0x2F0D, 'M', '冖'),
+    (0x2F0E, 'M', '冫'),
+    (0x2F0F, 'M', '几'),
+    (0x2F10, 'M', '凵'),
+    (0x2F11, 'M', '刀'),
+    (0x2F12, 'M', '力'),
+    (0x2F13, 'M', '勹'),
+    (0x2F14, 'M', '匕'),
+    (0x2F15, 'M', '匚'),
     ]
 
-def _seg_27():
+def _seg_27() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F12, 'M', u'力'),
-    (0x2F13, 'M', u'勹'),
-    (0x2F14, 'M', u'匕'),
-    (0x2F15, 'M', u'匚'),
-    (0x2F16, 'M', u'匸'),
-    (0x2F17, 'M', u'十'),
-    (0x2F18, 'M', u'卜'),
-    (0x2F19, 'M', u'卩'),
-    (0x2F1A, 'M', u'厂'),
-    (0x2F1B, 'M', u'厶'),
-    (0x2F1C, 'M', u'又'),
-    (0x2F1D, 'M', u'口'),
-    (0x2F1E, 'M', u'囗'),
-    (0x2F1F, 'M', u'土'),
-    (0x2F20, 'M', u'士'),
-    (0x2F21, 'M', u'夂'),
-    (0x2F22, 'M', u'夊'),
-    (0x2F23, 'M', u'夕'),
-    (0x2F24, 'M', u'大'),
-    (0x2F25, 'M', u'女'),
-    (0x2F26, 'M', u'子'),
-    (0x2F27, 'M', u'宀'),
-    (0x2F28, 'M', u'寸'),
-    (0x2F29, 'M', u'小'),
-    (0x2F2A, 'M', u'尢'),
-    (0x2F2B, 'M', u'尸'),
-    (0x2F2C, 'M', u'屮'),
-    (0x2F2D, 'M', u'山'),
-    (0x2F2E, 'M', u'巛'),
-    (0x2F2F, 'M', u'工'),
-    (0x2F30, 'M', u'己'),
-    (0x2F31, 'M', u'巾'),
-    (0x2F32, 'M', u'干'),
-    (0x2F33, 'M', u'幺'),
-    (0x2F34, 'M', u'广'),
-    (0x2F35, 'M', u'廴'),
-    (0x2F36, 'M', u'廾'),
-    (0x2F37, 'M', u'弋'),
-    (0x2F38, 'M', u'弓'),
-    (0x2F39, 'M', u'彐'),
-    (0x2F3A, 'M', u'彡'),
-    (0x2F3B, 'M', u'彳'),
-    (0x2F3C, 'M', u'心'),
-    (0x2F3D, 'M', u'戈'),
-    (0x2F3E, 'M', u'戶'),
-    (0x2F3F, 'M', u'手'),
-    (0x2F40, 'M', u'支'),
-    (0x2F41, 'M', u'攴'),
-    (0x2F42, 'M', u'文'),
-    (0x2F43, 'M', u'斗'),
-    (0x2F44, 'M', u'斤'),
-    (0x2F45, 'M', u'方'),
-    (0x2F46, 'M', u'无'),
-    (0x2F47, 'M', u'日'),
-    (0x2F48, 'M', u'曰'),
-    (0x2F49, 'M', u'月'),
-    (0x2F4A, 'M', u'木'),
-    (0x2F4B, 'M', u'欠'),
-    (0x2F4C, 'M', u'止'),
-    (0x2F4D, 'M', u'歹'),
-    (0x2F4E, 'M', u'殳'),
-    (0x2F4F, 'M', u'毋'),
-    (0x2F50, 'M', u'比'),
-    (0x2F51, 'M', u'毛'),
-    (0x2F52, 'M', u'氏'),
-    (0x2F53, 'M', u'气'),
-    (0x2F54, 'M', u'水'),
-    (0x2F55, 'M', u'火'),
-    (0x2F56, 'M', u'爪'),
-    (0x2F57, 'M', u'父'),
-    (0x2F58, 'M', u'爻'),
-    (0x2F59, 'M', u'爿'),
-    (0x2F5A, 'M', u'片'),
-    (0x2F5B, 'M', u'牙'),
-    (0x2F5C, 'M', u'牛'),
-    (0x2F5D, 'M', u'犬'),
-    (0x2F5E, 'M', u'玄'),
-    (0x2F5F, 'M', u'玉'),
-    (0x2F60, 'M', u'瓜'),
-    (0x2F61, 'M', u'瓦'),
-    (0x2F62, 'M', u'甘'),
-    (0x2F63, 'M', u'生'),
-    (0x2F64, 'M', u'用'),
-    (0x2F65, 'M', u'田'),
-    (0x2F66, 'M', u'疋'),
-    (0x2F67, 'M', u'疒'),
-    (0x2F68, 'M', u'癶'),
-    (0x2F69, 'M', u'白'),
-    (0x2F6A, 'M', u'皮'),
-    (0x2F6B, 'M', u'皿'),
-    (0x2F6C, 'M', u'目'),
-    (0x2F6D, 'M', u'矛'),
-    (0x2F6E, 'M', u'矢'),
-    (0x2F6F, 'M', u'石'),
-    (0x2F70, 'M', u'示'),
-    (0x2F71, 'M', u'禸'),
-    (0x2F72, 'M', u'禾'),
-    (0x2F73, 'M', u'穴'),
-    (0x2F74, 'M', u'立'),
-    (0x2F75, 'M', u'竹'),
+    (0x2F16, 'M', '匸'),
+    (0x2F17, 'M', '十'),
+    (0x2F18, 'M', '卜'),
+    (0x2F19, 'M', '卩'),
+    (0x2F1A, 'M', '厂'),
+    (0x2F1B, 'M', '厶'),
+    (0x2F1C, 'M', '又'),
+    (0x2F1D, 'M', '口'),
+    (0x2F1E, 'M', '囗'),
+    (0x2F1F, 'M', '土'),
+    (0x2F20, 'M', '士'),
+    (0x2F21, 'M', '夂'),
+    (0x2F22, 'M', '夊'),
+    (0x2F23, 'M', '夕'),
+    (0x2F24, 'M', '大'),
+    (0x2F25, 'M', '女'),
+    (0x2F26, 'M', '子'),
+    (0x2F27, 'M', '宀'),
+    (0x2F28, 'M', '寸'),
+    (0x2F29, 'M', '小'),
+    (0x2F2A, 'M', '尢'),
+    (0x2F2B, 'M', '尸'),
+    (0x2F2C, 'M', '屮'),
+    (0x2F2D, 'M', '山'),
+    (0x2F2E, 'M', '巛'),
+    (0x2F2F, 'M', '工'),
+    (0x2F30, 'M', '己'),
+    (0x2F31, 'M', '巾'),
+    (0x2F32, 'M', '干'),
+    (0x2F33, 'M', '幺'),
+    (0x2F34, 'M', '广'),
+    (0x2F35, 'M', '廴'),
+    (0x2F36, 'M', '廾'),
+    (0x2F37, 'M', '弋'),
+    (0x2F38, 'M', '弓'),
+    (0x2F39, 'M', '彐'),
+    (0x2F3A, 'M', '彡'),
+    (0x2F3B, 'M', '彳'),
+    (0x2F3C, 'M', '心'),
+    (0x2F3D, 'M', '戈'),
+    (0x2F3E, 'M', '戶'),
+    (0x2F3F, 'M', '手'),
+    (0x2F40, 'M', '支'),
+    (0x2F41, 'M', '攴'),
+    (0x2F42, 'M', '文'),
+    (0x2F43, 'M', '斗'),
+    (0x2F44, 'M', '斤'),
+    (0x2F45, 'M', '方'),
+    (0x2F46, 'M', '无'),
+    (0x2F47, 'M', '日'),
+    (0x2F48, 'M', '曰'),
+    (0x2F49, 'M', '月'),
+    (0x2F4A, 'M', '木'),
+    (0x2F4B, 'M', '欠'),
+    (0x2F4C, 'M', '止'),
+    (0x2F4D, 'M', '歹'),
+    (0x2F4E, 'M', '殳'),
+    (0x2F4F, 'M', '毋'),
+    (0x2F50, 'M', '比'),
+    (0x2F51, 'M', '毛'),
+    (0x2F52, 'M', '氏'),
+    (0x2F53, 'M', '气'),
+    (0x2F54, 'M', '水'),
+    (0x2F55, 'M', '火'),
+    (0x2F56, 'M', '爪'),
+    (0x2F57, 'M', '父'),
+    (0x2F58, 'M', '爻'),
+    (0x2F59, 'M', '爿'),
+    (0x2F5A, 'M', '片'),
+    (0x2F5B, 'M', '牙'),
+    (0x2F5C, 'M', '牛'),
+    (0x2F5D, 'M', '犬'),
+    (0x2F5E, 'M', '玄'),
+    (0x2F5F, 'M', '玉'),
+    (0x2F60, 'M', '瓜'),
+    (0x2F61, 'M', '瓦'),
+    (0x2F62, 'M', '甘'),
+    (0x2F63, 'M', '生'),
+    (0x2F64, 'M', '用'),
+    (0x2F65, 'M', '田'),
+    (0x2F66, 'M', '疋'),
+    (0x2F67, 'M', '疒'),
+    (0x2F68, 'M', '癶'),
+    (0x2F69, 'M', '白'),
+    (0x2F6A, 'M', '皮'),
+    (0x2F6B, 'M', '皿'),
+    (0x2F6C, 'M', '目'),
+    (0x2F6D, 'M', '矛'),
+    (0x2F6E, 'M', '矢'),
+    (0x2F6F, 'M', '石'),
+    (0x2F70, 'M', '示'),
+    (0x2F71, 'M', '禸'),
+    (0x2F72, 'M', '禾'),
+    (0x2F73, 'M', '穴'),
+    (0x2F74, 'M', '立'),
+    (0x2F75, 'M', '竹'),
+    (0x2F76, 'M', '米'),
+    (0x2F77, 'M', '糸'),
+    (0x2F78, 'M', '缶'),
+    (0x2F79, 'M', '网'),
     ]
 
-def _seg_28():
+def _seg_28() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F76, 'M', u'米'),
-    (0x2F77, 'M', u'糸'),
-    (0x2F78, 'M', u'缶'),
-    (0x2F79, 'M', u'网'),
-    (0x2F7A, 'M', u'羊'),
-    (0x2F7B, 'M', u'羽'),
-    (0x2F7C, 'M', u'老'),
-    (0x2F7D, 'M', u'而'),
-    (0x2F7E, 'M', u'耒'),
-    (0x2F7F, 'M', u'耳'),
-    (0x2F80, 'M', u'聿'),
-    (0x2F81, 'M', u'肉'),
-    (0x2F82, 'M', u'臣'),
-    (0x2F83, 'M', u'自'),
-    (0x2F84, 'M', u'至'),
-    (0x2F85, 'M', u'臼'),
-    (0x2F86, 'M', u'舌'),
-    (0x2F87, 'M', u'舛'),
-    (0x2F88, 'M', u'舟'),
-    (0x2F89, 'M', u'艮'),
-    (0x2F8A, 'M', u'色'),
-    (0x2F8B, 'M', u'艸'),
-    (0x2F8C, 'M', u'虍'),
-    (0x2F8D, 'M', u'虫'),
-    (0x2F8E, 'M', u'血'),
-    (0x2F8F, 'M', u'行'),
-    (0x2F90, 'M', u'衣'),
-    (0x2F91, 'M', u'襾'),
-    (0x2F92, 'M', u'見'),
-    (0x2F93, 'M', u'角'),
-    (0x2F94, 'M', u'言'),
-    (0x2F95, 'M', u'谷'),
-    (0x2F96, 'M', u'豆'),
-    (0x2F97, 'M', u'豕'),
-    (0x2F98, 'M', u'豸'),
-    (0x2F99, 'M', u'貝'),
-    (0x2F9A, 'M', u'赤'),
-    (0x2F9B, 'M', u'走'),
-    (0x2F9C, 'M', u'足'),
-    (0x2F9D, 'M', u'身'),
-    (0x2F9E, 'M', u'車'),
-    (0x2F9F, 'M', u'辛'),
-    (0x2FA0, 'M', u'辰'),
-    (0x2FA1, 'M', u'辵'),
-    (0x2FA2, 'M', u'邑'),
-    (0x2FA3, 'M', u'酉'),
-    (0x2FA4, 'M', u'釆'),
-    (0x2FA5, 'M', u'里'),
-    (0x2FA6, 'M', u'金'),
-    (0x2FA7, 'M', u'長'),
-    (0x2FA8, 'M', u'門'),
-    (0x2FA9, 'M', u'阜'),
-    (0x2FAA, 'M', u'隶'),
-    (0x2FAB, 'M', u'隹'),
-    (0x2FAC, 'M', u'雨'),
-    (0x2FAD, 'M', u'靑'),
-    (0x2FAE, 'M', u'非'),
-    (0x2FAF, 'M', u'面'),
-    (0x2FB0, 'M', u'革'),
-    (0x2FB1, 'M', u'韋'),
-    (0x2FB2, 'M', u'韭'),
-    (0x2FB3, 'M', u'音'),
-    (0x2FB4, 'M', u'頁'),
-    (0x2FB5, 'M', u'風'),
-    (0x2FB6, 'M', u'飛'),
-    (0x2FB7, 'M', u'食'),
-    (0x2FB8, 'M', u'首'),
-    (0x2FB9, 'M', u'香'),
-    (0x2FBA, 'M', u'馬'),
-    (0x2FBB, 'M', u'骨'),
-    (0x2FBC, 'M', u'高'),
-    (0x2FBD, 'M', u'髟'),
-    (0x2FBE, 'M', u'鬥'),
-    (0x2FBF, 'M', u'鬯'),
-    (0x2FC0, 'M', u'鬲'),
-    (0x2FC1, 'M', u'鬼'),
-    (0x2FC2, 'M', u'魚'),
-    (0x2FC3, 'M', u'鳥'),
-    (0x2FC4, 'M', u'鹵'),
-    (0x2FC5, 'M', u'鹿'),
-    (0x2FC6, 'M', u'麥'),
-    (0x2FC7, 'M', u'麻'),
-    (0x2FC8, 'M', u'黃'),
-    (0x2FC9, 'M', u'黍'),
-    (0x2FCA, 'M', u'黑'),
-    (0x2FCB, 'M', u'黹'),
-    (0x2FCC, 'M', u'黽'),
-    (0x2FCD, 'M', u'鼎'),
-    (0x2FCE, 'M', u'鼓'),
-    (0x2FCF, 'M', u'鼠'),
-    (0x2FD0, 'M', u'鼻'),
-    (0x2FD1, 'M', u'齊'),
-    (0x2FD2, 'M', u'齒'),
-    (0x2FD3, 'M', u'龍'),
-    (0x2FD4, 'M', u'龜'),
-    (0x2FD5, 'M', u'龠'),
+    (0x2F7A, 'M', '羊'),
+    (0x2F7B, 'M', '羽'),
+    (0x2F7C, 'M', '老'),
+    (0x2F7D, 'M', '而'),
+    (0x2F7E, 'M', '耒'),
+    (0x2F7F, 'M', '耳'),
+    (0x2F80, 'M', '聿'),
+    (0x2F81, 'M', '肉'),
+    (0x2F82, 'M', '臣'),
+    (0x2F83, 'M', '自'),
+    (0x2F84, 'M', '至'),
+    (0x2F85, 'M', '臼'),
+    (0x2F86, 'M', '舌'),
+    (0x2F87, 'M', '舛'),
+    (0x2F88, 'M', '舟'),
+    (0x2F89, 'M', '艮'),
+    (0x2F8A, 'M', '色'),
+    (0x2F8B, 'M', '艸'),
+    (0x2F8C, 'M', '虍'),
+    (0x2F8D, 'M', '虫'),
+    (0x2F8E, 'M', '血'),
+    (0x2F8F, 'M', '行'),
+    (0x2F90, 'M', '衣'),
+    (0x2F91, 'M', '襾'),
+    (0x2F92, 'M', '見'),
+    (0x2F93, 'M', '角'),
+    (0x2F94, 'M', '言'),
+    (0x2F95, 'M', '谷'),
+    (0x2F96, 'M', '豆'),
+    (0x2F97, 'M', '豕'),
+    (0x2F98, 'M', '豸'),
+    (0x2F99, 'M', '貝'),
+    (0x2F9A, 'M', '赤'),
+    (0x2F9B, 'M', '走'),
+    (0x2F9C, 'M', '足'),
+    (0x2F9D, 'M', '身'),
+    (0x2F9E, 'M', '車'),
+    (0x2F9F, 'M', '辛'),
+    (0x2FA0, 'M', '辰'),
+    (0x2FA1, 'M', '辵'),
+    (0x2FA2, 'M', '邑'),
+    (0x2FA3, 'M', '酉'),
+    (0x2FA4, 'M', '釆'),
+    (0x2FA5, 'M', '里'),
+    (0x2FA6, 'M', '金'),
+    (0x2FA7, 'M', '長'),
+    (0x2FA8, 'M', '門'),
+    (0x2FA9, 'M', '阜'),
+    (0x2FAA, 'M', '隶'),
+    (0x2FAB, 'M', '隹'),
+    (0x2FAC, 'M', '雨'),
+    (0x2FAD, 'M', '靑'),
+    (0x2FAE, 'M', '非'),
+    (0x2FAF, 'M', '面'),
+    (0x2FB0, 'M', '革'),
+    (0x2FB1, 'M', '韋'),
+    (0x2FB2, 'M', '韭'),
+    (0x2FB3, 'M', '音'),
+    (0x2FB4, 'M', '頁'),
+    (0x2FB5, 'M', '風'),
+    (0x2FB6, 'M', '飛'),
+    (0x2FB7, 'M', '食'),
+    (0x2FB8, 'M', '首'),
+    (0x2FB9, 'M', '香'),
+    (0x2FBA, 'M', '馬'),
+    (0x2FBB, 'M', '骨'),
+    (0x2FBC, 'M', '高'),
+    (0x2FBD, 'M', '髟'),
+    (0x2FBE, 'M', '鬥'),
+    (0x2FBF, 'M', '鬯'),
+    (0x2FC0, 'M', '鬲'),
+    (0x2FC1, 'M', '鬼'),
+    (0x2FC2, 'M', '魚'),
+    (0x2FC3, 'M', '鳥'),
+    (0x2FC4, 'M', '鹵'),
+    (0x2FC5, 'M', '鹿'),
+    (0x2FC6, 'M', '麥'),
+    (0x2FC7, 'M', '麻'),
+    (0x2FC8, 'M', '黃'),
+    (0x2FC9, 'M', '黍'),
+    (0x2FCA, 'M', '黑'),
+    (0x2FCB, 'M', '黹'),
+    (0x2FCC, 'M', '黽'),
+    (0x2FCD, 'M', '鼎'),
+    (0x2FCE, 'M', '鼓'),
+    (0x2FCF, 'M', '鼠'),
+    (0x2FD0, 'M', '鼻'),
+    (0x2FD1, 'M', '齊'),
+    (0x2FD2, 'M', '齒'),
+    (0x2FD3, 'M', '龍'),
+    (0x2FD4, 'M', '龜'),
+    (0x2FD5, 'M', '龠'),
     (0x2FD6, 'X'),
-    (0x3000, '3', u' '),
+    (0x3000, '3', ' '),
     (0x3001, 'V'),
-    (0x3002, 'M', u'.'),
+    (0x3002, 'M', '.'),
+    (0x3003, 'V'),
+    (0x3036, 'M', '〒'),
+    (0x3037, 'V'),
+    (0x3038, 'M', '十'),
     ]
 
-def _seg_29():
+def _seg_29() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x3003, 'V'),
-    (0x3036, 'M', u'〒'),
-    (0x3037, 'V'),
-    (0x3038, 'M', u'十'),
-    (0x3039, 'M', u'卄'),
-    (0x303A, 'M', u'卅'),
+    (0x3039, 'M', '卄'),
+    (0x303A, 'M', '卅'),
     (0x303B, 'V'),
     (0x3040, 'X'),
     (0x3041, 'V'),
     (0x3097, 'X'),
     (0x3099, 'V'),
-    (0x309B, '3', u' ゙'),
-    (0x309C, '3', u' ゚'),
+    (0x309B, '3', ' ゙'),
+    (0x309C, '3', ' ゚'),
     (0x309D, 'V'),
-    (0x309F, 'M', u'より'),
+    (0x309F, 'M', 'より'),
     (0x30A0, 'V'),
-    (0x30FF, 'M', u'コト'),
+    (0x30FF, 'M', 'コト'),
     (0x3100, 'X'),
     (0x3105, 'V'),
     (0x3130, 'X'),
-    (0x3131, 'M', u'ᄀ'),
-    (0x3132, 'M', u'ᄁ'),
-    (0x3133, 'M', u'ᆪ'),
-    (0x3134, 'M', u'ᄂ'),
-    (0x3135, 'M', u'ᆬ'),
-    (0x3136, 'M', u'ᆭ'),
-    (0x3137, 'M', u'ᄃ'),
-    (0x3138, 'M', u'ᄄ'),
-    (0x3139, 'M', u'ᄅ'),
-    (0x313A, 'M', u'ᆰ'),
-    (0x313B, 'M', u'ᆱ'),
-    (0x313C, 'M', u'ᆲ'),
-    (0x313D, 'M', u'ᆳ'),
-    (0x313E, 'M', u'ᆴ'),
-    (0x313F, 'M', u'ᆵ'),
-    (0x3140, 'M', u'ᄚ'),
-    (0x3141, 'M', u'ᄆ'),
-    (0x3142, 'M', u'ᄇ'),
-    (0x3143, 'M', u'ᄈ'),
-    (0x3144, 'M', u'ᄡ'),
-    (0x3145, 'M', u'ᄉ'),
-    (0x3146, 'M', u'ᄊ'),
-    (0x3147, 'M', u'ᄋ'),
-    (0x3148, 'M', u'ᄌ'),
-    (0x3149, 'M', u'ᄍ'),
-    (0x314A, 'M', u'ᄎ'),
-    (0x314B, 'M', u'ᄏ'),
-    (0x314C, 'M', u'ᄐ'),
-    (0x314D, 'M', u'ᄑ'),
-    (0x314E, 'M', u'ᄒ'),
-    (0x314F, 'M', u'ᅡ'),
-    (0x3150, 'M', u'ᅢ'),
-    (0x3151, 'M', u'ᅣ'),
-    (0x3152, 'M', u'ᅤ'),
-    (0x3153, 'M', u'ᅥ'),
-    (0x3154, 'M', u'ᅦ'),
-    (0x3155, 'M', u'ᅧ'),
-    (0x3156, 'M', u'ᅨ'),
-    (0x3157, 'M', u'ᅩ'),
-    (0x3158, 'M', u'ᅪ'),
-    (0x3159, 'M', u'ᅫ'),
-    (0x315A, 'M', u'ᅬ'),
-    (0x315B, 'M', u'ᅭ'),
-    (0x315C, 'M', u'ᅮ'),
-    (0x315D, 'M', u'ᅯ'),
-    (0x315E, 'M', u'ᅰ'),
-    (0x315F, 'M', u'ᅱ'),
-    (0x3160, 'M', u'ᅲ'),
-    (0x3161, 'M', u'ᅳ'),
-    (0x3162, 'M', u'ᅴ'),
-    (0x3163, 'M', u'ᅵ'),
+    (0x3131, 'M', 'ᄀ'),
+    (0x3132, 'M', 'ᄁ'),
+    (0x3133, 'M', 'ᆪ'),
+    (0x3134, 'M', 'ᄂ'),
+    (0x3135, 'M', 'ᆬ'),
+    (0x3136, 'M', 'ᆭ'),
+    (0x3137, 'M', 'ᄃ'),
+    (0x3138, 'M', 'ᄄ'),
+    (0x3139, 'M', 'ᄅ'),
+    (0x313A, 'M', 'ᆰ'),
+    (0x313B, 'M', 'ᆱ'),
+    (0x313C, 'M', 'ᆲ'),
+    (0x313D, 'M', 'ᆳ'),
+    (0x313E, 'M', 'ᆴ'),
+    (0x313F, 'M', 'ᆵ'),
+    (0x3140, 'M', 'ᄚ'),
+    (0x3141, 'M', 'ᄆ'),
+    (0x3142, 'M', 'ᄇ'),
+    (0x3143, 'M', 'ᄈ'),
+    (0x3144, 'M', 'ᄡ'),
+    (0x3145, 'M', 'ᄉ'),
+    (0x3146, 'M', 'ᄊ'),
+    (0x3147, 'M', 'ᄋ'),
+    (0x3148, 'M', 'ᄌ'),
+    (0x3149, 'M', 'ᄍ'),
+    (0x314A, 'M', 'ᄎ'),
+    (0x314B, 'M', 'ᄏ'),
+    (0x314C, 'M', 'ᄐ'),
+    (0x314D, 'M', 'ᄑ'),
+    (0x314E, 'M', 'ᄒ'),
+    (0x314F, 'M', 'ᅡ'),
+    (0x3150, 'M', 'ᅢ'),
+    (0x3151, 'M', 'ᅣ'),
+    (0x3152, 'M', 'ᅤ'),
+    (0x3153, 'M', 'ᅥ'),
+    (0x3154, 'M', 'ᅦ'),
+    (0x3155, 'M', 'ᅧ'),
+    (0x3156, 'M', 'ᅨ'),
+    (0x3157, 'M', 'ᅩ'),
+    (0x3158, 'M', 'ᅪ'),
+    (0x3159, 'M', 'ᅫ'),
+    (0x315A, 'M', 'ᅬ'),
+    (0x315B, 'M', 'ᅭ'),
+    (0x315C, 'M', 'ᅮ'),
+    (0x315D, 'M', 'ᅯ'),
+    (0x315E, 'M', 'ᅰ'),
+    (0x315F, 'M', 'ᅱ'),
+    (0x3160, 'M', 'ᅲ'),
+    (0x3161, 'M', 'ᅳ'),
+    (0x3162, 'M', 'ᅴ'),
+    (0x3163, 'M', 'ᅵ'),
     (0x3164, 'X'),
-    (0x3165, 'M', u'ᄔ'),
-    (0x3166, 'M', u'ᄕ'),
-    (0x3167, 'M', u'ᇇ'),
-    (0x3168, 'M', u'ᇈ'),
-    (0x3169, 'M', u'ᇌ'),
-    (0x316A, 'M', u'ᇎ'),
-    (0x316B, 'M', u'ᇓ'),
-    (0x316C, 'M', u'ᇗ'),
-    (0x316D, 'M', u'ᇙ'),
-    (0x316E, 'M', u'ᄜ'),
-    (0x316F, 'M', u'ᇝ'),
-    (0x3170, 'M', u'ᇟ'),
-    (0x3171, 'M', u'ᄝ'),
-    (0x3172, 'M', u'ᄞ'),
-    (0x3173, 'M', u'ᄠ'),
-    (0x3174, 'M', u'ᄢ'),
-    (0x3175, 'M', u'ᄣ'),
-    (0x3176, 'M', u'ᄧ'),
-    (0x3177, 'M', u'ᄩ'),
-    (0x3178, 'M', u'ᄫ'),
-    (0x3179, 'M', u'ᄬ'),
-    (0x317A, 'M', u'ᄭ'),
-    (0x317B, 'M', u'ᄮ'),
-    (0x317C, 'M', u'ᄯ'),
-    (0x317D, 'M', u'ᄲ'),
-    (0x317E, 'M', u'ᄶ'),
-    (0x317F, 'M', u'ᅀ'),
-    (0x3180, 'M', u'ᅇ'),
+    (0x3165, 'M', 'ᄔ'),
+    (0x3166, 'M', 'ᄕ'),
+    (0x3167, 'M', 'ᇇ'),
+    (0x3168, 'M', 'ᇈ'),
+    (0x3169, 'M', 'ᇌ'),
+    (0x316A, 'M', 'ᇎ'),
+    (0x316B, 'M', 'ᇓ'),
+    (0x316C, 'M', 'ᇗ'),
+    (0x316D, 'M', 'ᇙ'),
+    (0x316E, 'M', 'ᄜ'),
+    (0x316F, 'M', 'ᇝ'),
+    (0x3170, 'M', 'ᇟ'),
+    (0x3171, 'M', 'ᄝ'),
+    (0x3172, 'M', 'ᄞ'),
+    (0x3173, 'M', 'ᄠ'),
+    (0x3174, 'M', 'ᄢ'),
+    (0x3175, 'M', 'ᄣ'),
+    (0x3176, 'M', 'ᄧ'),
+    (0x3177, 'M', 'ᄩ'),
+    (0x3178, 'M', 'ᄫ'),
+    (0x3179, 'M', 'ᄬ'),
+    (0x317A, 'M', 'ᄭ'),
+    (0x317B, 'M', 'ᄮ'),
+    (0x317C, 'M', 'ᄯ'),
+    (0x317D, 'M', 'ᄲ'),
+    (0x317E, 'M', 'ᄶ'),
+    (0x317F, 'M', 'ᅀ'),
+    (0x3180, 'M', 'ᅇ'),
+    (0x3181, 'M', 'ᅌ'),
+    (0x3182, 'M', 'ᇱ'),
+    (0x3183, 'M', 'ᇲ'),
+    (0x3184, 'M', 'ᅗ'),
     ]
 
-def _seg_30():
+def _seg_30() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x3181, 'M', u'ᅌ'),
-    (0x3182, 'M', u'ᇱ'),
-    (0x3183, 'M', u'ᇲ'),
-    (0x3184, 'M', u'ᅗ'),
-    (0x3185, 'M', u'ᅘ'),
-    (0x3186, 'M', u'ᅙ'),
-    (0x3187, 'M', u'ᆄ'),
-    (0x3188, 'M', u'ᆅ'),
-    (0x3189, 'M', u'ᆈ'),
-    (0x318A, 'M', u'ᆑ'),
-    (0x318B, 'M', u'ᆒ'),
-    (0x318C, 'M', u'ᆔ'),
-    (0x318D, 'M', u'ᆞ'),
-    (0x318E, 'M', u'ᆡ'),
+    (0x3185, 'M', 'ᅘ'),
+    (0x3186, 'M', 'ᅙ'),
+    (0x3187, 'M', 'ᆄ'),
+    (0x3188, 'M', 'ᆅ'),
+    (0x3189, 'M', 'ᆈ'),
+    (0x318A, 'M', 'ᆑ'),
+    (0x318B, 'M', 'ᆒ'),
+    (0x318C, 'M', 'ᆔ'),
+    (0x318D, 'M', 'ᆞ'),
+    (0x318E, 'M', 'ᆡ'),
     (0x318F, 'X'),
     (0x3190, 'V'),
-    (0x3192, 'M', u'一'),
-    (0x3193, 'M', u'二'),
-    (0x3194, 'M', u'三'),
-    (0x3195, 'M', u'四'),
-    (0x3196, 'M', u'上'),
-    (0x3197, 'M', u'中'),
-    (0x3198, 'M', u'下'),
-    (0x3199, 'M', u'甲'),
-    (0x319A, 'M', u'乙'),
-    (0x319B, 'M', u'丙'),
-    (0x319C, 'M', u'丁'),
-    (0x319D, 'M', u'天'),
-    (0x319E, 'M', u'地'),
-    (0x319F, 'M', u'人'),
+    (0x3192, 'M', '一'),
+    (0x3193, 'M', '二'),
+    (0x3194, 'M', '三'),
+    (0x3195, 'M', '四'),
+    (0x3196, 'M', '上'),
+    (0x3197, 'M', '中'),
+    (0x3198, 'M', '下'),
+    (0x3199, 'M', '甲'),
+    (0x319A, 'M', '乙'),
+    (0x319B, 'M', '丙'),
+    (0x319C, 'M', '丁'),
+    (0x319D, 'M', '天'),
+    (0x319E, 'M', '地'),
+    (0x319F, 'M', '人'),
     (0x31A0, 'V'),
     (0x31E4, 'X'),
     (0x31F0, 'V'),
-    (0x3200, '3', u'(ᄀ)'),
-    (0x3201, '3', u'(ᄂ)'),
-    (0x3202, '3', u'(ᄃ)'),
-    (0x3203, '3', u'(ᄅ)'),
-    (0x3204, '3', u'(ᄆ)'),
-    (0x3205, '3', u'(ᄇ)'),
-    (0x3206, '3', u'(ᄉ)'),
-    (0x3207, '3', u'(ᄋ)'),
-    (0x3208, '3', u'(ᄌ)'),
-    (0x3209, '3', u'(ᄎ)'),
-    (0x320A, '3', u'(ᄏ)'),
-    (0x320B, '3', u'(ᄐ)'),
-    (0x320C, '3', u'(ᄑ)'),
-    (0x320D, '3', u'(ᄒ)'),
-    (0x320E, '3', u'(가)'),
-    (0x320F, '3', u'(나)'),
-    (0x3210, '3', u'(다)'),
-    (0x3211, '3', u'(라)'),
-    (0x3212, '3', u'(마)'),
-    (0x3213, '3', u'(바)'),
-    (0x3214, '3', u'(사)'),
-    (0x3215, '3', u'(아)'),
-    (0x3216, '3', u'(자)'),
-    (0x3217, '3', u'(차)'),
-    (0x3218, '3', u'(카)'),
-    (0x3219, '3', u'(타)'),
-    (0x321A, '3', u'(파)'),
-    (0x321B, '3', u'(하)'),
-    (0x321C, '3', u'(주)'),
-    (0x321D, '3', u'(오전)'),
-    (0x321E, '3', u'(오후)'),
+    (0x3200, '3', '(ᄀ)'),
+    (0x3201, '3', '(ᄂ)'),
+    (0x3202, '3', '(ᄃ)'),
+    (0x3203, '3', '(ᄅ)'),
+    (0x3204, '3', '(ᄆ)'),
+    (0x3205, '3', '(ᄇ)'),
+    (0x3206, '3', '(ᄉ)'),
+    (0x3207, '3', '(ᄋ)'),
+    (0x3208, '3', '(ᄌ)'),
+    (0x3209, '3', '(ᄎ)'),
+    (0x320A, '3', '(ᄏ)'),
+    (0x320B, '3', '(ᄐ)'),
+    (0x320C, '3', '(ᄑ)'),
+    (0x320D, '3', '(ᄒ)'),
+    (0x320E, '3', '(가)'),
+    (0x320F, '3', '(나)'),
+    (0x3210, '3', '(다)'),
+    (0x3211, '3', '(라)'),
+    (0x3212, '3', '(마)'),
+    (0x3213, '3', '(바)'),
+    (0x3214, '3', '(사)'),
+    (0x3215, '3', '(아)'),
+    (0x3216, '3', '(자)'),
+    (0x3217, '3', '(차)'),
+    (0x3218, '3', '(카)'),
+    (0x3219, '3', '(타)'),
+    (0x321A, '3', '(파)'),
+    (0x321B, '3', '(하)'),
+    (0x321C, '3', '(주)'),
+    (0x321D, '3', '(오전)'),
+    (0x321E, '3', '(오후)'),
     (0x321F, 'X'),
-    (0x3220, '3', u'(一)'),
-    (0x3221, '3', u'(二)'),
-    (0x3222, '3', u'(三)'),
-    (0x3223, '3', u'(四)'),
-    (0x3224, '3', u'(五)'),
-    (0x3225, '3', u'(六)'),
-    (0x3226, '3', u'(七)'),
-    (0x3227, '3', u'(八)'),
-    (0x3228, '3', u'(九)'),
-    (0x3229, '3', u'(十)'),
-    (0x322A, '3', u'(月)'),
-    (0x322B, '3', u'(火)'),
-    (0x322C, '3', u'(水)'),
-    (0x322D, '3', u'(木)'),
-    (0x322E, '3', u'(金)'),
-    (0x322F, '3', u'(土)'),
-    (0x3230, '3', u'(日)'),
-    (0x3231, '3', u'(株)'),
-    (0x3232, '3', u'(有)'),
-    (0x3233, '3', u'(社)'),
-    (0x3234, '3', u'(名)'),
-    (0x3235, '3', u'(特)'),
-    (0x3236, '3', u'(財)'),
-    (0x3237, '3', u'(祝)'),
-    (0x3238, '3', u'(労)'),
-    (0x3239, '3', u'(代)'),
-    (0x323A, '3', u'(呼)'),
-    (0x323B, '3', u'(学)'),
-    (0x323C, '3', u'(監)'),
-    (0x323D, '3', u'(企)'),
-    (0x323E, '3', u'(資)'),
-    (0x323F, '3', u'(協)'),
-    (0x3240, '3', u'(祭)'),
-    (0x3241, '3', u'(休)'),
-    (0x3242, '3', u'(自)'),
+    (0x3220, '3', '(一)'),
+    (0x3221, '3', '(二)'),
+    (0x3222, '3', '(三)'),
+    (0x3223, '3', '(四)'),
+    (0x3224, '3', '(五)'),
+    (0x3225, '3', '(六)'),
+    (0x3226, '3', '(七)'),
+    (0x3227, '3', '(八)'),
+    (0x3228, '3', '(九)'),
+    (0x3229, '3', '(十)'),
+    (0x322A, '3', '(月)'),
+    (0x322B, '3', '(火)'),
+    (0x322C, '3', '(水)'),
+    (0x322D, '3', '(木)'),
+    (0x322E, '3', '(金)'),
+    (0x322F, '3', '(土)'),
+    (0x3230, '3', '(日)'),
+    (0x3231, '3', '(株)'),
+    (0x3232, '3', '(有)'),
+    (0x3233, '3', '(社)'),
+    (0x3234, '3', '(名)'),
+    (0x3235, '3', '(特)'),
+    (0x3236, '3', '(財)'),
+    (0x3237, '3', '(祝)'),
+    (0x3238, '3', '(労)'),
+    (0x3239, '3', '(代)'),
+    (0x323A, '3', '(呼)'),
+    (0x323B, '3', '(学)'),
+    (0x323C, '3', '(監)'),
+    (0x323D, '3', '(企)'),
+    (0x323E, '3', '(資)'),
+    (0x323F, '3', '(協)'),
+    (0x3240, '3', '(祭)'),
+    (0x3241, '3', '(休)'),
+    (0x3242, '3', '(自)'),
+    (0x3243, '3', '(至)'),
+    (0x3244, 'M', '問'),
+    (0x3245, 'M', '幼'),
+    (0x3246, 'M', '文'),
     ]
 
-def _seg_31():
+def _seg_31() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x3243, '3', u'(至)'),
-    (0x3244, 'M', u'問'),
-    (0x3245, 'M', u'幼'),
-    (0x3246, 'M', u'文'),
-    (0x3247, 'M', u'箏'),
+    (0x3247, 'M', '箏'),
     (0x3248, 'V'),
-    (0x3250, 'M', u'pte'),
-    (0x3251, 'M', u'21'),
-    (0x3252, 'M', u'22'),
-    (0x3253, 'M', u'23'),
-    (0x3254, 'M', u'24'),
-    (0x3255, 'M', u'25'),
-    (0x3256, 'M', u'26'),
-    (0x3257, 'M', u'27'),
-    (0x3258, 'M', u'28'),
-    (0x3259, 'M', u'29'),
-    (0x325A, 'M', u'30'),
-    (0x325B, 'M', u'31'),
-    (0x325C, 'M', u'32'),
-    (0x325D, 'M', u'33'),
-    (0x325E, 'M', u'34'),
-    (0x325F, 'M', u'35'),
-    (0x3260, 'M', u'ᄀ'),
-    (0x3261, 'M', u'ᄂ'),
-    (0x3262, 'M', u'ᄃ'),
-    (0x3263, 'M', u'ᄅ'),
-    (0x3264, 'M', u'ᄆ'),
-    (0x3265, 'M', u'ᄇ'),
-    (0x3266, 'M', u'ᄉ'),
-    (0x3267, 'M', u'ᄋ'),
-    (0x3268, 'M', u'ᄌ'),
-    (0x3269, 'M', u'ᄎ'),
-    (0x326A, 'M', u'ᄏ'),
-    (0x326B, 'M', u'ᄐ'),
-    (0x326C, 'M', u'ᄑ'),
-    (0x326D, 'M', u'ᄒ'),
-    (0x326E, 'M', u'가'),
-    (0x326F, 'M', u'나'),
-    (0x3270, 'M', u'다'),
-    (0x3271, 'M', u'라'),
-    (0x3272, 'M', u'마'),
-    (0x3273, 'M', u'바'),
-    (0x3274, 'M', u'사'),
-    (0x3275, 'M', u'아'),
-    (0x3276, 'M', u'자'),
-    (0x3277, 'M', u'차'),
-    (0x3278, 'M', u'카'),
-    (0x3279, 'M', u'타'),
-    (0x327A, 'M', u'파'),
-    (0x327B, 'M', u'하'),
-    (0x327C, 'M', u'참고'),
-    (0x327D, 'M', u'주의'),
-    (0x327E, 'M', u'우'),
+    (0x3250, 'M', 'pte'),
+    (0x3251, 'M', '21'),
+    (0x3252, 'M', '22'),
+    (0x3253, 'M', '23'),
+    (0x3254, 'M', '24'),
+    (0x3255, 'M', '25'),
+    (0x3256, 'M', '26'),
+    (0x3257, 'M', '27'),
+    (0x3258, 'M', '28'),
+    (0x3259, 'M', '29'),
+    (0x325A, 'M', '30'),
+    (0x325B, 'M', '31'),
+    (0x325C, 'M', '32'),
+    (0x325D, 'M', '33'),
+    (0x325E, 'M', '34'),
+    (0x325F, 'M', '35'),
+    (0x3260, 'M', 'ᄀ'),
+    (0x3261, 'M', 'ᄂ'),
+    (0x3262, 'M', 'ᄃ'),
+    (0x3263, 'M', 'ᄅ'),
+    (0x3264, 'M', 'ᄆ'),
+    (0x3265, 'M', 'ᄇ'),
+    (0x3266, 'M', 'ᄉ'),
+    (0x3267, 'M', 'ᄋ'),
+    (0x3268, 'M', 'ᄌ'),
+    (0x3269, 'M', 'ᄎ'),
+    (0x326A, 'M', 'ᄏ'),
+    (0x326B, 'M', 'ᄐ'),
+    (0x326C, 'M', 'ᄑ'),
+    (0x326D, 'M', 'ᄒ'),
+    (0x326E, 'M', '가'),
+    (0x326F, 'M', '나'),
+    (0x3270, 'M', '다'),
+    (0x3271, 'M', '라'),
+    (0x3272, 'M', '마'),
+    (0x3273, 'M', '바'),
+    (0x3274, 'M', '사'),
+    (0x3275, 'M', '아'),
+    (0x3276, 'M', '자'),
+    (0x3277, 'M', '차'),
+    (0x3278, 'M', '카'),
+    (0x3279, 'M', '타'),
+    (0x327A, 'M', '파'),
+    (0x327B, 'M', '하'),
+    (0x327C, 'M', '참고'),
+    (0x327D, 'M', '주의'),
+    (0x327E, 'M', '우'),
     (0x327F, 'V'),
-    (0x3280, 'M', u'一'),
-    (0x3281, 'M', u'二'),
-    (0x3282, 'M', u'三'),
-    (0x3283, 'M', u'四'),
-    (0x3284, 'M', u'五'),
-    (0x3285, 'M', u'六'),
-    (0x3286, 'M', u'七'),
-    (0x3287, 'M', u'八'),
-    (0x3288, 'M', u'九'),
-    (0x3289, 'M', u'十'),
-    (0x328A, 'M', u'月'),
-    (0x328B, 'M', u'火'),
-    (0x328C, 'M', u'水'),
-    (0x328D, 'M', u'木'),
-    (0x328E, 'M', u'金'),
-    (0x328F, 'M', u'土'),
-    (0x3290, 'M', u'日'),
-    (0x3291, 'M', u'株'),
-    (0x3292, 'M', u'有'),
-    (0x3293, 'M', u'社'),
-    (0x3294, 'M', u'名'),
-    (0x3295, 'M', u'特'),
-    (0x3296, 'M', u'財'),
-    (0x3297, 'M', u'祝'),
-    (0x3298, 'M', u'労'),
-    (0x3299, 'M', u'秘'),
-    (0x329A, 'M', u'男'),
-    (0x329B, 'M', u'女'),
-    (0x329C, 'M', u'適'),
-    (0x329D, 'M', u'優'),
-    (0x329E, 'M', u'印'),
-    (0x329F, 'M', u'注'),
-    (0x32A0, 'M', u'項'),
-    (0x32A1, 'M', u'休'),
-    (0x32A2, 'M', u'写'),
-    (0x32A3, 'M', u'正'),
-    (0x32A4, 'M', u'上'),
-    (0x32A5, 'M', u'中'),
-    (0x32A6, 'M', u'下'),
-    (0x32A7, 'M', u'左'),
-    (0x32A8, 'M', u'右'),
-    (0x32A9, 'M', u'医'),
-    (0x32AA, 'M', u'宗'),
-    (0x32AB, 'M', u'学'),
-    (0x32AC, 'M', u'監'),
-    (0x32AD, 'M', u'企'),
+    (0x3280, 'M', '一'),
+    (0x3281, 'M', '二'),
+    (0x3282, 'M', '三'),
+    (0x3283, 'M', '四'),
+    (0x3284, 'M', '五'),
+    (0x3285, 'M', '六'),
+    (0x3286, 'M', '七'),
+    (0x3287, 'M', '八'),
+    (0x3288, 'M', '九'),
+    (0x3289, 'M', '十'),
+    (0x328A, 'M', '月'),
+    (0x328B, 'M', '火'),
+    (0x328C, 'M', '水'),
+    (0x328D, 'M', '木'),
+    (0x328E, 'M', '金'),
+    (0x328F, 'M', '土'),
+    (0x3290, 'M', '日'),
+    (0x3291, 'M', '株'),
+    (0x3292, 'M', '有'),
+    (0x3293, 'M', '社'),
+    (0x3294, 'M', '名'),
+    (0x3295, 'M', '特'),
+    (0x3296, 'M', '財'),
+    (0x3297, 'M', '祝'),
+    (0x3298, 'M', '労'),
+    (0x3299, 'M', '秘'),
+    (0x329A, 'M', '男'),
+    (0x329B, 'M', '女'),
+    (0x329C, 'M', '適'),
+    (0x329D, 'M', '優'),
+    (0x329E, 'M', '印'),
+    (0x329F, 'M', '注'),
+    (0x32A0, 'M', '項'),
+    (0x32A1, 'M', '休'),
+    (0x32A2, 'M', '写'),
+    (0x32A3, 'M', '正'),
+    (0x32A4, 'M', '上'),
+    (0x32A5, 'M', '中'),
+    (0x32A6, 'M', '下'),
+    (0x32A7, 'M', '左'),
+    (0x32A8, 'M', '右'),
+    (0x32A9, 'M', '医'),
+    (0x32AA, 'M', '宗'),
+    (0x32AB, 'M', '学'),
+    (0x32AC, 'M', '監'),
+    (0x32AD, 'M', '企'),
+    (0x32AE, 'M', '資'),
+    (0x32AF, 'M', '協'),
+    (0x32B0, 'M', '夜'),
+    (0x32B1, 'M', '36'),
     ]
 
-def _seg_32():
+def _seg_32() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x32AE, 'M', u'資'),
-    (0x32AF, 'M', u'協'),
-    (0x32B0, 'M', u'夜'),
-    (0x32B1, 'M', u'36'),
-    (0x32B2, 'M', u'37'),
-    (0x32B3, 'M', u'38'),
-    (0x32B4, 'M', u'39'),
-    (0x32B5, 'M', u'40'),
-    (0x32B6, 'M', u'41'),
-    (0x32B7, 'M', u'42'),
-    (0x32B8, 'M', u'43'),
-    (0x32B9, 'M', u'44'),
-    (0x32BA, 'M', u'45'),
-    (0x32BB, 'M', u'46'),
-    (0x32BC, 'M', u'47'),
-    (0x32BD, 'M', u'48'),
-    (0x32BE, 'M', u'49'),
-    (0x32BF, 'M', u'50'),
-    (0x32C0, 'M', u'1月'),
-    (0x32C1, 'M', u'2月'),
-    (0x32C2, 'M', u'3月'),
-    (0x32C3, 'M', u'4月'),
-    (0x32C4, 'M', u'5月'),
-    (0x32C5, 'M', u'6月'),
-    (0x32C6, 'M', u'7月'),
-    (0x32C7, 'M', u'8月'),
-    (0x32C8, 'M', u'9月'),
-    (0x32C9, 'M', u'10月'),
-    (0x32CA, 'M', u'11月'),
-    (0x32CB, 'M', u'12月'),
-    (0x32CC, 'M', u'hg'),
-    (0x32CD, 'M', u'erg'),
-    (0x32CE, 'M', u'ev'),
-    (0x32CF, 'M', u'ltd'),
-    (0x32D0, 'M', u'ア'),
-    (0x32D1, 'M', u'イ'),
-    (0x32D2, 'M', u'ウ'),
-    (0x32D3, 'M', u'エ'),
-    (0x32D4, 'M', u'オ'),
-    (0x32D5, 'M', u'カ'),
-    (0x32D6, 'M', u'キ'),
-    (0x32D7, 'M', u'ク'),
-    (0x32D8, 'M', u'ケ'),
-    (0x32D9, 'M', u'コ'),
-    (0x32DA, 'M', u'サ'),
-    (0x32DB, 'M', u'シ'),
-    (0x32DC, 'M', u'ス'),
-    (0x32DD, 'M', u'セ'),
-    (0x32DE, 'M', u'ソ'),
-    (0x32DF, 'M', u'タ'),
-    (0x32E0, 'M', u'チ'),
-    (0x32E1, 'M', u'ツ'),
-    (0x32E2, 'M', u'テ'),
-    (0x32E3, 'M', u'ト'),
-    (0x32E4, 'M', u'ナ'),
-    (0x32E5, 'M', u'ニ'),
-    (0x32E6, 'M', u'ヌ'),
-    (0x32E7, 'M', u'ネ'),
-    (0x32E8, 'M', u'ノ'),
-    (0x32E9, 'M', u'ハ'),
-    (0x32EA, 'M', u'ヒ'),
-    (0x32EB, 'M', u'フ'),
-    (0x32EC, 'M', u'ヘ'),
-    (0x32ED, 'M', u'ホ'),
-    (0x32EE, 'M', u'マ'),
-    (0x32EF, 'M', u'ミ'),
-    (0x32F0, 'M', u'ム'),
-    (0x32F1, 'M', u'メ'),
-    (0x32F2, 'M', u'モ'),
-    (0x32F3, 'M', u'ヤ'),
-    (0x32F4, 'M', u'ユ'),
-    (0x32F5, 'M', u'ヨ'),
-    (0x32F6, 'M', u'ラ'),
-    (0x32F7, 'M', u'リ'),
-    (0x32F8, 'M', u'ル'),
-    (0x32F9, 'M', u'レ'),
-    (0x32FA, 'M', u'ロ'),
-    (0x32FB, 'M', u'ワ'),
-    (0x32FC, 'M', u'ヰ'),
-    (0x32FD, 'M', u'ヱ'),
-    (0x32FE, 'M', u'ヲ'),
-    (0x32FF, 'M', u'令和'),
-    (0x3300, 'M', u'アパート'),
-    (0x3301, 'M', u'アルファ'),
-    (0x3302, 'M', u'アンペア'),
-    (0x3303, 'M', u'アール'),
-    (0x3304, 'M', u'イニング'),
-    (0x3305, 'M', u'インチ'),
-    (0x3306, 'M', u'ウォン'),
-    (0x3307, 'M', u'エスクード'),
-    (0x3308, 'M', u'エーカー'),
-    (0x3309, 'M', u'オンス'),
-    (0x330A, 'M', u'オーム'),
-    (0x330B, 'M', u'カイリ'),
-    (0x330C, 'M', u'カラット'),
-    (0x330D, 'M', u'カロリー'),
-    (0x330E, 'M', u'ガロン'),
-    (0x330F, 'M', u'ガンマ'),
-    (0x3310, 'M', u'ギガ'),
-    (0x3311, 'M', u'ギニー'),
+    (0x32B2, 'M', '37'),
+    (0x32B3, 'M', '38'),
+    (0x32B4, 'M', '39'),
+    (0x32B5, 'M', '40'),
+    (0x32B6, 'M', '41'),
+    (0x32B7, 'M', '42'),
+    (0x32B8, 'M', '43'),
+    (0x32B9, 'M', '44'),
+    (0x32BA, 'M', '45'),
+    (0x32BB, 'M', '46'),
+    (0x32BC, 'M', '47'),
+    (0x32BD, 'M', '48'),
+    (0x32BE, 'M', '49'),
+    (0x32BF, 'M', '50'),
+    (0x32C0, 'M', '1月'),
+    (0x32C1, 'M', '2月'),
+    (0x32C2, 'M', '3月'),
+    (0x32C3, 'M', '4月'),
+    (0x32C4, 'M', '5月'),
+    (0x32C5, 'M', '6月'),
+    (0x32C6, 'M', '7月'),
+    (0x32C7, 'M', '8月'),
+    (0x32C8, 'M', '9月'),
+    (0x32C9, 'M', '10月'),
+    (0x32CA, 'M', '11月'),
+    (0x32CB, 'M', '12月'),
+    (0x32CC, 'M', 'hg'),
+    (0x32CD, 'M', 'erg'),
+    (0x32CE, 'M', 'ev'),
+    (0x32CF, 'M', 'ltd'),
+    (0x32D0, 'M', 'ア'),
+    (0x32D1, 'M', 'イ'),
+    (0x32D2, 'M', 'ウ'),
+    (0x32D3, 'M', 'エ'),
+    (0x32D4, 'M', 'オ'),
+    (0x32D5, 'M', 'カ'),
+    (0x32D6, 'M', 'キ'),
+    (0x32D7, 'M', 'ク'),
+    (0x32D8, 'M', 'ケ'),
+    (0x32D9, 'M', 'コ'),
+    (0x32DA, 'M', 'サ'),
+    (0x32DB, 'M', 'シ'),
+    (0x32DC, 'M', 'ス'),
+    (0x32DD, 'M', 'セ'),
+    (0x32DE, 'M', 'ソ'),
+    (0x32DF, 'M', 'タ'),
+    (0x32E0, 'M', 'チ'),
+    (0x32E1, 'M', 'ツ'),
+    (0x32E2, 'M', 'テ'),
+    (0x32E3, 'M', 'ト'),
+    (0x32E4, 'M', 'ナ'),
+    (0x32E5, 'M', 'ニ'),
+    (0x32E6, 'M', 'ヌ'),
+    (0x32E7, 'M', 'ネ'),
+    (0x32E8, 'M', 'ノ'),
+    (0x32E9, 'M', 'ハ'),
+    (0x32EA, 'M', 'ヒ'),
+    (0x32EB, 'M', 'フ'),
+    (0x32EC, 'M', 'ヘ'),
+    (0x32ED, 'M', 'ホ'),
+    (0x32EE, 'M', 'マ'),
+    (0x32EF, 'M', 'ミ'),
+    (0x32F0, 'M', 'ム'),
+    (0x32F1, 'M', 'メ'),
+    (0x32F2, 'M', 'モ'),
+    (0x32F3, 'M', 'ヤ'),
+    (0x32F4, 'M', 'ユ'),
+    (0x32F5, 'M', 'ヨ'),
+    (0x32F6, 'M', 'ラ'),
+    (0x32F7, 'M', 'リ'),
+    (0x32F8, 'M', 'ル'),
+    (0x32F9, 'M', 'レ'),
+    (0x32FA, 'M', 'ロ'),
+    (0x32FB, 'M', 'ワ'),
+    (0x32FC, 'M', 'ヰ'),
+    (0x32FD, 'M', 'ヱ'),
+    (0x32FE, 'M', 'ヲ'),
+    (0x32FF, 'M', '令和'),
+    (0x3300, 'M', 'アパート'),
+    (0x3301, 'M', 'アルファ'),
+    (0x3302, 'M', 'アンペア'),
+    (0x3303, 'M', 'アール'),
+    (0x3304, 'M', 'イニング'),
+    (0x3305, 'M', 'インチ'),
+    (0x3306, 'M', 'ウォン'),
+    (0x3307, 'M', 'エスクード'),
+    (0x3308, 'M', 'エーカー'),
+    (0x3309, 'M', 'オンス'),
+    (0x330A, 'M', 'オーム'),
+    (0x330B, 'M', 'カイリ'),
+    (0x330C, 'M', 'カラット'),
+    (0x330D, 'M', 'カロリー'),
+    (0x330E, 'M', 'ガロン'),
+    (0x330F, 'M', 'ガンマ'),
+    (0x3310, 'M', 'ギガ'),
+    (0x3311, 'M', 'ギニー'),
+    (0x3312, 'M', 'キュリー'),
+    (0x3313, 'M', 'ギルダー'),
+    (0x3314, 'M', 'キロ'),
+    (0x3315, 'M', 'キログラム'),
     ]
 
-def _seg_33():
+def _seg_33() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x3312, 'M', u'キュリー'),
-    (0x3313, 'M', u'ギルダー'),
-    (0x3314, 'M', u'キロ'),
-    (0x3315, 'M', u'キログラム'),
-    (0x3316, 'M', u'キロメートル'),
-    (0x3317, 'M', u'キロワット'),
-    (0x3318, 'M', u'グラム'),
-    (0x3319, 'M', u'グラムトン'),
-    (0x331A, 'M', u'クルゼイロ'),
-    (0x331B, 'M', u'クローネ'),
-    (0x331C, 'M', u'ケース'),
-    (0x331D, 'M', u'コルナ'),
-    (0x331E, 'M', u'コーポ'),
-    (0x331F, 'M', u'サイクル'),
-    (0x3320, 'M', u'サンチーム'),
-    (0x3321, 'M', u'シリング'),
-    (0x3322, 'M', u'センチ'),
-    (0x3323, 'M', u'セント'),
-    (0x3324, 'M', u'ダース'),
-    (0x3325, 'M', u'デシ'),
-    (0x3326, 'M', u'ドル'),
-    (0x3327, 'M', u'トン'),
-    (0x3328, 'M', u'ナノ'),
-    (0x3329, 'M', u'ノット'),
-    (0x332A, 'M', u'ハイツ'),
-    (0x332B, 'M', u'パーセント'),
-    (0x332C, 'M', u'パーツ'),
-    (0x332D, 'M', u'バーレル'),
-    (0x332E, 'M', u'ピアストル'),
-    (0x332F, 'M', u'ピクル'),
-    (0x3330, 'M', u'ピコ'),
-    (0x3331, 'M', u'ビル'),
-    (0x3332, 'M', u'ファラッド'),
-    (0x3333, 'M', u'フィート'),
-    (0x3334, 'M', u'ブッシェル'),
-    (0x3335, 'M', u'フラン'),
-    (0x3336, 'M', u'ヘクタール'),
-    (0x3337, 'M', u'ペソ'),
-    (0x3338, 'M', u'ペニヒ'),
-    (0x3339, 'M', u'ヘルツ'),
-    (0x333A, 'M', u'ペンス'),
-    (0x333B, 'M', u'ページ'),
-    (0x333C, 'M', u'ベータ'),
-    (0x333D, 'M', u'ポイント'),
-    (0x333E, 'M', u'ボルト'),
-    (0x333F, 'M', u'ホン'),
-    (0x3340, 'M', u'ポンド'),
-    (0x3341, 'M', u'ホール'),
-    (0x3342, 'M', u'ホーン'),
-    (0x3343, 'M', u'マイクロ'),
-    (0x3344, 'M', u'マイル'),
-    (0x3345, 'M', u'マッハ'),
-    (0x3346, 'M', u'マルク'),
-    (0x3347, 'M', u'マンション'),
-    (0x3348, 'M', u'ミクロン'),
-    (0x3349, 'M', u'ミリ'),
-    (0x334A, 'M', u'ミリバール'),
-    (0x334B, 'M', u'メガ'),
-    (0x334C, 'M', u'メガトン'),
-    (0x334D, 'M', u'メートル'),
-    (0x334E, 'M', u'ヤード'),
-    (0x334F, 'M', u'ヤール'),
-    (0x3350, 'M', u'ユアン'),
-    (0x3351, 'M', u'リットル'),
-    (0x3352, 'M', u'リラ'),
-    (0x3353, 'M', u'ルピー'),
-    (0x3354, 'M', u'ルーブル'),
-    (0x3355, 'M', u'レム'),
-    (0x3356, 'M', u'レントゲン'),
-    (0x3357, 'M', u'ワット'),
-    (0x3358, 'M', u'0点'),
-    (0x3359, 'M', u'1点'),
-    (0x335A, 'M', u'2点'),
-    (0x335B, 'M', u'3点'),
-    (0x335C, 'M', u'4点'),
-    (0x335D, 'M', u'5点'),
-    (0x335E, 'M', u'6点'),
-    (0x335F, 'M', u'7点'),
-    (0x3360, 'M', u'8点'),
-    (0x3361, 'M', u'9点'),
-    (0x3362, 'M', u'10点'),
-    (0x3363, 'M', u'11点'),
-    (0x3364, 'M', u'12点'),
-    (0x3365, 'M', u'13点'),
-    (0x3366, 'M', u'14点'),
-    (0x3367, 'M', u'15点'),
-    (0x3368, 'M', u'16点'),
-    (0x3369, 'M', u'17点'),
-    (0x336A, 'M', u'18点'),
-    (0x336B, 'M', u'19点'),
-    (0x336C, 'M', u'20点'),
-    (0x336D, 'M', u'21点'),
-    (0x336E, 'M', u'22点'),
-    (0x336F, 'M', u'23点'),
-    (0x3370, 'M', u'24点'),
-    (0x3371, 'M', u'hpa'),
-    (0x3372, 'M', u'da'),
-    (0x3373, 'M', u'au'),
-    (0x3374, 'M', u'bar'),
-    (0x3375, 'M', u'ov'),
+    (0x3316, 'M', 'キロメートル'),
+    (0x3317, 'M', 'キロワット'),
+    (0x3318, 'M', 'グラム'),
+    (0x3319, 'M', 'グラムトン'),
+    (0x331A, 'M', 'クルゼイロ'),
+    (0x331B, 'M', 'クローネ'),
+    (0x331C, 'M', 'ケース'),
+    (0x331D, 'M', 'コルナ'),
+    (0x331E, 'M', 'コーポ'),
+    (0x331F, 'M', 'サイクル'),
+    (0x3320, 'M', 'サンチーム'),
+    (0x3321, 'M', 'シリング'),
+    (0x3322, 'M', 'センチ'),
+    (0x3323, 'M', 'セント'),
+    (0x3324, 'M', 'ダース'),
+    (0x3325, 'M', 'デシ'),
+    (0x3326, 'M', 'ドル'),
+    (0x3327, 'M', 'トン'),
+    (0x3328, 'M', 'ナノ'),
+    (0x3329, 'M', 'ノット'),
+    (0x332A, 'M', 'ハイツ'),
+    (0x332B, 'M', 'パーセント'),
+    (0x332C, 'M', 'パーツ'),
+    (0x332D, 'M', 'バーレル'),
+    (0x332E, 'M', 'ピアストル'),
+    (0x332F, 'M', 'ピクル'),
+    (0x3330, 'M', 'ピコ'),
+    (0x3331, 'M', 'ビル'),
+    (0x3332, 'M', 'ファラッド'),
+    (0x3333, 'M', 'フィート'),
+    (0x3334, 'M', 'ブッシェル'),
+    (0x3335, 'M', 'フラン'),
+    (0x3336, 'M', 'ヘクタール'),
+    (0x3337, 'M', 'ペソ'),
+    (0x3338, 'M', 'ペニヒ'),
+    (0x3339, 'M', 'ヘルツ'),
+    (0x333A, 'M', 'ペンス'),
+    (0x333B, 'M', 'ページ'),
+    (0x333C, 'M', 'ベータ'),
+    (0x333D, 'M', 'ポイント'),
+    (0x333E, 'M', 'ボルト'),
+    (0x333F, 'M', 'ホン'),
+    (0x3340, 'M', 'ポンド'),
+    (0x3341, 'M', 'ホール'),
+    (0x3342, 'M', 'ホーン'),
+    (0x3343, 'M', 'マイクロ'),
+    (0x3344, 'M', 'マイル'),
+    (0x3345, 'M', 'マッハ'),
+    (0x3346, 'M', 'マルク'),
+    (0x3347, 'M', 'マンション'),
+    (0x3348, 'M', 'ミクロン'),
+    (0x3349, 'M', 'ミリ'),
+    (0x334A, 'M', 'ミリバール'),
+    (0x334B, 'M', 'メガ'),
+    (0x334C, 'M', 'メガトン'),
+    (0x334D, 'M', 'メートル'),
+    (0x334E, 'M', 'ヤード'),
+    (0x334F, 'M', 'ヤール'),
+    (0x3350, 'M', 'ユアン'),
+    (0x3351, 'M', 'リットル'),
+    (0x3352, 'M', 'リラ'),
+    (0x3353, 'M', 'ルピー'),
+    (0x3354, 'M', 'ルーブル'),
+    (0x3355, 'M', 'レム'),
+    (0x3356, 'M', 'レントゲン'),
+    (0x3357, 'M', 'ワット'),
+    (0x3358, 'M', '0点'),
+    (0x3359, 'M', '1点'),
+    (0x335A, 'M', '2点'),
+    (0x335B, 'M', '3点'),
+    (0x335C, 'M', '4点'),
+    (0x335D, 'M', '5点'),
+    (0x335E, 'M', '6点'),
+    (0x335F, 'M', '7点'),
+    (0x3360, 'M', '8点'),
+    (0x3361, 'M', '9点'),
+    (0x3362, 'M', '10点'),
+    (0x3363, 'M', '11点'),
+    (0x3364, 'M', '12点'),
+    (0x3365, 'M', '13点'),
+    (0x3366, 'M', '14点'),
+    (0x3367, 'M', '15点'),
+    (0x3368, 'M', '16点'),
+    (0x3369, 'M', '17点'),
+    (0x336A, 'M', '18点'),
+    (0x336B, 'M', '19点'),
+    (0x336C, 'M', '20点'),
+    (0x336D, 'M', '21点'),
+    (0x336E, 'M', '22点'),
+    (0x336F, 'M', '23点'),
+    (0x3370, 'M', '24点'),
+    (0x3371, 'M', 'hpa'),
+    (0x3372, 'M', 'da'),
+    (0x3373, 'M', 'au'),
+    (0x3374, 'M', 'bar'),
+    (0x3375, 'M', 'ov'),
+    (0x3376, 'M', 'pc'),
+    (0x3377, 'M', 'dm'),
+    (0x3378, 'M', 'dm2'),
+    (0x3379, 'M', 'dm3'),
     ]
 
-def _seg_34():
+def _seg_34() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x3376, 'M', u'pc'),
-    (0x3377, 'M', u'dm'),
-    (0x3378, 'M', u'dm2'),
-    (0x3379, 'M', u'dm3'),
-    (0x337A, 'M', u'iu'),
-    (0x337B, 'M', u'平成'),
-    (0x337C, 'M', u'昭和'),
-    (0x337D, 'M', u'大正'),
-    (0x337E, 'M', u'明治'),
-    (0x337F, 'M', u'株式会社'),
-    (0x3380, 'M', u'pa'),
-    (0x3381, 'M', u'na'),
-    (0x3382, 'M', u'μa'),
-    (0x3383, 'M', u'ma'),
-    (0x3384, 'M', u'ka'),
-    (0x3385, 'M', u'kb'),
-    (0x3386, 'M', u'mb'),
-    (0x3387, 'M', u'gb'),
-    (0x3388, 'M', u'cal'),
-    (0x3389, 'M', u'kcal'),
-    (0x338A, 'M', u'pf'),
-    (0x338B, 'M', u'nf'),
-    (0x338C, 'M', u'μf'),
-    (0x338D, 'M', u'μg'),
-    (0x338E, 'M', u'mg'),
-    (0x338F, 'M', u'kg'),
-    (0x3390, 'M', u'hz'),
-    (0x3391, 'M', u'khz'),
-    (0x3392, 'M', u'mhz'),
-    (0x3393, 'M', u'ghz'),
-    (0x3394, 'M', u'thz'),
-    (0x3395, 'M', u'μl'),
-    (0x3396, 'M', u'ml'),
-    (0x3397, 'M', u'dl'),
-    (0x3398, 'M', u'kl'),
-    (0x3399, 'M', u'fm'),
-    (0x339A, 'M', u'nm'),
-    (0x339B, 'M', u'μm'),
-    (0x339C, 'M', u'mm'),
-    (0x339D, 'M', u'cm'),
-    (0x339E, 'M', u'km'),
-    (0x339F, 'M', u'mm2'),
-    (0x33A0, 'M', u'cm2'),
-    (0x33A1, 'M', u'm2'),
-    (0x33A2, 'M', u'km2'),
-    (0x33A3, 'M', u'mm3'),
-    (0x33A4, 'M', u'cm3'),
-    (0x33A5, 'M', u'm3'),
-    (0x33A6, 'M', u'km3'),
-    (0x33A7, 'M', u'm∕s'),
-    (0x33A8, 'M', u'm∕s2'),
-    (0x33A9, 'M', u'pa'),
-    (0x33AA, 'M', u'kpa'),
-    (0x33AB, 'M', u'mpa'),
-    (0x33AC, 'M', u'gpa'),
-    (0x33AD, 'M', u'rad'),
-    (0x33AE, 'M', u'rad∕s'),
-    (0x33AF, 'M', u'rad∕s2'),
-    (0x33B0, 'M', u'ps'),
-    (0x33B1, 'M', u'ns'),
-    (0x33B2, 'M', u'μs'),
-    (0x33B3, 'M', u'ms'),
-    (0x33B4, 'M', u'pv'),
-    (0x33B5, 'M', u'nv'),
-    (0x33B6, 'M', u'μv'),
-    (0x33B7, 'M', u'mv'),
-    (0x33B8, 'M', u'kv'),
-    (0x33B9, 'M', u'mv'),
-    (0x33BA, 'M', u'pw'),
-    (0x33BB, 'M', u'nw'),
-    (0x33BC, 'M', u'μw'),
-    (0x33BD, 'M', u'mw'),
-    (0x33BE, 'M', u'kw'),
-    (0x33BF, 'M', u'mw'),
-    (0x33C0, 'M', u'kω'),
-    (0x33C1, 'M', u'mω'),
+    (0x337A, 'M', 'iu'),
+    (0x337B, 'M', '平成'),
+    (0x337C, 'M', '昭和'),
+    (0x337D, 'M', '大正'),
+    (0x337E, 'M', '明治'),
+    (0x337F, 'M', '株式会社'),
+    (0x3380, 'M', 'pa'),
+    (0x3381, 'M', 'na'),
+    (0x3382, 'M', 'μa'),
+    (0x3383, 'M', 'ma'),
+    (0x3384, 'M', 'ka'),
+    (0x3385, 'M', 'kb'),
+    (0x3386, 'M', 'mb'),
+    (0x3387, 'M', 'gb'),
+    (0x3388, 'M', 'cal'),
+    (0x3389, 'M', 'kcal'),
+    (0x338A, 'M', 'pf'),
+    (0x338B, 'M', 'nf'),
+    (0x338C, 'M', 'μf'),
+    (0x338D, 'M', 'μg'),
+    (0x338E, 'M', 'mg'),
+    (0x338F, 'M', 'kg'),
+    (0x3390, 'M', 'hz'),
+    (0x3391, 'M', 'khz'),
+    (0x3392, 'M', 'mhz'),
+    (0x3393, 'M', 'ghz'),
+    (0x3394, 'M', 'thz'),
+    (0x3395, 'M', 'μl'),
+    (0x3396, 'M', 'ml'),
+    (0x3397, 'M', 'dl'),
+    (0x3398, 'M', 'kl'),
+    (0x3399, 'M', 'fm'),
+    (0x339A, 'M', 'nm'),
+    (0x339B, 'M', 'μm'),
+    (0x339C, 'M', 'mm'),
+    (0x339D, 'M', 'cm'),
+    (0x339E, 'M', 'km'),
+    (0x339F, 'M', 'mm2'),
+    (0x33A0, 'M', 'cm2'),
+    (0x33A1, 'M', 'm2'),
+    (0x33A2, 'M', 'km2'),
+    (0x33A3, 'M', 'mm3'),
+    (0x33A4, 'M', 'cm3'),
+    (0x33A5, 'M', 'm3'),
+    (0x33A6, 'M', 'km3'),
+    (0x33A7, 'M', 'm∕s'),
+    (0x33A8, 'M', 'm∕s2'),
+    (0x33A9, 'M', 'pa'),
+    (0x33AA, 'M', 'kpa'),
+    (0x33AB, 'M', 'mpa'),
+    (0x33AC, 'M', 'gpa'),
+    (0x33AD, 'M', 'rad'),
+    (0x33AE, 'M', 'rad∕s'),
+    (0x33AF, 'M', 'rad∕s2'),
+    (0x33B0, 'M', 'ps'),
+    (0x33B1, 'M', 'ns'),
+    (0x33B2, 'M', 'μs'),
+    (0x33B3, 'M', 'ms'),
+    (0x33B4, 'M', 'pv'),
+    (0x33B5, 'M', 'nv'),
+    (0x33B6, 'M', 'μv'),
+    (0x33B7, 'M', 'mv'),
+    (0x33B8, 'M', 'kv'),
+    (0x33B9, 'M', 'mv'),
+    (0x33BA, 'M', 'pw'),
+    (0x33BB, 'M', 'nw'),
+    (0x33BC, 'M', 'μw'),
+    (0x33BD, 'M', 'mw'),
+    (0x33BE, 'M', 'kw'),
+    (0x33BF, 'M', 'mw'),
+    (0x33C0, 'M', 'kω'),
+    (0x33C1, 'M', 'mω'),
     (0x33C2, 'X'),
-    (0x33C3, 'M', u'bq'),
-    (0x33C4, 'M', u'cc'),
-    (0x33C5, 'M', u'cd'),
-    (0x33C6, 'M', u'c∕kg'),
+    (0x33C3, 'M', 'bq'),
+    (0x33C4, 'M', 'cc'),
+    (0x33C5, 'M', 'cd'),
+    (0x33C6, 'M', 'c∕kg'),
     (0x33C7, 'X'),
-    (0x33C8, 'M', u'db'),
-    (0x33C9, 'M', u'gy'),
-    (0x33CA, 'M', u'ha'),
-    (0x33CB, 'M', u'hp'),
-    (0x33CC, 'M', u'in'),
-    (0x33CD, 'M', u'kk'),
-    (0x33CE, 'M', u'km'),
-    (0x33CF, 'M', u'kt'),
-    (0x33D0, 'M', u'lm'),
-    (0x33D1, 'M', u'ln'),
-    (0x33D2, 'M', u'log'),
-    (0x33D3, 'M', u'lx'),
-    (0x33D4, 'M', u'mb'),
-    (0x33D5, 'M', u'mil'),
-    (0x33D6, 'M', u'mol'),
-    (0x33D7, 'M', u'ph'),
+    (0x33C8, 'M', 'db'),
+    (0x33C9, 'M', 'gy'),
+    (0x33CA, 'M', 'ha'),
+    (0x33CB, 'M', 'hp'),
+    (0x33CC, 'M', 'in'),
+    (0x33CD, 'M', 'kk'),
+    (0x33CE, 'M', 'km'),
+    (0x33CF, 'M', 'kt'),
+    (0x33D0, 'M', 'lm'),
+    (0x33D1, 'M', 'ln'),
+    (0x33D2, 'M', 'log'),
+    (0x33D3, 'M', 'lx'),
+    (0x33D4, 'M', 'mb'),
+    (0x33D5, 'M', 'mil'),
+    (0x33D6, 'M', 'mol'),
+    (0x33D7, 'M', 'ph'),
     (0x33D8, 'X'),
-    (0x33D9, 'M', u'ppm'),
+    (0x33D9, 'M', 'ppm'),
+    (0x33DA, 'M', 'pr'),
+    (0x33DB, 'M', 'sr'),
+    (0x33DC, 'M', 'sv'),
+    (0x33DD, 'M', 'wb'),
     ]
 
-def _seg_35():
+def _seg_35() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x33DA, 'M', u'pr'),
-    (0x33DB, 'M', u'sr'),
-    (0x33DC, 'M', u'sv'),
-    (0x33DD, 'M', u'wb'),
-    (0x33DE, 'M', u'v∕m'),
-    (0x33DF, 'M', u'a∕m'),
-    (0x33E0, 'M', u'1日'),
-    (0x33E1, 'M', u'2日'),
-    (0x33E2, 'M', u'3日'),
-    (0x33E3, 'M', u'4日'),
-    (0x33E4, 'M', u'5日'),
-    (0x33E5, 'M', u'6日'),
-    (0x33E6, 'M', u'7日'),
-    (0x33E7, 'M', u'8日'),
-    (0x33E8, 'M', u'9日'),
-    (0x33E9, 'M', u'10日'),
-    (0x33EA, 'M', u'11日'),
-    (0x33EB, 'M', u'12日'),
-    (0x33EC, 'M', u'13日'),
-    (0x33ED, 'M', u'14日'),
-    (0x33EE, 'M', u'15日'),
-    (0x33EF, 'M', u'16日'),
-    (0x33F0, 'M', u'17日'),
-    (0x33F1, 'M', u'18日'),
-    (0x33F2, 'M', u'19日'),
-    (0x33F3, 'M', u'20日'),
-    (0x33F4, 'M', u'21日'),
-    (0x33F5, 'M', u'22日'),
-    (0x33F6, 'M', u'23日'),
-    (0x33F7, 'M', u'24日'),
-    (0x33F8, 'M', u'25日'),
-    (0x33F9, 'M', u'26日'),
-    (0x33FA, 'M', u'27日'),
-    (0x33FB, 'M', u'28日'),
-    (0x33FC, 'M', u'29日'),
-    (0x33FD, 'M', u'30日'),
-    (0x33FE, 'M', u'31日'),
-    (0x33FF, 'M', u'gal'),
+    (0x33DE, 'M', 'v∕m'),
+    (0x33DF, 'M', 'a∕m'),
+    (0x33E0, 'M', '1日'),
+    (0x33E1, 'M', '2日'),
+    (0x33E2, 'M', '3日'),
+    (0x33E3, 'M', '4日'),
+    (0x33E4, 'M', '5日'),
+    (0x33E5, 'M', '6日'),
+    (0x33E6, 'M', '7日'),
+    (0x33E7, 'M', '8日'),
+    (0x33E8, 'M', '9日'),
+    (0x33E9, 'M', '10日'),
+    (0x33EA, 'M', '11日'),
+    (0x33EB, 'M', '12日'),
+    (0x33EC, 'M', '13日'),
+    (0x33ED, 'M', '14日'),
+    (0x33EE, 'M', '15日'),
+    (0x33EF, 'M', '16日'),
+    (0x33F0, 'M', '17日'),
+    (0x33F1, 'M', '18日'),
+    (0x33F2, 'M', '19日'),
+    (0x33F3, 'M', '20日'),
+    (0x33F4, 'M', '21日'),
+    (0x33F5, 'M', '22日'),
+    (0x33F6, 'M', '23日'),
+    (0x33F7, 'M', '24日'),
+    (0x33F8, 'M', '25日'),
+    (0x33F9, 'M', '26日'),
+    (0x33FA, 'M', '27日'),
+    (0x33FB, 'M', '28日'),
+    (0x33FC, 'M', '29日'),
+    (0x33FD, 'M', '30日'),
+    (0x33FE, 'M', '31日'),
+    (0x33FF, 'M', 'gal'),
     (0x3400, 'V'),
-    (0x9FFD, 'X'),
-    (0xA000, 'V'),
     (0xA48D, 'X'),
     (0xA490, 'V'),
     (0xA4C7, 'X'),
     (0xA4D0, 'V'),
     (0xA62C, 'X'),
-    (0xA640, 'M', u'ꙁ'),
+    (0xA640, 'M', 'ꙁ'),
     (0xA641, 'V'),
-    (0xA642, 'M', u'ꙃ'),
+    (0xA642, 'M', 'ꙃ'),
     (0xA643, 'V'),
-    (0xA644, 'M', u'ꙅ'),
+    (0xA644, 'M', 'ꙅ'),
     (0xA645, 'V'),
-    (0xA646, 'M', u'ꙇ'),
+    (0xA646, 'M', 'ꙇ'),
     (0xA647, 'V'),
-    (0xA648, 'M', u'ꙉ'),
+    (0xA648, 'M', 'ꙉ'),
     (0xA649, 'V'),
-    (0xA64A, 'M', u'ꙋ'),
+    (0xA64A, 'M', 'ꙋ'),
     (0xA64B, 'V'),
-    (0xA64C, 'M', u'ꙍ'),
+    (0xA64C, 'M', 'ꙍ'),
     (0xA64D, 'V'),
-    (0xA64E, 'M', u'ꙏ'),
+    (0xA64E, 'M', 'ꙏ'),
     (0xA64F, 'V'),
-    (0xA650, 'M', u'ꙑ'),
+    (0xA650, 'M', 'ꙑ'),
     (0xA651, 'V'),
-    (0xA652, 'M', u'ꙓ'),
+    (0xA652, 'M', 'ꙓ'),
     (0xA653, 'V'),
-    (0xA654, 'M', u'ꙕ'),
+    (0xA654, 'M', 'ꙕ'),
     (0xA655, 'V'),
-    (0xA656, 'M', u'ꙗ'),
+    (0xA656, 'M', 'ꙗ'),
     (0xA657, 'V'),
-    (0xA658, 'M', u'ꙙ'),
+    (0xA658, 'M', 'ꙙ'),
     (0xA659, 'V'),
-    (0xA65A, 'M', u'ꙛ'),
+    (0xA65A, 'M', 'ꙛ'),
     (0xA65B, 'V'),
-    (0xA65C, 'M', u'ꙝ'),
+    (0xA65C, 'M', 'ꙝ'),
     (0xA65D, 'V'),
-    (0xA65E, 'M', u'ꙟ'),
+    (0xA65E, 'M', 'ꙟ'),
     (0xA65F, 'V'),
-    (0xA660, 'M', u'ꙡ'),
+    (0xA660, 'M', 'ꙡ'),
     (0xA661, 'V'),
-    (0xA662, 'M', u'ꙣ'),
+    (0xA662, 'M', 'ꙣ'),
     (0xA663, 'V'),
-    (0xA664, 'M', u'ꙥ'),
+    (0xA664, 'M', 'ꙥ'),
     (0xA665, 'V'),
-    (0xA666, 'M', u'ꙧ'),
+    (0xA666, 'M', 'ꙧ'),
     (0xA667, 'V'),
-    (0xA668, 'M', u'ꙩ'),
+    (0xA668, 'M', 'ꙩ'),
     (0xA669, 'V'),
-    (0xA66A, 'M', u'ꙫ'),
+    (0xA66A, 'M', 'ꙫ'),
     (0xA66B, 'V'),
-    (0xA66C, 'M', u'ꙭ'),
+    (0xA66C, 'M', 'ꙭ'),
     (0xA66D, 'V'),
-    (0xA680, 'M', u'ꚁ'),
+    (0xA680, 'M', 'ꚁ'),
     (0xA681, 'V'),
-    (0xA682, 'M', u'ꚃ'),
+    (0xA682, 'M', 'ꚃ'),
     (0xA683, 'V'),
-    (0xA684, 'M', u'ꚅ'),
+    (0xA684, 'M', 'ꚅ'),
     (0xA685, 'V'),
-    (0xA686, 'M', u'ꚇ'),
+    (0xA686, 'M', 'ꚇ'),
     (0xA687, 'V'),
-    ]
-
-def _seg_36():
-    return [
-    (0xA688, 'M', u'ꚉ'),
+    (0xA688, 'M', 'ꚉ'),
     (0xA689, 'V'),
-    (0xA68A, 'M', u'ꚋ'),
+    (0xA68A, 'M', 'ꚋ'),
     (0xA68B, 'V'),
-    (0xA68C, 'M', u'ꚍ'),
+    (0xA68C, 'M', 'ꚍ'),
     (0xA68D, 'V'),
-    (0xA68E, 'M', u'ꚏ'),
+    ]
+
+def _seg_36() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0xA68E, 'M', 'ꚏ'),
     (0xA68F, 'V'),
-    (0xA690, 'M', u'ꚑ'),
+    (0xA690, 'M', 'ꚑ'),
     (0xA691, 'V'),
-    (0xA692, 'M', u'ꚓ'),
+    (0xA692, 'M', 'ꚓ'),
     (0xA693, 'V'),
-    (0xA694, 'M', u'ꚕ'),
+    (0xA694, 'M', 'ꚕ'),
     (0xA695, 'V'),
-    (0xA696, 'M', u'ꚗ'),
+    (0xA696, 'M', 'ꚗ'),
     (0xA697, 'V'),
-    (0xA698, 'M', u'ꚙ'),
+    (0xA698, 'M', 'ꚙ'),
     (0xA699, 'V'),
-    (0xA69A, 'M', u'ꚛ'),
+    (0xA69A, 'M', 'ꚛ'),
     (0xA69B, 'V'),
-    (0xA69C, 'M', u'ъ'),
-    (0xA69D, 'M', u'ь'),
+    (0xA69C, 'M', 'ъ'),
+    (0xA69D, 'M', 'ь'),
     (0xA69E, 'V'),
     (0xA6F8, 'X'),
     (0xA700, 'V'),
-    (0xA722, 'M', u'ꜣ'),
+    (0xA722, 'M', 'ꜣ'),
     (0xA723, 'V'),
-    (0xA724, 'M', u'ꜥ'),
+    (0xA724, 'M', 'ꜥ'),
     (0xA725, 'V'),
-    (0xA726, 'M', u'ꜧ'),
+    (0xA726, 'M', 'ꜧ'),
     (0xA727, 'V'),
-    (0xA728, 'M', u'ꜩ'),
+    (0xA728, 'M', 'ꜩ'),
     (0xA729, 'V'),
-    (0xA72A, 'M', u'ꜫ'),
+    (0xA72A, 'M', 'ꜫ'),
     (0xA72B, 'V'),
-    (0xA72C, 'M', u'ꜭ'),
+    (0xA72C, 'M', 'ꜭ'),
     (0xA72D, 'V'),
-    (0xA72E, 'M', u'ꜯ'),
+    (0xA72E, 'M', 'ꜯ'),
     (0xA72F, 'V'),
-    (0xA732, 'M', u'ꜳ'),
+    (0xA732, 'M', 'ꜳ'),
     (0xA733, 'V'),
-    (0xA734, 'M', u'ꜵ'),
+    (0xA734, 'M', 'ꜵ'),
     (0xA735, 'V'),
-    (0xA736, 'M', u'ꜷ'),
+    (0xA736, 'M', 'ꜷ'),
     (0xA737, 'V'),
-    (0xA738, 'M', u'ꜹ'),
+    (0xA738, 'M', 'ꜹ'),
     (0xA739, 'V'),
-    (0xA73A, 'M', u'ꜻ'),
+    (0xA73A, 'M', 'ꜻ'),
     (0xA73B, 'V'),
-    (0xA73C, 'M', u'ꜽ'),
+    (0xA73C, 'M', 'ꜽ'),
     (0xA73D, 'V'),
-    (0xA73E, 'M', u'ꜿ'),
+    (0xA73E, 'M', 'ꜿ'),
     (0xA73F, 'V'),
-    (0xA740, 'M', u'ꝁ'),
+    (0xA740, 'M', 'ꝁ'),
     (0xA741, 'V'),
-    (0xA742, 'M', u'ꝃ'),
+    (0xA742, 'M', 'ꝃ'),
     (0xA743, 'V'),
-    (0xA744, 'M', u'ꝅ'),
+    (0xA744, 'M', 'ꝅ'),
     (0xA745, 'V'),
-    (0xA746, 'M', u'ꝇ'),
+    (0xA746, 'M', 'ꝇ'),
     (0xA747, 'V'),
-    (0xA748, 'M', u'ꝉ'),
+    (0xA748, 'M', 'ꝉ'),
     (0xA749, 'V'),
-    (0xA74A, 'M', u'ꝋ'),
+    (0xA74A, 'M', 'ꝋ'),
     (0xA74B, 'V'),
-    (0xA74C, 'M', u'ꝍ'),
+    (0xA74C, 'M', 'ꝍ'),
     (0xA74D, 'V'),
-    (0xA74E, 'M', u'ꝏ'),
+    (0xA74E, 'M', 'ꝏ'),
     (0xA74F, 'V'),
-    (0xA750, 'M', u'ꝑ'),
+    (0xA750, 'M', 'ꝑ'),
     (0xA751, 'V'),
-    (0xA752, 'M', u'ꝓ'),
+    (0xA752, 'M', 'ꝓ'),
     (0xA753, 'V'),
-    (0xA754, 'M', u'ꝕ'),
+    (0xA754, 'M', 'ꝕ'),
     (0xA755, 'V'),
-    (0xA756, 'M', u'ꝗ'),
+    (0xA756, 'M', 'ꝗ'),
     (0xA757, 'V'),
-    (0xA758, 'M', u'ꝙ'),
+    (0xA758, 'M', 'ꝙ'),
     (0xA759, 'V'),
-    (0xA75A, 'M', u'ꝛ'),
+    (0xA75A, 'M', 'ꝛ'),
     (0xA75B, 'V'),
-    (0xA75C, 'M', u'ꝝ'),
+    (0xA75C, 'M', 'ꝝ'),
     (0xA75D, 'V'),
-    (0xA75E, 'M', u'ꝟ'),
+    (0xA75E, 'M', 'ꝟ'),
     (0xA75F, 'V'),
-    (0xA760, 'M', u'ꝡ'),
+    (0xA760, 'M', 'ꝡ'),
     (0xA761, 'V'),
-    (0xA762, 'M', u'ꝣ'),
+    (0xA762, 'M', 'ꝣ'),
     (0xA763, 'V'),
-    (0xA764, 'M', u'ꝥ'),
+    (0xA764, 'M', 'ꝥ'),
     (0xA765, 'V'),
-    (0xA766, 'M', u'ꝧ'),
+    (0xA766, 'M', 'ꝧ'),
     (0xA767, 'V'),
-    (0xA768, 'M', u'ꝩ'),
+    (0xA768, 'M', 'ꝩ'),
     (0xA769, 'V'),
-    (0xA76A, 'M', u'ꝫ'),
+    (0xA76A, 'M', 'ꝫ'),
     (0xA76B, 'V'),
-    (0xA76C, 'M', u'ꝭ'),
+    (0xA76C, 'M', 'ꝭ'),
     (0xA76D, 'V'),
-    (0xA76E, 'M', u'ꝯ'),
-    ]
-
-def _seg_37():
-    return [
+    (0xA76E, 'M', 'ꝯ'),
     (0xA76F, 'V'),
-    (0xA770, 'M', u'ꝯ'),
+    (0xA770, 'M', 'ꝯ'),
     (0xA771, 'V'),
-    (0xA779, 'M', u'ꝺ'),
+    (0xA779, 'M', 'ꝺ'),
     (0xA77A, 'V'),
-    (0xA77B, 'M', u'ꝼ'),
+    (0xA77B, 'M', 'ꝼ'),
+    ]
+
+def _seg_37() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
     (0xA77C, 'V'),
-    (0xA77D, 'M', u'ᵹ'),
-    (0xA77E, 'M', u'ꝿ'),
+    (0xA77D, 'M', 'ᵹ'),
+    (0xA77E, 'M', 'ꝿ'),
     (0xA77F, 'V'),
-    (0xA780, 'M', u'ꞁ'),
+    (0xA780, 'M', 'ꞁ'),
     (0xA781, 'V'),
-    (0xA782, 'M', u'ꞃ'),
+    (0xA782, 'M', 'ꞃ'),
     (0xA783, 'V'),
-    (0xA784, 'M', u'ꞅ'),
+    (0xA784, 'M', 'ꞅ'),
     (0xA785, 'V'),
-    (0xA786, 'M', u'ꞇ'),
+    (0xA786, 'M', 'ꞇ'),
     (0xA787, 'V'),
-    (0xA78B, 'M', u'ꞌ'),
+    (0xA78B, 'M', 'ꞌ'),
     (0xA78C, 'V'),
-    (0xA78D, 'M', u'ɥ'),
+    (0xA78D, 'M', 'ɥ'),
     (0xA78E, 'V'),
-    (0xA790, 'M', u'ꞑ'),
+    (0xA790, 'M', 'ꞑ'),
     (0xA791, 'V'),
-    (0xA792, 'M', u'ꞓ'),
+    (0xA792, 'M', 'ꞓ'),
     (0xA793, 'V'),
-    (0xA796, 'M', u'ꞗ'),
+    (0xA796, 'M', 'ꞗ'),
     (0xA797, 'V'),
-    (0xA798, 'M', u'ꞙ'),
+    (0xA798, 'M', 'ꞙ'),
     (0xA799, 'V'),
-    (0xA79A, 'M', u'ꞛ'),
+    (0xA79A, 'M', 'ꞛ'),
     (0xA79B, 'V'),
-    (0xA79C, 'M', u'ꞝ'),
+    (0xA79C, 'M', 'ꞝ'),
     (0xA79D, 'V'),
-    (0xA79E, 'M', u'ꞟ'),
+    (0xA79E, 'M', 'ꞟ'),
     (0xA79F, 'V'),
-    (0xA7A0, 'M', u'ꞡ'),
+    (0xA7A0, 'M', 'ꞡ'),
     (0xA7A1, 'V'),
-    (0xA7A2, 'M', u'ꞣ'),
+    (0xA7A2, 'M', 'ꞣ'),
     (0xA7A3, 'V'),
-    (0xA7A4, 'M', u'ꞥ'),
+    (0xA7A4, 'M', 'ꞥ'),
     (0xA7A5, 'V'),
-    (0xA7A6, 'M', u'ꞧ'),
+    (0xA7A6, 'M', 'ꞧ'),
     (0xA7A7, 'V'),
-    (0xA7A8, 'M', u'ꞩ'),
+    (0xA7A8, 'M', 'ꞩ'),
     (0xA7A9, 'V'),
-    (0xA7AA, 'M', u'ɦ'),
-    (0xA7AB, 'M', u'ɜ'),
-    (0xA7AC, 'M', u'ɡ'),
-    (0xA7AD, 'M', u'ɬ'),
-    (0xA7AE, 'M', u'ɪ'),
+    (0xA7AA, 'M', 'ɦ'),
+    (0xA7AB, 'M', 'ɜ'),
+    (0xA7AC, 'M', 'ɡ'),
+    (0xA7AD, 'M', 'ɬ'),
+    (0xA7AE, 'M', 'ɪ'),
     (0xA7AF, 'V'),
-    (0xA7B0, 'M', u'ʞ'),
-    (0xA7B1, 'M', u'ʇ'),
-    (0xA7B2, 'M', u'ʝ'),
-    (0xA7B3, 'M', u'ꭓ'),
-    (0xA7B4, 'M', u'ꞵ'),
+    (0xA7B0, 'M', 'ʞ'),
+    (0xA7B1, 'M', 'ʇ'),
+    (0xA7B2, 'M', 'ʝ'),
+    (0xA7B3, 'M', 'ꭓ'),
+    (0xA7B4, 'M', 'ꞵ'),
     (0xA7B5, 'V'),
-    (0xA7B6, 'M', u'ꞷ'),
+    (0xA7B6, 'M', 'ꞷ'),
     (0xA7B7, 'V'),
-    (0xA7B8, 'M', u'ꞹ'),
+    (0xA7B8, 'M', 'ꞹ'),
     (0xA7B9, 'V'),
-    (0xA7BA, 'M', u'ꞻ'),
+    (0xA7BA, 'M', 'ꞻ'),
     (0xA7BB, 'V'),
-    (0xA7BC, 'M', u'ꞽ'),
+    (0xA7BC, 'M', 'ꞽ'),
     (0xA7BD, 'V'),
-    (0xA7BE, 'M', u'ꞿ'),
+    (0xA7BE, 'M', 'ꞿ'),
     (0xA7BF, 'V'),
-    (0xA7C0, 'X'),
-    (0xA7C2, 'M', u'ꟃ'),
+    (0xA7C0, 'M', 'ꟁ'),
+    (0xA7C1, 'V'),
+    (0xA7C2, 'M', 'ꟃ'),
     (0xA7C3, 'V'),
-    (0xA7C4, 'M', u'ꞔ'),
-    (0xA7C5, 'M', u'ʂ'),
-    (0xA7C6, 'M', u'ᶎ'),
-    (0xA7C7, 'M', u'ꟈ'),
+    (0xA7C4, 'M', 'ꞔ'),
+    (0xA7C5, 'M', 'ʂ'),
+    (0xA7C6, 'M', 'ᶎ'),
+    (0xA7C7, 'M', 'ꟈ'),
     (0xA7C8, 'V'),
-    (0xA7C9, 'M', u'ꟊ'),
+    (0xA7C9, 'M', 'ꟊ'),
     (0xA7CA, 'V'),
     (0xA7CB, 'X'),
-    (0xA7F5, 'M', u'ꟶ'),
+    (0xA7D0, 'M', 'ꟑ'),
+    (0xA7D1, 'V'),
+    (0xA7D2, 'X'),
+    (0xA7D3, 'V'),
+    (0xA7D4, 'X'),
+    (0xA7D5, 'V'),
+    (0xA7D6, 'M', 'ꟗ'),
+    (0xA7D7, 'V'),
+    (0xA7D8, 'M', 'ꟙ'),
+    (0xA7D9, 'V'),
+    (0xA7DA, 'X'),
+    (0xA7F2, 'M', 'c'),
+    (0xA7F3, 'M', 'f'),
+    (0xA7F4, 'M', 'q'),
+    (0xA7F5, 'M', 'ꟶ'),
     (0xA7F6, 'V'),
-    (0xA7F8, 'M', u'ħ'),
-    (0xA7F9, 'M', u'œ'),
+    (0xA7F8, 'M', 'ħ'),
+    (0xA7F9, 'M', 'œ'),
     (0xA7FA, 'V'),
     (0xA82D, 'X'),
     (0xA830, 'V'),
@@ -3946,6 +3958,10 @@ def _seg_37():
     (0xA878, 'X'),
     (0xA880, 'V'),
     (0xA8C6, 'X'),
+    ]
+
+def _seg_38() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
     (0xA8CE, 'V'),
     (0xA8DA, 'X'),
     (0xA8E0, 'V'),
@@ -3955,10 +3971,6 @@ def _seg_37():
     (0xA980, 'V'),
     (0xA9CE, 'X'),
     (0xA9CF, 'V'),
-    ]
-
-def _seg_38():
-    return [
     (0xA9DA, 'X'),
     (0xA9DE, 'V'),
     (0xA9FF, 'X'),
@@ -3983,98 +3995,98 @@ def _seg_38():
     (0xAB28, 'V'),
     (0xAB2F, 'X'),
     (0xAB30, 'V'),
-    (0xAB5C, 'M', u'ꜧ'),
-    (0xAB5D, 'M', u'ꬷ'),
-    (0xAB5E, 'M', u'ɫ'),
-    (0xAB5F, 'M', u'ꭒ'),
+    (0xAB5C, 'M', 'ꜧ'),
+    (0xAB5D, 'M', 'ꬷ'),
+    (0xAB5E, 'M', 'ɫ'),
+    (0xAB5F, 'M', 'ꭒ'),
     (0xAB60, 'V'),
-    (0xAB69, 'M', u'ʍ'),
+    (0xAB69, 'M', 'ʍ'),
     (0xAB6A, 'V'),
     (0xAB6C, 'X'),
-    (0xAB70, 'M', u'Ꭰ'),
-    (0xAB71, 'M', u'Ꭱ'),
-    (0xAB72, 'M', u'Ꭲ'),
-    (0xAB73, 'M', u'Ꭳ'),
-    (0xAB74, 'M', u'Ꭴ'),
-    (0xAB75, 'M', u'Ꭵ'),
-    (0xAB76, 'M', u'Ꭶ'),
-    (0xAB77, 'M', u'Ꭷ'),
-    (0xAB78, 'M', u'Ꭸ'),
-    (0xAB79, 'M', u'Ꭹ'),
-    (0xAB7A, 'M', u'Ꭺ'),
-    (0xAB7B, 'M', u'Ꭻ'),
-    (0xAB7C, 'M', u'Ꭼ'),
-    (0xAB7D, 'M', u'Ꭽ'),
-    (0xAB7E, 'M', u'Ꭾ'),
-    (0xAB7F, 'M', u'Ꭿ'),
-    (0xAB80, 'M', u'Ꮀ'),
-    (0xAB81, 'M', u'Ꮁ'),
-    (0xAB82, 'M', u'Ꮂ'),
-    (0xAB83, 'M', u'Ꮃ'),
-    (0xAB84, 'M', u'Ꮄ'),
-    (0xAB85, 'M', u'Ꮅ'),
-    (0xAB86, 'M', u'Ꮆ'),
-    (0xAB87, 'M', u'Ꮇ'),
-    (0xAB88, 'M', u'Ꮈ'),
-    (0xAB89, 'M', u'Ꮉ'),
-    (0xAB8A, 'M', u'Ꮊ'),
-    (0xAB8B, 'M', u'Ꮋ'),
-    (0xAB8C, 'M', u'Ꮌ'),
-    (0xAB8D, 'M', u'Ꮍ'),
-    (0xAB8E, 'M', u'Ꮎ'),
-    (0xAB8F, 'M', u'Ꮏ'),
-    (0xAB90, 'M', u'Ꮐ'),
-    (0xAB91, 'M', u'Ꮑ'),
-    (0xAB92, 'M', u'Ꮒ'),
-    (0xAB93, 'M', u'Ꮓ'),
-    (0xAB94, 'M', u'Ꮔ'),
-    (0xAB95, 'M', u'Ꮕ'),
-    (0xAB96, 'M', u'Ꮖ'),
-    (0xAB97, 'M', u'Ꮗ'),
-    (0xAB98, 'M', u'Ꮘ'),
-    (0xAB99, 'M', u'Ꮙ'),
-    (0xAB9A, 'M', u'Ꮚ'),
-    (0xAB9B, 'M', u'Ꮛ'),
-    (0xAB9C, 'M', u'Ꮜ'),
-    (0xAB9D, 'M', u'Ꮝ'),
-    (0xAB9E, 'M', u'Ꮞ'),
-    (0xAB9F, 'M', u'Ꮟ'),
-    (0xABA0, 'M', u'Ꮠ'),
-    (0xABA1, 'M', u'Ꮡ'),
-    (0xABA2, 'M', u'Ꮢ'),
-    (0xABA3, 'M', u'Ꮣ'),
-    (0xABA4, 'M', u'Ꮤ'),
-    (0xABA5, 'M', u'Ꮥ'),
-    (0xABA6, 'M', u'Ꮦ'),
-    (0xABA7, 'M', u'Ꮧ'),
-    (0xABA8, 'M', u'Ꮨ'),
-    (0xABA9, 'M', u'Ꮩ'),
-    (0xABAA, 'M', u'Ꮪ'),
-    (0xABAB, 'M', u'Ꮫ'),
-    (0xABAC, 'M', u'Ꮬ'),
-    (0xABAD, 'M', u'Ꮭ'),
-    (0xABAE, 'M', u'Ꮮ'),
-    (0xABAF, 'M', u'Ꮯ'),
-    (0xABB0, 'M', u'Ꮰ'),
-    (0xABB1, 'M', u'Ꮱ'),
-    (0xABB2, 'M', u'Ꮲ'),
-    (0xABB3, 'M', u'Ꮳ'),
+    (0xAB70, 'M', 'Ꭰ'),
+    (0xAB71, 'M', 'Ꭱ'),
+    (0xAB72, 'M', 'Ꭲ'),
+    (0xAB73, 'M', 'Ꭳ'),
+    (0xAB74, 'M', 'Ꭴ'),
+    (0xAB75, 'M', 'Ꭵ'),
+    (0xAB76, 'M', 'Ꭶ'),
+    (0xAB77, 'M', 'Ꭷ'),
+    (0xAB78, 'M', 'Ꭸ'),
+    (0xAB79, 'M', 'Ꭹ'),
+    (0xAB7A, 'M', 'Ꭺ'),
+    (0xAB7B, 'M', 'Ꭻ'),
+    (0xAB7C, 'M', 'Ꭼ'),
+    (0xAB7D, 'M', 'Ꭽ'),
+    (0xAB7E, 'M', 'Ꭾ'),
+    (0xAB7F, 'M', 'Ꭿ'),
+    (0xAB80, 'M', 'Ꮀ'),
+    (0xAB81, 'M', 'Ꮁ'),
+    (0xAB82, 'M', 'Ꮂ'),
+    (0xAB83, 'M', 'Ꮃ'),
+    (0xAB84, 'M', 'Ꮄ'),
+    (0xAB85, 'M', 'Ꮅ'),
+    (0xAB86, 'M', 'Ꮆ'),
+    (0xAB87, 'M', 'Ꮇ'),
+    (0xAB88, 'M', 'Ꮈ'),
+    (0xAB89, 'M', 'Ꮉ'),
+    (0xAB8A, 'M', 'Ꮊ'),
+    (0xAB8B, 'M', 'Ꮋ'),
+    (0xAB8C, 'M', 'Ꮌ'),
+    (0xAB8D, 'M', 'Ꮍ'),
+    (0xAB8E, 'M', 'Ꮎ'),
+    (0xAB8F, 'M', 'Ꮏ'),
+    (0xAB90, 'M', 'Ꮐ'),
+    (0xAB91, 'M', 'Ꮑ'),
+    (0xAB92, 'M', 'Ꮒ'),
+    (0xAB93, 'M', 'Ꮓ'),
+    (0xAB94, 'M', 'Ꮔ'),
+    (0xAB95, 'M', 'Ꮕ'),
+    (0xAB96, 'M', 'Ꮖ'),
+    (0xAB97, 'M', 'Ꮗ'),
+    (0xAB98, 'M', 'Ꮘ'),
+    (0xAB99, 'M', 'Ꮙ'),
+    (0xAB9A, 'M', 'Ꮚ'),
+    (0xAB9B, 'M', 'Ꮛ'),
+    (0xAB9C, 'M', 'Ꮜ'),
+    (0xAB9D, 'M', 'Ꮝ'),
+    (0xAB9E, 'M', 'Ꮞ'),
+    (0xAB9F, 'M', 'Ꮟ'),
+    (0xABA0, 'M', 'Ꮠ'),
+    (0xABA1, 'M', 'Ꮡ'),
+    (0xABA2, 'M', 'Ꮢ'),
+    (0xABA3, 'M', 'Ꮣ'),
+    (0xABA4, 'M', 'Ꮤ'),
+    (0xABA5, 'M', 'Ꮥ'),
+    (0xABA6, 'M', 'Ꮦ'),
+    (0xABA7, 'M', 'Ꮧ'),
+    (0xABA8, 'M', 'Ꮨ'),
+    (0xABA9, 'M', 'Ꮩ'),
+    (0xABAA, 'M', 'Ꮪ'),
     ]
 
-def _seg_39():
+def _seg_39() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xABB4, 'M', u'Ꮴ'),
-    (0xABB5, 'M', u'Ꮵ'),
-    (0xABB6, 'M', u'Ꮶ'),
-    (0xABB7, 'M', u'Ꮷ'),
-    (0xABB8, 'M', u'Ꮸ'),
-    (0xABB9, 'M', u'Ꮹ'),
-    (0xABBA, 'M', u'Ꮺ'),
-    (0xABBB, 'M', u'Ꮻ'),
-    (0xABBC, 'M', u'Ꮼ'),
-    (0xABBD, 'M', u'Ꮽ'),
-    (0xABBE, 'M', u'Ꮾ'),
-    (0xABBF, 'M', u'Ꮿ'),
+    (0xABAB, 'M', 'Ꮫ'),
+    (0xABAC, 'M', 'Ꮬ'),
+    (0xABAD, 'M', 'Ꮭ'),
+    (0xABAE, 'M', 'Ꮮ'),
+    (0xABAF, 'M', 'Ꮯ'),
+    (0xABB0, 'M', 'Ꮰ'),
+    (0xABB1, 'M', 'Ꮱ'),
+    (0xABB2, 'M', 'Ꮲ'),
+    (0xABB3, 'M', 'Ꮳ'),
+    (0xABB4, 'M', 'Ꮴ'),
+    (0xABB5, 'M', 'Ꮵ'),
+    (0xABB6, 'M', 'Ꮶ'),
+    (0xABB7, 'M', 'Ꮷ'),
+    (0xABB8, 'M', 'Ꮸ'),
+    (0xABB9, 'M', 'Ꮹ'),
+    (0xABBA, 'M', 'Ꮺ'),
+    (0xABBB, 'M', 'Ꮻ'),
+    (0xABBC, 'M', 'Ꮼ'),
+    (0xABBD, 'M', 'Ꮽ'),
+    (0xABBE, 'M', 'Ꮾ'),
+    (0xABBF, 'M', 'Ꮿ'),
     (0xABC0, 'V'),
     (0xABEE, 'X'),
     (0xABF0, 'V'),
@@ -4085,1440 +4097,1440 @@ def _seg_39():
     (0xD7C7, 'X'),
     (0xD7CB, 'V'),
     (0xD7FC, 'X'),
-    (0xF900, 'M', u'豈'),
-    (0xF901, 'M', u'更'),
-    (0xF902, 'M', u'車'),
-    (0xF903, 'M', u'賈'),
-    (0xF904, 'M', u'滑'),
-    (0xF905, 'M', u'串'),
-    (0xF906, 'M', u'句'),
-    (0xF907, 'M', u'龜'),
-    (0xF909, 'M', u'契'),
-    (0xF90A, 'M', u'金'),
-    (0xF90B, 'M', u'喇'),
-    (0xF90C, 'M', u'奈'),
-    (0xF90D, 'M', u'懶'),
-    (0xF90E, 'M', u'癩'),
-    (0xF90F, 'M', u'羅'),
-    (0xF910, 'M', u'蘿'),
-    (0xF911, 'M', u'螺'),
-    (0xF912, 'M', u'裸'),
-    (0xF913, 'M', u'邏'),
-    (0xF914, 'M', u'樂'),
-    (0xF915, 'M', u'洛'),
-    (0xF916, 'M', u'烙'),
-    (0xF917, 'M', u'珞'),
-    (0xF918, 'M', u'落'),
-    (0xF919, 'M', u'酪'),
-    (0xF91A, 'M', u'駱'),
-    (0xF91B, 'M', u'亂'),
-    (0xF91C, 'M', u'卵'),
-    (0xF91D, 'M', u'欄'),
-    (0xF91E, 'M', u'爛'),
-    (0xF91F, 'M', u'蘭'),
-    (0xF920, 'M', u'鸞'),
-    (0xF921, 'M', u'嵐'),
-    (0xF922, 'M', u'濫'),
-    (0xF923, 'M', u'藍'),
-    (0xF924, 'M', u'襤'),
-    (0xF925, 'M', u'拉'),
-    (0xF926, 'M', u'臘'),
-    (0xF927, 'M', u'蠟'),
-    (0xF928, 'M', u'廊'),
-    (0xF929, 'M', u'朗'),
-    (0xF92A, 'M', u'浪'),
-    (0xF92B, 'M', u'狼'),
-    (0xF92C, 'M', u'郎'),
-    (0xF92D, 'M', u'來'),
-    (0xF92E, 'M', u'冷'),
-    (0xF92F, 'M', u'勞'),
-    (0xF930, 'M', u'擄'),
-    (0xF931, 'M', u'櫓'),
-    (0xF932, 'M', u'爐'),
-    (0xF933, 'M', u'盧'),
-    (0xF934, 'M', u'老'),
-    (0xF935, 'M', u'蘆'),
-    (0xF936, 'M', u'虜'),
-    (0xF937, 'M', u'路'),
-    (0xF938, 'M', u'露'),
-    (0xF939, 'M', u'魯'),
-    (0xF93A, 'M', u'鷺'),
-    (0xF93B, 'M', u'碌'),
-    (0xF93C, 'M', u'祿'),
-    (0xF93D, 'M', u'綠'),
-    (0xF93E, 'M', u'菉'),
-    (0xF93F, 'M', u'錄'),
-    (0xF940, 'M', u'鹿'),
-    (0xF941, 'M', u'論'),
-    (0xF942, 'M', u'壟'),
-    (0xF943, 'M', u'弄'),
-    (0xF944, 'M', u'籠'),
-    (0xF945, 'M', u'聾'),
-    (0xF946, 'M', u'牢'),
-    (0xF947, 'M', u'磊'),
-    (0xF948, 'M', u'賂'),
-    (0xF949, 'M', u'雷'),
-    (0xF94A, 'M', u'壘'),
-    (0xF94B, 'M', u'屢'),
-    (0xF94C, 'M', u'樓'),
-    (0xF94D, 'M', u'淚'),
-    (0xF94E, 'M', u'漏'),
+    (0xF900, 'M', '豈'),
+    (0xF901, 'M', '更'),
+    (0xF902, 'M', '車'),
+    (0xF903, 'M', '賈'),
+    (0xF904, 'M', '滑'),
+    (0xF905, 'M', '串'),
+    (0xF906, 'M', '句'),
+    (0xF907, 'M', '龜'),
+    (0xF909, 'M', '契'),
+    (0xF90A, 'M', '金'),
+    (0xF90B, 'M', '喇'),
+    (0xF90C, 'M', '奈'),
+    (0xF90D, 'M', '懶'),
+    (0xF90E, 'M', '癩'),
+    (0xF90F, 'M', '羅'),
+    (0xF910, 'M', '蘿'),
+    (0xF911, 'M', '螺'),
+    (0xF912, 'M', '裸'),
+    (0xF913, 'M', '邏'),
+    (0xF914, 'M', '樂'),
+    (0xF915, 'M', '洛'),
+    (0xF916, 'M', '烙'),
+    (0xF917, 'M', '珞'),
+    (0xF918, 'M', '落'),
+    (0xF919, 'M', '酪'),
+    (0xF91A, 'M', '駱'),
+    (0xF91B, 'M', '亂'),
+    (0xF91C, 'M', '卵'),
+    (0xF91D, 'M', '欄'),
+    (0xF91E, 'M', '爛'),
+    (0xF91F, 'M', '蘭'),
+    (0xF920, 'M', '鸞'),
+    (0xF921, 'M', '嵐'),
+    (0xF922, 'M', '濫'),
+    (0xF923, 'M', '藍'),
+    (0xF924, 'M', '襤'),
+    (0xF925, 'M', '拉'),
+    (0xF926, 'M', '臘'),
+    (0xF927, 'M', '蠟'),
+    (0xF928, 'M', '廊'),
+    (0xF929, 'M', '朗'),
+    (0xF92A, 'M', '浪'),
+    (0xF92B, 'M', '狼'),
+    (0xF92C, 'M', '郎'),
+    (0xF92D, 'M', '來'),
+    (0xF92E, 'M', '冷'),
+    (0xF92F, 'M', '勞'),
+    (0xF930, 'M', '擄'),
+    (0xF931, 'M', '櫓'),
+    (0xF932, 'M', '爐'),
+    (0xF933, 'M', '盧'),
+    (0xF934, 'M', '老'),
+    (0xF935, 'M', '蘆'),
+    (0xF936, 'M', '虜'),
+    (0xF937, 'M', '路'),
+    (0xF938, 'M', '露'),
+    (0xF939, 'M', '魯'),
+    (0xF93A, 'M', '鷺'),
+    (0xF93B, 'M', '碌'),
+    (0xF93C, 'M', '祿'),
+    (0xF93D, 'M', '綠'),
+    (0xF93E, 'M', '菉'),
+    (0xF93F, 'M', '錄'),
+    (0xF940, 'M', '鹿'),
+    (0xF941, 'M', '論'),
+    (0xF942, 'M', '壟'),
+    (0xF943, 'M', '弄'),
+    (0xF944, 'M', '籠'),
+    (0xF945, 'M', '聾'),
     ]
 
-def _seg_40():
+def _seg_40() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xF94F, 'M', u'累'),
-    (0xF950, 'M', u'縷'),
-    (0xF951, 'M', u'陋'),
-    (0xF952, 'M', u'勒'),
-    (0xF953, 'M', u'肋'),
-    (0xF954, 'M', u'凜'),
-    (0xF955, 'M', u'凌'),
-    (0xF956, 'M', u'稜'),
-    (0xF957, 'M', u'綾'),
-    (0xF958, 'M', u'菱'),
-    (0xF959, 'M', u'陵'),
-    (0xF95A, 'M', u'讀'),
-    (0xF95B, 'M', u'拏'),
-    (0xF95C, 'M', u'樂'),
-    (0xF95D, 'M', u'諾'),
-    (0xF95E, 'M', u'丹'),
-    (0xF95F, 'M', u'寧'),
-    (0xF960, 'M', u'怒'),
-    (0xF961, 'M', u'率'),
-    (0xF962, 'M', u'異'),
-    (0xF963, 'M', u'北'),
-    (0xF964, 'M', u'磻'),
-    (0xF965, 'M', u'便'),
-    (0xF966, 'M', u'復'),
-    (0xF967, 'M', u'不'),
-    (0xF968, 'M', u'泌'),
-    (0xF969, 'M', u'數'),
-    (0xF96A, 'M', u'索'),
-    (0xF96B, 'M', u'參'),
-    (0xF96C, 'M', u'塞'),
-    (0xF96D, 'M', u'省'),
-    (0xF96E, 'M', u'葉'),
-    (0xF96F, 'M', u'說'),
-    (0xF970, 'M', u'殺'),
-    (0xF971, 'M', u'辰'),
-    (0xF972, 'M', u'沈'),
-    (0xF973, 'M', u'拾'),
-    (0xF974, 'M', u'若'),
-    (0xF975, 'M', u'掠'),
-    (0xF976, 'M', u'略'),
-    (0xF977, 'M', u'亮'),
-    (0xF978, 'M', u'兩'),
-    (0xF979, 'M', u'凉'),
-    (0xF97A, 'M', u'梁'),
-    (0xF97B, 'M', u'糧'),
-    (0xF97C, 'M', u'良'),
-    (0xF97D, 'M', u'諒'),
-    (0xF97E, 'M', u'量'),
-    (0xF97F, 'M', u'勵'),
-    (0xF980, 'M', u'呂'),
-    (0xF981, 'M', u'女'),
-    (0xF982, 'M', u'廬'),
-    (0xF983, 'M', u'旅'),
-    (0xF984, 'M', u'濾'),
-    (0xF985, 'M', u'礪'),
-    (0xF986, 'M', u'閭'),
-    (0xF987, 'M', u'驪'),
-    (0xF988, 'M', u'麗'),
-    (0xF989, 'M', u'黎'),
-    (0xF98A, 'M', u'力'),
-    (0xF98B, 'M', u'曆'),
-    (0xF98C, 'M', u'歷'),
-    (0xF98D, 'M', u'轢'),
-    (0xF98E, 'M', u'年'),
-    (0xF98F, 'M', u'憐'),
-    (0xF990, 'M', u'戀'),
-    (0xF991, 'M', u'撚'),
-    (0xF992, 'M', u'漣'),
-    (0xF993, 'M', u'煉'),
-    (0xF994, 'M', u'璉'),
-    (0xF995, 'M', u'秊'),
-    (0xF996, 'M', u'練'),
-    (0xF997, 'M', u'聯'),
-    (0xF998, 'M', u'輦'),
-    (0xF999, 'M', u'蓮'),
-    (0xF99A, 'M', u'連'),
-    (0xF99B, 'M', u'鍊'),
-    (0xF99C, 'M', u'列'),
-    (0xF99D, 'M', u'劣'),
-    (0xF99E, 'M', u'咽'),
-    (0xF99F, 'M', u'烈'),
-    (0xF9A0, 'M', u'裂'),
-    (0xF9A1, 'M', u'說'),
-    (0xF9A2, 'M', u'廉'),
-    (0xF9A3, 'M', u'念'),
-    (0xF9A4, 'M', u'捻'),
-    (0xF9A5, 'M', u'殮'),
-    (0xF9A6, 'M', u'簾'),
-    (0xF9A7, 'M', u'獵'),
-    (0xF9A8, 'M', u'令'),
-    (0xF9A9, 'M', u'囹'),
-    (0xF9AA, 'M', u'寧'),
-    (0xF9AB, 'M', u'嶺'),
-    (0xF9AC, 'M', u'怜'),
-    (0xF9AD, 'M', u'玲'),
-    (0xF9AE, 'M', u'瑩'),
-    (0xF9AF, 'M', u'羚'),
-    (0xF9B0, 'M', u'聆'),
-    (0xF9B1, 'M', u'鈴'),
-    (0xF9B2, 'M', u'零'),
+    (0xF946, 'M', '牢'),
+    (0xF947, 'M', '磊'),
+    (0xF948, 'M', '賂'),
+    (0xF949, 'M', '雷'),
+    (0xF94A, 'M', '壘'),
+    (0xF94B, 'M', '屢'),
+    (0xF94C, 'M', '樓'),
+    (0xF94D, 'M', '淚'),
+    (0xF94E, 'M', '漏'),
+    (0xF94F, 'M', '累'),
+    (0xF950, 'M', '縷'),
+    (0xF951, 'M', '陋'),
+    (0xF952, 'M', '勒'),
+    (0xF953, 'M', '肋'),
+    (0xF954, 'M', '凜'),
+    (0xF955, 'M', '凌'),
+    (0xF956, 'M', '稜'),
+    (0xF957, 'M', '綾'),
+    (0xF958, 'M', '菱'),
+    (0xF959, 'M', '陵'),
+    (0xF95A, 'M', '讀'),
+    (0xF95B, 'M', '拏'),
+    (0xF95C, 'M', '樂'),
+    (0xF95D, 'M', '諾'),
+    (0xF95E, 'M', '丹'),
+    (0xF95F, 'M', '寧'),
+    (0xF960, 'M', '怒'),
+    (0xF961, 'M', '率'),
+    (0xF962, 'M', '異'),
+    (0xF963, 'M', '北'),
+    (0xF964, 'M', '磻'),
+    (0xF965, 'M', '便'),
+    (0xF966, 'M', '復'),
+    (0xF967, 'M', '不'),
+    (0xF968, 'M', '泌'),
+    (0xF969, 'M', '數'),
+    (0xF96A, 'M', '索'),
+    (0xF96B, 'M', '參'),
+    (0xF96C, 'M', '塞'),
+    (0xF96D, 'M', '省'),
+    (0xF96E, 'M', '葉'),
+    (0xF96F, 'M', '說'),
+    (0xF970, 'M', '殺'),
+    (0xF971, 'M', '辰'),
+    (0xF972, 'M', '沈'),
+    (0xF973, 'M', '拾'),
+    (0xF974, 'M', '若'),
+    (0xF975, 'M', '掠'),
+    (0xF976, 'M', '略'),
+    (0xF977, 'M', '亮'),
+    (0xF978, 'M', '兩'),
+    (0xF979, 'M', '凉'),
+    (0xF97A, 'M', '梁'),
+    (0xF97B, 'M', '糧'),
+    (0xF97C, 'M', '良'),
+    (0xF97D, 'M', '諒'),
+    (0xF97E, 'M', '量'),
+    (0xF97F, 'M', '勵'),
+    (0xF980, 'M', '呂'),
+    (0xF981, 'M', '女'),
+    (0xF982, 'M', '廬'),
+    (0xF983, 'M', '旅'),
+    (0xF984, 'M', '濾'),
+    (0xF985, 'M', '礪'),
+    (0xF986, 'M', '閭'),
+    (0xF987, 'M', '驪'),
+    (0xF988, 'M', '麗'),
+    (0xF989, 'M', '黎'),
+    (0xF98A, 'M', '力'),
+    (0xF98B, 'M', '曆'),
+    (0xF98C, 'M', '歷'),
+    (0xF98D, 'M', '轢'),
+    (0xF98E, 'M', '年'),
+    (0xF98F, 'M', '憐'),
+    (0xF990, 'M', '戀'),
+    (0xF991, 'M', '撚'),
+    (0xF992, 'M', '漣'),
+    (0xF993, 'M', '煉'),
+    (0xF994, 'M', '璉'),
+    (0xF995, 'M', '秊'),
+    (0xF996, 'M', '練'),
+    (0xF997, 'M', '聯'),
+    (0xF998, 'M', '輦'),
+    (0xF999, 'M', '蓮'),
+    (0xF99A, 'M', '連'),
+    (0xF99B, 'M', '鍊'),
+    (0xF99C, 'M', '列'),
+    (0xF99D, 'M', '劣'),
+    (0xF99E, 'M', '咽'),
+    (0xF99F, 'M', '烈'),
+    (0xF9A0, 'M', '裂'),
+    (0xF9A1, 'M', '說'),
+    (0xF9A2, 'M', '廉'),
+    (0xF9A3, 'M', '念'),
+    (0xF9A4, 'M', '捻'),
+    (0xF9A5, 'M', '殮'),
+    (0xF9A6, 'M', '簾'),
+    (0xF9A7, 'M', '獵'),
+    (0xF9A8, 'M', '令'),
+    (0xF9A9, 'M', '囹'),
     ]
 
-def _seg_41():
+def _seg_41() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xF9B3, 'M', u'靈'),
-    (0xF9B4, 'M', u'領'),
-    (0xF9B5, 'M', u'例'),
-    (0xF9B6, 'M', u'禮'),
-    (0xF9B7, 'M', u'醴'),
-    (0xF9B8, 'M', u'隸'),
-    (0xF9B9, 'M', u'惡'),
-    (0xF9BA, 'M', u'了'),
-    (0xF9BB, 'M', u'僚'),
-    (0xF9BC, 'M', u'寮'),
-    (0xF9BD, 'M', u'尿'),
-    (0xF9BE, 'M', u'料'),
-    (0xF9BF, 'M', u'樂'),
-    (0xF9C0, 'M', u'燎'),
-    (0xF9C1, 'M', u'療'),
-    (0xF9C2, 'M', u'蓼'),
-    (0xF9C3, 'M', u'遼'),
-    (0xF9C4, 'M', u'龍'),
-    (0xF9C5, 'M', u'暈'),
-    (0xF9C6, 'M', u'阮'),
-    (0xF9C7, 'M', u'劉'),
-    (0xF9C8, 'M', u'杻'),
-    (0xF9C9, 'M', u'柳'),
-    (0xF9CA, 'M', u'流'),
-    (0xF9CB, 'M', u'溜'),
-    (0xF9CC, 'M', u'琉'),
-    (0xF9CD, 'M', u'留'),
-    (0xF9CE, 'M', u'硫'),
-    (0xF9CF, 'M', u'紐'),
-    (0xF9D0, 'M', u'類'),
-    (0xF9D1, 'M', u'六'),
-    (0xF9D2, 'M', u'戮'),
-    (0xF9D3, 'M', u'陸'),
-    (0xF9D4, 'M', u'倫'),
-    (0xF9D5, 'M', u'崙'),
-    (0xF9D6, 'M', u'淪'),
-    (0xF9D7, 'M', u'輪'),
-    (0xF9D8, 'M', u'律'),
-    (0xF9D9, 'M', u'慄'),
-    (0xF9DA, 'M', u'栗'),
-    (0xF9DB, 'M', u'率'),
-    (0xF9DC, 'M', u'隆'),
-    (0xF9DD, 'M', u'利'),
-    (0xF9DE, 'M', u'吏'),
-    (0xF9DF, 'M', u'履'),
-    (0xF9E0, 'M', u'易'),
-    (0xF9E1, 'M', u'李'),
-    (0xF9E2, 'M', u'梨'),
-    (0xF9E3, 'M', u'泥'),
-    (0xF9E4, 'M', u'理'),
-    (0xF9E5, 'M', u'痢'),
-    (0xF9E6, 'M', u'罹'),
-    (0xF9E7, 'M', u'裏'),
-    (0xF9E8, 'M', u'裡'),
-    (0xF9E9, 'M', u'里'),
-    (0xF9EA, 'M', u'離'),
-    (0xF9EB, 'M', u'匿'),
-    (0xF9EC, 'M', u'溺'),
-    (0xF9ED, 'M', u'吝'),
-    (0xF9EE, 'M', u'燐'),
-    (0xF9EF, 'M', u'璘'),
-    (0xF9F0, 'M', u'藺'),
-    (0xF9F1, 'M', u'隣'),
-    (0xF9F2, 'M', u'鱗'),
-    (0xF9F3, 'M', u'麟'),
-    (0xF9F4, 'M', u'林'),
-    (0xF9F5, 'M', u'淋'),
-    (0xF9F6, 'M', u'臨'),
-    (0xF9F7, 'M', u'立'),
-    (0xF9F8, 'M', u'笠'),
-    (0xF9F9, 'M', u'粒'),
-    (0xF9FA, 'M', u'狀'),
-    (0xF9FB, 'M', u'炙'),
-    (0xF9FC, 'M', u'識'),
-    (0xF9FD, 'M', u'什'),
-    (0xF9FE, 'M', u'茶'),
-    (0xF9FF, 'M', u'刺'),
-    (0xFA00, 'M', u'切'),
-    (0xFA01, 'M', u'度'),
-    (0xFA02, 'M', u'拓'),
-    (0xFA03, 'M', u'糖'),
-    (0xFA04, 'M', u'宅'),
-    (0xFA05, 'M', u'洞'),
-    (0xFA06, 'M', u'暴'),
-    (0xFA07, 'M', u'輻'),
-    (0xFA08, 'M', u'行'),
-    (0xFA09, 'M', u'降'),
-    (0xFA0A, 'M', u'見'),
-    (0xFA0B, 'M', u'廓'),
-    (0xFA0C, 'M', u'兀'),
-    (0xFA0D, 'M', u'嗀'),
-    (0xFA0E, 'V'),
-    (0xFA10, 'M', u'塚'),
-    (0xFA11, 'V'),
-    (0xFA12, 'M', u'晴'),
-    (0xFA13, 'V'),
-    (0xFA15, 'M', u'凞'),
-    (0xFA16, 'M', u'猪'),
-    (0xFA17, 'M', u'益'),
-    (0xFA18, 'M', u'礼'),
+    (0xF9AA, 'M', '寧'),
+    (0xF9AB, 'M', '嶺'),
+    (0xF9AC, 'M', '怜'),
+    (0xF9AD, 'M', '玲'),
+    (0xF9AE, 'M', '瑩'),
+    (0xF9AF, 'M', '羚'),
+    (0xF9B0, 'M', '聆'),
+    (0xF9B1, 'M', '鈴'),
+    (0xF9B2, 'M', '零'),
+    (0xF9B3, 'M', '靈'),
+    (0xF9B4, 'M', '領'),
+    (0xF9B5, 'M', '例'),
+    (0xF9B6, 'M', '禮'),
+    (0xF9B7, 'M', '醴'),
+    (0xF9B8, 'M', '隸'),
+    (0xF9B9, 'M', '惡'),
+    (0xF9BA, 'M', '了'),
+    (0xF9BB, 'M', '僚'),
+    (0xF9BC, 'M', '寮'),
+    (0xF9BD, 'M', '尿'),
+    (0xF9BE, 'M', '料'),
+    (0xF9BF, 'M', '樂'),
+    (0xF9C0, 'M', '燎'),
+    (0xF9C1, 'M', '療'),
+    (0xF9C2, 'M', '蓼'),
+    (0xF9C3, 'M', '遼'),
+    (0xF9C4, 'M', '龍'),
+    (0xF9C5, 'M', '暈'),
+    (0xF9C6, 'M', '阮'),
+    (0xF9C7, 'M', '劉'),
+    (0xF9C8, 'M', '杻'),
+    (0xF9C9, 'M', '柳'),
+    (0xF9CA, 'M', '流'),
+    (0xF9CB, 'M', '溜'),
+    (0xF9CC, 'M', '琉'),
+    (0xF9CD, 'M', '留'),
+    (0xF9CE, 'M', '硫'),
+    (0xF9CF, 'M', '紐'),
+    (0xF9D0, 'M', '類'),
+    (0xF9D1, 'M', '六'),
+    (0xF9D2, 'M', '戮'),
+    (0xF9D3, 'M', '陸'),
+    (0xF9D4, 'M', '倫'),
+    (0xF9D5, 'M', '崙'),
+    (0xF9D6, 'M', '淪'),
+    (0xF9D7, 'M', '輪'),
+    (0xF9D8, 'M', '律'),
+    (0xF9D9, 'M', '慄'),
+    (0xF9DA, 'M', '栗'),
+    (0xF9DB, 'M', '率'),
+    (0xF9DC, 'M', '隆'),
+    (0xF9DD, 'M', '利'),
+    (0xF9DE, 'M', '吏'),
+    (0xF9DF, 'M', '履'),
+    (0xF9E0, 'M', '易'),
+    (0xF9E1, 'M', '李'),
+    (0xF9E2, 'M', '梨'),
+    (0xF9E3, 'M', '泥'),
+    (0xF9E4, 'M', '理'),
+    (0xF9E5, 'M', '痢'),
+    (0xF9E6, 'M', '罹'),
+    (0xF9E7, 'M', '裏'),
+    (0xF9E8, 'M', '裡'),
+    (0xF9E9, 'M', '里'),
+    (0xF9EA, 'M', '離'),
+    (0xF9EB, 'M', '匿'),
+    (0xF9EC, 'M', '溺'),
+    (0xF9ED, 'M', '吝'),
+    (0xF9EE, 'M', '燐'),
+    (0xF9EF, 'M', '璘'),
+    (0xF9F0, 'M', '藺'),
+    (0xF9F1, 'M', '隣'),
+    (0xF9F2, 'M', '鱗'),
+    (0xF9F3, 'M', '麟'),
+    (0xF9F4, 'M', '林'),
+    (0xF9F5, 'M', '淋'),
+    (0xF9F6, 'M', '臨'),
+    (0xF9F7, 'M', '立'),
+    (0xF9F8, 'M', '笠'),
+    (0xF9F9, 'M', '粒'),
+    (0xF9FA, 'M', '狀'),
+    (0xF9FB, 'M', '炙'),
+    (0xF9FC, 'M', '識'),
+    (0xF9FD, 'M', '什'),
+    (0xF9FE, 'M', '茶'),
+    (0xF9FF, 'M', '刺'),
+    (0xFA00, 'M', '切'),
+    (0xFA01, 'M', '度'),
+    (0xFA02, 'M', '拓'),
+    (0xFA03, 'M', '糖'),
+    (0xFA04, 'M', '宅'),
+    (0xFA05, 'M', '洞'),
+    (0xFA06, 'M', '暴'),
+    (0xFA07, 'M', '輻'),
+    (0xFA08, 'M', '行'),
+    (0xFA09, 'M', '降'),
+    (0xFA0A, 'M', '見'),
+    (0xFA0B, 'M', '廓'),
+    (0xFA0C, 'M', '兀'),
+    (0xFA0D, 'M', '嗀'),
     ]
 
-def _seg_42():
+def _seg_42() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFA19, 'M', u'神'),
-    (0xFA1A, 'M', u'祥'),
-    (0xFA1B, 'M', u'福'),
-    (0xFA1C, 'M', u'靖'),
-    (0xFA1D, 'M', u'精'),
-    (0xFA1E, 'M', u'羽'),
+    (0xFA0E, 'V'),
+    (0xFA10, 'M', '塚'),
+    (0xFA11, 'V'),
+    (0xFA12, 'M', '晴'),
+    (0xFA13, 'V'),
+    (0xFA15, 'M', '凞'),
+    (0xFA16, 'M', '猪'),
+    (0xFA17, 'M', '益'),
+    (0xFA18, 'M', '礼'),
+    (0xFA19, 'M', '神'),
+    (0xFA1A, 'M', '祥'),
+    (0xFA1B, 'M', '福'),
+    (0xFA1C, 'M', '靖'),
+    (0xFA1D, 'M', '精'),
+    (0xFA1E, 'M', '羽'),
     (0xFA1F, 'V'),
-    (0xFA20, 'M', u'蘒'),
+    (0xFA20, 'M', '蘒'),
     (0xFA21, 'V'),
-    (0xFA22, 'M', u'諸'),
+    (0xFA22, 'M', '諸'),
     (0xFA23, 'V'),
-    (0xFA25, 'M', u'逸'),
-    (0xFA26, 'M', u'都'),
+    (0xFA25, 'M', '逸'),
+    (0xFA26, 'M', '都'),
     (0xFA27, 'V'),
-    (0xFA2A, 'M', u'飯'),
-    (0xFA2B, 'M', u'飼'),
-    (0xFA2C, 'M', u'館'),
-    (0xFA2D, 'M', u'鶴'),
-    (0xFA2E, 'M', u'郞'),
-    (0xFA2F, 'M', u'隷'),
-    (0xFA30, 'M', u'侮'),
-    (0xFA31, 'M', u'僧'),
-    (0xFA32, 'M', u'免'),
-    (0xFA33, 'M', u'勉'),
-    (0xFA34, 'M', u'勤'),
-    (0xFA35, 'M', u'卑'),
-    (0xFA36, 'M', u'喝'),
-    (0xFA37, 'M', u'嘆'),
-    (0xFA38, 'M', u'器'),
-    (0xFA39, 'M', u'塀'),
-    (0xFA3A, 'M', u'墨'),
-    (0xFA3B, 'M', u'層'),
-    (0xFA3C, 'M', u'屮'),
-    (0xFA3D, 'M', u'悔'),
-    (0xFA3E, 'M', u'慨'),
-    (0xFA3F, 'M', u'憎'),
-    (0xFA40, 'M', u'懲'),
-    (0xFA41, 'M', u'敏'),
-    (0xFA42, 'M', u'既'),
-    (0xFA43, 'M', u'暑'),
-    (0xFA44, 'M', u'梅'),
-    (0xFA45, 'M', u'海'),
-    (0xFA46, 'M', u'渚'),
-    (0xFA47, 'M', u'漢'),
-    (0xFA48, 'M', u'煮'),
-    (0xFA49, 'M', u'爫'),
-    (0xFA4A, 'M', u'琢'),
-    (0xFA4B, 'M', u'碑'),
-    (0xFA4C, 'M', u'社'),
-    (0xFA4D, 'M', u'祉'),
-    (0xFA4E, 'M', u'祈'),
-    (0xFA4F, 'M', u'祐'),
-    (0xFA50, 'M', u'祖'),
-    (0xFA51, 'M', u'祝'),
-    (0xFA52, 'M', u'禍'),
-    (0xFA53, 'M', u'禎'),
-    (0xFA54, 'M', u'穀'),
-    (0xFA55, 'M', u'突'),
-    (0xFA56, 'M', u'節'),
-    (0xFA57, 'M', u'練'),
-    (0xFA58, 'M', u'縉'),
-    (0xFA59, 'M', u'繁'),
-    (0xFA5A, 'M', u'署'),
-    (0xFA5B, 'M', u'者'),
-    (0xFA5C, 'M', u'臭'),
-    (0xFA5D, 'M', u'艹'),
-    (0xFA5F, 'M', u'著'),
-    (0xFA60, 'M', u'褐'),
-    (0xFA61, 'M', u'視'),
-    (0xFA62, 'M', u'謁'),
-    (0xFA63, 'M', u'謹'),
-    (0xFA64, 'M', u'賓'),
-    (0xFA65, 'M', u'贈'),
-    (0xFA66, 'M', u'辶'),
-    (0xFA67, 'M', u'逸'),
-    (0xFA68, 'M', u'難'),
-    (0xFA69, 'M', u'響'),
-    (0xFA6A, 'M', u'頻'),
-    (0xFA6B, 'M', u'恵'),
-    (0xFA6C, 'M', u'𤋮'),
-    (0xFA6D, 'M', u'舘'),
+    (0xFA2A, 'M', '飯'),
+    (0xFA2B, 'M', '飼'),
+    (0xFA2C, 'M', '館'),
+    (0xFA2D, 'M', '鶴'),
+    (0xFA2E, 'M', '郞'),
+    (0xFA2F, 'M', '隷'),
+    (0xFA30, 'M', '侮'),
+    (0xFA31, 'M', '僧'),
+    (0xFA32, 'M', '免'),
+    (0xFA33, 'M', '勉'),
+    (0xFA34, 'M', '勤'),
+    (0xFA35, 'M', '卑'),
+    (0xFA36, 'M', '喝'),
+    (0xFA37, 'M', '嘆'),
+    (0xFA38, 'M', '器'),
+    (0xFA39, 'M', '塀'),
+    (0xFA3A, 'M', '墨'),
+    (0xFA3B, 'M', '層'),
+    (0xFA3C, 'M', '屮'),
+    (0xFA3D, 'M', '悔'),
+    (0xFA3E, 'M', '慨'),
+    (0xFA3F, 'M', '憎'),
+    (0xFA40, 'M', '懲'),
+    (0xFA41, 'M', '敏'),
+    (0xFA42, 'M', '既'),
+    (0xFA43, 'M', '暑'),
+    (0xFA44, 'M', '梅'),
+    (0xFA45, 'M', '海'),
+    (0xFA46, 'M', '渚'),
+    (0xFA47, 'M', '漢'),
+    (0xFA48, 'M', '煮'),
+    (0xFA49, 'M', '爫'),
+    (0xFA4A, 'M', '琢'),
+    (0xFA4B, 'M', '碑'),
+    (0xFA4C, 'M', '社'),
+    (0xFA4D, 'M', '祉'),
+    (0xFA4E, 'M', '祈'),
+    (0xFA4F, 'M', '祐'),
+    (0xFA50, 'M', '祖'),
+    (0xFA51, 'M', '祝'),
+    (0xFA52, 'M', '禍'),
+    (0xFA53, 'M', '禎'),
+    (0xFA54, 'M', '穀'),
+    (0xFA55, 'M', '突'),
+    (0xFA56, 'M', '節'),
+    (0xFA57, 'M', '練'),
+    (0xFA58, 'M', '縉'),
+    (0xFA59, 'M', '繁'),
+    (0xFA5A, 'M', '署'),
+    (0xFA5B, 'M', '者'),
+    (0xFA5C, 'M', '臭'),
+    (0xFA5D, 'M', '艹'),
+    (0xFA5F, 'M', '著'),
+    (0xFA60, 'M', '褐'),
+    (0xFA61, 'M', '視'),
+    (0xFA62, 'M', '謁'),
+    (0xFA63, 'M', '謹'),
+    (0xFA64, 'M', '賓'),
+    (0xFA65, 'M', '贈'),
+    (0xFA66, 'M', '辶'),
+    (0xFA67, 'M', '逸'),
+    (0xFA68, 'M', '難'),
+    (0xFA69, 'M', '響'),
+    (0xFA6A, 'M', '頻'),
+    (0xFA6B, 'M', '恵'),
+    (0xFA6C, 'M', '𤋮'),
+    (0xFA6D, 'M', '舘'),
     (0xFA6E, 'X'),
-    (0xFA70, 'M', u'並'),
-    (0xFA71, 'M', u'况'),
-    (0xFA72, 'M', u'全'),
-    (0xFA73, 'M', u'侀'),
-    (0xFA74, 'M', u'充'),
-    (0xFA75, 'M', u'冀'),
-    (0xFA76, 'M', u'勇'),
-    (0xFA77, 'M', u'勺'),
-    (0xFA78, 'M', u'喝'),
-    (0xFA79, 'M', u'啕'),
-    (0xFA7A, 'M', u'喙'),
-    (0xFA7B, 'M', u'嗢'),
-    (0xFA7C, 'M', u'塚'),
-    (0xFA7D, 'M', u'墳'),
-    (0xFA7E, 'M', u'奄'),
-    (0xFA7F, 'M', u'奔'),
-    (0xFA80, 'M', u'婢'),
-    (0xFA81, 'M', u'嬨'),
+    (0xFA70, 'M', '並'),
+    (0xFA71, 'M', '况'),
+    (0xFA72, 'M', '全'),
+    (0xFA73, 'M', '侀'),
+    (0xFA74, 'M', '充'),
+    (0xFA75, 'M', '冀'),
+    (0xFA76, 'M', '勇'),
+    (0xFA77, 'M', '勺'),
+    (0xFA78, 'M', '喝'),
     ]
 
-def _seg_43():
+def _seg_43() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFA82, 'M', u'廒'),
-    (0xFA83, 'M', u'廙'),
-    (0xFA84, 'M', u'彩'),
-    (0xFA85, 'M', u'徭'),
-    (0xFA86, 'M', u'惘'),
-    (0xFA87, 'M', u'慎'),
-    (0xFA88, 'M', u'愈'),
-    (0xFA89, 'M', u'憎'),
-    (0xFA8A, 'M', u'慠'),
-    (0xFA8B, 'M', u'懲'),
-    (0xFA8C, 'M', u'戴'),
-    (0xFA8D, 'M', u'揄'),
-    (0xFA8E, 'M', u'搜'),
-    (0xFA8F, 'M', u'摒'),
-    (0xFA90, 'M', u'敖'),
-    (0xFA91, 'M', u'晴'),
-    (0xFA92, 'M', u'朗'),
-    (0xFA93, 'M', u'望'),
-    (0xFA94, 'M', u'杖'),
-    (0xFA95, 'M', u'歹'),
-    (0xFA96, 'M', u'殺'),
-    (0xFA97, 'M', u'流'),
-    (0xFA98, 'M', u'滛'),
-    (0xFA99, 'M', u'滋'),
-    (0xFA9A, 'M', u'漢'),
-    (0xFA9B, 'M', u'瀞'),
-    (0xFA9C, 'M', u'煮'),
-    (0xFA9D, 'M', u'瞧'),
-    (0xFA9E, 'M', u'爵'),
-    (0xFA9F, 'M', u'犯'),
-    (0xFAA0, 'M', u'猪'),
-    (0xFAA1, 'M', u'瑱'),
-    (0xFAA2, 'M', u'甆'),
-    (0xFAA3, 'M', u'画'),
-    (0xFAA4, 'M', u'瘝'),
-    (0xFAA5, 'M', u'瘟'),
-    (0xFAA6, 'M', u'益'),
-    (0xFAA7, 'M', u'盛'),
-    (0xFAA8, 'M', u'直'),
-    (0xFAA9, 'M', u'睊'),
-    (0xFAAA, 'M', u'着'),
-    (0xFAAB, 'M', u'磌'),
-    (0xFAAC, 'M', u'窱'),
-    (0xFAAD, 'M', u'節'),
-    (0xFAAE, 'M', u'类'),
-    (0xFAAF, 'M', u'絛'),
-    (0xFAB0, 'M', u'練'),
-    (0xFAB1, 'M', u'缾'),
-    (0xFAB2, 'M', u'者'),
-    (0xFAB3, 'M', u'荒'),
-    (0xFAB4, 'M', u'華'),
-    (0xFAB5, 'M', u'蝹'),
-    (0xFAB6, 'M', u'襁'),
-    (0xFAB7, 'M', u'覆'),
-    (0xFAB8, 'M', u'視'),
-    (0xFAB9, 'M', u'調'),
-    (0xFABA, 'M', u'諸'),
-    (0xFABB, 'M', u'請'),
-    (0xFABC, 'M', u'謁'),
-    (0xFABD, 'M', u'諾'),
-    (0xFABE, 'M', u'諭'),
-    (0xFABF, 'M', u'謹'),
-    (0xFAC0, 'M', u'變'),
-    (0xFAC1, 'M', u'贈'),
-    (0xFAC2, 'M', u'輸'),
-    (0xFAC3, 'M', u'遲'),
-    (0xFAC4, 'M', u'醙'),
-    (0xFAC5, 'M', u'鉶'),
-    (0xFAC6, 'M', u'陼'),
-    (0xFAC7, 'M', u'難'),
-    (0xFAC8, 'M', u'靖'),
-    (0xFAC9, 'M', u'韛'),
-    (0xFACA, 'M', u'響'),
-    (0xFACB, 'M', u'頋'),
-    (0xFACC, 'M', u'頻'),
-    (0xFACD, 'M', u'鬒'),
-    (0xFACE, 'M', u'龜'),
-    (0xFACF, 'M', u'𢡊'),
-    (0xFAD0, 'M', u'𢡄'),
-    (0xFAD1, 'M', u'𣏕'),
-    (0xFAD2, 'M', u'㮝'),
-    (0xFAD3, 'M', u'䀘'),
-    (0xFAD4, 'M', u'䀹'),
-    (0xFAD5, 'M', u'𥉉'),
-    (0xFAD6, 'M', u'𥳐'),
-    (0xFAD7, 'M', u'𧻓'),
-    (0xFAD8, 'M', u'齃'),
-    (0xFAD9, 'M', u'龎'),
+    (0xFA79, 'M', '啕'),
+    (0xFA7A, 'M', '喙'),
+    (0xFA7B, 'M', '嗢'),
+    (0xFA7C, 'M', '塚'),
+    (0xFA7D, 'M', '墳'),
+    (0xFA7E, 'M', '奄'),
+    (0xFA7F, 'M', '奔'),
+    (0xFA80, 'M', '婢'),
+    (0xFA81, 'M', '嬨'),
+    (0xFA82, 'M', '廒'),
+    (0xFA83, 'M', '廙'),
+    (0xFA84, 'M', '彩'),
+    (0xFA85, 'M', '徭'),
+    (0xFA86, 'M', '惘'),
+    (0xFA87, 'M', '慎'),
+    (0xFA88, 'M', '愈'),
+    (0xFA89, 'M', '憎'),
+    (0xFA8A, 'M', '慠'),
+    (0xFA8B, 'M', '懲'),
+    (0xFA8C, 'M', '戴'),
+    (0xFA8D, 'M', '揄'),
+    (0xFA8E, 'M', '搜'),
+    (0xFA8F, 'M', '摒'),
+    (0xFA90, 'M', '敖'),
+    (0xFA91, 'M', '晴'),
+    (0xFA92, 'M', '朗'),
+    (0xFA93, 'M', '望'),
+    (0xFA94, 'M', '杖'),
+    (0xFA95, 'M', '歹'),
+    (0xFA96, 'M', '殺'),
+    (0xFA97, 'M', '流'),
+    (0xFA98, 'M', '滛'),
+    (0xFA99, 'M', '滋'),
+    (0xFA9A, 'M', '漢'),
+    (0xFA9B, 'M', '瀞'),
+    (0xFA9C, 'M', '煮'),
+    (0xFA9D, 'M', '瞧'),
+    (0xFA9E, 'M', '爵'),
+    (0xFA9F, 'M', '犯'),
+    (0xFAA0, 'M', '猪'),
+    (0xFAA1, 'M', '瑱'),
+    (0xFAA2, 'M', '甆'),
+    (0xFAA3, 'M', '画'),
+    (0xFAA4, 'M', '瘝'),
+    (0xFAA5, 'M', '瘟'),
+    (0xFAA6, 'M', '益'),
+    (0xFAA7, 'M', '盛'),
+    (0xFAA8, 'M', '直'),
+    (0xFAA9, 'M', '睊'),
+    (0xFAAA, 'M', '着'),
+    (0xFAAB, 'M', '磌'),
+    (0xFAAC, 'M', '窱'),
+    (0xFAAD, 'M', '節'),
+    (0xFAAE, 'M', '类'),
+    (0xFAAF, 'M', '絛'),
+    (0xFAB0, 'M', '練'),
+    (0xFAB1, 'M', '缾'),
+    (0xFAB2, 'M', '者'),
+    (0xFAB3, 'M', '荒'),
+    (0xFAB4, 'M', '華'),
+    (0xFAB5, 'M', '蝹'),
+    (0xFAB6, 'M', '襁'),
+    (0xFAB7, 'M', '覆'),
+    (0xFAB8, 'M', '視'),
+    (0xFAB9, 'M', '調'),
+    (0xFABA, 'M', '諸'),
+    (0xFABB, 'M', '請'),
+    (0xFABC, 'M', '謁'),
+    (0xFABD, 'M', '諾'),
+    (0xFABE, 'M', '諭'),
+    (0xFABF, 'M', '謹'),
+    (0xFAC0, 'M', '變'),
+    (0xFAC1, 'M', '贈'),
+    (0xFAC2, 'M', '輸'),
+    (0xFAC3, 'M', '遲'),
+    (0xFAC4, 'M', '醙'),
+    (0xFAC5, 'M', '鉶'),
+    (0xFAC6, 'M', '陼'),
+    (0xFAC7, 'M', '難'),
+    (0xFAC8, 'M', '靖'),
+    (0xFAC9, 'M', '韛'),
+    (0xFACA, 'M', '響'),
+    (0xFACB, 'M', '頋'),
+    (0xFACC, 'M', '頻'),
+    (0xFACD, 'M', '鬒'),
+    (0xFACE, 'M', '龜'),
+    (0xFACF, 'M', '𢡊'),
+    (0xFAD0, 'M', '𢡄'),
+    (0xFAD1, 'M', '𣏕'),
+    (0xFAD2, 'M', '㮝'),
+    (0xFAD3, 'M', '䀘'),
+    (0xFAD4, 'M', '䀹'),
+    (0xFAD5, 'M', '𥉉'),
+    (0xFAD6, 'M', '𥳐'),
+    (0xFAD7, 'M', '𧻓'),
+    (0xFAD8, 'M', '齃'),
+    (0xFAD9, 'M', '龎'),
     (0xFADA, 'X'),
-    (0xFB00, 'M', u'ff'),
-    (0xFB01, 'M', u'fi'),
-    (0xFB02, 'M', u'fl'),
-    (0xFB03, 'M', u'ffi'),
-    (0xFB04, 'M', u'ffl'),
-    (0xFB05, 'M', u'st'),
-    (0xFB07, 'X'),
-    (0xFB13, 'M', u'մն'),
-    (0xFB14, 'M', u'մե'),
-    (0xFB15, 'M', u'մի'),
-    (0xFB16, 'M', u'վն'),
+    (0xFB00, 'M', 'ff'),
+    (0xFB01, 'M', 'fi'),
     ]
 
-def _seg_44():
+def _seg_44() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFB17, 'M', u'մխ'),
+    (0xFB02, 'M', 'fl'),
+    (0xFB03, 'M', 'ffi'),
+    (0xFB04, 'M', 'ffl'),
+    (0xFB05, 'M', 'st'),
+    (0xFB07, 'X'),
+    (0xFB13, 'M', 'մն'),
+    (0xFB14, 'M', 'մե'),
+    (0xFB15, 'M', 'մի'),
+    (0xFB16, 'M', 'վն'),
+    (0xFB17, 'M', 'մխ'),
     (0xFB18, 'X'),
-    (0xFB1D, 'M', u'יִ'),
+    (0xFB1D, 'M', 'יִ'),
     (0xFB1E, 'V'),
-    (0xFB1F, 'M', u'ײַ'),
-    (0xFB20, 'M', u'ע'),
-    (0xFB21, 'M', u'א'),
-    (0xFB22, 'M', u'ד'),
-    (0xFB23, 'M', u'ה'),
-    (0xFB24, 'M', u'כ'),
-    (0xFB25, 'M', u'ל'),
-    (0xFB26, 'M', u'ם'),
-    (0xFB27, 'M', u'ר'),
-    (0xFB28, 'M', u'ת'),
-    (0xFB29, '3', u'+'),
-    (0xFB2A, 'M', u'שׁ'),
-    (0xFB2B, 'M', u'שׂ'),
-    (0xFB2C, 'M', u'שּׁ'),
-    (0xFB2D, 'M', u'שּׂ'),
-    (0xFB2E, 'M', u'אַ'),
-    (0xFB2F, 'M', u'אָ'),
-    (0xFB30, 'M', u'אּ'),
-    (0xFB31, 'M', u'בּ'),
-    (0xFB32, 'M', u'גּ'),
-    (0xFB33, 'M', u'דּ'),
-    (0xFB34, 'M', u'הּ'),
-    (0xFB35, 'M', u'וּ'),
-    (0xFB36, 'M', u'זּ'),
+    (0xFB1F, 'M', 'ײַ'),
+    (0xFB20, 'M', 'ע'),
+    (0xFB21, 'M', 'א'),
+    (0xFB22, 'M', 'ד'),
+    (0xFB23, 'M', 'ה'),
+    (0xFB24, 'M', 'כ'),
+    (0xFB25, 'M', 'ל'),
+    (0xFB26, 'M', 'ם'),
+    (0xFB27, 'M', 'ר'),
+    (0xFB28, 'M', 'ת'),
+    (0xFB29, '3', '+'),
+    (0xFB2A, 'M', 'שׁ'),
+    (0xFB2B, 'M', 'שׂ'),
+    (0xFB2C, 'M', 'שּׁ'),
+    (0xFB2D, 'M', 'שּׂ'),
+    (0xFB2E, 'M', 'אַ'),
+    (0xFB2F, 'M', 'אָ'),
+    (0xFB30, 'M', 'אּ'),
+    (0xFB31, 'M', 'בּ'),
+    (0xFB32, 'M', 'גּ'),
+    (0xFB33, 'M', 'דּ'),
+    (0xFB34, 'M', 'הּ'),
+    (0xFB35, 'M', 'וּ'),
+    (0xFB36, 'M', 'זּ'),
     (0xFB37, 'X'),
-    (0xFB38, 'M', u'טּ'),
-    (0xFB39, 'M', u'יּ'),
-    (0xFB3A, 'M', u'ךּ'),
-    (0xFB3B, 'M', u'כּ'),
-    (0xFB3C, 'M', u'לּ'),
+    (0xFB38, 'M', 'טּ'),
+    (0xFB39, 'M', 'יּ'),
+    (0xFB3A, 'M', 'ךּ'),
+    (0xFB3B, 'M', 'כּ'),
+    (0xFB3C, 'M', 'לּ'),
     (0xFB3D, 'X'),
-    (0xFB3E, 'M', u'מּ'),
+    (0xFB3E, 'M', 'מּ'),
     (0xFB3F, 'X'),
-    (0xFB40, 'M', u'נּ'),
-    (0xFB41, 'M', u'סּ'),
+    (0xFB40, 'M', 'נּ'),
+    (0xFB41, 'M', 'סּ'),
     (0xFB42, 'X'),
-    (0xFB43, 'M', u'ףּ'),
-    (0xFB44, 'M', u'פּ'),
+    (0xFB43, 'M', 'ףּ'),
+    (0xFB44, 'M', 'פּ'),
     (0xFB45, 'X'),
-    (0xFB46, 'M', u'צּ'),
-    (0xFB47, 'M', u'קּ'),
-    (0xFB48, 'M', u'רּ'),
-    (0xFB49, 'M', u'שּ'),
-    (0xFB4A, 'M', u'תּ'),
-    (0xFB4B, 'M', u'וֹ'),
-    (0xFB4C, 'M', u'בֿ'),
-    (0xFB4D, 'M', u'כֿ'),
-    (0xFB4E, 'M', u'פֿ'),
-    (0xFB4F, 'M', u'אל'),
-    (0xFB50, 'M', u'ٱ'),
-    (0xFB52, 'M', u'ٻ'),
-    (0xFB56, 'M', u'پ'),
-    (0xFB5A, 'M', u'ڀ'),
-    (0xFB5E, 'M', u'ٺ'),
-    (0xFB62, 'M', u'ٿ'),
-    (0xFB66, 'M', u'ٹ'),
-    (0xFB6A, 'M', u'ڤ'),
-    (0xFB6E, 'M', u'ڦ'),
-    (0xFB72, 'M', u'ڄ'),
-    (0xFB76, 'M', u'ڃ'),
-    (0xFB7A, 'M', u'چ'),
-    (0xFB7E, 'M', u'ڇ'),
-    (0xFB82, 'M', u'ڍ'),
-    (0xFB84, 'M', u'ڌ'),
-    (0xFB86, 'M', u'ڎ'),
-    (0xFB88, 'M', u'ڈ'),
-    (0xFB8A, 'M', u'ژ'),
-    (0xFB8C, 'M', u'ڑ'),
-    (0xFB8E, 'M', u'ک'),
-    (0xFB92, 'M', u'گ'),
-    (0xFB96, 'M', u'ڳ'),
-    (0xFB9A, 'M', u'ڱ'),
-    (0xFB9E, 'M', u'ں'),
-    (0xFBA0, 'M', u'ڻ'),
-    (0xFBA4, 'M', u'ۀ'),
-    (0xFBA6, 'M', u'ہ'),
-    (0xFBAA, 'M', u'ھ'),
-    (0xFBAE, 'M', u'ے'),
-    (0xFBB0, 'M', u'ۓ'),
+    (0xFB46, 'M', 'צּ'),
+    (0xFB47, 'M', 'קּ'),
+    (0xFB48, 'M', 'רּ'),
+    (0xFB49, 'M', 'שּ'),
+    (0xFB4A, 'M', 'תּ'),
+    (0xFB4B, 'M', 'וֹ'),
+    (0xFB4C, 'M', 'בֿ'),
+    (0xFB4D, 'M', 'כֿ'),
+    (0xFB4E, 'M', 'פֿ'),
+    (0xFB4F, 'M', 'אל'),
+    (0xFB50, 'M', 'ٱ'),
+    (0xFB52, 'M', 'ٻ'),
+    (0xFB56, 'M', 'پ'),
+    (0xFB5A, 'M', 'ڀ'),
+    (0xFB5E, 'M', 'ٺ'),
+    (0xFB62, 'M', 'ٿ'),
+    (0xFB66, 'M', 'ٹ'),
+    (0xFB6A, 'M', 'ڤ'),
+    (0xFB6E, 'M', 'ڦ'),
+    (0xFB72, 'M', 'ڄ'),
+    (0xFB76, 'M', 'ڃ'),
+    (0xFB7A, 'M', 'چ'),
+    (0xFB7E, 'M', 'ڇ'),
+    (0xFB82, 'M', 'ڍ'),
+    (0xFB84, 'M', 'ڌ'),
+    (0xFB86, 'M', 'ڎ'),
+    (0xFB88, 'M', 'ڈ'),
+    (0xFB8A, 'M', 'ژ'),
+    (0xFB8C, 'M', 'ڑ'),
+    (0xFB8E, 'M', 'ک'),
+    (0xFB92, 'M', 'گ'),
+    (0xFB96, 'M', 'ڳ'),
+    (0xFB9A, 'M', 'ڱ'),
+    (0xFB9E, 'M', 'ں'),
+    (0xFBA0, 'M', 'ڻ'),
+    (0xFBA4, 'M', 'ۀ'),
+    (0xFBA6, 'M', 'ہ'),
+    (0xFBAA, 'M', 'ھ'),
+    (0xFBAE, 'M', 'ے'),
+    (0xFBB0, 'M', 'ۓ'),
     (0xFBB2, 'V'),
-    (0xFBC2, 'X'),
-    (0xFBD3, 'M', u'ڭ'),
-    (0xFBD7, 'M', u'ۇ'),
-    (0xFBD9, 'M', u'ۆ'),
-    (0xFBDB, 'M', u'ۈ'),
-    (0xFBDD, 'M', u'ۇٴ'),
-    (0xFBDE, 'M', u'ۋ'),
-    (0xFBE0, 'M', u'ۅ'),
-    (0xFBE2, 'M', u'ۉ'),
-    (0xFBE4, 'M', u'ې'),
-    (0xFBE8, 'M', u'ى'),
-    (0xFBEA, 'M', u'ئا'),
-    (0xFBEC, 'M', u'ئە'),
-    (0xFBEE, 'M', u'ئو'),
-    (0xFBF0, 'M', u'ئۇ'),
-    (0xFBF2, 'M', u'ئۆ'),
+    (0xFBC3, 'X'),
+    (0xFBD3, 'M', 'ڭ'),
+    (0xFBD7, 'M', 'ۇ'),
+    (0xFBD9, 'M', 'ۆ'),
+    (0xFBDB, 'M', 'ۈ'),
+    (0xFBDD, 'M', 'ۇٴ'),
+    (0xFBDE, 'M', 'ۋ'),
     ]
 
-def _seg_45():
+def _seg_45() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFBF4, 'M', u'ئۈ'),
-    (0xFBF6, 'M', u'ئې'),
-    (0xFBF9, 'M', u'ئى'),
-    (0xFBFC, 'M', u'ی'),
-    (0xFC00, 'M', u'ئج'),
-    (0xFC01, 'M', u'ئح'),
-    (0xFC02, 'M', u'ئم'),
-    (0xFC03, 'M', u'ئى'),
-    (0xFC04, 'M', u'ئي'),
-    (0xFC05, 'M', u'بج'),
-    (0xFC06, 'M', u'بح'),
-    (0xFC07, 'M', u'بخ'),
-    (0xFC08, 'M', u'بم'),
-    (0xFC09, 'M', u'بى'),
-    (0xFC0A, 'M', u'بي'),
-    (0xFC0B, 'M', u'تج'),
-    (0xFC0C, 'M', u'تح'),
-    (0xFC0D, 'M', u'تخ'),
-    (0xFC0E, 'M', u'تم'),
-    (0xFC0F, 'M', u'تى'),
-    (0xFC10, 'M', u'تي'),
-    (0xFC11, 'M', u'ثج'),
-    (0xFC12, 'M', u'ثم'),
-    (0xFC13, 'M', u'ثى'),
-    (0xFC14, 'M', u'ثي'),
-    (0xFC15, 'M', u'جح'),
-    (0xFC16, 'M', u'جم'),
-    (0xFC17, 'M', u'حج'),
-    (0xFC18, 'M', u'حم'),
-    (0xFC19, 'M', u'خج'),
-    (0xFC1A, 'M', u'خح'),
-    (0xFC1B, 'M', u'خم'),
-    (0xFC1C, 'M', u'سج'),
-    (0xFC1D, 'M', u'سح'),
-    (0xFC1E, 'M', u'سخ'),
-    (0xFC1F, 'M', u'سم'),
-    (0xFC20, 'M', u'صح'),
-    (0xFC21, 'M', u'صم'),
-    (0xFC22, 'M', u'ضج'),
-    (0xFC23, 'M', u'ضح'),
-    (0xFC24, 'M', u'ضخ'),
-    (0xFC25, 'M', u'ضم'),
-    (0xFC26, 'M', u'طح'),
-    (0xFC27, 'M', u'طم'),
-    (0xFC28, 'M', u'ظم'),
-    (0xFC29, 'M', u'عج'),
-    (0xFC2A, 'M', u'عم'),
-    (0xFC2B, 'M', u'غج'),
-    (0xFC2C, 'M', u'غم'),
-    (0xFC2D, 'M', u'فج'),
-    (0xFC2E, 'M', u'فح'),
-    (0xFC2F, 'M', u'فخ'),
-    (0xFC30, 'M', u'فم'),
-    (0xFC31, 'M', u'فى'),
-    (0xFC32, 'M', u'في'),
-    (0xFC33, 'M', u'قح'),
-    (0xFC34, 'M', u'قم'),
-    (0xFC35, 'M', u'قى'),
-    (0xFC36, 'M', u'قي'),
-    (0xFC37, 'M', u'كا'),
-    (0xFC38, 'M', u'كج'),
-    (0xFC39, 'M', u'كح'),
-    (0xFC3A, 'M', u'كخ'),
-    (0xFC3B, 'M', u'كل'),
-    (0xFC3C, 'M', u'كم'),
-    (0xFC3D, 'M', u'كى'),
-    (0xFC3E, 'M', u'كي'),
-    (0xFC3F, 'M', u'لج'),
-    (0xFC40, 'M', u'لح'),
-    (0xFC41, 'M', u'لخ'),
-    (0xFC42, 'M', u'لم'),
-    (0xFC43, 'M', u'لى'),
-    (0xFC44, 'M', u'لي'),
-    (0xFC45, 'M', u'مج'),
-    (0xFC46, 'M', u'مح'),
-    (0xFC47, 'M', u'مخ'),
-    (0xFC48, 'M', u'مم'),
-    (0xFC49, 'M', u'مى'),
-    (0xFC4A, 'M', u'مي'),
-    (0xFC4B, 'M', u'نج'),
-    (0xFC4C, 'M', u'نح'),
-    (0xFC4D, 'M', u'نخ'),
-    (0xFC4E, 'M', u'نم'),
-    (0xFC4F, 'M', u'نى'),
-    (0xFC50, 'M', u'ني'),
-    (0xFC51, 'M', u'هج'),
-    (0xFC52, 'M', u'هم'),
-    (0xFC53, 'M', u'هى'),
-    (0xFC54, 'M', u'هي'),
-    (0xFC55, 'M', u'يج'),
-    (0xFC56, 'M', u'يح'),
-    (0xFC57, 'M', u'يخ'),
-    (0xFC58, 'M', u'يم'),
-    (0xFC59, 'M', u'يى'),
-    (0xFC5A, 'M', u'يي'),
-    (0xFC5B, 'M', u'ذٰ'),
-    (0xFC5C, 'M', u'رٰ'),
-    (0xFC5D, 'M', u'ىٰ'),
-    (0xFC5E, '3', u' ٌّ'),
-    (0xFC5F, '3', u' ٍّ'),
+    (0xFBE0, 'M', 'ۅ'),
+    (0xFBE2, 'M', 'ۉ'),
+    (0xFBE4, 'M', 'ې'),
+    (0xFBE8, 'M', 'ى'),
+    (0xFBEA, 'M', 'ئا'),
+    (0xFBEC, 'M', 'ئە'),
+    (0xFBEE, 'M', 'ئو'),
+    (0xFBF0, 'M', 'ئۇ'),
+    (0xFBF2, 'M', 'ئۆ'),
+    (0xFBF4, 'M', 'ئۈ'),
+    (0xFBF6, 'M', 'ئې'),
+    (0xFBF9, 'M', 'ئى'),
+    (0xFBFC, 'M', 'ی'),
+    (0xFC00, 'M', 'ئج'),
+    (0xFC01, 'M', 'ئح'),
+    (0xFC02, 'M', 'ئم'),
+    (0xFC03, 'M', 'ئى'),
+    (0xFC04, 'M', 'ئي'),
+    (0xFC05, 'M', 'بج'),
+    (0xFC06, 'M', 'بح'),
+    (0xFC07, 'M', 'بخ'),
+    (0xFC08, 'M', 'بم'),
+    (0xFC09, 'M', 'بى'),
+    (0xFC0A, 'M', 'بي'),
+    (0xFC0B, 'M', 'تج'),
+    (0xFC0C, 'M', 'تح'),
+    (0xFC0D, 'M', 'تخ'),
+    (0xFC0E, 'M', 'تم'),
+    (0xFC0F, 'M', 'تى'),
+    (0xFC10, 'M', 'تي'),
+    (0xFC11, 'M', 'ثج'),
+    (0xFC12, 'M', 'ثم'),
+    (0xFC13, 'M', 'ثى'),
+    (0xFC14, 'M', 'ثي'),
+    (0xFC15, 'M', 'جح'),
+    (0xFC16, 'M', 'جم'),
+    (0xFC17, 'M', 'حج'),
+    (0xFC18, 'M', 'حم'),
+    (0xFC19, 'M', 'خج'),
+    (0xFC1A, 'M', 'خح'),
+    (0xFC1B, 'M', 'خم'),
+    (0xFC1C, 'M', 'سج'),
+    (0xFC1D, 'M', 'سح'),
+    (0xFC1E, 'M', 'سخ'),
+    (0xFC1F, 'M', 'سم'),
+    (0xFC20, 'M', 'صح'),
+    (0xFC21, 'M', 'صم'),
+    (0xFC22, 'M', 'ضج'),
+    (0xFC23, 'M', 'ضح'),
+    (0xFC24, 'M', 'ضخ'),
+    (0xFC25, 'M', 'ضم'),
+    (0xFC26, 'M', 'طح'),
+    (0xFC27, 'M', 'طم'),
+    (0xFC28, 'M', 'ظم'),
+    (0xFC29, 'M', 'عج'),
+    (0xFC2A, 'M', 'عم'),
+    (0xFC2B, 'M', 'غج'),
+    (0xFC2C, 'M', 'غم'),
+    (0xFC2D, 'M', 'فج'),
+    (0xFC2E, 'M', 'فح'),
+    (0xFC2F, 'M', 'فخ'),
+    (0xFC30, 'M', 'فم'),
+    (0xFC31, 'M', 'فى'),
+    (0xFC32, 'M', 'في'),
+    (0xFC33, 'M', 'قح'),
+    (0xFC34, 'M', 'قم'),
+    (0xFC35, 'M', 'قى'),
+    (0xFC36, 'M', 'قي'),
+    (0xFC37, 'M', 'كا'),
+    (0xFC38, 'M', 'كج'),
+    (0xFC39, 'M', 'كح'),
+    (0xFC3A, 'M', 'كخ'),
+    (0xFC3B, 'M', 'كل'),
+    (0xFC3C, 'M', 'كم'),
+    (0xFC3D, 'M', 'كى'),
+    (0xFC3E, 'M', 'كي'),
+    (0xFC3F, 'M', 'لج'),
+    (0xFC40, 'M', 'لح'),
+    (0xFC41, 'M', 'لخ'),
+    (0xFC42, 'M', 'لم'),
+    (0xFC43, 'M', 'لى'),
+    (0xFC44, 'M', 'لي'),
+    (0xFC45, 'M', 'مج'),
+    (0xFC46, 'M', 'مح'),
+    (0xFC47, 'M', 'مخ'),
+    (0xFC48, 'M', 'مم'),
+    (0xFC49, 'M', 'مى'),
+    (0xFC4A, 'M', 'مي'),
+    (0xFC4B, 'M', 'نج'),
+    (0xFC4C, 'M', 'نح'),
+    (0xFC4D, 'M', 'نخ'),
+    (0xFC4E, 'M', 'نم'),
+    (0xFC4F, 'M', 'نى'),
+    (0xFC50, 'M', 'ني'),
+    (0xFC51, 'M', 'هج'),
+    (0xFC52, 'M', 'هم'),
+    (0xFC53, 'M', 'هى'),
+    (0xFC54, 'M', 'هي'),
+    (0xFC55, 'M', 'يج'),
+    (0xFC56, 'M', 'يح'),
     ]
 
-def _seg_46():
+def _seg_46() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFC60, '3', u' َّ'),
-    (0xFC61, '3', u' ُّ'),
-    (0xFC62, '3', u' ِّ'),
-    (0xFC63, '3', u' ّٰ'),
-    (0xFC64, 'M', u'ئر'),
-    (0xFC65, 'M', u'ئز'),
-    (0xFC66, 'M', u'ئم'),
-    (0xFC67, 'M', u'ئن'),
-    (0xFC68, 'M', u'ئى'),
-    (0xFC69, 'M', u'ئي'),
-    (0xFC6A, 'M', u'بر'),
-    (0xFC6B, 'M', u'بز'),
-    (0xFC6C, 'M', u'بم'),
-    (0xFC6D, 'M', u'بن'),
-    (0xFC6E, 'M', u'بى'),
-    (0xFC6F, 'M', u'بي'),
-    (0xFC70, 'M', u'تر'),
-    (0xFC71, 'M', u'تز'),
-    (0xFC72, 'M', u'تم'),
-    (0xFC73, 'M', u'تن'),
-    (0xFC74, 'M', u'تى'),
-    (0xFC75, 'M', u'تي'),
-    (0xFC76, 'M', u'ثر'),
-    (0xFC77, 'M', u'ثز'),
-    (0xFC78, 'M', u'ثم'),
-    (0xFC79, 'M', u'ثن'),
-    (0xFC7A, 'M', u'ثى'),
-    (0xFC7B, 'M', u'ثي'),
-    (0xFC7C, 'M', u'فى'),
-    (0xFC7D, 'M', u'في'),
-    (0xFC7E, 'M', u'قى'),
-    (0xFC7F, 'M', u'قي'),
-    (0xFC80, 'M', u'كا'),
-    (0xFC81, 'M', u'كل'),
-    (0xFC82, 'M', u'كم'),
-    (0xFC83, 'M', u'كى'),
-    (0xFC84, 'M', u'كي'),
-    (0xFC85, 'M', u'لم'),
-    (0xFC86, 'M', u'لى'),
-    (0xFC87, 'M', u'لي'),
-    (0xFC88, 'M', u'ما'),
-    (0xFC89, 'M', u'مم'),
-    (0xFC8A, 'M', u'نر'),
-    (0xFC8B, 'M', u'نز'),
-    (0xFC8C, 'M', u'نم'),
-    (0xFC8D, 'M', u'نن'),
-    (0xFC8E, 'M', u'نى'),
-    (0xFC8F, 'M', u'ني'),
-    (0xFC90, 'M', u'ىٰ'),
-    (0xFC91, 'M', u'ير'),
-    (0xFC92, 'M', u'يز'),
-    (0xFC93, 'M', u'يم'),
-    (0xFC94, 'M', u'ين'),
-    (0xFC95, 'M', u'يى'),
-    (0xFC96, 'M', u'يي'),
-    (0xFC97, 'M', u'ئج'),
-    (0xFC98, 'M', u'ئح'),
-    (0xFC99, 'M', u'ئخ'),
-    (0xFC9A, 'M', u'ئم'),
-    (0xFC9B, 'M', u'ئه'),
-    (0xFC9C, 'M', u'بج'),
-    (0xFC9D, 'M', u'بح'),
-    (0xFC9E, 'M', u'بخ'),
-    (0xFC9F, 'M', u'بم'),
-    (0xFCA0, 'M', u'به'),
-    (0xFCA1, 'M', u'تج'),
-    (0xFCA2, 'M', u'تح'),
-    (0xFCA3, 'M', u'تخ'),
-    (0xFCA4, 'M', u'تم'),
-    (0xFCA5, 'M', u'ته'),
-    (0xFCA6, 'M', u'ثم'),
-    (0xFCA7, 'M', u'جح'),
-    (0xFCA8, 'M', u'جم'),
-    (0xFCA9, 'M', u'حج'),
-    (0xFCAA, 'M', u'حم'),
-    (0xFCAB, 'M', u'خج'),
-    (0xFCAC, 'M', u'خم'),
-    (0xFCAD, 'M', u'سج'),
-    (0xFCAE, 'M', u'سح'),
-    (0xFCAF, 'M', u'سخ'),
-    (0xFCB0, 'M', u'سم'),
-    (0xFCB1, 'M', u'صح'),
-    (0xFCB2, 'M', u'صخ'),
-    (0xFCB3, 'M', u'صم'),
-    (0xFCB4, 'M', u'ضج'),
-    (0xFCB5, 'M', u'ضح'),
-    (0xFCB6, 'M', u'ضخ'),
-    (0xFCB7, 'M', u'ضم'),
-    (0xFCB8, 'M', u'طح'),
-    (0xFCB9, 'M', u'ظم'),
-    (0xFCBA, 'M', u'عج'),
-    (0xFCBB, 'M', u'عم'),
-    (0xFCBC, 'M', u'غج'),
-    (0xFCBD, 'M', u'غم'),
-    (0xFCBE, 'M', u'فج'),
-    (0xFCBF, 'M', u'فح'),
-    (0xFCC0, 'M', u'فخ'),
-    (0xFCC1, 'M', u'فم'),
-    (0xFCC2, 'M', u'قح'),
-    (0xFCC3, 'M', u'قم'),
+    (0xFC57, 'M', 'يخ'),
+    (0xFC58, 'M', 'يم'),
+    (0xFC59, 'M', 'يى'),
+    (0xFC5A, 'M', 'يي'),
+    (0xFC5B, 'M', 'ذٰ'),
+    (0xFC5C, 'M', 'رٰ'),
+    (0xFC5D, 'M', 'ىٰ'),
+    (0xFC5E, '3', ' ٌّ'),
+    (0xFC5F, '3', ' ٍّ'),
+    (0xFC60, '3', ' َّ'),
+    (0xFC61, '3', ' ُّ'),
+    (0xFC62, '3', ' ِّ'),
+    (0xFC63, '3', ' ّٰ'),
+    (0xFC64, 'M', 'ئر'),
+    (0xFC65, 'M', 'ئز'),
+    (0xFC66, 'M', 'ئم'),
+    (0xFC67, 'M', 'ئن'),
+    (0xFC68, 'M', 'ئى'),
+    (0xFC69, 'M', 'ئي'),
+    (0xFC6A, 'M', 'بر'),
+    (0xFC6B, 'M', 'بز'),
+    (0xFC6C, 'M', 'بم'),
+    (0xFC6D, 'M', 'بن'),
+    (0xFC6E, 'M', 'بى'),
+    (0xFC6F, 'M', 'بي'),
+    (0xFC70, 'M', 'تر'),
+    (0xFC71, 'M', 'تز'),
+    (0xFC72, 'M', 'تم'),
+    (0xFC73, 'M', 'تن'),
+    (0xFC74, 'M', 'تى'),
+    (0xFC75, 'M', 'تي'),
+    (0xFC76, 'M', 'ثر'),
+    (0xFC77, 'M', 'ثز'),
+    (0xFC78, 'M', 'ثم'),
+    (0xFC79, 'M', 'ثن'),
+    (0xFC7A, 'M', 'ثى'),
+    (0xFC7B, 'M', 'ثي'),
+    (0xFC7C, 'M', 'فى'),
+    (0xFC7D, 'M', 'في'),
+    (0xFC7E, 'M', 'قى'),
+    (0xFC7F, 'M', 'قي'),
+    (0xFC80, 'M', 'كا'),
+    (0xFC81, 'M', 'كل'),
+    (0xFC82, 'M', 'كم'),
+    (0xFC83, 'M', 'كى'),
+    (0xFC84, 'M', 'كي'),
+    (0xFC85, 'M', 'لم'),
+    (0xFC86, 'M', 'لى'),
+    (0xFC87, 'M', 'لي'),
+    (0xFC88, 'M', 'ما'),
+    (0xFC89, 'M', 'مم'),
+    (0xFC8A, 'M', 'نر'),
+    (0xFC8B, 'M', 'نز'),
+    (0xFC8C, 'M', 'نم'),
+    (0xFC8D, 'M', 'نن'),
+    (0xFC8E, 'M', 'نى'),
+    (0xFC8F, 'M', 'ني'),
+    (0xFC90, 'M', 'ىٰ'),
+    (0xFC91, 'M', 'ير'),
+    (0xFC92, 'M', 'يز'),
+    (0xFC93, 'M', 'يم'),
+    (0xFC94, 'M', 'ين'),
+    (0xFC95, 'M', 'يى'),
+    (0xFC96, 'M', 'يي'),
+    (0xFC97, 'M', 'ئج'),
+    (0xFC98, 'M', 'ئح'),
+    (0xFC99, 'M', 'ئخ'),
+    (0xFC9A, 'M', 'ئم'),
+    (0xFC9B, 'M', 'ئه'),
+    (0xFC9C, 'M', 'بج'),
+    (0xFC9D, 'M', 'بح'),
+    (0xFC9E, 'M', 'بخ'),
+    (0xFC9F, 'M', 'بم'),
+    (0xFCA0, 'M', 'به'),
+    (0xFCA1, 'M', 'تج'),
+    (0xFCA2, 'M', 'تح'),
+    (0xFCA3, 'M', 'تخ'),
+    (0xFCA4, 'M', 'تم'),
+    (0xFCA5, 'M', 'ته'),
+    (0xFCA6, 'M', 'ثم'),
+    (0xFCA7, 'M', 'جح'),
+    (0xFCA8, 'M', 'جم'),
+    (0xFCA9, 'M', 'حج'),
+    (0xFCAA, 'M', 'حم'),
+    (0xFCAB, 'M', 'خج'),
+    (0xFCAC, 'M', 'خم'),
+    (0xFCAD, 'M', 'سج'),
+    (0xFCAE, 'M', 'سح'),
+    (0xFCAF, 'M', 'سخ'),
+    (0xFCB0, 'M', 'سم'),
+    (0xFCB1, 'M', 'صح'),
+    (0xFCB2, 'M', 'صخ'),
+    (0xFCB3, 'M', 'صم'),
+    (0xFCB4, 'M', 'ضج'),
+    (0xFCB5, 'M', 'ضح'),
+    (0xFCB6, 'M', 'ضخ'),
+    (0xFCB7, 'M', 'ضم'),
+    (0xFCB8, 'M', 'طح'),
+    (0xFCB9, 'M', 'ظم'),
+    (0xFCBA, 'M', 'عج'),
     ]
 
-def _seg_47():
+def _seg_47() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFCC4, 'M', u'كج'),
-    (0xFCC5, 'M', u'كح'),
-    (0xFCC6, 'M', u'كخ'),
-    (0xFCC7, 'M', u'كل'),
-    (0xFCC8, 'M', u'كم'),
-    (0xFCC9, 'M', u'لج'),
-    (0xFCCA, 'M', u'لح'),
-    (0xFCCB, 'M', u'لخ'),
-    (0xFCCC, 'M', u'لم'),
-    (0xFCCD, 'M', u'له'),
-    (0xFCCE, 'M', u'مج'),
-    (0xFCCF, 'M', u'مح'),
-    (0xFCD0, 'M', u'مخ'),
-    (0xFCD1, 'M', u'مم'),
-    (0xFCD2, 'M', u'نج'),
-    (0xFCD3, 'M', u'نح'),
-    (0xFCD4, 'M', u'نخ'),
-    (0xFCD5, 'M', u'نم'),
-    (0xFCD6, 'M', u'نه'),
-    (0xFCD7, 'M', u'هج'),
-    (0xFCD8, 'M', u'هم'),
-    (0xFCD9, 'M', u'هٰ'),
-    (0xFCDA, 'M', u'يج'),
-    (0xFCDB, 'M', u'يح'),
-    (0xFCDC, 'M', u'يخ'),
-    (0xFCDD, 'M', u'يم'),
-    (0xFCDE, 'M', u'يه'),
-    (0xFCDF, 'M', u'ئم'),
-    (0xFCE0, 'M', u'ئه'),
-    (0xFCE1, 'M', u'بم'),
-    (0xFCE2, 'M', u'به'),
-    (0xFCE3, 'M', u'تم'),
-    (0xFCE4, 'M', u'ته'),
-    (0xFCE5, 'M', u'ثم'),
-    (0xFCE6, 'M', u'ثه'),
-    (0xFCE7, 'M', u'سم'),
-    (0xFCE8, 'M', u'سه'),
-    (0xFCE9, 'M', u'شم'),
-    (0xFCEA, 'M', u'شه'),
-    (0xFCEB, 'M', u'كل'),
-    (0xFCEC, 'M', u'كم'),
-    (0xFCED, 'M', u'لم'),
-    (0xFCEE, 'M', u'نم'),
-    (0xFCEF, 'M', u'نه'),
-    (0xFCF0, 'M', u'يم'),
-    (0xFCF1, 'M', u'يه'),
-    (0xFCF2, 'M', u'ـَّ'),
-    (0xFCF3, 'M', u'ـُّ'),
-    (0xFCF4, 'M', u'ـِّ'),
-    (0xFCF5, 'M', u'طى'),
-    (0xFCF6, 'M', u'طي'),
-    (0xFCF7, 'M', u'عى'),
-    (0xFCF8, 'M', u'عي'),
-    (0xFCF9, 'M', u'غى'),
-    (0xFCFA, 'M', u'غي'),
-    (0xFCFB, 'M', u'سى'),
-    (0xFCFC, 'M', u'سي'),
-    (0xFCFD, 'M', u'شى'),
-    (0xFCFE, 'M', u'شي'),
-    (0xFCFF, 'M', u'حى'),
-    (0xFD00, 'M', u'حي'),
-    (0xFD01, 'M', u'جى'),
-    (0xFD02, 'M', u'جي'),
-    (0xFD03, 'M', u'خى'),
-    (0xFD04, 'M', u'خي'),
-    (0xFD05, 'M', u'صى'),
-    (0xFD06, 'M', u'صي'),
-    (0xFD07, 'M', u'ضى'),
-    (0xFD08, 'M', u'ضي'),
-    (0xFD09, 'M', u'شج'),
-    (0xFD0A, 'M', u'شح'),
-    (0xFD0B, 'M', u'شخ'),
-    (0xFD0C, 'M', u'شم'),
-    (0xFD0D, 'M', u'شر'),
-    (0xFD0E, 'M', u'سر'),
-    (0xFD0F, 'M', u'صر'),
-    (0xFD10, 'M', u'ضر'),
-    (0xFD11, 'M', u'طى'),
-    (0xFD12, 'M', u'طي'),
-    (0xFD13, 'M', u'عى'),
-    (0xFD14, 'M', u'عي'),
-    (0xFD15, 'M', u'غى'),
-    (0xFD16, 'M', u'غي'),
-    (0xFD17, 'M', u'سى'),
-    (0xFD18, 'M', u'سي'),
-    (0xFD19, 'M', u'شى'),
-    (0xFD1A, 'M', u'شي'),
-    (0xFD1B, 'M', u'حى'),
-    (0xFD1C, 'M', u'حي'),
-    (0xFD1D, 'M', u'جى'),
-    (0xFD1E, 'M', u'جي'),
-    (0xFD1F, 'M', u'خى'),
-    (0xFD20, 'M', u'خي'),
-    (0xFD21, 'M', u'صى'),
-    (0xFD22, 'M', u'صي'),
-    (0xFD23, 'M', u'ضى'),
-    (0xFD24, 'M', u'ضي'),
-    (0xFD25, 'M', u'شج'),
-    (0xFD26, 'M', u'شح'),
-    (0xFD27, 'M', u'شخ'),
+    (0xFCBB, 'M', 'عم'),
+    (0xFCBC, 'M', 'غج'),
+    (0xFCBD, 'M', 'غم'),
+    (0xFCBE, 'M', 'فج'),
+    (0xFCBF, 'M', 'فح'),
+    (0xFCC0, 'M', 'فخ'),
+    (0xFCC1, 'M', 'فم'),
+    (0xFCC2, 'M', 'قح'),
+    (0xFCC3, 'M', 'قم'),
+    (0xFCC4, 'M', 'كج'),
+    (0xFCC5, 'M', 'كح'),
+    (0xFCC6, 'M', 'كخ'),
+    (0xFCC7, 'M', 'كل'),
+    (0xFCC8, 'M', 'كم'),
+    (0xFCC9, 'M', 'لج'),
+    (0xFCCA, 'M', 'لح'),
+    (0xFCCB, 'M', 'لخ'),
+    (0xFCCC, 'M', 'لم'),
+    (0xFCCD, 'M', 'له'),
+    (0xFCCE, 'M', 'مج'),
+    (0xFCCF, 'M', 'مح'),
+    (0xFCD0, 'M', 'مخ'),
+    (0xFCD1, 'M', 'مم'),
+    (0xFCD2, 'M', 'نج'),
+    (0xFCD3, 'M', 'نح'),
+    (0xFCD4, 'M', 'نخ'),
+    (0xFCD5, 'M', 'نم'),
+    (0xFCD6, 'M', 'نه'),
+    (0xFCD7, 'M', 'هج'),
+    (0xFCD8, 'M', 'هم'),
+    (0xFCD9, 'M', 'هٰ'),
+    (0xFCDA, 'M', 'يج'),
+    (0xFCDB, 'M', 'يح'),
+    (0xFCDC, 'M', 'يخ'),
+    (0xFCDD, 'M', 'يم'),
+    (0xFCDE, 'M', 'يه'),
+    (0xFCDF, 'M', 'ئم'),
+    (0xFCE0, 'M', 'ئه'),
+    (0xFCE1, 'M', 'بم'),
+    (0xFCE2, 'M', 'به'),
+    (0xFCE3, 'M', 'تم'),
+    (0xFCE4, 'M', 'ته'),
+    (0xFCE5, 'M', 'ثم'),
+    (0xFCE6, 'M', 'ثه'),
+    (0xFCE7, 'M', 'سم'),
+    (0xFCE8, 'M', 'سه'),
+    (0xFCE9, 'M', 'شم'),
+    (0xFCEA, 'M', 'شه'),
+    (0xFCEB, 'M', 'كل'),
+    (0xFCEC, 'M', 'كم'),
+    (0xFCED, 'M', 'لم'),
+    (0xFCEE, 'M', 'نم'),
+    (0xFCEF, 'M', 'نه'),
+    (0xFCF0, 'M', 'يم'),
+    (0xFCF1, 'M', 'يه'),
+    (0xFCF2, 'M', 'ـَّ'),
+    (0xFCF3, 'M', 'ـُّ'),
+    (0xFCF4, 'M', 'ـِّ'),
+    (0xFCF5, 'M', 'طى'),
+    (0xFCF6, 'M', 'طي'),
+    (0xFCF7, 'M', 'عى'),
+    (0xFCF8, 'M', 'عي'),
+    (0xFCF9, 'M', 'غى'),
+    (0xFCFA, 'M', 'غي'),
+    (0xFCFB, 'M', 'سى'),
+    (0xFCFC, 'M', 'سي'),
+    (0xFCFD, 'M', 'شى'),
+    (0xFCFE, 'M', 'شي'),
+    (0xFCFF, 'M', 'حى'),
+    (0xFD00, 'M', 'حي'),
+    (0xFD01, 'M', 'جى'),
+    (0xFD02, 'M', 'جي'),
+    (0xFD03, 'M', 'خى'),
+    (0xFD04, 'M', 'خي'),
+    (0xFD05, 'M', 'صى'),
+    (0xFD06, 'M', 'صي'),
+    (0xFD07, 'M', 'ضى'),
+    (0xFD08, 'M', 'ضي'),
+    (0xFD09, 'M', 'شج'),
+    (0xFD0A, 'M', 'شح'),
+    (0xFD0B, 'M', 'شخ'),
+    (0xFD0C, 'M', 'شم'),
+    (0xFD0D, 'M', 'شر'),
+    (0xFD0E, 'M', 'سر'),
+    (0xFD0F, 'M', 'صر'),
+    (0xFD10, 'M', 'ضر'),
+    (0xFD11, 'M', 'طى'),
+    (0xFD12, 'M', 'طي'),
+    (0xFD13, 'M', 'عى'),
+    (0xFD14, 'M', 'عي'),
+    (0xFD15, 'M', 'غى'),
+    (0xFD16, 'M', 'غي'),
+    (0xFD17, 'M', 'سى'),
+    (0xFD18, 'M', 'سي'),
+    (0xFD19, 'M', 'شى'),
+    (0xFD1A, 'M', 'شي'),
+    (0xFD1B, 'M', 'حى'),
+    (0xFD1C, 'M', 'حي'),
+    (0xFD1D, 'M', 'جى'),
+    (0xFD1E, 'M', 'جي'),
     ]
 
-def _seg_48():
+def _seg_48() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFD28, 'M', u'شم'),
-    (0xFD29, 'M', u'شر'),
-    (0xFD2A, 'M', u'سر'),
-    (0xFD2B, 'M', u'صر'),
-    (0xFD2C, 'M', u'ضر'),
-    (0xFD2D, 'M', u'شج'),
-    (0xFD2E, 'M', u'شح'),
-    (0xFD2F, 'M', u'شخ'),
-    (0xFD30, 'M', u'شم'),
-    (0xFD31, 'M', u'سه'),
-    (0xFD32, 'M', u'شه'),
-    (0xFD33, 'M', u'طم'),
-    (0xFD34, 'M', u'سج'),
-    (0xFD35, 'M', u'سح'),
-    (0xFD36, 'M', u'سخ'),
-    (0xFD37, 'M', u'شج'),
-    (0xFD38, 'M', u'شح'),
-    (0xFD39, 'M', u'شخ'),
-    (0xFD3A, 'M', u'طم'),
-    (0xFD3B, 'M', u'ظم'),
-    (0xFD3C, 'M', u'اً'),
+    (0xFD1F, 'M', 'خى'),
+    (0xFD20, 'M', 'خي'),
+    (0xFD21, 'M', 'صى'),
+    (0xFD22, 'M', 'صي'),
+    (0xFD23, 'M', 'ضى'),
+    (0xFD24, 'M', 'ضي'),
+    (0xFD25, 'M', 'شج'),
+    (0xFD26, 'M', 'شح'),
+    (0xFD27, 'M', 'شخ'),
+    (0xFD28, 'M', 'شم'),
+    (0xFD29, 'M', 'شر'),
+    (0xFD2A, 'M', 'سر'),
+    (0xFD2B, 'M', 'صر'),
+    (0xFD2C, 'M', 'ضر'),
+    (0xFD2D, 'M', 'شج'),
+    (0xFD2E, 'M', 'شح'),
+    (0xFD2F, 'M', 'شخ'),
+    (0xFD30, 'M', 'شم'),
+    (0xFD31, 'M', 'سه'),
+    (0xFD32, 'M', 'شه'),
+    (0xFD33, 'M', 'طم'),
+    (0xFD34, 'M', 'سج'),
+    (0xFD35, 'M', 'سح'),
+    (0xFD36, 'M', 'سخ'),
+    (0xFD37, 'M', 'شج'),
+    (0xFD38, 'M', 'شح'),
+    (0xFD39, 'M', 'شخ'),
+    (0xFD3A, 'M', 'طم'),
+    (0xFD3B, 'M', 'ظم'),
+    (0xFD3C, 'M', 'اً'),
     (0xFD3E, 'V'),
-    (0xFD40, 'X'),
-    (0xFD50, 'M', u'تجم'),
-    (0xFD51, 'M', u'تحج'),
-    (0xFD53, 'M', u'تحم'),
-    (0xFD54, 'M', u'تخم'),
-    (0xFD55, 'M', u'تمج'),
-    (0xFD56, 'M', u'تمح'),
-    (0xFD57, 'M', u'تمخ'),
-    (0xFD58, 'M', u'جمح'),
-    (0xFD5A, 'M', u'حمي'),
-    (0xFD5B, 'M', u'حمى'),
-    (0xFD5C, 'M', u'سحج'),
-    (0xFD5D, 'M', u'سجح'),
-    (0xFD5E, 'M', u'سجى'),
-    (0xFD5F, 'M', u'سمح'),
-    (0xFD61, 'M', u'سمج'),
-    (0xFD62, 'M', u'سمم'),
-    (0xFD64, 'M', u'صحح'),
-    (0xFD66, 'M', u'صمم'),
-    (0xFD67, 'M', u'شحم'),
-    (0xFD69, 'M', u'شجي'),
-    (0xFD6A, 'M', u'شمخ'),
-    (0xFD6C, 'M', u'شمم'),
-    (0xFD6E, 'M', u'ضحى'),
-    (0xFD6F, 'M', u'ضخم'),
-    (0xFD71, 'M', u'طمح'),
-    (0xFD73, 'M', u'طمم'),
-    (0xFD74, 'M', u'طمي'),
-    (0xFD75, 'M', u'عجم'),
-    (0xFD76, 'M', u'عمم'),
-    (0xFD78, 'M', u'عمى'),
-    (0xFD79, 'M', u'غمم'),
-    (0xFD7A, 'M', u'غمي'),
-    (0xFD7B, 'M', u'غمى'),
-    (0xFD7C, 'M', u'فخم'),
-    (0xFD7E, 'M', u'قمح'),
-    (0xFD7F, 'M', u'قمم'),
-    (0xFD80, 'M', u'لحم'),
-    (0xFD81, 'M', u'لحي'),
-    (0xFD82, 'M', u'لحى'),
-    (0xFD83, 'M', u'لجج'),
-    (0xFD85, 'M', u'لخم'),
-    (0xFD87, 'M', u'لمح'),
-    (0xFD89, 'M', u'محج'),
-    (0xFD8A, 'M', u'محم'),
-    (0xFD8B, 'M', u'محي'),
-    (0xFD8C, 'M', u'مجح'),
-    (0xFD8D, 'M', u'مجم'),
-    (0xFD8E, 'M', u'مخج'),
-    (0xFD8F, 'M', u'مخم'),
+    (0xFD50, 'M', 'تجم'),
+    (0xFD51, 'M', 'تحج'),
+    (0xFD53, 'M', 'تحم'),
+    (0xFD54, 'M', 'تخم'),
+    (0xFD55, 'M', 'تمج'),
+    (0xFD56, 'M', 'تمح'),
+    (0xFD57, 'M', 'تمخ'),
+    (0xFD58, 'M', 'جمح'),
+    (0xFD5A, 'M', 'حمي'),
+    (0xFD5B, 'M', 'حمى'),
+    (0xFD5C, 'M', 'سحج'),
+    (0xFD5D, 'M', 'سجح'),
+    (0xFD5E, 'M', 'سجى'),
+    (0xFD5F, 'M', 'سمح'),
+    (0xFD61, 'M', 'سمج'),
+    (0xFD62, 'M', 'سمم'),
+    (0xFD64, 'M', 'صحح'),
+    (0xFD66, 'M', 'صمم'),
+    (0xFD67, 'M', 'شحم'),
+    (0xFD69, 'M', 'شجي'),
+    (0xFD6A, 'M', 'شمخ'),
+    (0xFD6C, 'M', 'شمم'),
+    (0xFD6E, 'M', 'ضحى'),
+    (0xFD6F, 'M', 'ضخم'),
+    (0xFD71, 'M', 'طمح'),
+    (0xFD73, 'M', 'طمم'),
+    (0xFD74, 'M', 'طمي'),
+    (0xFD75, 'M', 'عجم'),
+    (0xFD76, 'M', 'عمم'),
+    (0xFD78, 'M', 'عمى'),
+    (0xFD79, 'M', 'غمم'),
+    (0xFD7A, 'M', 'غمي'),
+    (0xFD7B, 'M', 'غمى'),
+    (0xFD7C, 'M', 'فخم'),
+    (0xFD7E, 'M', 'قمح'),
+    (0xFD7F, 'M', 'قمم'),
+    (0xFD80, 'M', 'لحم'),
+    (0xFD81, 'M', 'لحي'),
+    (0xFD82, 'M', 'لحى'),
+    (0xFD83, 'M', 'لجج'),
+    (0xFD85, 'M', 'لخم'),
+    (0xFD87, 'M', 'لمح'),
+    (0xFD89, 'M', 'محج'),
+    (0xFD8A, 'M', 'محم'),
+    (0xFD8B, 'M', 'محي'),
+    (0xFD8C, 'M', 'مجح'),
+    (0xFD8D, 'M', 'مجم'),
+    (0xFD8E, 'M', 'مخج'),
+    (0xFD8F, 'M', 'مخم'),
     (0xFD90, 'X'),
-    (0xFD92, 'M', u'مجخ'),
-    (0xFD93, 'M', u'همج'),
-    (0xFD94, 'M', u'همم'),
-    (0xFD95, 'M', u'نحم'),
-    (0xFD96, 'M', u'نحى'),
-    (0xFD97, 'M', u'نجم'),
-    (0xFD99, 'M', u'نجى'),
-    (0xFD9A, 'M', u'نمي'),
-    (0xFD9B, 'M', u'نمى'),
-    (0xFD9C, 'M', u'يمم'),
-    (0xFD9E, 'M', u'بخي'),
-    (0xFD9F, 'M', u'تجي'),
-    (0xFDA0, 'M', u'تجى'),
-    (0xFDA1, 'M', u'تخي'),
-    (0xFDA2, 'M', u'تخى'),
-    (0xFDA3, 'M', u'تمي'),
-    (0xFDA4, 'M', u'تمى'),
-    (0xFDA5, 'M', u'جمي'),
-    (0xFDA6, 'M', u'جحى'),
-    (0xFDA7, 'M', u'جمى'),
-    (0xFDA8, 'M', u'سخى'),
-    (0xFDA9, 'M', u'صحي'),
-    (0xFDAA, 'M', u'شحي'),
-    (0xFDAB, 'M', u'ضحي'),
-    (0xFDAC, 'M', u'لجي'),
-    (0xFDAD, 'M', u'لمي'),
-    (0xFDAE, 'M', u'يحي'),
+    (0xFD92, 'M', 'مجخ'),
+    (0xFD93, 'M', 'همج'),
+    (0xFD94, 'M', 'همم'),
+    (0xFD95, 'M', 'نحم'),
+    (0xFD96, 'M', 'نحى'),
+    (0xFD97, 'M', 'نجم'),
+    (0xFD99, 'M', 'نجى'),
+    (0xFD9A, 'M', 'نمي'),
+    (0xFD9B, 'M', 'نمى'),
+    (0xFD9C, 'M', 'يمم'),
+    (0xFD9E, 'M', 'بخي'),
+    (0xFD9F, 'M', 'تجي'),
+    (0xFDA0, 'M', 'تجى'),
+    (0xFDA1, 'M', 'تخي'),
+    (0xFDA2, 'M', 'تخى'),
+    (0xFDA3, 'M', 'تمي'),
+    (0xFDA4, 'M', 'تمى'),
+    (0xFDA5, 'M', 'جمي'),
+    (0xFDA6, 'M', 'جحى'),
     ]
 
-def _seg_49():
+def _seg_49() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFDAF, 'M', u'يجي'),
-    (0xFDB0, 'M', u'يمي'),
-    (0xFDB1, 'M', u'ممي'),
-    (0xFDB2, 'M', u'قمي'),
-    (0xFDB3, 'M', u'نحي'),
-    (0xFDB4, 'M', u'قمح'),
-    (0xFDB5, 'M', u'لحم'),
-    (0xFDB6, 'M', u'عمي'),
-    (0xFDB7, 'M', u'كمي'),
-    (0xFDB8, 'M', u'نجح'),
-    (0xFDB9, 'M', u'مخي'),
-    (0xFDBA, 'M', u'لجم'),
-    (0xFDBB, 'M', u'كمم'),
-    (0xFDBC, 'M', u'لجم'),
-    (0xFDBD, 'M', u'نجح'),
-    (0xFDBE, 'M', u'جحي'),
-    (0xFDBF, 'M', u'حجي'),
-    (0xFDC0, 'M', u'مجي'),
-    (0xFDC1, 'M', u'فمي'),
-    (0xFDC2, 'M', u'بحي'),
-    (0xFDC3, 'M', u'كمم'),
-    (0xFDC4, 'M', u'عجم'),
-    (0xFDC5, 'M', u'صمم'),
-    (0xFDC6, 'M', u'سخي'),
-    (0xFDC7, 'M', u'نجي'),
+    (0xFDA7, 'M', 'جمى'),
+    (0xFDA8, 'M', 'سخى'),
+    (0xFDA9, 'M', 'صحي'),
+    (0xFDAA, 'M', 'شحي'),
+    (0xFDAB, 'M', 'ضحي'),
+    (0xFDAC, 'M', 'لجي'),
+    (0xFDAD, 'M', 'لمي'),
+    (0xFDAE, 'M', 'يحي'),
+    (0xFDAF, 'M', 'يجي'),
+    (0xFDB0, 'M', 'يمي'),
+    (0xFDB1, 'M', 'ممي'),
+    (0xFDB2, 'M', 'قمي'),
+    (0xFDB3, 'M', 'نحي'),
+    (0xFDB4, 'M', 'قمح'),
+    (0xFDB5, 'M', 'لحم'),
+    (0xFDB6, 'M', 'عمي'),
+    (0xFDB7, 'M', 'كمي'),
+    (0xFDB8, 'M', 'نجح'),
+    (0xFDB9, 'M', 'مخي'),
+    (0xFDBA, 'M', 'لجم'),
+    (0xFDBB, 'M', 'كمم'),
+    (0xFDBC, 'M', 'لجم'),
+    (0xFDBD, 'M', 'نجح'),
+    (0xFDBE, 'M', 'جحي'),
+    (0xFDBF, 'M', 'حجي'),
+    (0xFDC0, 'M', 'مجي'),
+    (0xFDC1, 'M', 'فمي'),
+    (0xFDC2, 'M', 'بحي'),
+    (0xFDC3, 'M', 'كمم'),
+    (0xFDC4, 'M', 'عجم'),
+    (0xFDC5, 'M', 'صمم'),
+    (0xFDC6, 'M', 'سخي'),
+    (0xFDC7, 'M', 'نجي'),
     (0xFDC8, 'X'),
-    (0xFDF0, 'M', u'صلے'),
-    (0xFDF1, 'M', u'قلے'),
-    (0xFDF2, 'M', u'الله'),
-    (0xFDF3, 'M', u'اكبر'),
-    (0xFDF4, 'M', u'محمد'),
-    (0xFDF5, 'M', u'صلعم'),
-    (0xFDF6, 'M', u'رسول'),
-    (0xFDF7, 'M', u'عليه'),
-    (0xFDF8, 'M', u'وسلم'),
-    (0xFDF9, 'M', u'صلى'),
-    (0xFDFA, '3', u'صلى الله عليه وسلم'),
-    (0xFDFB, '3', u'جل جلاله'),
-    (0xFDFC, 'M', u'ریال'),
+    (0xFDCF, 'V'),
+    (0xFDD0, 'X'),
+    (0xFDF0, 'M', 'صلے'),
+    (0xFDF1, 'M', 'قلے'),
+    (0xFDF2, 'M', 'الله'),
+    (0xFDF3, 'M', 'اكبر'),
+    (0xFDF4, 'M', 'محمد'),
+    (0xFDF5, 'M', 'صلعم'),
+    (0xFDF6, 'M', 'رسول'),
+    (0xFDF7, 'M', 'عليه'),
+    (0xFDF8, 'M', 'وسلم'),
+    (0xFDF9, 'M', 'صلى'),
+    (0xFDFA, '3', 'صلى الله عليه وسلم'),
+    (0xFDFB, '3', 'جل جلاله'),
+    (0xFDFC, 'M', 'ریال'),
     (0xFDFD, 'V'),
-    (0xFDFE, 'X'),
     (0xFE00, 'I'),
-    (0xFE10, '3', u','),
-    (0xFE11, 'M', u'、'),
+    (0xFE10, '3', ','),
+    (0xFE11, 'M', '、'),
     (0xFE12, 'X'),
-    (0xFE13, '3', u':'),
-    (0xFE14, '3', u';'),
-    (0xFE15, '3', u'!'),
-    (0xFE16, '3', u'?'),
-    (0xFE17, 'M', u'〖'),
-    (0xFE18, 'M', u'〗'),
+    (0xFE13, '3', ':'),
+    (0xFE14, '3', ';'),
+    (0xFE15, '3', '!'),
+    (0xFE16, '3', '?'),
+    (0xFE17, 'M', '〖'),
+    (0xFE18, 'M', '〗'),
     (0xFE19, 'X'),
     (0xFE20, 'V'),
     (0xFE30, 'X'),
-    (0xFE31, 'M', u'—'),
-    (0xFE32, 'M', u'–'),
-    (0xFE33, '3', u'_'),
-    (0xFE35, '3', u'('),
-    (0xFE36, '3', u')'),
-    (0xFE37, '3', u'{'),
-    (0xFE38, '3', u'}'),
-    (0xFE39, 'M', u'〔'),
-    (0xFE3A, 'M', u'〕'),
-    (0xFE3B, 'M', u'【'),
-    (0xFE3C, 'M', u'】'),
-    (0xFE3D, 'M', u'《'),
-    (0xFE3E, 'M', u'》'),
-    (0xFE3F, 'M', u'〈'),
-    (0xFE40, 'M', u'〉'),
-    (0xFE41, 'M', u'「'),
-    (0xFE42, 'M', u'」'),
-    (0xFE43, 'M', u'『'),
-    (0xFE44, 'M', u'』'),
+    (0xFE31, 'M', '—'),
+    (0xFE32, 'M', '–'),
+    (0xFE33, '3', '_'),
+    (0xFE35, '3', '('),
+    (0xFE36, '3', ')'),
+    (0xFE37, '3', '{'),
+    (0xFE38, '3', '}'),
+    (0xFE39, 'M', '〔'),
+    (0xFE3A, 'M', '〕'),
+    (0xFE3B, 'M', '【'),
+    (0xFE3C, 'M', '】'),
+    (0xFE3D, 'M', '《'),
+    (0xFE3E, 'M', '》'),
+    (0xFE3F, 'M', '〈'),
+    (0xFE40, 'M', '〉'),
+    (0xFE41, 'M', '「'),
+    (0xFE42, 'M', '」'),
+    (0xFE43, 'M', '『'),
+    (0xFE44, 'M', '』'),
     (0xFE45, 'V'),
-    (0xFE47, '3', u'['),
-    (0xFE48, '3', u']'),
-    (0xFE49, '3', u' ̅'),
-    (0xFE4D, '3', u'_'),
-    (0xFE50, '3', u','),
-    (0xFE51, 'M', u'、'),
+    (0xFE47, '3', '['),
+    (0xFE48, '3', ']'),
+    (0xFE49, '3', ' ̅'),
+    (0xFE4D, '3', '_'),
+    (0xFE50, '3', ','),
+    (0xFE51, 'M', '、'),
     (0xFE52, 'X'),
-    (0xFE54, '3', u';'),
-    (0xFE55, '3', u':'),
-    (0xFE56, '3', u'?'),
-    (0xFE57, '3', u'!'),
-    (0xFE58, 'M', u'—'),
-    (0xFE59, '3', u'('),
-    (0xFE5A, '3', u')'),
-    (0xFE5B, '3', u'{'),
-    (0xFE5C, '3', u'}'),
-    (0xFE5D, 'M', u'〔'),
-    (0xFE5E, 'M', u'〕'),
-    (0xFE5F, '3', u'#'),
-    (0xFE60, '3', u'&'),
-    (0xFE61, '3', u'*'),
-    (0xFE62, '3', u'+'),
-    (0xFE63, 'M', u'-'),
-    (0xFE64, '3', u'<'),
-    (0xFE65, '3', u'>'),
-    (0xFE66, '3', u'='),
+    (0xFE54, '3', ';'),
+    (0xFE55, '3', ':'),
+    (0xFE56, '3', '?'),
+    (0xFE57, '3', '!'),
+    (0xFE58, 'M', '—'),
+    (0xFE59, '3', '('),
+    (0xFE5A, '3', ')'),
+    (0xFE5B, '3', '{'),
+    (0xFE5C, '3', '}'),
+    (0xFE5D, 'M', '〔'),
     ]
 
-def _seg_50():
+def _seg_50() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
+    (0xFE5E, 'M', '〕'),
+    (0xFE5F, '3', '#'),
+    (0xFE60, '3', '&'),
+    (0xFE61, '3', '*'),
+    (0xFE62, '3', '+'),
+    (0xFE63, 'M', '-'),
+    (0xFE64, '3', '<'),
+    (0xFE65, '3', '>'),
+    (0xFE66, '3', '='),
     (0xFE67, 'X'),
-    (0xFE68, '3', u'\\'),
-    (0xFE69, '3', u'$'),
-    (0xFE6A, '3', u'%'),
-    (0xFE6B, '3', u'@'),
+    (0xFE68, '3', '\\'),
+    (0xFE69, '3', '$'),
+    (0xFE6A, '3', '%'),
+    (0xFE6B, '3', '@'),
     (0xFE6C, 'X'),
-    (0xFE70, '3', u' ً'),
-    (0xFE71, 'M', u'ـً'),
-    (0xFE72, '3', u' ٌ'),
+    (0xFE70, '3', ' ً'),
+    (0xFE71, 'M', 'ـً'),
+    (0xFE72, '3', ' ٌ'),
     (0xFE73, 'V'),
-    (0xFE74, '3', u' ٍ'),
+    (0xFE74, '3', ' ٍ'),
     (0xFE75, 'X'),
-    (0xFE76, '3', u' َ'),
-    (0xFE77, 'M', u'ـَ'),
-    (0xFE78, '3', u' ُ'),
-    (0xFE79, 'M', u'ـُ'),
-    (0xFE7A, '3', u' ِ'),
-    (0xFE7B, 'M', u'ـِ'),
-    (0xFE7C, '3', u' ّ'),
-    (0xFE7D, 'M', u'ـّ'),
-    (0xFE7E, '3', u' ْ'),
-    (0xFE7F, 'M', u'ـْ'),
-    (0xFE80, 'M', u'ء'),
-    (0xFE81, 'M', u'آ'),
-    (0xFE83, 'M', u'أ'),
-    (0xFE85, 'M', u'ؤ'),
-    (0xFE87, 'M', u'إ'),
-    (0xFE89, 'M', u'ئ'),
-    (0xFE8D, 'M', u'ا'),
-    (0xFE8F, 'M', u'ب'),
-    (0xFE93, 'M', u'ة'),
-    (0xFE95, 'M', u'ت'),
-    (0xFE99, 'M', u'ث'),
-    (0xFE9D, 'M', u'ج'),
-    (0xFEA1, 'M', u'ح'),
-    (0xFEA5, 'M', u'خ'),
-    (0xFEA9, 'M', u'د'),
-    (0xFEAB, 'M', u'ذ'),
-    (0xFEAD, 'M', u'ر'),
-    (0xFEAF, 'M', u'ز'),
-    (0xFEB1, 'M', u'س'),
-    (0xFEB5, 'M', u'ش'),
-    (0xFEB9, 'M', u'ص'),
-    (0xFEBD, 'M', u'ض'),
-    (0xFEC1, 'M', u'ط'),
-    (0xFEC5, 'M', u'ظ'),
-    (0xFEC9, 'M', u'ع'),
-    (0xFECD, 'M', u'غ'),
-    (0xFED1, 'M', u'ف'),
-    (0xFED5, 'M', u'ق'),
-    (0xFED9, 'M', u'ك'),
-    (0xFEDD, 'M', u'ل'),
-    (0xFEE1, 'M', u'م'),
-    (0xFEE5, 'M', u'ن'),
-    (0xFEE9, 'M', u'ه'),
-    (0xFEED, 'M', u'و'),
-    (0xFEEF, 'M', u'ى'),
-    (0xFEF1, 'M', u'ي'),
-    (0xFEF5, 'M', u'لآ'),
-    (0xFEF7, 'M', u'لأ'),
-    (0xFEF9, 'M', u'لإ'),
-    (0xFEFB, 'M', u'لا'),
+    (0xFE76, '3', ' َ'),
+    (0xFE77, 'M', 'ـَ'),
+    (0xFE78, '3', ' ُ'),
+    (0xFE79, 'M', 'ـُ'),
+    (0xFE7A, '3', ' ِ'),
+    (0xFE7B, 'M', 'ـِ'),
+    (0xFE7C, '3', ' ّ'),
+    (0xFE7D, 'M', 'ـّ'),
+    (0xFE7E, '3', ' ْ'),
+    (0xFE7F, 'M', 'ـْ'),
+    (0xFE80, 'M', 'ء'),
+    (0xFE81, 'M', 'آ'),
+    (0xFE83, 'M', 'أ'),
+    (0xFE85, 'M', 'ؤ'),
+    (0xFE87, 'M', 'إ'),
+    (0xFE89, 'M', 'ئ'),
+    (0xFE8D, 'M', 'ا'),
+    (0xFE8F, 'M', 'ب'),
+    (0xFE93, 'M', 'ة'),
+    (0xFE95, 'M', 'ت'),
+    (0xFE99, 'M', 'ث'),
+    (0xFE9D, 'M', 'ج'),
+    (0xFEA1, 'M', 'ح'),
+    (0xFEA5, 'M', 'خ'),
+    (0xFEA9, 'M', 'د'),
+    (0xFEAB, 'M', 'ذ'),
+    (0xFEAD, 'M', 'ر'),
+    (0xFEAF, 'M', 'ز'),
+    (0xFEB1, 'M', 'س'),
+    (0xFEB5, 'M', 'ش'),
+    (0xFEB9, 'M', 'ص'),
+    (0xFEBD, 'M', 'ض'),
+    (0xFEC1, 'M', 'ط'),
+    (0xFEC5, 'M', 'ظ'),
+    (0xFEC9, 'M', 'ع'),
+    (0xFECD, 'M', 'غ'),
+    (0xFED1, 'M', 'ف'),
+    (0xFED5, 'M', 'ق'),
+    (0xFED9, 'M', 'ك'),
+    (0xFEDD, 'M', 'ل'),
+    (0xFEE1, 'M', 'م'),
+    (0xFEE5, 'M', 'ن'),
+    (0xFEE9, 'M', 'ه'),
+    (0xFEED, 'M', 'و'),
+    (0xFEEF, 'M', 'ى'),
+    (0xFEF1, 'M', 'ي'),
+    (0xFEF5, 'M', 'لآ'),
+    (0xFEF7, 'M', 'لأ'),
+    (0xFEF9, 'M', 'لإ'),
+    (0xFEFB, 'M', 'لا'),
     (0xFEFD, 'X'),
     (0xFEFF, 'I'),
     (0xFF00, 'X'),
-    (0xFF01, '3', u'!'),
-    (0xFF02, '3', u'"'),
-    (0xFF03, '3', u'#'),
-    (0xFF04, '3', u'$'),
-    (0xFF05, '3', u'%'),
-    (0xFF06, '3', u'&'),
-    (0xFF07, '3', u'\''),
-    (0xFF08, '3', u'('),
-    (0xFF09, '3', u')'),
-    (0xFF0A, '3', u'*'),
-    (0xFF0B, '3', u'+'),
-    (0xFF0C, '3', u','),
-    (0xFF0D, 'M', u'-'),
-    (0xFF0E, 'M', u'.'),
-    (0xFF0F, '3', u'/'),
-    (0xFF10, 'M', u'0'),
-    (0xFF11, 'M', u'1'),
-    (0xFF12, 'M', u'2'),
-    (0xFF13, 'M', u'3'),
-    (0xFF14, 'M', u'4'),
-    (0xFF15, 'M', u'5'),
-    (0xFF16, 'M', u'6'),
-    (0xFF17, 'M', u'7'),
-    (0xFF18, 'M', u'8'),
-    (0xFF19, 'M', u'9'),
-    (0xFF1A, '3', u':'),
-    (0xFF1B, '3', u';'),
-    (0xFF1C, '3', u'<'),
-    (0xFF1D, '3', u'='),
-    (0xFF1E, '3', u'>'),
-    (0xFF1F, '3', u'?'),
-    (0xFF20, '3', u'@'),
-    (0xFF21, 'M', u'a'),
-    (0xFF22, 'M', u'b'),
-    (0xFF23, 'M', u'c'),
+    (0xFF01, '3', '!'),
+    (0xFF02, '3', '"'),
+    (0xFF03, '3', '#'),
+    (0xFF04, '3', '$'),
+    (0xFF05, '3', '%'),
+    (0xFF06, '3', '&'),
+    (0xFF07, '3', '\''),
+    (0xFF08, '3', '('),
+    (0xFF09, '3', ')'),
+    (0xFF0A, '3', '*'),
+    (0xFF0B, '3', '+'),
+    (0xFF0C, '3', ','),
+    (0xFF0D, 'M', '-'),
+    (0xFF0E, 'M', '.'),
+    (0xFF0F, '3', '/'),
+    (0xFF10, 'M', '0'),
+    (0xFF11, 'M', '1'),
+    (0xFF12, 'M', '2'),
+    (0xFF13, 'M', '3'),
+    (0xFF14, 'M', '4'),
+    (0xFF15, 'M', '5'),
+    (0xFF16, 'M', '6'),
+    (0xFF17, 'M', '7'),
+    (0xFF18, 'M', '8'),
+    (0xFF19, 'M', '9'),
+    (0xFF1A, '3', ':'),
     ]
 
-def _seg_51():
+def _seg_51() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFF24, 'M', u'd'),
-    (0xFF25, 'M', u'e'),
-    (0xFF26, 'M', u'f'),
-    (0xFF27, 'M', u'g'),
-    (0xFF28, 'M', u'h'),
-    (0xFF29, 'M', u'i'),
-    (0xFF2A, 'M', u'j'),
-    (0xFF2B, 'M', u'k'),
-    (0xFF2C, 'M', u'l'),
-    (0xFF2D, 'M', u'm'),
-    (0xFF2E, 'M', u'n'),
-    (0xFF2F, 'M', u'o'),
-    (0xFF30, 'M', u'p'),
-    (0xFF31, 'M', u'q'),
-    (0xFF32, 'M', u'r'),
-    (0xFF33, 'M', u's'),
-    (0xFF34, 'M', u't'),
-    (0xFF35, 'M', u'u'),
-    (0xFF36, 'M', u'v'),
-    (0xFF37, 'M', u'w'),
-    (0xFF38, 'M', u'x'),
-    (0xFF39, 'M', u'y'),
-    (0xFF3A, 'M', u'z'),
-    (0xFF3B, '3', u'['),
-    (0xFF3C, '3', u'\\'),
-    (0xFF3D, '3', u']'),
-    (0xFF3E, '3', u'^'),
-    (0xFF3F, '3', u'_'),
-    (0xFF40, '3', u'`'),
-    (0xFF41, 'M', u'a'),
-    (0xFF42, 'M', u'b'),
-    (0xFF43, 'M', u'c'),
-    (0xFF44, 'M', u'd'),
-    (0xFF45, 'M', u'e'),
-    (0xFF46, 'M', u'f'),
-    (0xFF47, 'M', u'g'),
-    (0xFF48, 'M', u'h'),
-    (0xFF49, 'M', u'i'),
-    (0xFF4A, 'M', u'j'),
-    (0xFF4B, 'M', u'k'),
-    (0xFF4C, 'M', u'l'),
-    (0xFF4D, 'M', u'm'),
-    (0xFF4E, 'M', u'n'),
-    (0xFF4F, 'M', u'o'),
-    (0xFF50, 'M', u'p'),
-    (0xFF51, 'M', u'q'),
-    (0xFF52, 'M', u'r'),
-    (0xFF53, 'M', u's'),
-    (0xFF54, 'M', u't'),
-    (0xFF55, 'M', u'u'),
-    (0xFF56, 'M', u'v'),
-    (0xFF57, 'M', u'w'),
-    (0xFF58, 'M', u'x'),
-    (0xFF59, 'M', u'y'),
-    (0xFF5A, 'M', u'z'),
-    (0xFF5B, '3', u'{'),
-    (0xFF5C, '3', u'|'),
-    (0xFF5D, '3', u'}'),
-    (0xFF5E, '3', u'~'),
-    (0xFF5F, 'M', u'⦅'),
-    (0xFF60, 'M', u'⦆'),
-    (0xFF61, 'M', u'.'),
-    (0xFF62, 'M', u'「'),
-    (0xFF63, 'M', u'」'),
-    (0xFF64, 'M', u'、'),
-    (0xFF65, 'M', u'・'),
-    (0xFF66, 'M', u'ヲ'),
-    (0xFF67, 'M', u'ァ'),
-    (0xFF68, 'M', u'ィ'),
-    (0xFF69, 'M', u'ゥ'),
-    (0xFF6A, 'M', u'ェ'),
-    (0xFF6B, 'M', u'ォ'),
-    (0xFF6C, 'M', u'ャ'),
-    (0xFF6D, 'M', u'ュ'),
-    (0xFF6E, 'M', u'ョ'),
-    (0xFF6F, 'M', u'ッ'),
-    (0xFF70, 'M', u'ー'),
-    (0xFF71, 'M', u'ア'),
-    (0xFF72, 'M', u'イ'),
-    (0xFF73, 'M', u'ウ'),
-    (0xFF74, 'M', u'エ'),
-    (0xFF75, 'M', u'オ'),
-    (0xFF76, 'M', u'カ'),
-    (0xFF77, 'M', u'キ'),
-    (0xFF78, 'M', u'ク'),
-    (0xFF79, 'M', u'ケ'),
-    (0xFF7A, 'M', u'コ'),
-    (0xFF7B, 'M', u'サ'),
-    (0xFF7C, 'M', u'シ'),
-    (0xFF7D, 'M', u'ス'),
-    (0xFF7E, 'M', u'セ'),
-    (0xFF7F, 'M', u'ソ'),
-    (0xFF80, 'M', u'タ'),
-    (0xFF81, 'M', u'チ'),
-    (0xFF82, 'M', u'ツ'),
-    (0xFF83, 'M', u'テ'),
-    (0xFF84, 'M', u'ト'),
-    (0xFF85, 'M', u'ナ'),
-    (0xFF86, 'M', u'ニ'),
-    (0xFF87, 'M', u'ヌ'),
+    (0xFF1B, '3', ';'),
+    (0xFF1C, '3', '<'),
+    (0xFF1D, '3', '='),
+    (0xFF1E, '3', '>'),
+    (0xFF1F, '3', '?'),
+    (0xFF20, '3', '@'),
+    (0xFF21, 'M', 'a'),
+    (0xFF22, 'M', 'b'),
+    (0xFF23, 'M', 'c'),
+    (0xFF24, 'M', 'd'),
+    (0xFF25, 'M', 'e'),
+    (0xFF26, 'M', 'f'),
+    (0xFF27, 'M', 'g'),
+    (0xFF28, 'M', 'h'),
+    (0xFF29, 'M', 'i'),
+    (0xFF2A, 'M', 'j'),
+    (0xFF2B, 'M', 'k'),
+    (0xFF2C, 'M', 'l'),
+    (0xFF2D, 'M', 'm'),
+    (0xFF2E, 'M', 'n'),
+    (0xFF2F, 'M', 'o'),
+    (0xFF30, 'M', 'p'),
+    (0xFF31, 'M', 'q'),
+    (0xFF32, 'M', 'r'),
+    (0xFF33, 'M', 's'),
+    (0xFF34, 'M', 't'),
+    (0xFF35, 'M', 'u'),
+    (0xFF36, 'M', 'v'),
+    (0xFF37, 'M', 'w'),
+    (0xFF38, 'M', 'x'),
+    (0xFF39, 'M', 'y'),
+    (0xFF3A, 'M', 'z'),
+    (0xFF3B, '3', '['),
+    (0xFF3C, '3', '\\'),
+    (0xFF3D, '3', ']'),
+    (0xFF3E, '3', '^'),
+    (0xFF3F, '3', '_'),
+    (0xFF40, '3', '`'),
+    (0xFF41, 'M', 'a'),
+    (0xFF42, 'M', 'b'),
+    (0xFF43, 'M', 'c'),
+    (0xFF44, 'M', 'd'),
+    (0xFF45, 'M', 'e'),
+    (0xFF46, 'M', 'f'),
+    (0xFF47, 'M', 'g'),
+    (0xFF48, 'M', 'h'),
+    (0xFF49, 'M', 'i'),
+    (0xFF4A, 'M', 'j'),
+    (0xFF4B, 'M', 'k'),
+    (0xFF4C, 'M', 'l'),
+    (0xFF4D, 'M', 'm'),
+    (0xFF4E, 'M', 'n'),
+    (0xFF4F, 'M', 'o'),
+    (0xFF50, 'M', 'p'),
+    (0xFF51, 'M', 'q'),
+    (0xFF52, 'M', 'r'),
+    (0xFF53, 'M', 's'),
+    (0xFF54, 'M', 't'),
+    (0xFF55, 'M', 'u'),
+    (0xFF56, 'M', 'v'),
+    (0xFF57, 'M', 'w'),
+    (0xFF58, 'M', 'x'),
+    (0xFF59, 'M', 'y'),
+    (0xFF5A, 'M', 'z'),
+    (0xFF5B, '3', '{'),
+    (0xFF5C, '3', '|'),
+    (0xFF5D, '3', '}'),
+    (0xFF5E, '3', '~'),
+    (0xFF5F, 'M', '⦅'),
+    (0xFF60, 'M', '⦆'),
+    (0xFF61, 'M', '.'),
+    (0xFF62, 'M', '「'),
+    (0xFF63, 'M', '」'),
+    (0xFF64, 'M', '、'),
+    (0xFF65, 'M', '・'),
+    (0xFF66, 'M', 'ヲ'),
+    (0xFF67, 'M', 'ァ'),
+    (0xFF68, 'M', 'ィ'),
+    (0xFF69, 'M', 'ゥ'),
+    (0xFF6A, 'M', 'ェ'),
+    (0xFF6B, 'M', 'ォ'),
+    (0xFF6C, 'M', 'ャ'),
+    (0xFF6D, 'M', 'ュ'),
+    (0xFF6E, 'M', 'ョ'),
+    (0xFF6F, 'M', 'ッ'),
+    (0xFF70, 'M', 'ー'),
+    (0xFF71, 'M', 'ア'),
+    (0xFF72, 'M', 'イ'),
+    (0xFF73, 'M', 'ウ'),
+    (0xFF74, 'M', 'エ'),
+    (0xFF75, 'M', 'オ'),
+    (0xFF76, 'M', 'カ'),
+    (0xFF77, 'M', 'キ'),
+    (0xFF78, 'M', 'ク'),
+    (0xFF79, 'M', 'ケ'),
+    (0xFF7A, 'M', 'コ'),
+    (0xFF7B, 'M', 'サ'),
+    (0xFF7C, 'M', 'シ'),
+    (0xFF7D, 'M', 'ス'),
+    (0xFF7E, 'M', 'セ'),
     ]
 
-def _seg_52():
+def _seg_52() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0xFF88, 'M', u'ネ'),
-    (0xFF89, 'M', u'ノ'),
-    (0xFF8A, 'M', u'ハ'),
-    (0xFF8B, 'M', u'ヒ'),
-    (0xFF8C, 'M', u'フ'),
-    (0xFF8D, 'M', u'ヘ'),
-    (0xFF8E, 'M', u'ホ'),
-    (0xFF8F, 'M', u'マ'),
-    (0xFF90, 'M', u'ミ'),
-    (0xFF91, 'M', u'ム'),
-    (0xFF92, 'M', u'メ'),
-    (0xFF93, 'M', u'モ'),
-    (0xFF94, 'M', u'ヤ'),
-    (0xFF95, 'M', u'ユ'),
-    (0xFF96, 'M', u'ヨ'),
-    (0xFF97, 'M', u'ラ'),
-    (0xFF98, 'M', u'リ'),
-    (0xFF99, 'M', u'ル'),
-    (0xFF9A, 'M', u'レ'),
-    (0xFF9B, 'M', u'ロ'),
-    (0xFF9C, 'M', u'ワ'),
-    (0xFF9D, 'M', u'ン'),
-    (0xFF9E, 'M', u'゙'),
-    (0xFF9F, 'M', u'゚'),
+    (0xFF7F, 'M', 'ソ'),
+    (0xFF80, 'M', 'タ'),
+    (0xFF81, 'M', 'チ'),
+    (0xFF82, 'M', 'ツ'),
+    (0xFF83, 'M', 'テ'),
+    (0xFF84, 'M', 'ト'),
+    (0xFF85, 'M', 'ナ'),
+    (0xFF86, 'M', 'ニ'),
+    (0xFF87, 'M', 'ヌ'),
+    (0xFF88, 'M', 'ネ'),
+    (0xFF89, 'M', 'ノ'),
+    (0xFF8A, 'M', 'ハ'),
+    (0xFF8B, 'M', 'ヒ'),
+    (0xFF8C, 'M', 'フ'),
+    (0xFF8D, 'M', 'ヘ'),
+    (0xFF8E, 'M', 'ホ'),
+    (0xFF8F, 'M', 'マ'),
+    (0xFF90, 'M', 'ミ'),
+    (0xFF91, 'M', 'ム'),
+    (0xFF92, 'M', 'メ'),
+    (0xFF93, 'M', 'モ'),
+    (0xFF94, 'M', 'ヤ'),
+    (0xFF95, 'M', 'ユ'),
+    (0xFF96, 'M', 'ヨ'),
+    (0xFF97, 'M', 'ラ'),
+    (0xFF98, 'M', 'リ'),
+    (0xFF99, 'M', 'ル'),
+    (0xFF9A, 'M', 'レ'),
+    (0xFF9B, 'M', 'ロ'),
+    (0xFF9C, 'M', 'ワ'),
+    (0xFF9D, 'M', 'ン'),
+    (0xFF9E, 'M', '゙'),
+    (0xFF9F, 'M', '゚'),
     (0xFFA0, 'X'),
-    (0xFFA1, 'M', u'ᄀ'),
-    (0xFFA2, 'M', u'ᄁ'),
-    (0xFFA3, 'M', u'ᆪ'),
-    (0xFFA4, 'M', u'ᄂ'),
-    (0xFFA5, 'M', u'ᆬ'),
-    (0xFFA6, 'M', u'ᆭ'),
-    (0xFFA7, 'M', u'ᄃ'),
-    (0xFFA8, 'M', u'ᄄ'),
-    (0xFFA9, 'M', u'ᄅ'),
-    (0xFFAA, 'M', u'ᆰ'),
-    (0xFFAB, 'M', u'ᆱ'),
-    (0xFFAC, 'M', u'ᆲ'),
-    (0xFFAD, 'M', u'ᆳ'),
-    (0xFFAE, 'M', u'ᆴ'),
-    (0xFFAF, 'M', u'ᆵ'),
-    (0xFFB0, 'M', u'ᄚ'),
-    (0xFFB1, 'M', u'ᄆ'),
-    (0xFFB2, 'M', u'ᄇ'),
-    (0xFFB3, 'M', u'ᄈ'),
-    (0xFFB4, 'M', u'ᄡ'),
-    (0xFFB5, 'M', u'ᄉ'),
-    (0xFFB6, 'M', u'ᄊ'),
-    (0xFFB7, 'M', u'ᄋ'),
-    (0xFFB8, 'M', u'ᄌ'),
-    (0xFFB9, 'M', u'ᄍ'),
-    (0xFFBA, 'M', u'ᄎ'),
-    (0xFFBB, 'M', u'ᄏ'),
-    (0xFFBC, 'M', u'ᄐ'),
-    (0xFFBD, 'M', u'ᄑ'),
-    (0xFFBE, 'M', u'ᄒ'),
+    (0xFFA1, 'M', 'ᄀ'),
+    (0xFFA2, 'M', 'ᄁ'),
+    (0xFFA3, 'M', 'ᆪ'),
+    (0xFFA4, 'M', 'ᄂ'),
+    (0xFFA5, 'M', 'ᆬ'),
+    (0xFFA6, 'M', 'ᆭ'),
+    (0xFFA7, 'M', 'ᄃ'),
+    (0xFFA8, 'M', 'ᄄ'),
+    (0xFFA9, 'M', 'ᄅ'),
+    (0xFFAA, 'M', 'ᆰ'),
+    (0xFFAB, 'M', 'ᆱ'),
+    (0xFFAC, 'M', 'ᆲ'),
+    (0xFFAD, 'M', 'ᆳ'),
+    (0xFFAE, 'M', 'ᆴ'),
+    (0xFFAF, 'M', 'ᆵ'),
+    (0xFFB0, 'M', 'ᄚ'),
+    (0xFFB1, 'M', 'ᄆ'),
+    (0xFFB2, 'M', 'ᄇ'),
+    (0xFFB3, 'M', 'ᄈ'),
+    (0xFFB4, 'M', 'ᄡ'),
+    (0xFFB5, 'M', 'ᄉ'),
+    (0xFFB6, 'M', 'ᄊ'),
+    (0xFFB7, 'M', 'ᄋ'),
+    (0xFFB8, 'M', 'ᄌ'),
+    (0xFFB9, 'M', 'ᄍ'),
+    (0xFFBA, 'M', 'ᄎ'),
+    (0xFFBB, 'M', 'ᄏ'),
+    (0xFFBC, 'M', 'ᄐ'),
+    (0xFFBD, 'M', 'ᄑ'),
+    (0xFFBE, 'M', 'ᄒ'),
     (0xFFBF, 'X'),
-    (0xFFC2, 'M', u'ᅡ'),
-    (0xFFC3, 'M', u'ᅢ'),
-    (0xFFC4, 'M', u'ᅣ'),
-    (0xFFC5, 'M', u'ᅤ'),
-    (0xFFC6, 'M', u'ᅥ'),
-    (0xFFC7, 'M', u'ᅦ'),
+    (0xFFC2, 'M', 'ᅡ'),
+    (0xFFC3, 'M', 'ᅢ'),
+    (0xFFC4, 'M', 'ᅣ'),
+    (0xFFC5, 'M', 'ᅤ'),
+    (0xFFC6, 'M', 'ᅥ'),
+    (0xFFC7, 'M', 'ᅦ'),
     (0xFFC8, 'X'),
-    (0xFFCA, 'M', u'ᅧ'),
-    (0xFFCB, 'M', u'ᅨ'),
-    (0xFFCC, 'M', u'ᅩ'),
-    (0xFFCD, 'M', u'ᅪ'),
-    (0xFFCE, 'M', u'ᅫ'),
-    (0xFFCF, 'M', u'ᅬ'),
+    (0xFFCA, 'M', 'ᅧ'),
+    (0xFFCB, 'M', 'ᅨ'),
+    (0xFFCC, 'M', 'ᅩ'),
+    (0xFFCD, 'M', 'ᅪ'),
+    (0xFFCE, 'M', 'ᅫ'),
+    (0xFFCF, 'M', 'ᅬ'),
     (0xFFD0, 'X'),
-    (0xFFD2, 'M', u'ᅭ'),
-    (0xFFD3, 'M', u'ᅮ'),
-    (0xFFD4, 'M', u'ᅯ'),
-    (0xFFD5, 'M', u'ᅰ'),
-    (0xFFD6, 'M', u'ᅱ'),
-    (0xFFD7, 'M', u'ᅲ'),
+    (0xFFD2, 'M', 'ᅭ'),
+    (0xFFD3, 'M', 'ᅮ'),
+    (0xFFD4, 'M', 'ᅯ'),
+    (0xFFD5, 'M', 'ᅰ'),
+    (0xFFD6, 'M', 'ᅱ'),
+    (0xFFD7, 'M', 'ᅲ'),
     (0xFFD8, 'X'),
-    (0xFFDA, 'M', u'ᅳ'),
-    (0xFFDB, 'M', u'ᅴ'),
-    (0xFFDC, 'M', u'ᅵ'),
+    (0xFFDA, 'M', 'ᅳ'),
+    (0xFFDB, 'M', 'ᅴ'),
+    (0xFFDC, 'M', 'ᅵ'),
     (0xFFDD, 'X'),
-    (0xFFE0, 'M', u'¢'),
-    (0xFFE1, 'M', u'£'),
-    (0xFFE2, 'M', u'¬'),
-    (0xFFE3, '3', u' ̄'),
-    (0xFFE4, 'M', u'¦'),
-    (0xFFE5, 'M', u'¥'),
-    (0xFFE6, 'M', u'₩'),
+    (0xFFE0, 'M', '¢'),
+    (0xFFE1, 'M', '£'),
+    (0xFFE2, 'M', '¬'),
+    (0xFFE3, '3', ' ̄'),
+    (0xFFE4, 'M', '¦'),
+    (0xFFE5, 'M', '¥'),
+    (0xFFE6, 'M', '₩'),
     (0xFFE7, 'X'),
-    (0xFFE8, 'M', u'│'),
-    (0xFFE9, 'M', u'←'),
-    (0xFFEA, 'M', u'↑'),
-    (0xFFEB, 'M', u'→'),
-    (0xFFEC, 'M', u'↓'),
-    (0xFFED, 'M', u'■'),
-    (0xFFEE, 'M', u'○'),
+    (0xFFE8, 'M', '│'),
+    (0xFFE9, 'M', '←'),
+    ]
+
+def _seg_53() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0xFFEA, 'M', '↑'),
+    (0xFFEB, 'M', '→'),
+    (0xFFEC, 'M', '↓'),
+    (0xFFED, 'M', '■'),
+    (0xFFEE, 'M', '○'),
     (0xFFEF, 'X'),
     (0x10000, 'V'),
     (0x1000C, 'X'),
     (0x1000D, 'V'),
-    ]
-
-def _seg_53():
-    return [
     (0x10027, 'X'),
     (0x10028, 'V'),
     (0x1003B, 'X'),
@@ -5560,90 +5572,90 @@ def _seg_53():
     (0x103C4, 'X'),
     (0x103C8, 'V'),
     (0x103D6, 'X'),
-    (0x10400, 'M', u'𐐨'),
-    (0x10401, 'M', u'𐐩'),
-    (0x10402, 'M', u'𐐪'),
-    (0x10403, 'M', u'𐐫'),
-    (0x10404, 'M', u'𐐬'),
-    (0x10405, 'M', u'𐐭'),
-    (0x10406, 'M', u'𐐮'),
-    (0x10407, 'M', u'𐐯'),
-    (0x10408, 'M', u'𐐰'),
-    (0x10409, 'M', u'𐐱'),
-    (0x1040A, 'M', u'𐐲'),
-    (0x1040B, 'M', u'𐐳'),
-    (0x1040C, 'M', u'𐐴'),
-    (0x1040D, 'M', u'𐐵'),
-    (0x1040E, 'M', u'𐐶'),
-    (0x1040F, 'M', u'𐐷'),
-    (0x10410, 'M', u'𐐸'),
-    (0x10411, 'M', u'𐐹'),
-    (0x10412, 'M', u'𐐺'),
-    (0x10413, 'M', u'𐐻'),
-    (0x10414, 'M', u'𐐼'),
-    (0x10415, 'M', u'𐐽'),
-    (0x10416, 'M', u'𐐾'),
-    (0x10417, 'M', u'𐐿'),
-    (0x10418, 'M', u'𐑀'),
-    (0x10419, 'M', u'𐑁'),
-    (0x1041A, 'M', u'𐑂'),
-    (0x1041B, 'M', u'𐑃'),
-    (0x1041C, 'M', u'𐑄'),
-    (0x1041D, 'M', u'𐑅'),
-    (0x1041E, 'M', u'𐑆'),
-    (0x1041F, 'M', u'𐑇'),
-    (0x10420, 'M', u'𐑈'),
-    (0x10421, 'M', u'𐑉'),
-    (0x10422, 'M', u'𐑊'),
-    (0x10423, 'M', u'𐑋'),
-    (0x10424, 'M', u'𐑌'),
-    (0x10425, 'M', u'𐑍'),
-    (0x10426, 'M', u'𐑎'),
-    (0x10427, 'M', u'𐑏'),
+    (0x10400, 'M', '𐐨'),
+    (0x10401, 'M', '𐐩'),
+    (0x10402, 'M', '𐐪'),
+    (0x10403, 'M', '𐐫'),
+    (0x10404, 'M', '𐐬'),
+    (0x10405, 'M', '𐐭'),
+    (0x10406, 'M', '𐐮'),
+    (0x10407, 'M', '𐐯'),
+    (0x10408, 'M', '𐐰'),
+    (0x10409, 'M', '𐐱'),
+    (0x1040A, 'M', '𐐲'),
+    (0x1040B, 'M', '𐐳'),
+    (0x1040C, 'M', '𐐴'),
+    (0x1040D, 'M', '𐐵'),
+    (0x1040E, 'M', '𐐶'),
+    (0x1040F, 'M', '𐐷'),
+    (0x10410, 'M', '𐐸'),
+    (0x10411, 'M', '𐐹'),
+    (0x10412, 'M', '𐐺'),
+    (0x10413, 'M', '𐐻'),
+    (0x10414, 'M', '𐐼'),
+    (0x10415, 'M', '𐐽'),
+    (0x10416, 'M', '𐐾'),
+    (0x10417, 'M', '𐐿'),
+    (0x10418, 'M', '𐑀'),
+    (0x10419, 'M', '𐑁'),
+    (0x1041A, 'M', '𐑂'),
+    (0x1041B, 'M', '𐑃'),
+    (0x1041C, 'M', '𐑄'),
+    (0x1041D, 'M', '𐑅'),
+    (0x1041E, 'M', '𐑆'),
+    (0x1041F, 'M', '𐑇'),
+    (0x10420, 'M', '𐑈'),
+    (0x10421, 'M', '𐑉'),
+    (0x10422, 'M', '𐑊'),
+    (0x10423, 'M', '𐑋'),
+    (0x10424, 'M', '𐑌'),
+    (0x10425, 'M', '𐑍'),
+    (0x10426, 'M', '𐑎'),
+    (0x10427, 'M', '𐑏'),
     (0x10428, 'V'),
     (0x1049E, 'X'),
     (0x104A0, 'V'),
     (0x104AA, 'X'),
-    (0x104B0, 'M', u'𐓘'),
-    (0x104B1, 'M', u'𐓙'),
-    (0x104B2, 'M', u'𐓚'),
-    (0x104B3, 'M', u'𐓛'),
-    (0x104B4, 'M', u'𐓜'),
-    (0x104B5, 'M', u'𐓝'),
-    (0x104B6, 'M', u'𐓞'),
-    (0x104B7, 'M', u'𐓟'),
-    (0x104B8, 'M', u'𐓠'),
-    (0x104B9, 'M', u'𐓡'),
-    (0x104BA, 'M', u'𐓢'),
-    (0x104BB, 'M', u'𐓣'),
-    (0x104BC, 'M', u'𐓤'),
-    (0x104BD, 'M', u'𐓥'),
-    (0x104BE, 'M', u'𐓦'),
+    (0x104B0, 'M', '𐓘'),
+    (0x104B1, 'M', '𐓙'),
+    (0x104B2, 'M', '𐓚'),
+    (0x104B3, 'M', '𐓛'),
+    (0x104B4, 'M', '𐓜'),
+    (0x104B5, 'M', '𐓝'),
     ]
 
-def _seg_54():
+def _seg_54() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x104BF, 'M', u'𐓧'),
-    (0x104C0, 'M', u'𐓨'),
-    (0x104C1, 'M', u'𐓩'),
-    (0x104C2, 'M', u'𐓪'),
-    (0x104C3, 'M', u'𐓫'),
-    (0x104C4, 'M', u'𐓬'),
-    (0x104C5, 'M', u'𐓭'),
-    (0x104C6, 'M', u'𐓮'),
-    (0x104C7, 'M', u'𐓯'),
-    (0x104C8, 'M', u'𐓰'),
-    (0x104C9, 'M', u'𐓱'),
-    (0x104CA, 'M', u'𐓲'),
-    (0x104CB, 'M', u'𐓳'),
-    (0x104CC, 'M', u'𐓴'),
-    (0x104CD, 'M', u'𐓵'),
-    (0x104CE, 'M', u'𐓶'),
-    (0x104CF, 'M', u'𐓷'),
-    (0x104D0, 'M', u'𐓸'),
-    (0x104D1, 'M', u'𐓹'),
-    (0x104D2, 'M', u'𐓺'),
-    (0x104D3, 'M', u'𐓻'),
+    (0x104B6, 'M', '𐓞'),
+    (0x104B7, 'M', '𐓟'),
+    (0x104B8, 'M', '𐓠'),
+    (0x104B9, 'M', '𐓡'),
+    (0x104BA, 'M', '𐓢'),
+    (0x104BB, 'M', '𐓣'),
+    (0x104BC, 'M', '𐓤'),
+    (0x104BD, 'M', '𐓥'),
+    (0x104BE, 'M', '𐓦'),
+    (0x104BF, 'M', '𐓧'),
+    (0x104C0, 'M', '𐓨'),
+    (0x104C1, 'M', '𐓩'),
+    (0x104C2, 'M', '𐓪'),
+    (0x104C3, 'M', '𐓫'),
+    (0x104C4, 'M', '𐓬'),
+    (0x104C5, 'M', '𐓭'),
+    (0x104C6, 'M', '𐓮'),
+    (0x104C7, 'M', '𐓯'),
+    (0x104C8, 'M', '𐓰'),
+    (0x104C9, 'M', '𐓱'),
+    (0x104CA, 'M', '𐓲'),
+    (0x104CB, 'M', '𐓳'),
+    (0x104CC, 'M', '𐓴'),
+    (0x104CD, 'M', '𐓵'),
+    (0x104CE, 'M', '𐓶'),
+    (0x104CF, 'M', '𐓷'),
+    (0x104D0, 'M', '𐓸'),
+    (0x104D1, 'M', '𐓹'),
+    (0x104D2, 'M', '𐓺'),
+    (0x104D3, 'M', '𐓻'),
     (0x104D4, 'X'),
     (0x104D8, 'V'),
     (0x104FC, 'X'),
@@ -5652,13 +5664,123 @@ def _seg_54():
     (0x10530, 'V'),
     (0x10564, 'X'),
     (0x1056F, 'V'),
-    (0x10570, 'X'),
+    (0x10570, 'M', '𐖗'),
+    (0x10571, 'M', '𐖘'),
+    (0x10572, 'M', '𐖙'),
+    (0x10573, 'M', '𐖚'),
+    (0x10574, 'M', '𐖛'),
+    (0x10575, 'M', '𐖜'),
+    (0x10576, 'M', '𐖝'),
+    (0x10577, 'M', '𐖞'),
+    (0x10578, 'M', '𐖟'),
+    (0x10579, 'M', '𐖠'),
+    (0x1057A, 'M', '𐖡'),
+    (0x1057B, 'X'),
+    (0x1057C, 'M', '𐖣'),
+    (0x1057D, 'M', '𐖤'),
+    (0x1057E, 'M', '𐖥'),
+    (0x1057F, 'M', '𐖦'),
+    (0x10580, 'M', '𐖧'),
+    (0x10581, 'M', '𐖨'),
+    (0x10582, 'M', '𐖩'),
+    (0x10583, 'M', '𐖪'),
+    (0x10584, 'M', '𐖫'),
+    (0x10585, 'M', '𐖬'),
+    (0x10586, 'M', '𐖭'),
+    (0x10587, 'M', '𐖮'),
+    (0x10588, 'M', '𐖯'),
+    (0x10589, 'M', '𐖰'),
+    (0x1058A, 'M', '𐖱'),
+    (0x1058B, 'X'),
+    (0x1058C, 'M', '𐖳'),
+    (0x1058D, 'M', '𐖴'),
+    (0x1058E, 'M', '𐖵'),
+    (0x1058F, 'M', '𐖶'),
+    (0x10590, 'M', '𐖷'),
+    (0x10591, 'M', '𐖸'),
+    (0x10592, 'M', '𐖹'),
+    (0x10593, 'X'),
+    (0x10594, 'M', '𐖻'),
+    (0x10595, 'M', '𐖼'),
+    (0x10596, 'X'),
+    (0x10597, 'V'),
+    (0x105A2, 'X'),
+    (0x105A3, 'V'),
+    (0x105B2, 'X'),
+    (0x105B3, 'V'),
+    (0x105BA, 'X'),
+    (0x105BB, 'V'),
+    (0x105BD, 'X'),
     (0x10600, 'V'),
     (0x10737, 'X'),
     (0x10740, 'V'),
     (0x10756, 'X'),
     (0x10760, 'V'),
     (0x10768, 'X'),
+    (0x10780, 'V'),
+    (0x10781, 'M', 'ː'),
+    (0x10782, 'M', 'ˑ'),
+    (0x10783, 'M', 'æ'),
+    (0x10784, 'M', 'ʙ'),
+    (0x10785, 'M', 'ɓ'),
+    (0x10786, 'X'),
+    (0x10787, 'M', 'ʣ'),
+    (0x10788, 'M', 'ꭦ'),
+    ]
+
+def _seg_55() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x10789, 'M', 'ʥ'),
+    (0x1078A, 'M', 'ʤ'),
+    (0x1078B, 'M', 'ɖ'),
+    (0x1078C, 'M', 'ɗ'),
+    (0x1078D, 'M', 'ᶑ'),
+    (0x1078E, 'M', 'ɘ'),
+    (0x1078F, 'M', 'ɞ'),
+    (0x10790, 'M', 'ʩ'),
+    (0x10791, 'M', 'ɤ'),
+    (0x10792, 'M', 'ɢ'),
+    (0x10793, 'M', 'ɠ'),
+    (0x10794, 'M', 'ʛ'),
+    (0x10795, 'M', 'ħ'),
+    (0x10796, 'M', 'ʜ'),
+    (0x10797, 'M', 'ɧ'),
+    (0x10798, 'M', 'ʄ'),
+    (0x10799, 'M', 'ʪ'),
+    (0x1079A, 'M', 'ʫ'),
+    (0x1079B, 'M', 'ɬ'),
+    (0x1079C, 'M', '𝼄'),
+    (0x1079D, 'M', 'ꞎ'),
+    (0x1079E, 'M', 'ɮ'),
+    (0x1079F, 'M', '𝼅'),
+    (0x107A0, 'M', 'ʎ'),
+    (0x107A1, 'M', '𝼆'),
+    (0x107A2, 'M', 'ø'),
+    (0x107A3, 'M', 'ɶ'),
+    (0x107A4, 'M', 'ɷ'),
+    (0x107A5, 'M', 'q'),
+    (0x107A6, 'M', 'ɺ'),
+    (0x107A7, 'M', '𝼈'),
+    (0x107A8, 'M', 'ɽ'),
+    (0x107A9, 'M', 'ɾ'),
+    (0x107AA, 'M', 'ʀ'),
+    (0x107AB, 'M', 'ʨ'),
+    (0x107AC, 'M', 'ʦ'),
+    (0x107AD, 'M', 'ꭧ'),
+    (0x107AE, 'M', 'ʧ'),
+    (0x107AF, 'M', 'ʈ'),
+    (0x107B0, 'M', 'ⱱ'),
+    (0x107B1, 'X'),
+    (0x107B2, 'M', 'ʏ'),
+    (0x107B3, 'M', 'ʡ'),
+    (0x107B4, 'M', 'ʢ'),
+    (0x107B5, 'M', 'ʘ'),
+    (0x107B6, 'M', 'ǀ'),
+    (0x107B7, 'M', 'ǁ'),
+    (0x107B8, 'M', 'ǂ'),
+    (0x107B9, 'M', '𝼊'),
+    (0x107BA, 'M', '𝼞'),
+    (0x107BB, 'X'),
     (0x10800, 'V'),
     (0x10806, 'X'),
     (0x10808, 'V'),
@@ -5708,6 +5830,10 @@ def _seg_54():
     (0x10A60, 'V'),
     (0x10AA0, 'X'),
     (0x10AC0, 'V'),
+    ]
+
+def _seg_56() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
     (0x10AE7, 'X'),
     (0x10AEB, 'V'),
     (0x10AF7, 'X'),
@@ -5723,63 +5849,59 @@ def _seg_54():
     (0x10B9D, 'X'),
     (0x10BA9, 'V'),
     (0x10BB0, 'X'),
-    ]
-
-def _seg_55():
-    return [
     (0x10C00, 'V'),
     (0x10C49, 'X'),
-    (0x10C80, 'M', u'𐳀'),
-    (0x10C81, 'M', u'𐳁'),
-    (0x10C82, 'M', u'𐳂'),
-    (0x10C83, 'M', u'𐳃'),
-    (0x10C84, 'M', u'𐳄'),
-    (0x10C85, 'M', u'𐳅'),
-    (0x10C86, 'M', u'𐳆'),
-    (0x10C87, 'M', u'𐳇'),
-    (0x10C88, 'M', u'𐳈'),
-    (0x10C89, 'M', u'𐳉'),
-    (0x10C8A, 'M', u'𐳊'),
-    (0x10C8B, 'M', u'𐳋'),
-    (0x10C8C, 'M', u'𐳌'),
-    (0x10C8D, 'M', u'𐳍'),
-    (0x10C8E, 'M', u'𐳎'),
-    (0x10C8F, 'M', u'𐳏'),
-    (0x10C90, 'M', u'𐳐'),
-    (0x10C91, 'M', u'𐳑'),
-    (0x10C92, 'M', u'𐳒'),
-    (0x10C93, 'M', u'𐳓'),
-    (0x10C94, 'M', u'𐳔'),
-    (0x10C95, 'M', u'𐳕'),
-    (0x10C96, 'M', u'𐳖'),
-    (0x10C97, 'M', u'𐳗'),
-    (0x10C98, 'M', u'𐳘'),
-    (0x10C99, 'M', u'𐳙'),
-    (0x10C9A, 'M', u'𐳚'),
-    (0x10C9B, 'M', u'𐳛'),
-    (0x10C9C, 'M', u'𐳜'),
-    (0x10C9D, 'M', u'𐳝'),
-    (0x10C9E, 'M', u'𐳞'),
-    (0x10C9F, 'M', u'𐳟'),
-    (0x10CA0, 'M', u'𐳠'),
-    (0x10CA1, 'M', u'𐳡'),
-    (0x10CA2, 'M', u'𐳢'),
-    (0x10CA3, 'M', u'𐳣'),
-    (0x10CA4, 'M', u'𐳤'),
-    (0x10CA5, 'M', u'𐳥'),
-    (0x10CA6, 'M', u'𐳦'),
-    (0x10CA7, 'M', u'𐳧'),
-    (0x10CA8, 'M', u'𐳨'),
-    (0x10CA9, 'M', u'𐳩'),
-    (0x10CAA, 'M', u'𐳪'),
-    (0x10CAB, 'M', u'𐳫'),
-    (0x10CAC, 'M', u'𐳬'),
-    (0x10CAD, 'M', u'𐳭'),
-    (0x10CAE, 'M', u'𐳮'),
-    (0x10CAF, 'M', u'𐳯'),
-    (0x10CB0, 'M', u'𐳰'),
-    (0x10CB1, 'M', u'𐳱'),
-    (0x10CB2, 'M', u'𐳲'),
+    (0x10C80, 'M', '𐳀'),
+    (0x10C81, 'M', '𐳁'),
+    (0x10C82, 'M', '𐳂'),
+    (0x10C83, 'M', '𐳃'),
+    (0x10C84, 'M', '𐳄'),
+    (0x10C85, 'M', '𐳅'),
+    (0x10C86, 'M', '𐳆'),
+    (0x10C87, 'M', '𐳇'),
+    (0x10C88, 'M', '𐳈'),
+    (0x10C89, 'M', '𐳉'),
+    (0x10C8A, 'M', '𐳊'),
+    (0x10C8B, 'M', '𐳋'),
+    (0x10C8C, 'M', '𐳌'),
+    (0x10C8D, 'M', '𐳍'),
+    (0x10C8E, 'M', '𐳎'),
+    (0x10C8F, 'M', '𐳏'),
+    (0x10C90, 'M', '𐳐'),
+    (0x10C91, 'M', '𐳑'),
+    (0x10C92, 'M', '𐳒'),
+    (0x10C93, 'M', '𐳓'),
+    (0x10C94, 'M', '𐳔'),
+    (0x10C95, 'M', '𐳕'),
+    (0x10C96, 'M', '𐳖'),
+    (0x10C97, 'M', '𐳗'),
+    (0x10C98, 'M', '𐳘'),
+    (0x10C99, 'M', '𐳙'),
+    (0x10C9A, 'M', '𐳚'),
+    (0x10C9B, 'M', '𐳛'),
+    (0x10C9C, 'M', '𐳜'),
+    (0x10C9D, 'M', '𐳝'),
+    (0x10C9E, 'M', '𐳞'),
+    (0x10C9F, 'M', '𐳟'),
+    (0x10CA0, 'M', '𐳠'),
+    (0x10CA1, 'M', '𐳡'),
+    (0x10CA2, 'M', '𐳢'),
+    (0x10CA3, 'M', '𐳣'),
+    (0x10CA4, 'M', '𐳤'),
+    (0x10CA5, 'M', '𐳥'),
+    (0x10CA6, 'M', '𐳦'),
+    (0x10CA7, 'M', '𐳧'),
+    (0x10CA8, 'M', '𐳨'),
+    (0x10CA9, 'M', '𐳩'),
+    (0x10CAA, 'M', '𐳪'),
+    (0x10CAB, 'M', '𐳫'),
+    (0x10CAC, 'M', '𐳬'),
+    (0x10CAD, 'M', '𐳭'),
+    (0x10CAE, 'M', '𐳮'),
+    (0x10CAF, 'M', '𐳯'),
+    (0x10CB0, 'M', '𐳰'),
+    (0x10CB1, 'M', '𐳱'),
+    (0x10CB2, 'M', '𐳲'),
     (0x10CB3, 'X'),
     (0x10CC0, 'V'),
     (0x10CF3, 'X'),
@@ -5799,6 +5921,8 @@ def _seg_55():
     (0x10F28, 'X'),
     (0x10F30, 'V'),
     (0x10F5A, 'X'),
+    (0x10F70, 'V'),
+    (0x10F8A, 'X'),
     (0x10FB0, 'V'),
     (0x10FCC, 'X'),
     (0x10FE0, 'V'),
@@ -5806,11 +5930,15 @@ def _seg_55():
     (0x11000, 'V'),
     (0x1104E, 'X'),
     (0x11052, 'V'),
-    (0x11070, 'X'),
+    (0x11076, 'X'),
     (0x1107F, 'V'),
     (0x110BD, 'X'),
     (0x110BE, 'V'),
-    (0x110C2, 'X'),
+    ]
+
+def _seg_57() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x110C3, 'X'),
     (0x110D0, 'V'),
     (0x110E9, 'X'),
     (0x110F0, 'V'),
@@ -5827,10 +5955,6 @@ def _seg_55():
     (0x111F5, 'X'),
     (0x11200, 'V'),
     (0x11212, 'X'),
-    ]
-
-def _seg_56():
-    return [
     (0x11213, 'V'),
     (0x1123F, 'X'),
     (0x11280, 'V'),
@@ -5896,7 +6020,7 @@ def _seg_56():
     (0x11660, 'V'),
     (0x1166D, 'X'),
     (0x11680, 'V'),
-    (0x116B9, 'X'),
+    (0x116BA, 'X'),
     (0x116C0, 'V'),
     (0x116CA, 'X'),
     (0x11700, 'V'),
@@ -5904,45 +6028,45 @@ def _seg_56():
     (0x1171D, 'V'),
     (0x1172C, 'X'),
     (0x11730, 'V'),
-    (0x11740, 'X'),
+    (0x11747, 'X'),
     (0x11800, 'V'),
     (0x1183C, 'X'),
-    (0x118A0, 'M', u'𑣀'),
-    (0x118A1, 'M', u'𑣁'),
-    (0x118A2, 'M', u'𑣂'),
-    (0x118A3, 'M', u'𑣃'),
-    (0x118A4, 'M', u'𑣄'),
-    (0x118A5, 'M', u'𑣅'),
-    (0x118A6, 'M', u'𑣆'),
-    (0x118A7, 'M', u'𑣇'),
-    (0x118A8, 'M', u'𑣈'),
-    (0x118A9, 'M', u'𑣉'),
-    (0x118AA, 'M', u'𑣊'),
-    (0x118AB, 'M', u'𑣋'),
-    (0x118AC, 'M', u'𑣌'),
-    (0x118AD, 'M', u'𑣍'),
-    (0x118AE, 'M', u'𑣎'),
-    (0x118AF, 'M', u'𑣏'),
-    (0x118B0, 'M', u'𑣐'),
-    (0x118B1, 'M', u'𑣑'),
-    (0x118B2, 'M', u'𑣒'),
-    (0x118B3, 'M', u'𑣓'),
-    (0x118B4, 'M', u'𑣔'),
-    (0x118B5, 'M', u'𑣕'),
-    (0x118B6, 'M', u'𑣖'),
-    (0x118B7, 'M', u'𑣗'),
+    (0x118A0, 'M', '𑣀'),
+    (0x118A1, 'M', '𑣁'),
+    (0x118A2, 'M', '𑣂'),
+    (0x118A3, 'M', '𑣃'),
+    (0x118A4, 'M', '𑣄'),
+    (0x118A5, 'M', '𑣅'),
+    (0x118A6, 'M', '𑣆'),
     ]
 
-def _seg_57():
+def _seg_58() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x118B8, 'M', u'𑣘'),
-    (0x118B9, 'M', u'𑣙'),
-    (0x118BA, 'M', u'𑣚'),
-    (0x118BB, 'M', u'𑣛'),
-    (0x118BC, 'M', u'𑣜'),
-    (0x118BD, 'M', u'𑣝'),
-    (0x118BE, 'M', u'𑣞'),
-    (0x118BF, 'M', u'𑣟'),
+    (0x118A7, 'M', '𑣇'),
+    (0x118A8, 'M', '𑣈'),
+    (0x118A9, 'M', '𑣉'),
+    (0x118AA, 'M', '𑣊'),
+    (0x118AB, 'M', '𑣋'),
+    (0x118AC, 'M', '𑣌'),
+    (0x118AD, 'M', '𑣍'),
+    (0x118AE, 'M', '𑣎'),
+    (0x118AF, 'M', '𑣏'),
+    (0x118B0, 'M', '𑣐'),
+    (0x118B1, 'M', '𑣑'),
+    (0x118B2, 'M', '𑣒'),
+    (0x118B3, 'M', '𑣓'),
+    (0x118B4, 'M', '𑣔'),
+    (0x118B5, 'M', '𑣕'),
+    (0x118B6, 'M', '𑣖'),
+    (0x118B7, 'M', '𑣗'),
+    (0x118B8, 'M', '𑣘'),
+    (0x118B9, 'M', '𑣙'),
+    (0x118BA, 'M', '𑣚'),
+    (0x118BB, 'M', '𑣛'),
+    (0x118BC, 'M', '𑣜'),
+    (0x118BD, 'M', '𑣝'),
+    (0x118BE, 'M', '𑣞'),
+    (0x118BF, 'M', '𑣟'),
     (0x118C0, 'V'),
     (0x118F3, 'X'),
     (0x118FF, 'V'),
@@ -5971,7 +6095,7 @@ def _seg_57():
     (0x11A48, 'X'),
     (0x11A50, 'V'),
     (0x11AA3, 'X'),
-    (0x11AC0, 'V'),
+    (0x11AB0, 'V'),
     (0x11AF9, 'X'),
     (0x11C00, 'V'),
     (0x11C09, 'X'),
@@ -6018,6 +6142,10 @@ def _seg_57():
     (0x11FB0, 'V'),
     (0x11FB1, 'X'),
     (0x11FC0, 'V'),
+    ]
+
+def _seg_59() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
     (0x11FF2, 'X'),
     (0x11FFF, 'V'),
     (0x1239A, 'X'),
@@ -6027,6 +6155,8 @@ def _seg_57():
     (0x12475, 'X'),
     (0x12480, 'V'),
     (0x12544, 'X'),
+    (0x12F90, 'V'),
+    (0x12FF3, 'X'),
     (0x13000, 'V'),
     (0x1342F, 'X'),
     (0x14400, 'V'),
@@ -6035,14 +6165,12 @@ def _seg_57():
     (0x16A39, 'X'),
     (0x16A40, 'V'),
     (0x16A5F, 'X'),
-    ]
-
-def _seg_58():
-    return [
     (0x16A60, 'V'),
     (0x16A6A, 'X'),
     (0x16A6E, 'V'),
-    (0x16A70, 'X'),
+    (0x16ABF, 'X'),
+    (0x16AC0, 'V'),
+    (0x16ACA, 'X'),
     (0x16AD0, 'V'),
     (0x16AEE, 'X'),
     (0x16AF0, 'V'),
@@ -6057,38 +6185,38 @@ def _seg_58():
     (0x16B78, 'X'),
     (0x16B7D, 'V'),
     (0x16B90, 'X'),
-    (0x16E40, 'M', u'𖹠'),
-    (0x16E41, 'M', u'𖹡'),
-    (0x16E42, 'M', u'𖹢'),
-    (0x16E43, 'M', u'𖹣'),
-    (0x16E44, 'M', u'𖹤'),
-    (0x16E45, 'M', u'𖹥'),
-    (0x16E46, 'M', u'𖹦'),
-    (0x16E47, 'M', u'𖹧'),
-    (0x16E48, 'M', u'𖹨'),
-    (0x16E49, 'M', u'𖹩'),
-    (0x16E4A, 'M', u'𖹪'),
-    (0x16E4B, 'M', u'𖹫'),
-    (0x16E4C, 'M', u'𖹬'),
-    (0x16E4D, 'M', u'𖹭'),
-    (0x16E4E, 'M', u'𖹮'),
-    (0x16E4F, 'M', u'𖹯'),
-    (0x16E50, 'M', u'𖹰'),
-    (0x16E51, 'M', u'𖹱'),
-    (0x16E52, 'M', u'𖹲'),
-    (0x16E53, 'M', u'𖹳'),
-    (0x16E54, 'M', u'𖹴'),
-    (0x16E55, 'M', u'𖹵'),
-    (0x16E56, 'M', u'𖹶'),
-    (0x16E57, 'M', u'𖹷'),
-    (0x16E58, 'M', u'𖹸'),
-    (0x16E59, 'M', u'𖹹'),
-    (0x16E5A, 'M', u'𖹺'),
-    (0x16E5B, 'M', u'𖹻'),
-    (0x16E5C, 'M', u'𖹼'),
-    (0x16E5D, 'M', u'𖹽'),
-    (0x16E5E, 'M', u'𖹾'),
-    (0x16E5F, 'M', u'𖹿'),
+    (0x16E40, 'M', '𖹠'),
+    (0x16E41, 'M', '𖹡'),
+    (0x16E42, 'M', '𖹢'),
+    (0x16E43, 'M', '𖹣'),
+    (0x16E44, 'M', '𖹤'),
+    (0x16E45, 'M', '𖹥'),
+    (0x16E46, 'M', '𖹦'),
+    (0x16E47, 'M', '𖹧'),
+    (0x16E48, 'M', '𖹨'),
+    (0x16E49, 'M', '𖹩'),
+    (0x16E4A, 'M', '𖹪'),
+    (0x16E4B, 'M', '𖹫'),
+    (0x16E4C, 'M', '𖹬'),
+    (0x16E4D, 'M', '𖹭'),
+    (0x16E4E, 'M', '𖹮'),
+    (0x16E4F, 'M', '𖹯'),
+    (0x16E50, 'M', '𖹰'),
+    (0x16E51, 'M', '𖹱'),
+    (0x16E52, 'M', '𖹲'),
+    (0x16E53, 'M', '𖹳'),
+    (0x16E54, 'M', '𖹴'),
+    (0x16E55, 'M', '𖹵'),
+    (0x16E56, 'M', '𖹶'),
+    (0x16E57, 'M', '𖹷'),
+    (0x16E58, 'M', '𖹸'),
+    (0x16E59, 'M', '𖹹'),
+    (0x16E5A, 'M', '𖹺'),
+    (0x16E5B, 'M', '𖹻'),
+    (0x16E5C, 'M', '𖹼'),
+    (0x16E5D, 'M', '𖹽'),
+    (0x16E5E, 'M', '𖹾'),
+    (0x16E5F, 'M', '𖹿'),
     (0x16E60, 'V'),
     (0x16E9B, 'X'),
     (0x16F00, 'V'),
@@ -6107,11 +6235,21 @@ def _seg_58():
     (0x18CD6, 'X'),
     (0x18D00, 'V'),
     (0x18D09, 'X'),
+    (0x1AFF0, 'V'),
+    (0x1AFF4, 'X'),
+    (0x1AFF5, 'V'),
+    (0x1AFFC, 'X'),
+    (0x1AFFD, 'V'),
+    (0x1AFFF, 'X'),
     (0x1B000, 'V'),
-    (0x1B11F, 'X'),
+    (0x1B123, 'X'),
     (0x1B150, 'V'),
     (0x1B153, 'X'),
     (0x1B164, 'V'),
+    ]
+
+def _seg_60() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
     (0x1B168, 'X'),
     (0x1B170, 'V'),
     (0x1B2FC, 'X'),
@@ -6126,33 +6264,35 @@ def _seg_58():
     (0x1BC9C, 'V'),
     (0x1BCA0, 'I'),
     (0x1BCA4, 'X'),
+    (0x1CF00, 'V'),
+    (0x1CF2E, 'X'),
+    (0x1CF30, 'V'),
+    (0x1CF47, 'X'),
+    (0x1CF50, 'V'),
+    (0x1CFC4, 'X'),
     (0x1D000, 'V'),
     (0x1D0F6, 'X'),
     (0x1D100, 'V'),
     (0x1D127, 'X'),
     (0x1D129, 'V'),
-    (0x1D15E, 'M', u'𝅗𝅥'),
-    (0x1D15F, 'M', u'𝅘𝅥'),
-    (0x1D160, 'M', u'𝅘𝅥𝅮'),
-    (0x1D161, 'M', u'𝅘𝅥𝅯'),
-    (0x1D162, 'M', u'𝅘𝅥𝅰'),
-    (0x1D163, 'M', u'𝅘𝅥𝅱'),
-    (0x1D164, 'M', u'𝅘𝅥𝅲'),
+    (0x1D15E, 'M', '𝅗𝅥'),
+    (0x1D15F, 'M', '𝅘𝅥'),
+    (0x1D160, 'M', '𝅘𝅥𝅮'),
+    (0x1D161, 'M', '𝅘𝅥𝅯'),
+    (0x1D162, 'M', '𝅘𝅥𝅰'),
+    (0x1D163, 'M', '𝅘𝅥𝅱'),
+    (0x1D164, 'M', '𝅘𝅥𝅲'),
     (0x1D165, 'V'),
-    ]
-
-def _seg_59():
-    return [
     (0x1D173, 'X'),
     (0x1D17B, 'V'),
-    (0x1D1BB, 'M', u'𝆹𝅥'),
-    (0x1D1BC, 'M', u'𝆺𝅥'),
-    (0x1D1BD, 'M', u'𝆹𝅥𝅮'),
-    (0x1D1BE, 'M', u'𝆺𝅥𝅮'),
-    (0x1D1BF, 'M', u'𝆹𝅥𝅯'),
-    (0x1D1C0, 'M', u'𝆺𝅥𝅯'),
+    (0x1D1BB, 'M', '𝆹𝅥'),
+    (0x1D1BC, 'M', '𝆺𝅥'),
+    (0x1D1BD, 'M', '𝆹𝅥𝅮'),
+    (0x1D1BE, 'M', '𝆺𝅥𝅮'),
+    (0x1D1BF, 'M', '𝆹𝅥𝅯'),
+    (0x1D1C0, 'M', '𝆺𝅥𝅯'),
     (0x1D1C1, 'V'),
-    (0x1D1E9, 'X'),
+    (0x1D1EB, 'X'),
     (0x1D200, 'V'),
     (0x1D246, 'X'),
     (0x1D2E0, 'V'),
@@ -6161,1062 +6301,1064 @@ def _seg_59():
     (0x1D357, 'X'),
     (0x1D360, 'V'),
     (0x1D379, 'X'),
-    (0x1D400, 'M', u'a'),
-    (0x1D401, 'M', u'b'),
-    (0x1D402, 'M', u'c'),
-    (0x1D403, 'M', u'd'),
-    (0x1D404, 'M', u'e'),
-    (0x1D405, 'M', u'f'),
-    (0x1D406, 'M', u'g'),
-    (0x1D407, 'M', u'h'),
-    (0x1D408, 'M', u'i'),
-    (0x1D409, 'M', u'j'),
-    (0x1D40A, 'M', u'k'),
-    (0x1D40B, 'M', u'l'),
-    (0x1D40C, 'M', u'm'),
-    (0x1D40D, 'M', u'n'),
-    (0x1D40E, 'M', u'o'),
-    (0x1D40F, 'M', u'p'),
-    (0x1D410, 'M', u'q'),
-    (0x1D411, 'M', u'r'),
-    (0x1D412, 'M', u's'),
-    (0x1D413, 'M', u't'),
-    (0x1D414, 'M', u'u'),
-    (0x1D415, 'M', u'v'),
-    (0x1D416, 'M', u'w'),
-    (0x1D417, 'M', u'x'),
-    (0x1D418, 'M', u'y'),
-    (0x1D419, 'M', u'z'),
-    (0x1D41A, 'M', u'a'),
-    (0x1D41B, 'M', u'b'),
-    (0x1D41C, 'M', u'c'),
-    (0x1D41D, 'M', u'd'),
-    (0x1D41E, 'M', u'e'),
-    (0x1D41F, 'M', u'f'),
-    (0x1D420, 'M', u'g'),
-    (0x1D421, 'M', u'h'),
-    (0x1D422, 'M', u'i'),
-    (0x1D423, 'M', u'j'),
-    (0x1D424, 'M', u'k'),
-    (0x1D425, 'M', u'l'),
-    (0x1D426, 'M', u'm'),
-    (0x1D427, 'M', u'n'),
-    (0x1D428, 'M', u'o'),
-    (0x1D429, 'M', u'p'),
-    (0x1D42A, 'M', u'q'),
-    (0x1D42B, 'M', u'r'),
-    (0x1D42C, 'M', u's'),
-    (0x1D42D, 'M', u't'),
-    (0x1D42E, 'M', u'u'),
-    (0x1D42F, 'M', u'v'),
-    (0x1D430, 'M', u'w'),
-    (0x1D431, 'M', u'x'),
-    (0x1D432, 'M', u'y'),
-    (0x1D433, 'M', u'z'),
-    (0x1D434, 'M', u'a'),
-    (0x1D435, 'M', u'b'),
-    (0x1D436, 'M', u'c'),
-    (0x1D437, 'M', u'd'),
-    (0x1D438, 'M', u'e'),
-    (0x1D439, 'M', u'f'),
-    (0x1D43A, 'M', u'g'),
-    (0x1D43B, 'M', u'h'),
-    (0x1D43C, 'M', u'i'),
-    (0x1D43D, 'M', u'j'),
-    (0x1D43E, 'M', u'k'),
-    (0x1D43F, 'M', u'l'),
-    (0x1D440, 'M', u'm'),
-    (0x1D441, 'M', u'n'),
-    (0x1D442, 'M', u'o'),
-    (0x1D443, 'M', u'p'),
-    (0x1D444, 'M', u'q'),
-    (0x1D445, 'M', u'r'),
-    (0x1D446, 'M', u's'),
-    (0x1D447, 'M', u't'),
-    (0x1D448, 'M', u'u'),
-    (0x1D449, 'M', u'v'),
-    (0x1D44A, 'M', u'w'),
-    (0x1D44B, 'M', u'x'),
-    (0x1D44C, 'M', u'y'),
-    (0x1D44D, 'M', u'z'),
-    (0x1D44E, 'M', u'a'),
-    (0x1D44F, 'M', u'b'),
-    (0x1D450, 'M', u'c'),
-    (0x1D451, 'M', u'd'),
+    (0x1D400, 'M', 'a'),
+    (0x1D401, 'M', 'b'),
+    (0x1D402, 'M', 'c'),
+    (0x1D403, 'M', 'd'),
+    (0x1D404, 'M', 'e'),
+    (0x1D405, 'M', 'f'),
+    (0x1D406, 'M', 'g'),
+    (0x1D407, 'M', 'h'),
+    (0x1D408, 'M', 'i'),
+    (0x1D409, 'M', 'j'),
+    (0x1D40A, 'M', 'k'),
+    (0x1D40B, 'M', 'l'),
+    (0x1D40C, 'M', 'm'),
+    (0x1D40D, 'M', 'n'),
+    (0x1D40E, 'M', 'o'),
+    (0x1D40F, 'M', 'p'),
+    (0x1D410, 'M', 'q'),
+    (0x1D411, 'M', 'r'),
+    (0x1D412, 'M', 's'),
+    (0x1D413, 'M', 't'),
+    (0x1D414, 'M', 'u'),
+    (0x1D415, 'M', 'v'),
+    (0x1D416, 'M', 'w'),
+    (0x1D417, 'M', 'x'),
+    (0x1D418, 'M', 'y'),
+    (0x1D419, 'M', 'z'),
+    (0x1D41A, 'M', 'a'),
+    (0x1D41B, 'M', 'b'),
+    (0x1D41C, 'M', 'c'),
+    (0x1D41D, 'M', 'd'),
+    (0x1D41E, 'M', 'e'),
+    (0x1D41F, 'M', 'f'),
+    (0x1D420, 'M', 'g'),
+    (0x1D421, 'M', 'h'),
+    (0x1D422, 'M', 'i'),
+    (0x1D423, 'M', 'j'),
+    (0x1D424, 'M', 'k'),
+    (0x1D425, 'M', 'l'),
+    (0x1D426, 'M', 'm'),
+    (0x1D427, 'M', 'n'),
+    (0x1D428, 'M', 'o'),
+    (0x1D429, 'M', 'p'),
+    (0x1D42A, 'M', 'q'),
+    (0x1D42B, 'M', 'r'),
+    (0x1D42C, 'M', 's'),
+    (0x1D42D, 'M', 't'),
+    (0x1D42E, 'M', 'u'),
+    (0x1D42F, 'M', 'v'),
+    (0x1D430, 'M', 'w'),
     ]
 
-def _seg_60():
+def _seg_61() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D452, 'M', u'e'),
-    (0x1D453, 'M', u'f'),
-    (0x1D454, 'M', u'g'),
+    (0x1D431, 'M', 'x'),
+    (0x1D432, 'M', 'y'),
+    (0x1D433, 'M', 'z'),
+    (0x1D434, 'M', 'a'),
+    (0x1D435, 'M', 'b'),
+    (0x1D436, 'M', 'c'),
+    (0x1D437, 'M', 'd'),
+    (0x1D438, 'M', 'e'),
+    (0x1D439, 'M', 'f'),
+    (0x1D43A, 'M', 'g'),
+    (0x1D43B, 'M', 'h'),
+    (0x1D43C, 'M', 'i'),
+    (0x1D43D, 'M', 'j'),
+    (0x1D43E, 'M', 'k'),
+    (0x1D43F, 'M', 'l'),
+    (0x1D440, 'M', 'm'),
+    (0x1D441, 'M', 'n'),
+    (0x1D442, 'M', 'o'),
+    (0x1D443, 'M', 'p'),
+    (0x1D444, 'M', 'q'),
+    (0x1D445, 'M', 'r'),
+    (0x1D446, 'M', 's'),
+    (0x1D447, 'M', 't'),
+    (0x1D448, 'M', 'u'),
+    (0x1D449, 'M', 'v'),
+    (0x1D44A, 'M', 'w'),
+    (0x1D44B, 'M', 'x'),
+    (0x1D44C, 'M', 'y'),
+    (0x1D44D, 'M', 'z'),
+    (0x1D44E, 'M', 'a'),
+    (0x1D44F, 'M', 'b'),
+    (0x1D450, 'M', 'c'),
+    (0x1D451, 'M', 'd'),
+    (0x1D452, 'M', 'e'),
+    (0x1D453, 'M', 'f'),
+    (0x1D454, 'M', 'g'),
     (0x1D455, 'X'),
-    (0x1D456, 'M', u'i'),
-    (0x1D457, 'M', u'j'),
-    (0x1D458, 'M', u'k'),
-    (0x1D459, 'M', u'l'),
-    (0x1D45A, 'M', u'm'),
-    (0x1D45B, 'M', u'n'),
-    (0x1D45C, 'M', u'o'),
-    (0x1D45D, 'M', u'p'),
-    (0x1D45E, 'M', u'q'),
-    (0x1D45F, 'M', u'r'),
-    (0x1D460, 'M', u's'),
-    (0x1D461, 'M', u't'),
-    (0x1D462, 'M', u'u'),
-    (0x1D463, 'M', u'v'),
-    (0x1D464, 'M', u'w'),
-    (0x1D465, 'M', u'x'),
-    (0x1D466, 'M', u'y'),
-    (0x1D467, 'M', u'z'),
-    (0x1D468, 'M', u'a'),
-    (0x1D469, 'M', u'b'),
-    (0x1D46A, 'M', u'c'),
-    (0x1D46B, 'M', u'd'),
-    (0x1D46C, 'M', u'e'),
-    (0x1D46D, 'M', u'f'),
-    (0x1D46E, 'M', u'g'),
-    (0x1D46F, 'M', u'h'),
-    (0x1D470, 'M', u'i'),
-    (0x1D471, 'M', u'j'),
-    (0x1D472, 'M', u'k'),
-    (0x1D473, 'M', u'l'),
-    (0x1D474, 'M', u'm'),
-    (0x1D475, 'M', u'n'),
-    (0x1D476, 'M', u'o'),
-    (0x1D477, 'M', u'p'),
-    (0x1D478, 'M', u'q'),
-    (0x1D479, 'M', u'r'),
-    (0x1D47A, 'M', u's'),
-    (0x1D47B, 'M', u't'),
-    (0x1D47C, 'M', u'u'),
-    (0x1D47D, 'M', u'v'),
-    (0x1D47E, 'M', u'w'),
-    (0x1D47F, 'M', u'x'),
-    (0x1D480, 'M', u'y'),
-    (0x1D481, 'M', u'z'),
-    (0x1D482, 'M', u'a'),
-    (0x1D483, 'M', u'b'),
-    (0x1D484, 'M', u'c'),
-    (0x1D485, 'M', u'd'),
-    (0x1D486, 'M', u'e'),
-    (0x1D487, 'M', u'f'),
-    (0x1D488, 'M', u'g'),
-    (0x1D489, 'M', u'h'),
-    (0x1D48A, 'M', u'i'),
-    (0x1D48B, 'M', u'j'),
-    (0x1D48C, 'M', u'k'),
-    (0x1D48D, 'M', u'l'),
-    (0x1D48E, 'M', u'm'),
-    (0x1D48F, 'M', u'n'),
-    (0x1D490, 'M', u'o'),
-    (0x1D491, 'M', u'p'),
-    (0x1D492, 'M', u'q'),
-    (0x1D493, 'M', u'r'),
-    (0x1D494, 'M', u's'),
-    (0x1D495, 'M', u't'),
-    (0x1D496, 'M', u'u'),
-    (0x1D497, 'M', u'v'),
-    (0x1D498, 'M', u'w'),
-    (0x1D499, 'M', u'x'),
-    (0x1D49A, 'M', u'y'),
-    (0x1D49B, 'M', u'z'),
-    (0x1D49C, 'M', u'a'),
+    (0x1D456, 'M', 'i'),
+    (0x1D457, 'M', 'j'),
+    (0x1D458, 'M', 'k'),
+    (0x1D459, 'M', 'l'),
+    (0x1D45A, 'M', 'm'),
+    (0x1D45B, 'M', 'n'),
+    (0x1D45C, 'M', 'o'),
+    (0x1D45D, 'M', 'p'),
+    (0x1D45E, 'M', 'q'),
+    (0x1D45F, 'M', 'r'),
+    (0x1D460, 'M', 's'),
+    (0x1D461, 'M', 't'),
+    (0x1D462, 'M', 'u'),
+    (0x1D463, 'M', 'v'),
+    (0x1D464, 'M', 'w'),
+    (0x1D465, 'M', 'x'),
+    (0x1D466, 'M', 'y'),
+    (0x1D467, 'M', 'z'),
+    (0x1D468, 'M', 'a'),
+    (0x1D469, 'M', 'b'),
+    (0x1D46A, 'M', 'c'),
+    (0x1D46B, 'M', 'd'),
+    (0x1D46C, 'M', 'e'),
+    (0x1D46D, 'M', 'f'),
+    (0x1D46E, 'M', 'g'),
+    (0x1D46F, 'M', 'h'),
+    (0x1D470, 'M', 'i'),
+    (0x1D471, 'M', 'j'),
+    (0x1D472, 'M', 'k'),
+    (0x1D473, 'M', 'l'),
+    (0x1D474, 'M', 'm'),
+    (0x1D475, 'M', 'n'),
+    (0x1D476, 'M', 'o'),
+    (0x1D477, 'M', 'p'),
+    (0x1D478, 'M', 'q'),
+    (0x1D479, 'M', 'r'),
+    (0x1D47A, 'M', 's'),
+    (0x1D47B, 'M', 't'),
+    (0x1D47C, 'M', 'u'),
+    (0x1D47D, 'M', 'v'),
+    (0x1D47E, 'M', 'w'),
+    (0x1D47F, 'M', 'x'),
+    (0x1D480, 'M', 'y'),
+    (0x1D481, 'M', 'z'),
+    (0x1D482, 'M', 'a'),
+    (0x1D483, 'M', 'b'),
+    (0x1D484, 'M', 'c'),
+    (0x1D485, 'M', 'd'),
+    (0x1D486, 'M', 'e'),
+    (0x1D487, 'M', 'f'),
+    (0x1D488, 'M', 'g'),
+    (0x1D489, 'M', 'h'),
+    (0x1D48A, 'M', 'i'),
+    (0x1D48B, 'M', 'j'),
+    (0x1D48C, 'M', 'k'),
+    (0x1D48D, 'M', 'l'),
+    (0x1D48E, 'M', 'm'),
+    (0x1D48F, 'M', 'n'),
+    (0x1D490, 'M', 'o'),
+    (0x1D491, 'M', 'p'),
+    (0x1D492, 'M', 'q'),
+    (0x1D493, 'M', 'r'),
+    (0x1D494, 'M', 's'),
+    ]
+
+def _seg_62() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x1D495, 'M', 't'),
+    (0x1D496, 'M', 'u'),
+    (0x1D497, 'M', 'v'),
+    (0x1D498, 'M', 'w'),
+    (0x1D499, 'M', 'x'),
+    (0x1D49A, 'M', 'y'),
+    (0x1D49B, 'M', 'z'),
+    (0x1D49C, 'M', 'a'),
     (0x1D49D, 'X'),
-    (0x1D49E, 'M', u'c'),
-    (0x1D49F, 'M', u'd'),
+    (0x1D49E, 'M', 'c'),
+    (0x1D49F, 'M', 'd'),
     (0x1D4A0, 'X'),
-    (0x1D4A2, 'M', u'g'),
+    (0x1D4A2, 'M', 'g'),
     (0x1D4A3, 'X'),
-    (0x1D4A5, 'M', u'j'),
-    (0x1D4A6, 'M', u'k'),
+    (0x1D4A5, 'M', 'j'),
+    (0x1D4A6, 'M', 'k'),
     (0x1D4A7, 'X'),
-    (0x1D4A9, 'M', u'n'),
-    (0x1D4AA, 'M', u'o'),
-    (0x1D4AB, 'M', u'p'),
-    (0x1D4AC, 'M', u'q'),
+    (0x1D4A9, 'M', 'n'),
+    (0x1D4AA, 'M', 'o'),
+    (0x1D4AB, 'M', 'p'),
+    (0x1D4AC, 'M', 'q'),
     (0x1D4AD, 'X'),
-    (0x1D4AE, 'M', u's'),
-    (0x1D4AF, 'M', u't'),
-    (0x1D4B0, 'M', u'u'),
-    (0x1D4B1, 'M', u'v'),
-    (0x1D4B2, 'M', u'w'),
-    (0x1D4B3, 'M', u'x'),
-    (0x1D4B4, 'M', u'y'),
-    (0x1D4B5, 'M', u'z'),
-    (0x1D4B6, 'M', u'a'),
-    (0x1D4B7, 'M', u'b'),
-    (0x1D4B8, 'M', u'c'),
-    ]
-
-def _seg_61():
-    return [
-    (0x1D4B9, 'M', u'd'),
+    (0x1D4AE, 'M', 's'),
+    (0x1D4AF, 'M', 't'),
+    (0x1D4B0, 'M', 'u'),
+    (0x1D4B1, 'M', 'v'),
+    (0x1D4B2, 'M', 'w'),
+    (0x1D4B3, 'M', 'x'),
+    (0x1D4B4, 'M', 'y'),
+    (0x1D4B5, 'M', 'z'),
+    (0x1D4B6, 'M', 'a'),
+    (0x1D4B7, 'M', 'b'),
+    (0x1D4B8, 'M', 'c'),
+    (0x1D4B9, 'M', 'd'),
     (0x1D4BA, 'X'),
-    (0x1D4BB, 'M', u'f'),
+    (0x1D4BB, 'M', 'f'),
     (0x1D4BC, 'X'),
-    (0x1D4BD, 'M', u'h'),
-    (0x1D4BE, 'M', u'i'),
-    (0x1D4BF, 'M', u'j'),
-    (0x1D4C0, 'M', u'k'),
-    (0x1D4C1, 'M', u'l'),
-    (0x1D4C2, 'M', u'm'),
-    (0x1D4C3, 'M', u'n'),
+    (0x1D4BD, 'M', 'h'),
+    (0x1D4BE, 'M', 'i'),
+    (0x1D4BF, 'M', 'j'),
+    (0x1D4C0, 'M', 'k'),
+    (0x1D4C1, 'M', 'l'),
+    (0x1D4C2, 'M', 'm'),
+    (0x1D4C3, 'M', 'n'),
     (0x1D4C4, 'X'),
-    (0x1D4C5, 'M', u'p'),
-    (0x1D4C6, 'M', u'q'),
-    (0x1D4C7, 'M', u'r'),
-    (0x1D4C8, 'M', u's'),
-    (0x1D4C9, 'M', u't'),
-    (0x1D4CA, 'M', u'u'),
-    (0x1D4CB, 'M', u'v'),
-    (0x1D4CC, 'M', u'w'),
-    (0x1D4CD, 'M', u'x'),
-    (0x1D4CE, 'M', u'y'),
-    (0x1D4CF, 'M', u'z'),
-    (0x1D4D0, 'M', u'a'),
-    (0x1D4D1, 'M', u'b'),
-    (0x1D4D2, 'M', u'c'),
-    (0x1D4D3, 'M', u'd'),
-    (0x1D4D4, 'M', u'e'),
-    (0x1D4D5, 'M', u'f'),
-    (0x1D4D6, 'M', u'g'),
-    (0x1D4D7, 'M', u'h'),
-    (0x1D4D8, 'M', u'i'),
-    (0x1D4D9, 'M', u'j'),
-    (0x1D4DA, 'M', u'k'),
-    (0x1D4DB, 'M', u'l'),
-    (0x1D4DC, 'M', u'm'),
-    (0x1D4DD, 'M', u'n'),
-    (0x1D4DE, 'M', u'o'),
-    (0x1D4DF, 'M', u'p'),
-    (0x1D4E0, 'M', u'q'),
-    (0x1D4E1, 'M', u'r'),
-    (0x1D4E2, 'M', u's'),
-    (0x1D4E3, 'M', u't'),
-    (0x1D4E4, 'M', u'u'),
-    (0x1D4E5, 'M', u'v'),
-    (0x1D4E6, 'M', u'w'),
-    (0x1D4E7, 'M', u'x'),
-    (0x1D4E8, 'M', u'y'),
-    (0x1D4E9, 'M', u'z'),
-    (0x1D4EA, 'M', u'a'),
-    (0x1D4EB, 'M', u'b'),
-    (0x1D4EC, 'M', u'c'),
-    (0x1D4ED, 'M', u'd'),
-    (0x1D4EE, 'M', u'e'),
-    (0x1D4EF, 'M', u'f'),
-    (0x1D4F0, 'M', u'g'),
-    (0x1D4F1, 'M', u'h'),
-    (0x1D4F2, 'M', u'i'),
-    (0x1D4F3, 'M', u'j'),
-    (0x1D4F4, 'M', u'k'),
-    (0x1D4F5, 'M', u'l'),
-    (0x1D4F6, 'M', u'm'),
-    (0x1D4F7, 'M', u'n'),
-    (0x1D4F8, 'M', u'o'),
-    (0x1D4F9, 'M', u'p'),
-    (0x1D4FA, 'M', u'q'),
-    (0x1D4FB, 'M', u'r'),
-    (0x1D4FC, 'M', u's'),
-    (0x1D4FD, 'M', u't'),
-    (0x1D4FE, 'M', u'u'),
-    (0x1D4FF, 'M', u'v'),
-    (0x1D500, 'M', u'w'),
-    (0x1D501, 'M', u'x'),
-    (0x1D502, 'M', u'y'),
-    (0x1D503, 'M', u'z'),
-    (0x1D504, 'M', u'a'),
-    (0x1D505, 'M', u'b'),
+    (0x1D4C5, 'M', 'p'),
+    (0x1D4C6, 'M', 'q'),
+    (0x1D4C7, 'M', 'r'),
+    (0x1D4C8, 'M', 's'),
+    (0x1D4C9, 'M', 't'),
+    (0x1D4CA, 'M', 'u'),
+    (0x1D4CB, 'M', 'v'),
+    (0x1D4CC, 'M', 'w'),
+    (0x1D4CD, 'M', 'x'),
+    (0x1D4CE, 'M', 'y'),
+    (0x1D4CF, 'M', 'z'),
+    (0x1D4D0, 'M', 'a'),
+    (0x1D4D1, 'M', 'b'),
+    (0x1D4D2, 'M', 'c'),
+    (0x1D4D3, 'M', 'd'),
+    (0x1D4D4, 'M', 'e'),
+    (0x1D4D5, 'M', 'f'),
+    (0x1D4D6, 'M', 'g'),
+    (0x1D4D7, 'M', 'h'),
+    (0x1D4D8, 'M', 'i'),
+    (0x1D4D9, 'M', 'j'),
+    (0x1D4DA, 'M', 'k'),
+    (0x1D4DB, 'M', 'l'),
+    (0x1D4DC, 'M', 'm'),
+    (0x1D4DD, 'M', 'n'),
+    (0x1D4DE, 'M', 'o'),
+    (0x1D4DF, 'M', 'p'),
+    (0x1D4E0, 'M', 'q'),
+    (0x1D4E1, 'M', 'r'),
+    (0x1D4E2, 'M', 's'),
+    (0x1D4E3, 'M', 't'),
+    (0x1D4E4, 'M', 'u'),
+    (0x1D4E5, 'M', 'v'),
+    (0x1D4E6, 'M', 'w'),
+    (0x1D4E7, 'M', 'x'),
+    (0x1D4E8, 'M', 'y'),
+    (0x1D4E9, 'M', 'z'),
+    (0x1D4EA, 'M', 'a'),
+    (0x1D4EB, 'M', 'b'),
+    (0x1D4EC, 'M', 'c'),
+    (0x1D4ED, 'M', 'd'),
+    (0x1D4EE, 'M', 'e'),
+    (0x1D4EF, 'M', 'f'),
+    (0x1D4F0, 'M', 'g'),
+    (0x1D4F1, 'M', 'h'),
+    (0x1D4F2, 'M', 'i'),
+    (0x1D4F3, 'M', 'j'),
+    (0x1D4F4, 'M', 'k'),
+    (0x1D4F5, 'M', 'l'),
+    (0x1D4F6, 'M', 'm'),
+    (0x1D4F7, 'M', 'n'),
+    (0x1D4F8, 'M', 'o'),
+    (0x1D4F9, 'M', 'p'),
+    (0x1D4FA, 'M', 'q'),
+    (0x1D4FB, 'M', 'r'),
+    ]
+
+def _seg_63() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x1D4FC, 'M', 's'),
+    (0x1D4FD, 'M', 't'),
+    (0x1D4FE, 'M', 'u'),
+    (0x1D4FF, 'M', 'v'),
+    (0x1D500, 'M', 'w'),
+    (0x1D501, 'M', 'x'),
+    (0x1D502, 'M', 'y'),
+    (0x1D503, 'M', 'z'),
+    (0x1D504, 'M', 'a'),
+    (0x1D505, 'M', 'b'),
     (0x1D506, 'X'),
-    (0x1D507, 'M', u'd'),
-    (0x1D508, 'M', u'e'),
-    (0x1D509, 'M', u'f'),
-    (0x1D50A, 'M', u'g'),
+    (0x1D507, 'M', 'd'),
+    (0x1D508, 'M', 'e'),
+    (0x1D509, 'M', 'f'),
+    (0x1D50A, 'M', 'g'),
     (0x1D50B, 'X'),
-    (0x1D50D, 'M', u'j'),
-    (0x1D50E, 'M', u'k'),
-    (0x1D50F, 'M', u'l'),
-    (0x1D510, 'M', u'm'),
-    (0x1D511, 'M', u'n'),
-    (0x1D512, 'M', u'o'),
-    (0x1D513, 'M', u'p'),
-    (0x1D514, 'M', u'q'),
+    (0x1D50D, 'M', 'j'),
+    (0x1D50E, 'M', 'k'),
+    (0x1D50F, 'M', 'l'),
+    (0x1D510, 'M', 'm'),
+    (0x1D511, 'M', 'n'),
+    (0x1D512, 'M', 'o'),
+    (0x1D513, 'M', 'p'),
+    (0x1D514, 'M', 'q'),
     (0x1D515, 'X'),
-    (0x1D516, 'M', u's'),
-    (0x1D517, 'M', u't'),
-    (0x1D518, 'M', u'u'),
-    (0x1D519, 'M', u'v'),
-    (0x1D51A, 'M', u'w'),
-    (0x1D51B, 'M', u'x'),
-    (0x1D51C, 'M', u'y'),
+    (0x1D516, 'M', 's'),
+    (0x1D517, 'M', 't'),
+    (0x1D518, 'M', 'u'),
+    (0x1D519, 'M', 'v'),
+    (0x1D51A, 'M', 'w'),
+    (0x1D51B, 'M', 'x'),
+    (0x1D51C, 'M', 'y'),
     (0x1D51D, 'X'),
-    ]
-
-def _seg_62():
-    return [
-    (0x1D51E, 'M', u'a'),
-    (0x1D51F, 'M', u'b'),
-    (0x1D520, 'M', u'c'),
-    (0x1D521, 'M', u'd'),
-    (0x1D522, 'M', u'e'),
-    (0x1D523, 'M', u'f'),
-    (0x1D524, 'M', u'g'),
-    (0x1D525, 'M', u'h'),
-    (0x1D526, 'M', u'i'),
-    (0x1D527, 'M', u'j'),
-    (0x1D528, 'M', u'k'),
-    (0x1D529, 'M', u'l'),
-    (0x1D52A, 'M', u'm'),
-    (0x1D52B, 'M', u'n'),
-    (0x1D52C, 'M', u'o'),
-    (0x1D52D, 'M', u'p'),
-    (0x1D52E, 'M', u'q'),
-    (0x1D52F, 'M', u'r'),
-    (0x1D530, 'M', u's'),
-    (0x1D531, 'M', u't'),
-    (0x1D532, 'M', u'u'),
-    (0x1D533, 'M', u'v'),
-    (0x1D534, 'M', u'w'),
-    (0x1D535, 'M', u'x'),
-    (0x1D536, 'M', u'y'),
-    (0x1D537, 'M', u'z'),
-    (0x1D538, 'M', u'a'),
-    (0x1D539, 'M', u'b'),
+    (0x1D51E, 'M', 'a'),
+    (0x1D51F, 'M', 'b'),
+    (0x1D520, 'M', 'c'),
+    (0x1D521, 'M', 'd'),
+    (0x1D522, 'M', 'e'),
+    (0x1D523, 'M', 'f'),
+    (0x1D524, 'M', 'g'),
+    (0x1D525, 'M', 'h'),
+    (0x1D526, 'M', 'i'),
+    (0x1D527, 'M', 'j'),
+    (0x1D528, 'M', 'k'),
+    (0x1D529, 'M', 'l'),
+    (0x1D52A, 'M', 'm'),
+    (0x1D52B, 'M', 'n'),
+    (0x1D52C, 'M', 'o'),
+    (0x1D52D, 'M', 'p'),
+    (0x1D52E, 'M', 'q'),
+    (0x1D52F, 'M', 'r'),
+    (0x1D530, 'M', 's'),
+    (0x1D531, 'M', 't'),
+    (0x1D532, 'M', 'u'),
+    (0x1D533, 'M', 'v'),
+    (0x1D534, 'M', 'w'),
+    (0x1D535, 'M', 'x'),
+    (0x1D536, 'M', 'y'),
+    (0x1D537, 'M', 'z'),
+    (0x1D538, 'M', 'a'),
+    (0x1D539, 'M', 'b'),
     (0x1D53A, 'X'),
-    (0x1D53B, 'M', u'd'),
-    (0x1D53C, 'M', u'e'),
-    (0x1D53D, 'M', u'f'),
-    (0x1D53E, 'M', u'g'),
+    (0x1D53B, 'M', 'd'),
+    (0x1D53C, 'M', 'e'),
+    (0x1D53D, 'M', 'f'),
+    (0x1D53E, 'M', 'g'),
     (0x1D53F, 'X'),
-    (0x1D540, 'M', u'i'),
-    (0x1D541, 'M', u'j'),
-    (0x1D542, 'M', u'k'),
-    (0x1D543, 'M', u'l'),
-    (0x1D544, 'M', u'm'),
+    (0x1D540, 'M', 'i'),
+    (0x1D541, 'M', 'j'),
+    (0x1D542, 'M', 'k'),
+    (0x1D543, 'M', 'l'),
+    (0x1D544, 'M', 'm'),
     (0x1D545, 'X'),
-    (0x1D546, 'M', u'o'),
+    (0x1D546, 'M', 'o'),
     (0x1D547, 'X'),
-    (0x1D54A, 'M', u's'),
-    (0x1D54B, 'M', u't'),
-    (0x1D54C, 'M', u'u'),
-    (0x1D54D, 'M', u'v'),
-    (0x1D54E, 'M', u'w'),
-    (0x1D54F, 'M', u'x'),
-    (0x1D550, 'M', u'y'),
+    (0x1D54A, 'M', 's'),
+    (0x1D54B, 'M', 't'),
+    (0x1D54C, 'M', 'u'),
+    (0x1D54D, 'M', 'v'),
+    (0x1D54E, 'M', 'w'),
+    (0x1D54F, 'M', 'x'),
+    (0x1D550, 'M', 'y'),
     (0x1D551, 'X'),
-    (0x1D552, 'M', u'a'),
-    (0x1D553, 'M', u'b'),
-    (0x1D554, 'M', u'c'),
-    (0x1D555, 'M', u'd'),
-    (0x1D556, 'M', u'e'),
-    (0x1D557, 'M', u'f'),
-    (0x1D558, 'M', u'g'),
-    (0x1D559, 'M', u'h'),
-    (0x1D55A, 'M', u'i'),
-    (0x1D55B, 'M', u'j'),
-    (0x1D55C, 'M', u'k'),
-    (0x1D55D, 'M', u'l'),
-    (0x1D55E, 'M', u'm'),
-    (0x1D55F, 'M', u'n'),
-    (0x1D560, 'M', u'o'),
-    (0x1D561, 'M', u'p'),
-    (0x1D562, 'M', u'q'),
-    (0x1D563, 'M', u'r'),
-    (0x1D564, 'M', u's'),
-    (0x1D565, 'M', u't'),
-    (0x1D566, 'M', u'u'),
-    (0x1D567, 'M', u'v'),
-    (0x1D568, 'M', u'w'),
-    (0x1D569, 'M', u'x'),
-    (0x1D56A, 'M', u'y'),
-    (0x1D56B, 'M', u'z'),
-    (0x1D56C, 'M', u'a'),
-    (0x1D56D, 'M', u'b'),
-    (0x1D56E, 'M', u'c'),
-    (0x1D56F, 'M', u'd'),
-    (0x1D570, 'M', u'e'),
-    (0x1D571, 'M', u'f'),
-    (0x1D572, 'M', u'g'),
-    (0x1D573, 'M', u'h'),
-    (0x1D574, 'M', u'i'),
-    (0x1D575, 'M', u'j'),
-    (0x1D576, 'M', u'k'),
-    (0x1D577, 'M', u'l'),
-    (0x1D578, 'M', u'm'),
-    (0x1D579, 'M', u'n'),
-    (0x1D57A, 'M', u'o'),
-    (0x1D57B, 'M', u'p'),
-    (0x1D57C, 'M', u'q'),
-    (0x1D57D, 'M', u'r'),
-    (0x1D57E, 'M', u's'),
-    (0x1D57F, 'M', u't'),
-    (0x1D580, 'M', u'u'),
-    (0x1D581, 'M', u'v'),
-    (0x1D582, 'M', u'w'),
-    (0x1D583, 'M', u'x'),
+    (0x1D552, 'M', 'a'),
+    (0x1D553, 'M', 'b'),
+    (0x1D554, 'M', 'c'),
+    (0x1D555, 'M', 'd'),
+    (0x1D556, 'M', 'e'),
+    (0x1D557, 'M', 'f'),
+    (0x1D558, 'M', 'g'),
+    (0x1D559, 'M', 'h'),
+    (0x1D55A, 'M', 'i'),
+    (0x1D55B, 'M', 'j'),
+    (0x1D55C, 'M', 'k'),
+    (0x1D55D, 'M', 'l'),
+    (0x1D55E, 'M', 'm'),
+    (0x1D55F, 'M', 'n'),
+    (0x1D560, 'M', 'o'),
+    (0x1D561, 'M', 'p'),
+    (0x1D562, 'M', 'q'),
     ]
 
-def _seg_63():
+def _seg_64() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D584, 'M', u'y'),
-    (0x1D585, 'M', u'z'),
-    (0x1D586, 'M', u'a'),
-    (0x1D587, 'M', u'b'),
-    (0x1D588, 'M', u'c'),
-    (0x1D589, 'M', u'd'),
-    (0x1D58A, 'M', u'e'),
-    (0x1D58B, 'M', u'f'),
-    (0x1D58C, 'M', u'g'),
-    (0x1D58D, 'M', u'h'),
-    (0x1D58E, 'M', u'i'),
-    (0x1D58F, 'M', u'j'),
-    (0x1D590, 'M', u'k'),
-    (0x1D591, 'M', u'l'),
-    (0x1D592, 'M', u'm'),
-    (0x1D593, 'M', u'n'),
-    (0x1D594, 'M', u'o'),
-    (0x1D595, 'M', u'p'),
-    (0x1D596, 'M', u'q'),
-    (0x1D597, 'M', u'r'),
-    (0x1D598, 'M', u's'),
-    (0x1D599, 'M', u't'),
-    (0x1D59A, 'M', u'u'),
-    (0x1D59B, 'M', u'v'),
-    (0x1D59C, 'M', u'w'),
-    (0x1D59D, 'M', u'x'),
-    (0x1D59E, 'M', u'y'),
-    (0x1D59F, 'M', u'z'),
-    (0x1D5A0, 'M', u'a'),
-    (0x1D5A1, 'M', u'b'),
-    (0x1D5A2, 'M', u'c'),
-    (0x1D5A3, 'M', u'd'),
-    (0x1D5A4, 'M', u'e'),
-    (0x1D5A5, 'M', u'f'),
-    (0x1D5A6, 'M', u'g'),
-    (0x1D5A7, 'M', u'h'),
-    (0x1D5A8, 'M', u'i'),
-    (0x1D5A9, 'M', u'j'),
-    (0x1D5AA, 'M', u'k'),
-    (0x1D5AB, 'M', u'l'),
-    (0x1D5AC, 'M', u'm'),
-    (0x1D5AD, 'M', u'n'),
-    (0x1D5AE, 'M', u'o'),
-    (0x1D5AF, 'M', u'p'),
-    (0x1D5B0, 'M', u'q'),
-    (0x1D5B1, 'M', u'r'),
-    (0x1D5B2, 'M', u's'),
-    (0x1D5B3, 'M', u't'),
-    (0x1D5B4, 'M', u'u'),
-    (0x1D5B5, 'M', u'v'),
-    (0x1D5B6, 'M', u'w'),
-    (0x1D5B7, 'M', u'x'),
-    (0x1D5B8, 'M', u'y'),
-    (0x1D5B9, 'M', u'z'),
-    (0x1D5BA, 'M', u'a'),
-    (0x1D5BB, 'M', u'b'),
-    (0x1D5BC, 'M', u'c'),
-    (0x1D5BD, 'M', u'd'),
-    (0x1D5BE, 'M', u'e'),
-    (0x1D5BF, 'M', u'f'),
-    (0x1D5C0, 'M', u'g'),
-    (0x1D5C1, 'M', u'h'),
-    (0x1D5C2, 'M', u'i'),
-    (0x1D5C3, 'M', u'j'),
-    (0x1D5C4, 'M', u'k'),
-    (0x1D5C5, 'M', u'l'),
-    (0x1D5C6, 'M', u'm'),
-    (0x1D5C7, 'M', u'n'),
-    (0x1D5C8, 'M', u'o'),
-    (0x1D5C9, 'M', u'p'),
-    (0x1D5CA, 'M', u'q'),
-    (0x1D5CB, 'M', u'r'),
-    (0x1D5CC, 'M', u's'),
-    (0x1D5CD, 'M', u't'),
-    (0x1D5CE, 'M', u'u'),
-    (0x1D5CF, 'M', u'v'),
-    (0x1D5D0, 'M', u'w'),
-    (0x1D5D1, 'M', u'x'),
-    (0x1D5D2, 'M', u'y'),
-    (0x1D5D3, 'M', u'z'),
-    (0x1D5D4, 'M', u'a'),
-    (0x1D5D5, 'M', u'b'),
-    (0x1D5D6, 'M', u'c'),
-    (0x1D5D7, 'M', u'd'),
-    (0x1D5D8, 'M', u'e'),
-    (0x1D5D9, 'M', u'f'),
-    (0x1D5DA, 'M', u'g'),
-    (0x1D5DB, 'M', u'h'),
-    (0x1D5DC, 'M', u'i'),
-    (0x1D5DD, 'M', u'j'),
-    (0x1D5DE, 'M', u'k'),
-    (0x1D5DF, 'M', u'l'),
-    (0x1D5E0, 'M', u'm'),
-    (0x1D5E1, 'M', u'n'),
-    (0x1D5E2, 'M', u'o'),
-    (0x1D5E3, 'M', u'p'),
-    (0x1D5E4, 'M', u'q'),
-    (0x1D5E5, 'M', u'r'),
-    (0x1D5E6, 'M', u's'),
-    (0x1D5E7, 'M', u't'),
+    (0x1D563, 'M', 'r'),
+    (0x1D564, 'M', 's'),
+    (0x1D565, 'M', 't'),
+    (0x1D566, 'M', 'u'),
+    (0x1D567, 'M', 'v'),
+    (0x1D568, 'M', 'w'),
+    (0x1D569, 'M', 'x'),
+    (0x1D56A, 'M', 'y'),
+    (0x1D56B, 'M', 'z'),
+    (0x1D56C, 'M', 'a'),
+    (0x1D56D, 'M', 'b'),
+    (0x1D56E, 'M', 'c'),
+    (0x1D56F, 'M', 'd'),
+    (0x1D570, 'M', 'e'),
+    (0x1D571, 'M', 'f'),
+    (0x1D572, 'M', 'g'),
+    (0x1D573, 'M', 'h'),
+    (0x1D574, 'M', 'i'),
+    (0x1D575, 'M', 'j'),
+    (0x1D576, 'M', 'k'),
+    (0x1D577, 'M', 'l'),
+    (0x1D578, 'M', 'm'),
+    (0x1D579, 'M', 'n'),
+    (0x1D57A, 'M', 'o'),
+    (0x1D57B, 'M', 'p'),
+    (0x1D57C, 'M', 'q'),
+    (0x1D57D, 'M', 'r'),
+    (0x1D57E, 'M', 's'),
+    (0x1D57F, 'M', 't'),
+    (0x1D580, 'M', 'u'),
+    (0x1D581, 'M', 'v'),
+    (0x1D582, 'M', 'w'),
+    (0x1D583, 'M', 'x'),
+    (0x1D584, 'M', 'y'),
+    (0x1D585, 'M', 'z'),
+    (0x1D586, 'M', 'a'),
+    (0x1D587, 'M', 'b'),
+    (0x1D588, 'M', 'c'),
+    (0x1D589, 'M', 'd'),
+    (0x1D58A, 'M', 'e'),
+    (0x1D58B, 'M', 'f'),
+    (0x1D58C, 'M', 'g'),
+    (0x1D58D, 'M', 'h'),
+    (0x1D58E, 'M', 'i'),
+    (0x1D58F, 'M', 'j'),
+    (0x1D590, 'M', 'k'),
+    (0x1D591, 'M', 'l'),
+    (0x1D592, 'M', 'm'),
+    (0x1D593, 'M', 'n'),
+    (0x1D594, 'M', 'o'),
+    (0x1D595, 'M', 'p'),
+    (0x1D596, 'M', 'q'),
+    (0x1D597, 'M', 'r'),
+    (0x1D598, 'M', 's'),
+    (0x1D599, 'M', 't'),
+    (0x1D59A, 'M', 'u'),
+    (0x1D59B, 'M', 'v'),
+    (0x1D59C, 'M', 'w'),
+    (0x1D59D, 'M', 'x'),
+    (0x1D59E, 'M', 'y'),
+    (0x1D59F, 'M', 'z'),
+    (0x1D5A0, 'M', 'a'),
+    (0x1D5A1, 'M', 'b'),
+    (0x1D5A2, 'M', 'c'),
+    (0x1D5A3, 'M', 'd'),
+    (0x1D5A4, 'M', 'e'),
+    (0x1D5A5, 'M', 'f'),
+    (0x1D5A6, 'M', 'g'),
+    (0x1D5A7, 'M', 'h'),
+    (0x1D5A8, 'M', 'i'),
+    (0x1D5A9, 'M', 'j'),
+    (0x1D5AA, 'M', 'k'),
+    (0x1D5AB, 'M', 'l'),
+    (0x1D5AC, 'M', 'm'),
+    (0x1D5AD, 'M', 'n'),
+    (0x1D5AE, 'M', 'o'),
+    (0x1D5AF, 'M', 'p'),
+    (0x1D5B0, 'M', 'q'),
+    (0x1D5B1, 'M', 'r'),
+    (0x1D5B2, 'M', 's'),
+    (0x1D5B3, 'M', 't'),
+    (0x1D5B4, 'M', 'u'),
+    (0x1D5B5, 'M', 'v'),
+    (0x1D5B6, 'M', 'w'),
+    (0x1D5B7, 'M', 'x'),
+    (0x1D5B8, 'M', 'y'),
+    (0x1D5B9, 'M', 'z'),
+    (0x1D5BA, 'M', 'a'),
+    (0x1D5BB, 'M', 'b'),
+    (0x1D5BC, 'M', 'c'),
+    (0x1D5BD, 'M', 'd'),
+    (0x1D5BE, 'M', 'e'),
+    (0x1D5BF, 'M', 'f'),
+    (0x1D5C0, 'M', 'g'),
+    (0x1D5C1, 'M', 'h'),
+    (0x1D5C2, 'M', 'i'),
+    (0x1D5C3, 'M', 'j'),
+    (0x1D5C4, 'M', 'k'),
+    (0x1D5C5, 'M', 'l'),
+    (0x1D5C6, 'M', 'm'),
     ]
 
-def _seg_64():
+def _seg_65() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D5E8, 'M', u'u'),
-    (0x1D5E9, 'M', u'v'),
-    (0x1D5EA, 'M', u'w'),
-    (0x1D5EB, 'M', u'x'),
-    (0x1D5EC, 'M', u'y'),
-    (0x1D5ED, 'M', u'z'),
-    (0x1D5EE, 'M', u'a'),
-    (0x1D5EF, 'M', u'b'),
-    (0x1D5F0, 'M', u'c'),
-    (0x1D5F1, 'M', u'd'),
-    (0x1D5F2, 'M', u'e'),
-    (0x1D5F3, 'M', u'f'),
-    (0x1D5F4, 'M', u'g'),
-    (0x1D5F5, 'M', u'h'),
-    (0x1D5F6, 'M', u'i'),
-    (0x1D5F7, 'M', u'j'),
-    (0x1D5F8, 'M', u'k'),
-    (0x1D5F9, 'M', u'l'),
-    (0x1D5FA, 'M', u'm'),
-    (0x1D5FB, 'M', u'n'),
-    (0x1D5FC, 'M', u'o'),
-    (0x1D5FD, 'M', u'p'),
-    (0x1D5FE, 'M', u'q'),
-    (0x1D5FF, 'M', u'r'),
-    (0x1D600, 'M', u's'),
-    (0x1D601, 'M', u't'),
-    (0x1D602, 'M', u'u'),
-    (0x1D603, 'M', u'v'),
-    (0x1D604, 'M', u'w'),
-    (0x1D605, 'M', u'x'),
-    (0x1D606, 'M', u'y'),
-    (0x1D607, 'M', u'z'),
-    (0x1D608, 'M', u'a'),
-    (0x1D609, 'M', u'b'),
-    (0x1D60A, 'M', u'c'),
-    (0x1D60B, 'M', u'd'),
-    (0x1D60C, 'M', u'e'),
-    (0x1D60D, 'M', u'f'),
-    (0x1D60E, 'M', u'g'),
-    (0x1D60F, 'M', u'h'),
-    (0x1D610, 'M', u'i'),
-    (0x1D611, 'M', u'j'),
-    (0x1D612, 'M', u'k'),
-    (0x1D613, 'M', u'l'),
-    (0x1D614, 'M', u'm'),
-    (0x1D615, 'M', u'n'),
-    (0x1D616, 'M', u'o'),
-    (0x1D617, 'M', u'p'),
-    (0x1D618, 'M', u'q'),
-    (0x1D619, 'M', u'r'),
-    (0x1D61A, 'M', u's'),
-    (0x1D61B, 'M', u't'),
-    (0x1D61C, 'M', u'u'),
-    (0x1D61D, 'M', u'v'),
-    (0x1D61E, 'M', u'w'),
-    (0x1D61F, 'M', u'x'),
-    (0x1D620, 'M', u'y'),
-    (0x1D621, 'M', u'z'),
-    (0x1D622, 'M', u'a'),
-    (0x1D623, 'M', u'b'),
-    (0x1D624, 'M', u'c'),
-    (0x1D625, 'M', u'd'),
-    (0x1D626, 'M', u'e'),
-    (0x1D627, 'M', u'f'),
-    (0x1D628, 'M', u'g'),
-    (0x1D629, 'M', u'h'),
-    (0x1D62A, 'M', u'i'),
-    (0x1D62B, 'M', u'j'),
-    (0x1D62C, 'M', u'k'),
-    (0x1D62D, 'M', u'l'),
-    (0x1D62E, 'M', u'm'),
-    (0x1D62F, 'M', u'n'),
-    (0x1D630, 'M', u'o'),
-    (0x1D631, 'M', u'p'),
-    (0x1D632, 'M', u'q'),
-    (0x1D633, 'M', u'r'),
-    (0x1D634, 'M', u's'),
-    (0x1D635, 'M', u't'),
-    (0x1D636, 'M', u'u'),
-    (0x1D637, 'M', u'v'),
-    (0x1D638, 'M', u'w'),
-    (0x1D639, 'M', u'x'),
-    (0x1D63A, 'M', u'y'),
-    (0x1D63B, 'M', u'z'),
-    (0x1D63C, 'M', u'a'),
-    (0x1D63D, 'M', u'b'),
-    (0x1D63E, 'M', u'c'),
-    (0x1D63F, 'M', u'd'),
-    (0x1D640, 'M', u'e'),
-    (0x1D641, 'M', u'f'),
-    (0x1D642, 'M', u'g'),
-    (0x1D643, 'M', u'h'),
-    (0x1D644, 'M', u'i'),
-    (0x1D645, 'M', u'j'),
-    (0x1D646, 'M', u'k'),
-    (0x1D647, 'M', u'l'),
-    (0x1D648, 'M', u'm'),
-    (0x1D649, 'M', u'n'),
-    (0x1D64A, 'M', u'o'),
-    (0x1D64B, 'M', u'p'),
+    (0x1D5C7, 'M', 'n'),
+    (0x1D5C8, 'M', 'o'),
+    (0x1D5C9, 'M', 'p'),
+    (0x1D5CA, 'M', 'q'),
+    (0x1D5CB, 'M', 'r'),
+    (0x1D5CC, 'M', 's'),
+    (0x1D5CD, 'M', 't'),
+    (0x1D5CE, 'M', 'u'),
+    (0x1D5CF, 'M', 'v'),
+    (0x1D5D0, 'M', 'w'),
+    (0x1D5D1, 'M', 'x'),
+    (0x1D5D2, 'M', 'y'),
+    (0x1D5D3, 'M', 'z'),
+    (0x1D5D4, 'M', 'a'),
+    (0x1D5D5, 'M', 'b'),
+    (0x1D5D6, 'M', 'c'),
+    (0x1D5D7, 'M', 'd'),
+    (0x1D5D8, 'M', 'e'),
+    (0x1D5D9, 'M', 'f'),
+    (0x1D5DA, 'M', 'g'),
+    (0x1D5DB, 'M', 'h'),
+    (0x1D5DC, 'M', 'i'),
+    (0x1D5DD, 'M', 'j'),
+    (0x1D5DE, 'M', 'k'),
+    (0x1D5DF, 'M', 'l'),
+    (0x1D5E0, 'M', 'm'),
+    (0x1D5E1, 'M', 'n'),
+    (0x1D5E2, 'M', 'o'),
+    (0x1D5E3, 'M', 'p'),
+    (0x1D5E4, 'M', 'q'),
+    (0x1D5E5, 'M', 'r'),
+    (0x1D5E6, 'M', 's'),
+    (0x1D5E7, 'M', 't'),
+    (0x1D5E8, 'M', 'u'),
+    (0x1D5E9, 'M', 'v'),
+    (0x1D5EA, 'M', 'w'),
+    (0x1D5EB, 'M', 'x'),
+    (0x1D5EC, 'M', 'y'),
+    (0x1D5ED, 'M', 'z'),
+    (0x1D5EE, 'M', 'a'),
+    (0x1D5EF, 'M', 'b'),
+    (0x1D5F0, 'M', 'c'),
+    (0x1D5F1, 'M', 'd'),
+    (0x1D5F2, 'M', 'e'),
+    (0x1D5F3, 'M', 'f'),
+    (0x1D5F4, 'M', 'g'),
+    (0x1D5F5, 'M', 'h'),
+    (0x1D5F6, 'M', 'i'),
+    (0x1D5F7, 'M', 'j'),
+    (0x1D5F8, 'M', 'k'),
+    (0x1D5F9, 'M', 'l'),
+    (0x1D5FA, 'M', 'm'),
+    (0x1D5FB, 'M', 'n'),
+    (0x1D5FC, 'M', 'o'),
+    (0x1D5FD, 'M', 'p'),
+    (0x1D5FE, 'M', 'q'),
+    (0x1D5FF, 'M', 'r'),
+    (0x1D600, 'M', 's'),
+    (0x1D601, 'M', 't'),
+    (0x1D602, 'M', 'u'),
+    (0x1D603, 'M', 'v'),
+    (0x1D604, 'M', 'w'),
+    (0x1D605, 'M', 'x'),
+    (0x1D606, 'M', 'y'),
+    (0x1D607, 'M', 'z'),
+    (0x1D608, 'M', 'a'),
+    (0x1D609, 'M', 'b'),
+    (0x1D60A, 'M', 'c'),
+    (0x1D60B, 'M', 'd'),
+    (0x1D60C, 'M', 'e'),
+    (0x1D60D, 'M', 'f'),
+    (0x1D60E, 'M', 'g'),
+    (0x1D60F, 'M', 'h'),
+    (0x1D610, 'M', 'i'),
+    (0x1D611, 'M', 'j'),
+    (0x1D612, 'M', 'k'),
+    (0x1D613, 'M', 'l'),
+    (0x1D614, 'M', 'm'),
+    (0x1D615, 'M', 'n'),
+    (0x1D616, 'M', 'o'),
+    (0x1D617, 'M', 'p'),
+    (0x1D618, 'M', 'q'),
+    (0x1D619, 'M', 'r'),
+    (0x1D61A, 'M', 's'),
+    (0x1D61B, 'M', 't'),
+    (0x1D61C, 'M', 'u'),
+    (0x1D61D, 'M', 'v'),
+    (0x1D61E, 'M', 'w'),
+    (0x1D61F, 'M', 'x'),
+    (0x1D620, 'M', 'y'),
+    (0x1D621, 'M', 'z'),
+    (0x1D622, 'M', 'a'),
+    (0x1D623, 'M', 'b'),
+    (0x1D624, 'M', 'c'),
+    (0x1D625, 'M', 'd'),
+    (0x1D626, 'M', 'e'),
+    (0x1D627, 'M', 'f'),
+    (0x1D628, 'M', 'g'),
+    (0x1D629, 'M', 'h'),
+    (0x1D62A, 'M', 'i'),
     ]
 
-def _seg_65():
+def _seg_66() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D64C, 'M', u'q'),
-    (0x1D64D, 'M', u'r'),
-    (0x1D64E, 'M', u's'),
-    (0x1D64F, 'M', u't'),
-    (0x1D650, 'M', u'u'),
-    (0x1D651, 'M', u'v'),
-    (0x1D652, 'M', u'w'),
-    (0x1D653, 'M', u'x'),
-    (0x1D654, 'M', u'y'),
-    (0x1D655, 'M', u'z'),
-    (0x1D656, 'M', u'a'),
-    (0x1D657, 'M', u'b'),
-    (0x1D658, 'M', u'c'),
-    (0x1D659, 'M', u'd'),
-    (0x1D65A, 'M', u'e'),
-    (0x1D65B, 'M', u'f'),
-    (0x1D65C, 'M', u'g'),
-    (0x1D65D, 'M', u'h'),
-    (0x1D65E, 'M', u'i'),
-    (0x1D65F, 'M', u'j'),
-    (0x1D660, 'M', u'k'),
-    (0x1D661, 'M', u'l'),
-    (0x1D662, 'M', u'm'),
-    (0x1D663, 'M', u'n'),
-    (0x1D664, 'M', u'o'),
-    (0x1D665, 'M', u'p'),
-    (0x1D666, 'M', u'q'),
-    (0x1D667, 'M', u'r'),
-    (0x1D668, 'M', u's'),
-    (0x1D669, 'M', u't'),
-    (0x1D66A, 'M', u'u'),
-    (0x1D66B, 'M', u'v'),
-    (0x1D66C, 'M', u'w'),
-    (0x1D66D, 'M', u'x'),
-    (0x1D66E, 'M', u'y'),
-    (0x1D66F, 'M', u'z'),
-    (0x1D670, 'M', u'a'),
-    (0x1D671, 'M', u'b'),
-    (0x1D672, 'M', u'c'),
-    (0x1D673, 'M', u'd'),
-    (0x1D674, 'M', u'e'),
-    (0x1D675, 'M', u'f'),
-    (0x1D676, 'M', u'g'),
-    (0x1D677, 'M', u'h'),
-    (0x1D678, 'M', u'i'),
-    (0x1D679, 'M', u'j'),
-    (0x1D67A, 'M', u'k'),
-    (0x1D67B, 'M', u'l'),
-    (0x1D67C, 'M', u'm'),
-    (0x1D67D, 'M', u'n'),
-    (0x1D67E, 'M', u'o'),
-    (0x1D67F, 'M', u'p'),
-    (0x1D680, 'M', u'q'),
-    (0x1D681, 'M', u'r'),
-    (0x1D682, 'M', u's'),
-    (0x1D683, 'M', u't'),
-    (0x1D684, 'M', u'u'),
-    (0x1D685, 'M', u'v'),
-    (0x1D686, 'M', u'w'),
-    (0x1D687, 'M', u'x'),
-    (0x1D688, 'M', u'y'),
-    (0x1D689, 'M', u'z'),
-    (0x1D68A, 'M', u'a'),
-    (0x1D68B, 'M', u'b'),
-    (0x1D68C, 'M', u'c'),
-    (0x1D68D, 'M', u'd'),
-    (0x1D68E, 'M', u'e'),
-    (0x1D68F, 'M', u'f'),
-    (0x1D690, 'M', u'g'),
-    (0x1D691, 'M', u'h'),
-    (0x1D692, 'M', u'i'),
-    (0x1D693, 'M', u'j'),
-    (0x1D694, 'M', u'k'),
-    (0x1D695, 'M', u'l'),
-    (0x1D696, 'M', u'm'),
-    (0x1D697, 'M', u'n'),
-    (0x1D698, 'M', u'o'),
-    (0x1D699, 'M', u'p'),
-    (0x1D69A, 'M', u'q'),
-    (0x1D69B, 'M', u'r'),
-    (0x1D69C, 'M', u's'),
-    (0x1D69D, 'M', u't'),
-    (0x1D69E, 'M', u'u'),
-    (0x1D69F, 'M', u'v'),
-    (0x1D6A0, 'M', u'w'),
-    (0x1D6A1, 'M', u'x'),
-    (0x1D6A2, 'M', u'y'),
-    (0x1D6A3, 'M', u'z'),
-    (0x1D6A4, 'M', u'ı'),
-    (0x1D6A5, 'M', u'ȷ'),
-    (0x1D6A6, 'X'),
-    (0x1D6A8, 'M', u'α'),
-    (0x1D6A9, 'M', u'β'),
-    (0x1D6AA, 'M', u'γ'),
-    (0x1D6AB, 'M', u'δ'),
-    (0x1D6AC, 'M', u'ε'),
-    (0x1D6AD, 'M', u'ζ'),
-    (0x1D6AE, 'M', u'η'),
-    (0x1D6AF, 'M', u'θ'),
-    (0x1D6B0, 'M', u'ι'),
+    (0x1D62B, 'M', 'j'),
+    (0x1D62C, 'M', 'k'),
+    (0x1D62D, 'M', 'l'),
+    (0x1D62E, 'M', 'm'),
+    (0x1D62F, 'M', 'n'),
+    (0x1D630, 'M', 'o'),
+    (0x1D631, 'M', 'p'),
+    (0x1D632, 'M', 'q'),
+    (0x1D633, 'M', 'r'),
+    (0x1D634, 'M', 's'),
+    (0x1D635, 'M', 't'),
+    (0x1D636, 'M', 'u'),
+    (0x1D637, 'M', 'v'),
+    (0x1D638, 'M', 'w'),
+    (0x1D639, 'M', 'x'),
+    (0x1D63A, 'M', 'y'),
+    (0x1D63B, 'M', 'z'),
+    (0x1D63C, 'M', 'a'),
+    (0x1D63D, 'M', 'b'),
+    (0x1D63E, 'M', 'c'),
+    (0x1D63F, 'M', 'd'),
+    (0x1D640, 'M', 'e'),
+    (0x1D641, 'M', 'f'),
+    (0x1D642, 'M', 'g'),
+    (0x1D643, 'M', 'h'),
+    (0x1D644, 'M', 'i'),
+    (0x1D645, 'M', 'j'),
+    (0x1D646, 'M', 'k'),
+    (0x1D647, 'M', 'l'),
+    (0x1D648, 'M', 'm'),
+    (0x1D649, 'M', 'n'),
+    (0x1D64A, 'M', 'o'),
+    (0x1D64B, 'M', 'p'),
+    (0x1D64C, 'M', 'q'),
+    (0x1D64D, 'M', 'r'),
+    (0x1D64E, 'M', 's'),
+    (0x1D64F, 'M', 't'),
+    (0x1D650, 'M', 'u'),
+    (0x1D651, 'M', 'v'),
+    (0x1D652, 'M', 'w'),
+    (0x1D653, 'M', 'x'),
+    (0x1D654, 'M', 'y'),
+    (0x1D655, 'M', 'z'),
+    (0x1D656, 'M', 'a'),
+    (0x1D657, 'M', 'b'),
+    (0x1D658, 'M', 'c'),
+    (0x1D659, 'M', 'd'),
+    (0x1D65A, 'M', 'e'),
+    (0x1D65B, 'M', 'f'),
+    (0x1D65C, 'M', 'g'),
+    (0x1D65D, 'M', 'h'),
+    (0x1D65E, 'M', 'i'),
+    (0x1D65F, 'M', 'j'),
+    (0x1D660, 'M', 'k'),
+    (0x1D661, 'M', 'l'),
+    (0x1D662, 'M', 'm'),
+    (0x1D663, 'M', 'n'),
+    (0x1D664, 'M', 'o'),
+    (0x1D665, 'M', 'p'),
+    (0x1D666, 'M', 'q'),
+    (0x1D667, 'M', 'r'),
+    (0x1D668, 'M', 's'),
+    (0x1D669, 'M', 't'),
+    (0x1D66A, 'M', 'u'),
+    (0x1D66B, 'M', 'v'),
+    (0x1D66C, 'M', 'w'),
+    (0x1D66D, 'M', 'x'),
+    (0x1D66E, 'M', 'y'),
+    (0x1D66F, 'M', 'z'),
+    (0x1D670, 'M', 'a'),
+    (0x1D671, 'M', 'b'),
+    (0x1D672, 'M', 'c'),
+    (0x1D673, 'M', 'd'),
+    (0x1D674, 'M', 'e'),
+    (0x1D675, 'M', 'f'),
+    (0x1D676, 'M', 'g'),
+    (0x1D677, 'M', 'h'),
+    (0x1D678, 'M', 'i'),
+    (0x1D679, 'M', 'j'),
+    (0x1D67A, 'M', 'k'),
+    (0x1D67B, 'M', 'l'),
+    (0x1D67C, 'M', 'm'),
+    (0x1D67D, 'M', 'n'),
+    (0x1D67E, 'M', 'o'),
+    (0x1D67F, 'M', 'p'),
+    (0x1D680, 'M', 'q'),
+    (0x1D681, 'M', 'r'),
+    (0x1D682, 'M', 's'),
+    (0x1D683, 'M', 't'),
+    (0x1D684, 'M', 'u'),
+    (0x1D685, 'M', 'v'),
+    (0x1D686, 'M', 'w'),
+    (0x1D687, 'M', 'x'),
+    (0x1D688, 'M', 'y'),
+    (0x1D689, 'M', 'z'),
+    (0x1D68A, 'M', 'a'),
+    (0x1D68B, 'M', 'b'),
+    (0x1D68C, 'M', 'c'),
+    (0x1D68D, 'M', 'd'),
+    (0x1D68E, 'M', 'e'),
     ]
 
-def _seg_66():
+def _seg_67() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D6B1, 'M', u'κ'),
-    (0x1D6B2, 'M', u'λ'),
-    (0x1D6B3, 'M', u'μ'),
-    (0x1D6B4, 'M', u'ν'),
-    (0x1D6B5, 'M', u'ξ'),
-    (0x1D6B6, 'M', u'ο'),
-    (0x1D6B7, 'M', u'π'),
-    (0x1D6B8, 'M', u'ρ'),
-    (0x1D6B9, 'M', u'θ'),
-    (0x1D6BA, 'M', u'σ'),
-    (0x1D6BB, 'M', u'τ'),
-    (0x1D6BC, 'M', u'υ'),
-    (0x1D6BD, 'M', u'φ'),
-    (0x1D6BE, 'M', u'χ'),
-    (0x1D6BF, 'M', u'ψ'),
-    (0x1D6C0, 'M', u'ω'),
-    (0x1D6C1, 'M', u'∇'),
-    (0x1D6C2, 'M', u'α'),
-    (0x1D6C3, 'M', u'β'),
-    (0x1D6C4, 'M', u'γ'),
-    (0x1D6C5, 'M', u'δ'),
-    (0x1D6C6, 'M', u'ε'),
-    (0x1D6C7, 'M', u'ζ'),
-    (0x1D6C8, 'M', u'η'),
-    (0x1D6C9, 'M', u'θ'),
-    (0x1D6CA, 'M', u'ι'),
-    (0x1D6CB, 'M', u'κ'),
-    (0x1D6CC, 'M', u'λ'),
-    (0x1D6CD, 'M', u'μ'),
-    (0x1D6CE, 'M', u'ν'),
-    (0x1D6CF, 'M', u'ξ'),
-    (0x1D6D0, 'M', u'ο'),
-    (0x1D6D1, 'M', u'π'),
-    (0x1D6D2, 'M', u'ρ'),
-    (0x1D6D3, 'M', u'σ'),
-    (0x1D6D5, 'M', u'τ'),
-    (0x1D6D6, 'M', u'υ'),
-    (0x1D6D7, 'M', u'φ'),
-    (0x1D6D8, 'M', u'χ'),
-    (0x1D6D9, 'M', u'ψ'),
-    (0x1D6DA, 'M', u'ω'),
-    (0x1D6DB, 'M', u'∂'),
-    (0x1D6DC, 'M', u'ε'),
-    (0x1D6DD, 'M', u'θ'),
-    (0x1D6DE, 'M', u'κ'),
-    (0x1D6DF, 'M', u'φ'),
-    (0x1D6E0, 'M', u'ρ'),
-    (0x1D6E1, 'M', u'π'),
-    (0x1D6E2, 'M', u'α'),
-    (0x1D6E3, 'M', u'β'),
-    (0x1D6E4, 'M', u'γ'),
-    (0x1D6E5, 'M', u'δ'),
-    (0x1D6E6, 'M', u'ε'),
-    (0x1D6E7, 'M', u'ζ'),
-    (0x1D6E8, 'M', u'η'),
-    (0x1D6E9, 'M', u'θ'),
-    (0x1D6EA, 'M', u'ι'),
-    (0x1D6EB, 'M', u'κ'),
-    (0x1D6EC, 'M', u'λ'),
-    (0x1D6ED, 'M', u'μ'),
-    (0x1D6EE, 'M', u'ν'),
-    (0x1D6EF, 'M', u'ξ'),
-    (0x1D6F0, 'M', u'ο'),
-    (0x1D6F1, 'M', u'π'),
-    (0x1D6F2, 'M', u'ρ'),
-    (0x1D6F3, 'M', u'θ'),
-    (0x1D6F4, 'M', u'σ'),
-    (0x1D6F5, 'M', u'τ'),
-    (0x1D6F6, 'M', u'υ'),
-    (0x1D6F7, 'M', u'φ'),
-    (0x1D6F8, 'M', u'χ'),
-    (0x1D6F9, 'M', u'ψ'),
-    (0x1D6FA, 'M', u'ω'),
-    (0x1D6FB, 'M', u'∇'),
-    (0x1D6FC, 'M', u'α'),
-    (0x1D6FD, 'M', u'β'),
-    (0x1D6FE, 'M', u'γ'),
-    (0x1D6FF, 'M', u'δ'),
-    (0x1D700, 'M', u'ε'),
-    (0x1D701, 'M', u'ζ'),
-    (0x1D702, 'M', u'η'),
-    (0x1D703, 'M', u'θ'),
-    (0x1D704, 'M', u'ι'),
-    (0x1D705, 'M', u'κ'),
-    (0x1D706, 'M', u'λ'),
-    (0x1D707, 'M', u'μ'),
-    (0x1D708, 'M', u'ν'),
-    (0x1D709, 'M', u'ξ'),
-    (0x1D70A, 'M', u'ο'),
-    (0x1D70B, 'M', u'π'),
-    (0x1D70C, 'M', u'ρ'),
-    (0x1D70D, 'M', u'σ'),
-    (0x1D70F, 'M', u'τ'),
-    (0x1D710, 'M', u'υ'),
-    (0x1D711, 'M', u'φ'),
-    (0x1D712, 'M', u'χ'),
-    (0x1D713, 'M', u'ψ'),
-    (0x1D714, 'M', u'ω'),
-    (0x1D715, 'M', u'∂'),
-    (0x1D716, 'M', u'ε'),
+    (0x1D68F, 'M', 'f'),
+    (0x1D690, 'M', 'g'),
+    (0x1D691, 'M', 'h'),
+    (0x1D692, 'M', 'i'),
+    (0x1D693, 'M', 'j'),
+    (0x1D694, 'M', 'k'),
+    (0x1D695, 'M', 'l'),
+    (0x1D696, 'M', 'm'),
+    (0x1D697, 'M', 'n'),
+    (0x1D698, 'M', 'o'),
+    (0x1D699, 'M', 'p'),
+    (0x1D69A, 'M', 'q'),
+    (0x1D69B, 'M', 'r'),
+    (0x1D69C, 'M', 's'),
+    (0x1D69D, 'M', 't'),
+    (0x1D69E, 'M', 'u'),
+    (0x1D69F, 'M', 'v'),
+    (0x1D6A0, 'M', 'w'),
+    (0x1D6A1, 'M', 'x'),
+    (0x1D6A2, 'M', 'y'),
+    (0x1D6A3, 'M', 'z'),
+    (0x1D6A4, 'M', 'ı'),
+    (0x1D6A5, 'M', 'ȷ'),
+    (0x1D6A6, 'X'),
+    (0x1D6A8, 'M', 'α'),
+    (0x1D6A9, 'M', 'β'),
+    (0x1D6AA, 'M', 'γ'),
+    (0x1D6AB, 'M', 'δ'),
+    (0x1D6AC, 'M', 'ε'),
+    (0x1D6AD, 'M', 'ζ'),
+    (0x1D6AE, 'M', 'η'),
+    (0x1D6AF, 'M', 'θ'),
+    (0x1D6B0, 'M', 'ι'),
+    (0x1D6B1, 'M', 'κ'),
+    (0x1D6B2, 'M', 'λ'),
+    (0x1D6B3, 'M', 'μ'),
+    (0x1D6B4, 'M', 'ν'),
+    (0x1D6B5, 'M', 'ξ'),
+    (0x1D6B6, 'M', 'ο'),
+    (0x1D6B7, 'M', 'π'),
+    (0x1D6B8, 'M', 'ρ'),
+    (0x1D6B9, 'M', 'θ'),
+    (0x1D6BA, 'M', 'σ'),
+    (0x1D6BB, 'M', 'τ'),
+    (0x1D6BC, 'M', 'υ'),
+    (0x1D6BD, 'M', 'φ'),
+    (0x1D6BE, 'M', 'χ'),
+    (0x1D6BF, 'M', 'ψ'),
+    (0x1D6C0, 'M', 'ω'),
+    (0x1D6C1, 'M', '∇'),
+    (0x1D6C2, 'M', 'α'),
+    (0x1D6C3, 'M', 'β'),
+    (0x1D6C4, 'M', 'γ'),
+    (0x1D6C5, 'M', 'δ'),
+    (0x1D6C6, 'M', 'ε'),
+    (0x1D6C7, 'M', 'ζ'),
+    (0x1D6C8, 'M', 'η'),
+    (0x1D6C9, 'M', 'θ'),
+    (0x1D6CA, 'M', 'ι'),
+    (0x1D6CB, 'M', 'κ'),
+    (0x1D6CC, 'M', 'λ'),
+    (0x1D6CD, 'M', 'μ'),
+    (0x1D6CE, 'M', 'ν'),
+    (0x1D6CF, 'M', 'ξ'),
+    (0x1D6D0, 'M', 'ο'),
+    (0x1D6D1, 'M', 'π'),
+    (0x1D6D2, 'M', 'ρ'),
+    (0x1D6D3, 'M', 'σ'),
+    (0x1D6D5, 'M', 'τ'),
+    (0x1D6D6, 'M', 'υ'),
+    (0x1D6D7, 'M', 'φ'),
+    (0x1D6D8, 'M', 'χ'),
+    (0x1D6D9, 'M', 'ψ'),
+    (0x1D6DA, 'M', 'ω'),
+    (0x1D6DB, 'M', '∂'),
+    (0x1D6DC, 'M', 'ε'),
+    (0x1D6DD, 'M', 'θ'),
+    (0x1D6DE, 'M', 'κ'),
+    (0x1D6DF, 'M', 'φ'),
+    (0x1D6E0, 'M', 'ρ'),
+    (0x1D6E1, 'M', 'π'),
+    (0x1D6E2, 'M', 'α'),
+    (0x1D6E3, 'M', 'β'),
+    (0x1D6E4, 'M', 'γ'),
+    (0x1D6E5, 'M', 'δ'),
+    (0x1D6E6, 'M', 'ε'),
+    (0x1D6E7, 'M', 'ζ'),
+    (0x1D6E8, 'M', 'η'),
+    (0x1D6E9, 'M', 'θ'),
+    (0x1D6EA, 'M', 'ι'),
+    (0x1D6EB, 'M', 'κ'),
+    (0x1D6EC, 'M', 'λ'),
+    (0x1D6ED, 'M', 'μ'),
+    (0x1D6EE, 'M', 'ν'),
+    (0x1D6EF, 'M', 'ξ'),
+    (0x1D6F0, 'M', 'ο'),
+    (0x1D6F1, 'M', 'π'),
+    (0x1D6F2, 'M', 'ρ'),
+    (0x1D6F3, 'M', 'θ'),
+    (0x1D6F4, 'M', 'σ'),
     ]
 
-def _seg_67():
+def _seg_68() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D717, 'M', u'θ'),
-    (0x1D718, 'M', u'κ'),
-    (0x1D719, 'M', u'φ'),
-    (0x1D71A, 'M', u'ρ'),
-    (0x1D71B, 'M', u'π'),
-    (0x1D71C, 'M', u'α'),
-    (0x1D71D, 'M', u'β'),
-    (0x1D71E, 'M', u'γ'),
-    (0x1D71F, 'M', u'δ'),
-    (0x1D720, 'M', u'ε'),
-    (0x1D721, 'M', u'ζ'),
-    (0x1D722, 'M', u'η'),
-    (0x1D723, 'M', u'θ'),
-    (0x1D724, 'M', u'ι'),
-    (0x1D725, 'M', u'κ'),
-    (0x1D726, 'M', u'λ'),
-    (0x1D727, 'M', u'μ'),
-    (0x1D728, 'M', u'ν'),
-    (0x1D729, 'M', u'ξ'),
-    (0x1D72A, 'M', u'ο'),
-    (0x1D72B, 'M', u'π'),
-    (0x1D72C, 'M', u'ρ'),
-    (0x1D72D, 'M', u'θ'),
-    (0x1D72E, 'M', u'σ'),
-    (0x1D72F, 'M', u'τ'),
-    (0x1D730, 'M', u'υ'),
-    (0x1D731, 'M', u'φ'),
-    (0x1D732, 'M', u'χ'),
-    (0x1D733, 'M', u'ψ'),
-    (0x1D734, 'M', u'ω'),
-    (0x1D735, 'M', u'∇'),
-    (0x1D736, 'M', u'α'),
-    (0x1D737, 'M', u'β'),
-    (0x1D738, 'M', u'γ'),
-    (0x1D739, 'M', u'δ'),
-    (0x1D73A, 'M', u'ε'),
-    (0x1D73B, 'M', u'ζ'),
-    (0x1D73C, 'M', u'η'),
-    (0x1D73D, 'M', u'θ'),
-    (0x1D73E, 'M', u'ι'),
-    (0x1D73F, 'M', u'κ'),
-    (0x1D740, 'M', u'λ'),
-    (0x1D741, 'M', u'μ'),
-    (0x1D742, 'M', u'ν'),
-    (0x1D743, 'M', u'ξ'),
-    (0x1D744, 'M', u'ο'),
-    (0x1D745, 'M', u'π'),
-    (0x1D746, 'M', u'ρ'),
-    (0x1D747, 'M', u'σ'),
-    (0x1D749, 'M', u'τ'),
-    (0x1D74A, 'M', u'υ'),
-    (0x1D74B, 'M', u'φ'),
-    (0x1D74C, 'M', u'χ'),
-    (0x1D74D, 'M', u'ψ'),
-    (0x1D74E, 'M', u'ω'),
-    (0x1D74F, 'M', u'∂'),
-    (0x1D750, 'M', u'ε'),
-    (0x1D751, 'M', u'θ'),
-    (0x1D752, 'M', u'κ'),
-    (0x1D753, 'M', u'φ'),
-    (0x1D754, 'M', u'ρ'),
-    (0x1D755, 'M', u'π'),
-    (0x1D756, 'M', u'α'),
-    (0x1D757, 'M', u'β'),
-    (0x1D758, 'M', u'γ'),
-    (0x1D759, 'M', u'δ'),
-    (0x1D75A, 'M', u'ε'),
-    (0x1D75B, 'M', u'ζ'),
-    (0x1D75C, 'M', u'η'),
-    (0x1D75D, 'M', u'θ'),
-    (0x1D75E, 'M', u'ι'),
-    (0x1D75F, 'M', u'κ'),
-    (0x1D760, 'M', u'λ'),
-    (0x1D761, 'M', u'μ'),
-    (0x1D762, 'M', u'ν'),
-    (0x1D763, 'M', u'ξ'),
-    (0x1D764, 'M', u'ο'),
-    (0x1D765, 'M', u'π'),
-    (0x1D766, 'M', u'ρ'),
-    (0x1D767, 'M', u'θ'),
-    (0x1D768, 'M', u'σ'),
-    (0x1D769, 'M', u'τ'),
-    (0x1D76A, 'M', u'υ'),
-    (0x1D76B, 'M', u'φ'),
-    (0x1D76C, 'M', u'χ'),
-    (0x1D76D, 'M', u'ψ'),
-    (0x1D76E, 'M', u'ω'),
-    (0x1D76F, 'M', u'∇'),
-    (0x1D770, 'M', u'α'),
-    (0x1D771, 'M', u'β'),
-    (0x1D772, 'M', u'γ'),
-    (0x1D773, 'M', u'δ'),
-    (0x1D774, 'M', u'ε'),
-    (0x1D775, 'M', u'ζ'),
-    (0x1D776, 'M', u'η'),
-    (0x1D777, 'M', u'θ'),
-    (0x1D778, 'M', u'ι'),
-    (0x1D779, 'M', u'κ'),
-    (0x1D77A, 'M', u'λ'),
-    (0x1D77B, 'M', u'μ'),
+    (0x1D6F5, 'M', 'τ'),
+    (0x1D6F6, 'M', 'υ'),
+    (0x1D6F7, 'M', 'φ'),
+    (0x1D6F8, 'M', 'χ'),
+    (0x1D6F9, 'M', 'ψ'),
+    (0x1D6FA, 'M', 'ω'),
+    (0x1D6FB, 'M', '∇'),
+    (0x1D6FC, 'M', 'α'),
+    (0x1D6FD, 'M', 'β'),
+    (0x1D6FE, 'M', 'γ'),
+    (0x1D6FF, 'M', 'δ'),
+    (0x1D700, 'M', 'ε'),
+    (0x1D701, 'M', 'ζ'),
+    (0x1D702, 'M', 'η'),
+    (0x1D703, 'M', 'θ'),
+    (0x1D704, 'M', 'ι'),
+    (0x1D705, 'M', 'κ'),
+    (0x1D706, 'M', 'λ'),
+    (0x1D707, 'M', 'μ'),
+    (0x1D708, 'M', 'ν'),
+    (0x1D709, 'M', 'ξ'),
+    (0x1D70A, 'M', 'ο'),
+    (0x1D70B, 'M', 'π'),
+    (0x1D70C, 'M', 'ρ'),
+    (0x1D70D, 'M', 'σ'),
+    (0x1D70F, 'M', 'τ'),
+    (0x1D710, 'M', 'υ'),
+    (0x1D711, 'M', 'φ'),
+    (0x1D712, 'M', 'χ'),
+    (0x1D713, 'M', 'ψ'),
+    (0x1D714, 'M', 'ω'),
+    (0x1D715, 'M', '∂'),
+    (0x1D716, 'M', 'ε'),
+    (0x1D717, 'M', 'θ'),
+    (0x1D718, 'M', 'κ'),
+    (0x1D719, 'M', 'φ'),
+    (0x1D71A, 'M', 'ρ'),
+    (0x1D71B, 'M', 'π'),
+    (0x1D71C, 'M', 'α'),
+    (0x1D71D, 'M', 'β'),
+    (0x1D71E, 'M', 'γ'),
+    (0x1D71F, 'M', 'δ'),
+    (0x1D720, 'M', 'ε'),
+    (0x1D721, 'M', 'ζ'),
+    (0x1D722, 'M', 'η'),
+    (0x1D723, 'M', 'θ'),
+    (0x1D724, 'M', 'ι'),
+    (0x1D725, 'M', 'κ'),
+    (0x1D726, 'M', 'λ'),
+    (0x1D727, 'M', 'μ'),
+    (0x1D728, 'M', 'ν'),
+    (0x1D729, 'M', 'ξ'),
+    (0x1D72A, 'M', 'ο'),
+    (0x1D72B, 'M', 'π'),
+    (0x1D72C, 'M', 'ρ'),
+    (0x1D72D, 'M', 'θ'),
+    (0x1D72E, 'M', 'σ'),
+    (0x1D72F, 'M', 'τ'),
+    (0x1D730, 'M', 'υ'),
+    (0x1D731, 'M', 'φ'),
+    (0x1D732, 'M', 'χ'),
+    (0x1D733, 'M', 'ψ'),
+    (0x1D734, 'M', 'ω'),
+    (0x1D735, 'M', '∇'),
+    (0x1D736, 'M', 'α'),
+    (0x1D737, 'M', 'β'),
+    (0x1D738, 'M', 'γ'),
+    (0x1D739, 'M', 'δ'),
+    (0x1D73A, 'M', 'ε'),
+    (0x1D73B, 'M', 'ζ'),
+    (0x1D73C, 'M', 'η'),
+    (0x1D73D, 'M', 'θ'),
+    (0x1D73E, 'M', 'ι'),
+    (0x1D73F, 'M', 'κ'),
+    (0x1D740, 'M', 'λ'),
+    (0x1D741, 'M', 'μ'),
+    (0x1D742, 'M', 'ν'),
+    (0x1D743, 'M', 'ξ'),
+    (0x1D744, 'M', 'ο'),
+    (0x1D745, 'M', 'π'),
+    (0x1D746, 'M', 'ρ'),
+    (0x1D747, 'M', 'σ'),
+    (0x1D749, 'M', 'τ'),
+    (0x1D74A, 'M', 'υ'),
+    (0x1D74B, 'M', 'φ'),
+    (0x1D74C, 'M', 'χ'),
+    (0x1D74D, 'M', 'ψ'),
+    (0x1D74E, 'M', 'ω'),
+    (0x1D74F, 'M', '∂'),
+    (0x1D750, 'M', 'ε'),
+    (0x1D751, 'M', 'θ'),
+    (0x1D752, 'M', 'κ'),
+    (0x1D753, 'M', 'φ'),
+    (0x1D754, 'M', 'ρ'),
+    (0x1D755, 'M', 'π'),
+    (0x1D756, 'M', 'α'),
+    (0x1D757, 'M', 'β'),
+    (0x1D758, 'M', 'γ'),
+    (0x1D759, 'M', 'δ'),
+    (0x1D75A, 'M', 'ε'),
     ]
 
-def _seg_68():
+def _seg_69() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D77C, 'M', u'ν'),
-    (0x1D77D, 'M', u'ξ'),
-    (0x1D77E, 'M', u'ο'),
-    (0x1D77F, 'M', u'π'),
-    (0x1D780, 'M', u'ρ'),
-    (0x1D781, 'M', u'σ'),
-    (0x1D783, 'M', u'τ'),
-    (0x1D784, 'M', u'υ'),
-    (0x1D785, 'M', u'φ'),
-    (0x1D786, 'M', u'χ'),
-    (0x1D787, 'M', u'ψ'),
-    (0x1D788, 'M', u'ω'),
-    (0x1D789, 'M', u'∂'),
-    (0x1D78A, 'M', u'ε'),
-    (0x1D78B, 'M', u'θ'),
-    (0x1D78C, 'M', u'κ'),
-    (0x1D78D, 'M', u'φ'),
-    (0x1D78E, 'M', u'ρ'),
-    (0x1D78F, 'M', u'π'),
-    (0x1D790, 'M', u'α'),
-    (0x1D791, 'M', u'β'),
-    (0x1D792, 'M', u'γ'),
-    (0x1D793, 'M', u'δ'),
-    (0x1D794, 'M', u'ε'),
-    (0x1D795, 'M', u'ζ'),
-    (0x1D796, 'M', u'η'),
-    (0x1D797, 'M', u'θ'),
-    (0x1D798, 'M', u'ι'),
-    (0x1D799, 'M', u'κ'),
-    (0x1D79A, 'M', u'λ'),
-    (0x1D79B, 'M', u'μ'),
-    (0x1D79C, 'M', u'ν'),
-    (0x1D79D, 'M', u'ξ'),
-    (0x1D79E, 'M', u'ο'),
-    (0x1D79F, 'M', u'π'),
-    (0x1D7A0, 'M', u'ρ'),
-    (0x1D7A1, 'M', u'θ'),
-    (0x1D7A2, 'M', u'σ'),
-    (0x1D7A3, 'M', u'τ'),
-    (0x1D7A4, 'M', u'υ'),
-    (0x1D7A5, 'M', u'φ'),
-    (0x1D7A6, 'M', u'χ'),
-    (0x1D7A7, 'M', u'ψ'),
-    (0x1D7A8, 'M', u'ω'),
-    (0x1D7A9, 'M', u'∇'),
-    (0x1D7AA, 'M', u'α'),
-    (0x1D7AB, 'M', u'β'),
-    (0x1D7AC, 'M', u'γ'),
-    (0x1D7AD, 'M', u'δ'),
-    (0x1D7AE, 'M', u'ε'),
-    (0x1D7AF, 'M', u'ζ'),
-    (0x1D7B0, 'M', u'η'),
-    (0x1D7B1, 'M', u'θ'),
-    (0x1D7B2, 'M', u'ι'),
-    (0x1D7B3, 'M', u'κ'),
-    (0x1D7B4, 'M', u'λ'),
-    (0x1D7B5, 'M', u'μ'),
-    (0x1D7B6, 'M', u'ν'),
-    (0x1D7B7, 'M', u'ξ'),
-    (0x1D7B8, 'M', u'ο'),
-    (0x1D7B9, 'M', u'π'),
-    (0x1D7BA, 'M', u'ρ'),
-    (0x1D7BB, 'M', u'σ'),
-    (0x1D7BD, 'M', u'τ'),
-    (0x1D7BE, 'M', u'υ'),
-    (0x1D7BF, 'M', u'φ'),
-    (0x1D7C0, 'M', u'χ'),
-    (0x1D7C1, 'M', u'ψ'),
-    (0x1D7C2, 'M', u'ω'),
-    (0x1D7C3, 'M', u'∂'),
-    (0x1D7C4, 'M', u'ε'),
-    (0x1D7C5, 'M', u'θ'),
-    (0x1D7C6, 'M', u'κ'),
-    (0x1D7C7, 'M', u'φ'),
-    (0x1D7C8, 'M', u'ρ'),
-    (0x1D7C9, 'M', u'π'),
-    (0x1D7CA, 'M', u'ϝ'),
-    (0x1D7CC, 'X'),
-    (0x1D7CE, 'M', u'0'),
-    (0x1D7CF, 'M', u'1'),
-    (0x1D7D0, 'M', u'2'),
-    (0x1D7D1, 'M', u'3'),
-    (0x1D7D2, 'M', u'4'),
-    (0x1D7D3, 'M', u'5'),
-    (0x1D7D4, 'M', u'6'),
-    (0x1D7D5, 'M', u'7'),
-    (0x1D7D6, 'M', u'8'),
-    (0x1D7D7, 'M', u'9'),
-    (0x1D7D8, 'M', u'0'),
-    (0x1D7D9, 'M', u'1'),
-    (0x1D7DA, 'M', u'2'),
-    (0x1D7DB, 'M', u'3'),
-    (0x1D7DC, 'M', u'4'),
-    (0x1D7DD, 'M', u'5'),
-    (0x1D7DE, 'M', u'6'),
-    (0x1D7DF, 'M', u'7'),
-    (0x1D7E0, 'M', u'8'),
-    (0x1D7E1, 'M', u'9'),
-    (0x1D7E2, 'M', u'0'),
-    (0x1D7E3, 'M', u'1'),
+    (0x1D75B, 'M', 'ζ'),
+    (0x1D75C, 'M', 'η'),
+    (0x1D75D, 'M', 'θ'),
+    (0x1D75E, 'M', 'ι'),
+    (0x1D75F, 'M', 'κ'),
+    (0x1D760, 'M', 'λ'),
+    (0x1D761, 'M', 'μ'),
+    (0x1D762, 'M', 'ν'),
+    (0x1D763, 'M', 'ξ'),
+    (0x1D764, 'M', 'ο'),
+    (0x1D765, 'M', 'π'),
+    (0x1D766, 'M', 'ρ'),
+    (0x1D767, 'M', 'θ'),
+    (0x1D768, 'M', 'σ'),
+    (0x1D769, 'M', 'τ'),
+    (0x1D76A, 'M', 'υ'),
+    (0x1D76B, 'M', 'φ'),
+    (0x1D76C, 'M', 'χ'),
+    (0x1D76D, 'M', 'ψ'),
+    (0x1D76E, 'M', 'ω'),
+    (0x1D76F, 'M', '∇'),
+    (0x1D770, 'M', 'α'),
+    (0x1D771, 'M', 'β'),
+    (0x1D772, 'M', 'γ'),
+    (0x1D773, 'M', 'δ'),
+    (0x1D774, 'M', 'ε'),
+    (0x1D775, 'M', 'ζ'),
+    (0x1D776, 'M', 'η'),
+    (0x1D777, 'M', 'θ'),
+    (0x1D778, 'M', 'ι'),
+    (0x1D779, 'M', 'κ'),
+    (0x1D77A, 'M', 'λ'),
+    (0x1D77B, 'M', 'μ'),
+    (0x1D77C, 'M', 'ν'),
+    (0x1D77D, 'M', 'ξ'),
+    (0x1D77E, 'M', 'ο'),
+    (0x1D77F, 'M', 'π'),
+    (0x1D780, 'M', 'ρ'),
+    (0x1D781, 'M', 'σ'),
+    (0x1D783, 'M', 'τ'),
+    (0x1D784, 'M', 'υ'),
+    (0x1D785, 'M', 'φ'),
+    (0x1D786, 'M', 'χ'),
+    (0x1D787, 'M', 'ψ'),
+    (0x1D788, 'M', 'ω'),
+    (0x1D789, 'M', '∂'),
+    (0x1D78A, 'M', 'ε'),
+    (0x1D78B, 'M', 'θ'),
+    (0x1D78C, 'M', 'κ'),
+    (0x1D78D, 'M', 'φ'),
+    (0x1D78E, 'M', 'ρ'),
+    (0x1D78F, 'M', 'π'),
+    (0x1D790, 'M', 'α'),
+    (0x1D791, 'M', 'β'),
+    (0x1D792, 'M', 'γ'),
+    (0x1D793, 'M', 'δ'),
+    (0x1D794, 'M', 'ε'),
+    (0x1D795, 'M', 'ζ'),
+    (0x1D796, 'M', 'η'),
+    (0x1D797, 'M', 'θ'),
+    (0x1D798, 'M', 'ι'),
+    (0x1D799, 'M', 'κ'),
+    (0x1D79A, 'M', 'λ'),
+    (0x1D79B, 'M', 'μ'),
+    (0x1D79C, 'M', 'ν'),
+    (0x1D79D, 'M', 'ξ'),
+    (0x1D79E, 'M', 'ο'),
+    (0x1D79F, 'M', 'π'),
+    (0x1D7A0, 'M', 'ρ'),
+    (0x1D7A1, 'M', 'θ'),
+    (0x1D7A2, 'M', 'σ'),
+    (0x1D7A3, 'M', 'τ'),
+    (0x1D7A4, 'M', 'υ'),
+    (0x1D7A5, 'M', 'φ'),
+    (0x1D7A6, 'M', 'χ'),
+    (0x1D7A7, 'M', 'ψ'),
+    (0x1D7A8, 'M', 'ω'),
+    (0x1D7A9, 'M', '∇'),
+    (0x1D7AA, 'M', 'α'),
+    (0x1D7AB, 'M', 'β'),
+    (0x1D7AC, 'M', 'γ'),
+    (0x1D7AD, 'M', 'δ'),
+    (0x1D7AE, 'M', 'ε'),
+    (0x1D7AF, 'M', 'ζ'),
+    (0x1D7B0, 'M', 'η'),
+    (0x1D7B1, 'M', 'θ'),
+    (0x1D7B2, 'M', 'ι'),
+    (0x1D7B3, 'M', 'κ'),
+    (0x1D7B4, 'M', 'λ'),
+    (0x1D7B5, 'M', 'μ'),
+    (0x1D7B6, 'M', 'ν'),
+    (0x1D7B7, 'M', 'ξ'),
+    (0x1D7B8, 'M', 'ο'),
+    (0x1D7B9, 'M', 'π'),
+    (0x1D7BA, 'M', 'ρ'),
+    (0x1D7BB, 'M', 'σ'),
+    (0x1D7BD, 'M', 'τ'),
+    (0x1D7BE, 'M', 'υ'),
+    (0x1D7BF, 'M', 'φ'),
+    (0x1D7C0, 'M', 'χ'),
     ]
 
-def _seg_69():
+def _seg_70() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1D7E4, 'M', u'2'),
-    (0x1D7E5, 'M', u'3'),
-    (0x1D7E6, 'M', u'4'),
-    (0x1D7E7, 'M', u'5'),
-    (0x1D7E8, 'M', u'6'),
-    (0x1D7E9, 'M', u'7'),
-    (0x1D7EA, 'M', u'8'),
-    (0x1D7EB, 'M', u'9'),
-    (0x1D7EC, 'M', u'0'),
-    (0x1D7ED, 'M', u'1'),
-    (0x1D7EE, 'M', u'2'),
-    (0x1D7EF, 'M', u'3'),
-    (0x1D7F0, 'M', u'4'),
-    (0x1D7F1, 'M', u'5'),
-    (0x1D7F2, 'M', u'6'),
-    (0x1D7F3, 'M', u'7'),
-    (0x1D7F4, 'M', u'8'),
-    (0x1D7F5, 'M', u'9'),
-    (0x1D7F6, 'M', u'0'),
-    (0x1D7F7, 'M', u'1'),
-    (0x1D7F8, 'M', u'2'),
-    (0x1D7F9, 'M', u'3'),
-    (0x1D7FA, 'M', u'4'),
-    (0x1D7FB, 'M', u'5'),
-    (0x1D7FC, 'M', u'6'),
-    (0x1D7FD, 'M', u'7'),
-    (0x1D7FE, 'M', u'8'),
-    (0x1D7FF, 'M', u'9'),
+    (0x1D7C1, 'M', 'ψ'),
+    (0x1D7C2, 'M', 'ω'),
+    (0x1D7C3, 'M', '∂'),
+    (0x1D7C4, 'M', 'ε'),
+    (0x1D7C5, 'M', 'θ'),
+    (0x1D7C6, 'M', 'κ'),
+    (0x1D7C7, 'M', 'φ'),
+    (0x1D7C8, 'M', 'ρ'),
+    (0x1D7C9, 'M', 'π'),
+    (0x1D7CA, 'M', 'ϝ'),
+    (0x1D7CC, 'X'),
+    (0x1D7CE, 'M', '0'),
+    (0x1D7CF, 'M', '1'),
+    (0x1D7D0, 'M', '2'),
+    (0x1D7D1, 'M', '3'),
+    (0x1D7D2, 'M', '4'),
+    (0x1D7D3, 'M', '5'),
+    (0x1D7D4, 'M', '6'),
+    (0x1D7D5, 'M', '7'),
+    (0x1D7D6, 'M', '8'),
+    (0x1D7D7, 'M', '9'),
+    (0x1D7D8, 'M', '0'),
+    (0x1D7D9, 'M', '1'),
+    (0x1D7DA, 'M', '2'),
+    (0x1D7DB, 'M', '3'),
+    (0x1D7DC, 'M', '4'),
+    (0x1D7DD, 'M', '5'),
+    (0x1D7DE, 'M', '6'),
+    (0x1D7DF, 'M', '7'),
+    (0x1D7E0, 'M', '8'),
+    (0x1D7E1, 'M', '9'),
+    (0x1D7E2, 'M', '0'),
+    (0x1D7E3, 'M', '1'),
+    (0x1D7E4, 'M', '2'),
+    (0x1D7E5, 'M', '3'),
+    (0x1D7E6, 'M', '4'),
+    (0x1D7E7, 'M', '5'),
+    (0x1D7E8, 'M', '6'),
+    (0x1D7E9, 'M', '7'),
+    (0x1D7EA, 'M', '8'),
+    (0x1D7EB, 'M', '9'),
+    (0x1D7EC, 'M', '0'),
+    (0x1D7ED, 'M', '1'),
+    (0x1D7EE, 'M', '2'),
+    (0x1D7EF, 'M', '3'),
+    (0x1D7F0, 'M', '4'),
+    (0x1D7F1, 'M', '5'),
+    (0x1D7F2, 'M', '6'),
+    (0x1D7F3, 'M', '7'),
+    (0x1D7F4, 'M', '8'),
+    (0x1D7F5, 'M', '9'),
+    (0x1D7F6, 'M', '0'),
+    (0x1D7F7, 'M', '1'),
+    (0x1D7F8, 'M', '2'),
+    (0x1D7F9, 'M', '3'),
+    (0x1D7FA, 'M', '4'),
+    (0x1D7FB, 'M', '5'),
+    (0x1D7FC, 'M', '6'),
+    (0x1D7FD, 'M', '7'),
+    (0x1D7FE, 'M', '8'),
+    (0x1D7FF, 'M', '9'),
     (0x1D800, 'V'),
     (0x1DA8C, 'X'),
     (0x1DA9B, 'V'),
     (0x1DAA0, 'X'),
     (0x1DAA1, 'V'),
     (0x1DAB0, 'X'),
+    (0x1DF00, 'V'),
+    (0x1DF1F, 'X'),
     (0x1E000, 'V'),
     (0x1E007, 'X'),
     (0x1E008, 'V'),
@@ -7235,239 +7377,253 @@ def _seg_69():
     (0x1E14A, 'X'),
     (0x1E14E, 'V'),
     (0x1E150, 'X'),
+    (0x1E290, 'V'),
+    (0x1E2AF, 'X'),
     (0x1E2C0, 'V'),
     (0x1E2FA, 'X'),
     (0x1E2FF, 'V'),
     (0x1E300, 'X'),
+    (0x1E7E0, 'V'),
+    (0x1E7E7, 'X'),
+    (0x1E7E8, 'V'),
+    (0x1E7EC, 'X'),
+    (0x1E7ED, 'V'),
+    (0x1E7EF, 'X'),
+    (0x1E7F0, 'V'),
+    ]
+
+def _seg_71() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x1E7FF, 'X'),
     (0x1E800, 'V'),
     (0x1E8C5, 'X'),
     (0x1E8C7, 'V'),
     (0x1E8D7, 'X'),
-    (0x1E900, 'M', u'𞤢'),
-    (0x1E901, 'M', u'𞤣'),
-    (0x1E902, 'M', u'𞤤'),
-    (0x1E903, 'M', u'𞤥'),
-    (0x1E904, 'M', u'𞤦'),
-    (0x1E905, 'M', u'𞤧'),
-    (0x1E906, 'M', u'𞤨'),
-    (0x1E907, 'M', u'𞤩'),
-    (0x1E908, 'M', u'𞤪'),
-    (0x1E909, 'M', u'𞤫'),
-    (0x1E90A, 'M', u'𞤬'),
-    (0x1E90B, 'M', u'𞤭'),
-    (0x1E90C, 'M', u'𞤮'),
-    (0x1E90D, 'M', u'𞤯'),
-    (0x1E90E, 'M', u'𞤰'),
-    (0x1E90F, 'M', u'𞤱'),
-    (0x1E910, 'M', u'𞤲'),
-    (0x1E911, 'M', u'𞤳'),
-    (0x1E912, 'M', u'𞤴'),
-    (0x1E913, 'M', u'𞤵'),
-    (0x1E914, 'M', u'𞤶'),
-    (0x1E915, 'M', u'𞤷'),
-    (0x1E916, 'M', u'𞤸'),
-    (0x1E917, 'M', u'𞤹'),
-    (0x1E918, 'M', u'𞤺'),
-    (0x1E919, 'M', u'𞤻'),
-    (0x1E91A, 'M', u'𞤼'),
-    (0x1E91B, 'M', u'𞤽'),
-    (0x1E91C, 'M', u'𞤾'),
-    (0x1E91D, 'M', u'𞤿'),
-    (0x1E91E, 'M', u'𞥀'),
-    (0x1E91F, 'M', u'𞥁'),
-    (0x1E920, 'M', u'𞥂'),
-    (0x1E921, 'M', u'𞥃'),
+    (0x1E900, 'M', '𞤢'),
+    (0x1E901, 'M', '𞤣'),
+    (0x1E902, 'M', '𞤤'),
+    (0x1E903, 'M', '𞤥'),
+    (0x1E904, 'M', '𞤦'),
+    (0x1E905, 'M', '𞤧'),
+    (0x1E906, 'M', '𞤨'),
+    (0x1E907, 'M', '𞤩'),
+    (0x1E908, 'M', '𞤪'),
+    (0x1E909, 'M', '𞤫'),
+    (0x1E90A, 'M', '𞤬'),
+    (0x1E90B, 'M', '𞤭'),
+    (0x1E90C, 'M', '𞤮'),
+    (0x1E90D, 'M', '𞤯'),
+    (0x1E90E, 'M', '𞤰'),
+    (0x1E90F, 'M', '𞤱'),
+    (0x1E910, 'M', '𞤲'),
+    (0x1E911, 'M', '𞤳'),
+    (0x1E912, 'M', '𞤴'),
+    (0x1E913, 'M', '𞤵'),
+    (0x1E914, 'M', '𞤶'),
+    (0x1E915, 'M', '𞤷'),
+    (0x1E916, 'M', '𞤸'),
+    (0x1E917, 'M', '𞤹'),
+    (0x1E918, 'M', '𞤺'),
+    (0x1E919, 'M', '𞤻'),
+    (0x1E91A, 'M', '𞤼'),
+    (0x1E91B, 'M', '𞤽'),
+    (0x1E91C, 'M', '𞤾'),
+    (0x1E91D, 'M', '𞤿'),
+    (0x1E91E, 'M', '𞥀'),
+    (0x1E91F, 'M', '𞥁'),
+    (0x1E920, 'M', '𞥂'),
+    (0x1E921, 'M', '𞥃'),
     (0x1E922, 'V'),
     (0x1E94C, 'X'),
     (0x1E950, 'V'),
     (0x1E95A, 'X'),
     (0x1E95E, 'V'),
     (0x1E960, 'X'),
-    ]
-
-def _seg_70():
-    return [
     (0x1EC71, 'V'),
     (0x1ECB5, 'X'),
     (0x1ED01, 'V'),
     (0x1ED3E, 'X'),
-    (0x1EE00, 'M', u'ا'),
-    (0x1EE01, 'M', u'ب'),
-    (0x1EE02, 'M', u'ج'),
-    (0x1EE03, 'M', u'د'),
+    (0x1EE00, 'M', 'ا'),
+    (0x1EE01, 'M', 'ب'),
+    (0x1EE02, 'M', 'ج'),
+    (0x1EE03, 'M', 'د'),
     (0x1EE04, 'X'),
-    (0x1EE05, 'M', u'و'),
-    (0x1EE06, 'M', u'ز'),
-    (0x1EE07, 'M', u'ح'),
-    (0x1EE08, 'M', u'ط'),
-    (0x1EE09, 'M', u'ي'),
-    (0x1EE0A, 'M', u'ك'),
-    (0x1EE0B, 'M', u'ل'),
-    (0x1EE0C, 'M', u'م'),
-    (0x1EE0D, 'M', u'ن'),
-    (0x1EE0E, 'M', u'س'),
-    (0x1EE0F, 'M', u'ع'),
-    (0x1EE10, 'M', u'ف'),
-    (0x1EE11, 'M', u'ص'),
-    (0x1EE12, 'M', u'ق'),
-    (0x1EE13, 'M', u'ر'),
-    (0x1EE14, 'M', u'ش'),
-    (0x1EE15, 'M', u'ت'),
-    (0x1EE16, 'M', u'ث'),
-    (0x1EE17, 'M', u'خ'),
-    (0x1EE18, 'M', u'ذ'),
-    (0x1EE19, 'M', u'ض'),
-    (0x1EE1A, 'M', u'ظ'),
-    (0x1EE1B, 'M', u'غ'),
-    (0x1EE1C, 'M', u'ٮ'),
-    (0x1EE1D, 'M', u'ں'),
-    (0x1EE1E, 'M', u'ڡ'),
-    (0x1EE1F, 'M', u'ٯ'),
+    (0x1EE05, 'M', 'و'),
+    (0x1EE06, 'M', 'ز'),
+    (0x1EE07, 'M', 'ح'),
+    (0x1EE08, 'M', 'ط'),
+    (0x1EE09, 'M', 'ي'),
+    (0x1EE0A, 'M', 'ك'),
+    (0x1EE0B, 'M', 'ل'),
+    (0x1EE0C, 'M', 'م'),
+    (0x1EE0D, 'M', 'ن'),
+    (0x1EE0E, 'M', 'س'),
+    (0x1EE0F, 'M', 'ع'),
+    (0x1EE10, 'M', 'ف'),
+    (0x1EE11, 'M', 'ص'),
+    (0x1EE12, 'M', 'ق'),
+    (0x1EE13, 'M', 'ر'),
+    (0x1EE14, 'M', 'ش'),
+    (0x1EE15, 'M', 'ت'),
+    (0x1EE16, 'M', 'ث'),
+    (0x1EE17, 'M', 'خ'),
+    (0x1EE18, 'M', 'ذ'),
+    (0x1EE19, 'M', 'ض'),
+    (0x1EE1A, 'M', 'ظ'),
+    (0x1EE1B, 'M', 'غ'),
+    (0x1EE1C, 'M', 'ٮ'),
+    (0x1EE1D, 'M', 'ں'),
+    (0x1EE1E, 'M', 'ڡ'),
+    (0x1EE1F, 'M', 'ٯ'),
     (0x1EE20, 'X'),
-    (0x1EE21, 'M', u'ب'),
-    (0x1EE22, 'M', u'ج'),
+    (0x1EE21, 'M', 'ب'),
+    (0x1EE22, 'M', 'ج'),
     (0x1EE23, 'X'),
-    (0x1EE24, 'M', u'ه'),
+    (0x1EE24, 'M', 'ه'),
     (0x1EE25, 'X'),
-    (0x1EE27, 'M', u'ح'),
+    (0x1EE27, 'M', 'ح'),
     (0x1EE28, 'X'),
-    (0x1EE29, 'M', u'ي'),
-    (0x1EE2A, 'M', u'ك'),
-    (0x1EE2B, 'M', u'ل'),
-    (0x1EE2C, 'M', u'م'),
-    (0x1EE2D, 'M', u'ن'),
-    (0x1EE2E, 'M', u'س'),
-    (0x1EE2F, 'M', u'ع'),
-    (0x1EE30, 'M', u'ف'),
-    (0x1EE31, 'M', u'ص'),
-    (0x1EE32, 'M', u'ق'),
+    (0x1EE29, 'M', 'ي'),
+    (0x1EE2A, 'M', 'ك'),
+    (0x1EE2B, 'M', 'ل'),
+    (0x1EE2C, 'M', 'م'),
+    (0x1EE2D, 'M', 'ن'),
+    (0x1EE2E, 'M', 'س'),
+    (0x1EE2F, 'M', 'ع'),
+    (0x1EE30, 'M', 'ف'),
+    (0x1EE31, 'M', 'ص'),
+    (0x1EE32, 'M', 'ق'),
     (0x1EE33, 'X'),
-    (0x1EE34, 'M', u'ش'),
-    (0x1EE35, 'M', u'ت'),
-    (0x1EE36, 'M', u'ث'),
-    (0x1EE37, 'M', u'خ'),
+    ]
+
+def _seg_72() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x1EE34, 'M', 'ش'),
+    (0x1EE35, 'M', 'ت'),
+    (0x1EE36, 'M', 'ث'),
+    (0x1EE37, 'M', 'خ'),
     (0x1EE38, 'X'),
-    (0x1EE39, 'M', u'ض'),
+    (0x1EE39, 'M', 'ض'),
     (0x1EE3A, 'X'),
-    (0x1EE3B, 'M', u'غ'),
+    (0x1EE3B, 'M', 'غ'),
     (0x1EE3C, 'X'),
-    (0x1EE42, 'M', u'ج'),
+    (0x1EE42, 'M', 'ج'),
     (0x1EE43, 'X'),
-    (0x1EE47, 'M', u'ح'),
+    (0x1EE47, 'M', 'ح'),
     (0x1EE48, 'X'),
-    (0x1EE49, 'M', u'ي'),
+    (0x1EE49, 'M', 'ي'),
     (0x1EE4A, 'X'),
-    (0x1EE4B, 'M', u'ل'),
+    (0x1EE4B, 'M', 'ل'),
     (0x1EE4C, 'X'),
-    (0x1EE4D, 'M', u'ن'),
-    (0x1EE4E, 'M', u'س'),
-    (0x1EE4F, 'M', u'ع'),
+    (0x1EE4D, 'M', 'ن'),
+    (0x1EE4E, 'M', 'س'),
+    (0x1EE4F, 'M', 'ع'),
     (0x1EE50, 'X'),
-    (0x1EE51, 'M', u'ص'),
-    (0x1EE52, 'M', u'ق'),
+    (0x1EE51, 'M', 'ص'),
+    (0x1EE52, 'M', 'ق'),
     (0x1EE53, 'X'),
-    (0x1EE54, 'M', u'ش'),
+    (0x1EE54, 'M', 'ش'),
     (0x1EE55, 'X'),
-    (0x1EE57, 'M', u'خ'),
+    (0x1EE57, 'M', 'خ'),
     (0x1EE58, 'X'),
-    (0x1EE59, 'M', u'ض'),
+    (0x1EE59, 'M', 'ض'),
     (0x1EE5A, 'X'),
-    (0x1EE5B, 'M', u'غ'),
+    (0x1EE5B, 'M', 'غ'),
     (0x1EE5C, 'X'),
-    (0x1EE5D, 'M', u'ں'),
+    (0x1EE5D, 'M', 'ں'),
     (0x1EE5E, 'X'),
-    (0x1EE5F, 'M', u'ٯ'),
+    (0x1EE5F, 'M', 'ٯ'),
     (0x1EE60, 'X'),
-    (0x1EE61, 'M', u'ب'),
-    (0x1EE62, 'M', u'ج'),
+    (0x1EE61, 'M', 'ب'),
+    (0x1EE62, 'M', 'ج'),
     (0x1EE63, 'X'),
-    (0x1EE64, 'M', u'ه'),
+    (0x1EE64, 'M', 'ه'),
     (0x1EE65, 'X'),
-    (0x1EE67, 'M', u'ح'),
-    (0x1EE68, 'M', u'ط'),
-    (0x1EE69, 'M', u'ي'),
-    (0x1EE6A, 'M', u'ك'),
-    ]
-
-def _seg_71():
-    return [
+    (0x1EE67, 'M', 'ح'),
+    (0x1EE68, 'M', 'ط'),
+    (0x1EE69, 'M', 'ي'),
+    (0x1EE6A, 'M', 'ك'),
     (0x1EE6B, 'X'),
-    (0x1EE6C, 'M', u'م'),
-    (0x1EE6D, 'M', u'ن'),
-    (0x1EE6E, 'M', u'س'),
-    (0x1EE6F, 'M', u'ع'),
-    (0x1EE70, 'M', u'ف'),
-    (0x1EE71, 'M', u'ص'),
-    (0x1EE72, 'M', u'ق'),
+    (0x1EE6C, 'M', 'م'),
+    (0x1EE6D, 'M', 'ن'),
+    (0x1EE6E, 'M', 'س'),
+    (0x1EE6F, 'M', 'ع'),
+    (0x1EE70, 'M', 'ف'),
+    (0x1EE71, 'M', 'ص'),
+    (0x1EE72, 'M', 'ق'),
     (0x1EE73, 'X'),
-    (0x1EE74, 'M', u'ش'),
-    (0x1EE75, 'M', u'ت'),
-    (0x1EE76, 'M', u'ث'),
-    (0x1EE77, 'M', u'خ'),
+    (0x1EE74, 'M', 'ش'),
+    (0x1EE75, 'M', 'ت'),
+    (0x1EE76, 'M', 'ث'),
+    (0x1EE77, 'M', 'خ'),
     (0x1EE78, 'X'),
-    (0x1EE79, 'M', u'ض'),
-    (0x1EE7A, 'M', u'ظ'),
-    (0x1EE7B, 'M', u'غ'),
-    (0x1EE7C, 'M', u'ٮ'),
+    (0x1EE79, 'M', 'ض'),
+    (0x1EE7A, 'M', 'ظ'),
+    (0x1EE7B, 'M', 'غ'),
+    (0x1EE7C, 'M', 'ٮ'),
     (0x1EE7D, 'X'),
-    (0x1EE7E, 'M', u'ڡ'),
+    (0x1EE7E, 'M', 'ڡ'),
     (0x1EE7F, 'X'),
-    (0x1EE80, 'M', u'ا'),
-    (0x1EE81, 'M', u'ب'),
-    (0x1EE82, 'M', u'ج'),
-    (0x1EE83, 'M', u'د'),
-    (0x1EE84, 'M', u'ه'),
-    (0x1EE85, 'M', u'و'),
-    (0x1EE86, 'M', u'ز'),
-    (0x1EE87, 'M', u'ح'),
-    (0x1EE88, 'M', u'ط'),
-    (0x1EE89, 'M', u'ي'),
+    (0x1EE80, 'M', 'ا'),
+    (0x1EE81, 'M', 'ب'),
+    (0x1EE82, 'M', 'ج'),
+    (0x1EE83, 'M', 'د'),
+    (0x1EE84, 'M', 'ه'),
+    (0x1EE85, 'M', 'و'),
+    (0x1EE86, 'M', 'ز'),
+    (0x1EE87, 'M', 'ح'),
+    (0x1EE88, 'M', 'ط'),
+    (0x1EE89, 'M', 'ي'),
     (0x1EE8A, 'X'),
-    (0x1EE8B, 'M', u'ل'),
-    (0x1EE8C, 'M', u'م'),
-    (0x1EE8D, 'M', u'ن'),
-    (0x1EE8E, 'M', u'س'),
-    (0x1EE8F, 'M', u'ع'),
-    (0x1EE90, 'M', u'ف'),
-    (0x1EE91, 'M', u'ص'),
-    (0x1EE92, 'M', u'ق'),
-    (0x1EE93, 'M', u'ر'),
-    (0x1EE94, 'M', u'ش'),
-    (0x1EE95, 'M', u'ت'),
-    (0x1EE96, 'M', u'ث'),
-    (0x1EE97, 'M', u'خ'),
-    (0x1EE98, 'M', u'ذ'),
-    (0x1EE99, 'M', u'ض'),
-    (0x1EE9A, 'M', u'ظ'),
-    (0x1EE9B, 'M', u'غ'),
+    (0x1EE8B, 'M', 'ل'),
+    (0x1EE8C, 'M', 'م'),
+    (0x1EE8D, 'M', 'ن'),
+    (0x1EE8E, 'M', 'س'),
+    (0x1EE8F, 'M', 'ع'),
+    (0x1EE90, 'M', 'ف'),
+    (0x1EE91, 'M', 'ص'),
+    (0x1EE92, 'M', 'ق'),
+    (0x1EE93, 'M', 'ر'),
+    (0x1EE94, 'M', 'ش'),
+    (0x1EE95, 'M', 'ت'),
+    (0x1EE96, 'M', 'ث'),
+    (0x1EE97, 'M', 'خ'),
+    (0x1EE98, 'M', 'ذ'),
+    (0x1EE99, 'M', 'ض'),
+    (0x1EE9A, 'M', 'ظ'),
+    (0x1EE9B, 'M', 'غ'),
     (0x1EE9C, 'X'),
-    (0x1EEA1, 'M', u'ب'),
-    (0x1EEA2, 'M', u'ج'),
-    (0x1EEA3, 'M', u'د'),
+    (0x1EEA1, 'M', 'ب'),
+    (0x1EEA2, 'M', 'ج'),
+    (0x1EEA3, 'M', 'د'),
     (0x1EEA4, 'X'),
-    (0x1EEA5, 'M', u'و'),
-    (0x1EEA6, 'M', u'ز'),
-    (0x1EEA7, 'M', u'ح'),
-    (0x1EEA8, 'M', u'ط'),
-    (0x1EEA9, 'M', u'ي'),
+    (0x1EEA5, 'M', 'و'),
+    ]
+
+def _seg_73() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
+    (0x1EEA6, 'M', 'ز'),
+    (0x1EEA7, 'M', 'ح'),
+    (0x1EEA8, 'M', 'ط'),
+    (0x1EEA9, 'M', 'ي'),
     (0x1EEAA, 'X'),
-    (0x1EEAB, 'M', u'ل'),
-    (0x1EEAC, 'M', u'م'),
-    (0x1EEAD, 'M', u'ن'),
-    (0x1EEAE, 'M', u'س'),
-    (0x1EEAF, 'M', u'ع'),
-    (0x1EEB0, 'M', u'ف'),
-    (0x1EEB1, 'M', u'ص'),
-    (0x1EEB2, 'M', u'ق'),
-    (0x1EEB3, 'M', u'ر'),
-    (0x1EEB4, 'M', u'ش'),
-    (0x1EEB5, 'M', u'ت'),
-    (0x1EEB6, 'M', u'ث'),
-    (0x1EEB7, 'M', u'خ'),
-    (0x1EEB8, 'M', u'ذ'),
-    (0x1EEB9, 'M', u'ض'),
-    (0x1EEBA, 'M', u'ظ'),
-    (0x1EEBB, 'M', u'غ'),
+    (0x1EEAB, 'M', 'ل'),
+    (0x1EEAC, 'M', 'م'),
+    (0x1EEAD, 'M', 'ن'),
+    (0x1EEAE, 'M', 'س'),
+    (0x1EEAF, 'M', 'ع'),
+    (0x1EEB0, 'M', 'ف'),
+    (0x1EEB1, 'M', 'ص'),
+    (0x1EEB2, 'M', 'ق'),
+    (0x1EEB3, 'M', 'ر'),
+    (0x1EEB4, 'M', 'ش'),
+    (0x1EEB5, 'M', 'ت'),
+    (0x1EEB6, 'M', 'ث'),
+    (0x1EEB7, 'M', 'خ'),
+    (0x1EEB8, 'M', 'ذ'),
+    (0x1EEB9, 'M', 'ض'),
+    (0x1EEBA, 'M', 'ظ'),
+    (0x1EEBB, 'M', 'غ'),
     (0x1EEBC, 'X'),
     (0x1EEF0, 'V'),
     (0x1EEF2, 'X'),
@@ -7483,165 +7639,161 @@ def _seg_71():
     (0x1F0D0, 'X'),
     (0x1F0D1, 'V'),
     (0x1F0F6, 'X'),
-    (0x1F101, '3', u'0,'),
-    (0x1F102, '3', u'1,'),
-    (0x1F103, '3', u'2,'),
-    (0x1F104, '3', u'3,'),
-    (0x1F105, '3', u'4,'),
-    (0x1F106, '3', u'5,'),
-    (0x1F107, '3', u'6,'),
-    (0x1F108, '3', u'7,'),
+    (0x1F101, '3', '0,'),
+    (0x1F102, '3', '1,'),
+    (0x1F103, '3', '2,'),
+    (0x1F104, '3', '3,'),
+    (0x1F105, '3', '4,'),
+    (0x1F106, '3', '5,'),
+    (0x1F107, '3', '6,'),
+    (0x1F108, '3', '7,'),
+    (0x1F109, '3', '8,'),
+    (0x1F10A, '3', '9,'),
+    (0x1F10B, 'V'),
+    (0x1F110, '3', '(a)'),
+    (0x1F111, '3', '(b)'),
+    (0x1F112, '3', '(c)'),
+    (0x1F113, '3', '(d)'),
+    (0x1F114, '3', '(e)'),
+    (0x1F115, '3', '(f)'),
+    (0x1F116, '3', '(g)'),
+    (0x1F117, '3', '(h)'),
+    (0x1F118, '3', '(i)'),
+    (0x1F119, '3', '(j)'),
+    (0x1F11A, '3', '(k)'),
+    (0x1F11B, '3', '(l)'),
+    (0x1F11C, '3', '(m)'),
+    (0x1F11D, '3', '(n)'),
+    (0x1F11E, '3', '(o)'),
+    (0x1F11F, '3', '(p)'),
+    (0x1F120, '3', '(q)'),
+    (0x1F121, '3', '(r)'),
+    (0x1F122, '3', '(s)'),
+    (0x1F123, '3', '(t)'),
+    (0x1F124, '3', '(u)'),
+    (0x1F125, '3', '(v)'),
+    (0x1F126, '3', '(w)'),
+    (0x1F127, '3', '(x)'),
+    (0x1F128, '3', '(y)'),
+    (0x1F129, '3', '(z)'),
+    (0x1F12A, 'M', '〔s〕'),
+    (0x1F12B, 'M', 'c'),
+    (0x1F12C, 'M', 'r'),
+    (0x1F12D, 'M', 'cd'),
+    (0x1F12E, 'M', 'wz'),
+    (0x1F12F, 'V'),
+    (0x1F130, 'M', 'a'),
+    (0x1F131, 'M', 'b'),
+    (0x1F132, 'M', 'c'),
+    (0x1F133, 'M', 'd'),
+    (0x1F134, 'M', 'e'),
+    (0x1F135, 'M', 'f'),
+    (0x1F136, 'M', 'g'),
+    (0x1F137, 'M', 'h'),
+    (0x1F138, 'M', 'i'),
+    (0x1F139, 'M', 'j'),
+    (0x1F13A, 'M', 'k'),
+    (0x1F13B, 'M', 'l'),
+    (0x1F13C, 'M', 'm'),
+    (0x1F13D, 'M', 'n'),
+    (0x1F13E, 'M', 'o'),
+    (0x1F13F, 'M', 'p'),
+    (0x1F140, 'M', 'q'),
+    (0x1F141, 'M', 'r'),
+    (0x1F142, 'M', 's'),
+    (0x1F143, 'M', 't'),
     ]
 
-def _seg_72():
+def _seg_74() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x1F109, '3', u'8,'),
-    (0x1F10A, '3', u'9,'),
-    (0x1F10B, 'V'),
-    (0x1F110, '3', u'(a)'),
-    (0x1F111, '3', u'(b)'),
-    (0x1F112, '3', u'(c)'),
-    (0x1F113, '3', u'(d)'),
-    (0x1F114, '3', u'(e)'),
-    (0x1F115, '3', u'(f)'),
-    (0x1F116, '3', u'(g)'),
-    (0x1F117, '3', u'(h)'),
-    (0x1F118, '3', u'(i)'),
-    (0x1F119, '3', u'(j)'),
-    (0x1F11A, '3', u'(k)'),
-    (0x1F11B, '3', u'(l)'),
-    (0x1F11C, '3', u'(m)'),
-    (0x1F11D, '3', u'(n)'),
-    (0x1F11E, '3', u'(o)'),
-    (0x1F11F, '3', u'(p)'),
-    (0x1F120, '3', u'(q)'),
-    (0x1F121, '3', u'(r)'),
-    (0x1F122, '3', u'(s)'),
-    (0x1F123, '3', u'(t)'),
-    (0x1F124, '3', u'(u)'),
-    (0x1F125, '3', u'(v)'),
-    (0x1F126, '3', u'(w)'),
-    (0x1F127, '3', u'(x)'),
-    (0x1F128, '3', u'(y)'),
-    (0x1F129, '3', u'(z)'),
-    (0x1F12A, 'M', u'〔s〕'),
-    (0x1F12B, 'M', u'c'),
-    (0x1F12C, 'M', u'r'),
-    (0x1F12D, 'M', u'cd'),
-    (0x1F12E, 'M', u'wz'),
-    (0x1F12F, 'V'),
-    (0x1F130, 'M', u'a'),
-    (0x1F131, 'M', u'b'),
-    (0x1F132, 'M', u'c'),
-    (0x1F133, 'M', u'd'),
-    (0x1F134, 'M', u'e'),
-    (0x1F135, 'M', u'f'),
-    (0x1F136, 'M', u'g'),
-    (0x1F137, 'M', u'h'),
-    (0x1F138, 'M', u'i'),
-    (0x1F139, 'M', u'j'),
-    (0x1F13A, 'M', u'k'),
-    (0x1F13B, 'M', u'l'),
-    (0x1F13C, 'M', u'm'),
-    (0x1F13D, 'M', u'n'),
-    (0x1F13E, 'M', u'o'),
-    (0x1F13F, 'M', u'p'),
-    (0x1F140, 'M', u'q'),
-    (0x1F141, 'M', u'r'),
-    (0x1F142, 'M', u's'),
-    (0x1F143, 'M', u't'),
-    (0x1F144, 'M', u'u'),
-    (0x1F145, 'M', u'v'),
-    (0x1F146, 'M', u'w'),
-    (0x1F147, 'M', u'x'),
-    (0x1F148, 'M', u'y'),
-    (0x1F149, 'M', u'z'),
-    (0x1F14A, 'M', u'hv'),
-    (0x1F14B, 'M', u'mv'),
-    (0x1F14C, 'M', u'sd'),
-    (0x1F14D, 'M', u'ss'),
-    (0x1F14E, 'M', u'ppv'),
-    (0x1F14F, 'M', u'wc'),
+    (0x1F144, 'M', 'u'),
+    (0x1F145, 'M', 'v'),
+    (0x1F146, 'M', 'w'),
+    (0x1F147, 'M', 'x'),
+    (0x1F148, 'M', 'y'),
+    (0x1F149, 'M', 'z'),
+    (0x1F14A, 'M', 'hv'),
+    (0x1F14B, 'M', 'mv'),
+    (0x1F14C, 'M', 'sd'),
+    (0x1F14D, 'M', 'ss'),
+    (0x1F14E, 'M', 'ppv'),
+    (0x1F14F, 'M', 'wc'),
     (0x1F150, 'V'),
-    (0x1F16A, 'M', u'mc'),
-    (0x1F16B, 'M', u'md'),
-    (0x1F16C, 'M', u'mr'),
+    (0x1F16A, 'M', 'mc'),
+    (0x1F16B, 'M', 'md'),
+    (0x1F16C, 'M', 'mr'),
     (0x1F16D, 'V'),
-    (0x1F190, 'M', u'dj'),
+    (0x1F190, 'M', 'dj'),
     (0x1F191, 'V'),
     (0x1F1AE, 'X'),
     (0x1F1E6, 'V'),
-    (0x1F200, 'M', u'ほか'),
-    (0x1F201, 'M', u'ココ'),
-    (0x1F202, 'M', u'サ'),
+    (0x1F200, 'M', 'ほか'),
+    (0x1F201, 'M', 'ココ'),
+    (0x1F202, 'M', 'サ'),
     (0x1F203, 'X'),
-    (0x1F210, 'M', u'手'),
-    (0x1F211, 'M', u'字'),
-    (0x1F212, 'M', u'双'),
-    (0x1F213, 'M', u'デ'),
-    (0x1F214, 'M', u'二'),
-    (0x1F215, 'M', u'多'),
-    (0x1F216, 'M', u'解'),
-    (0x1F217, 'M', u'天'),
-    (0x1F218, 'M', u'交'),
-    (0x1F219, 'M', u'映'),
-    (0x1F21A, 'M', u'無'),
-    (0x1F21B, 'M', u'料'),
-    (0x1F21C, 'M', u'前'),
-    (0x1F21D, 'M', u'後'),
-    (0x1F21E, 'M', u'再'),
-    (0x1F21F, 'M', u'新'),
-    (0x1F220, 'M', u'初'),
-    (0x1F221, 'M', u'終'),
-    (0x1F222, 'M', u'生'),
-    (0x1F223, 'M', u'販'),
-    ]
-
-def _seg_73():
-    return [
-    (0x1F224, 'M', u'声'),
-    (0x1F225, 'M', u'吹'),
-    (0x1F226, 'M', u'演'),
-    (0x1F227, 'M', u'投'),
-    (0x1F228, 'M', u'捕'),
-    (0x1F229, 'M', u'一'),
-    (0x1F22A, 'M', u'三'),
-    (0x1F22B, 'M', u'遊'),
-    (0x1F22C, 'M', u'左'),
-    (0x1F22D, 'M', u'中'),
-    (0x1F22E, 'M', u'右'),
-    (0x1F22F, 'M', u'指'),
-    (0x1F230, 'M', u'走'),
-    (0x1F231, 'M', u'打'),
-    (0x1F232, 'M', u'禁'),
-    (0x1F233, 'M', u'空'),
-    (0x1F234, 'M', u'合'),
-    (0x1F235, 'M', u'満'),
-    (0x1F236, 'M', u'有'),
-    (0x1F237, 'M', u'月'),
-    (0x1F238, 'M', u'申'),
-    (0x1F239, 'M', u'割'),
-    (0x1F23A, 'M', u'営'),
-    (0x1F23B, 'M', u'配'),
+    (0x1F210, 'M', '手'),
+    (0x1F211, 'M', '字'),
+    (0x1F212, 'M', '双'),
+    (0x1F213, 'M', 'デ'),
+    (0x1F214, 'M', '二'),
+    (0x1F215, 'M', '多'),
+    (0x1F216, 'M', '解'),
+    (0x1F217, 'M', '天'),
+    (0x1F218, 'M', '交'),
+    (0x1F219, 'M', '映'),
+    (0x1F21A, 'M', '無'),
+    (0x1F21B, 'M', '料'),
+    (0x1F21C, 'M', '前'),
+    (0x1F21D, 'M', '後'),
+    (0x1F21E, 'M', '再'),
+    (0x1F21F, 'M', '新'),
+    (0x1F220, 'M', '初'),
+    (0x1F221, 'M', '終'),
+    (0x1F222, 'M', '生'),
+    (0x1F223, 'M', '販'),
+    (0x1F224, 'M', '声'),
+    (0x1F225, 'M', '吹'),
+    (0x1F226, 'M', '演'),
+    (0x1F227, 'M', '投'),
+    (0x1F228, 'M', '捕'),
+    (0x1F229, 'M', '一'),
+    (0x1F22A, 'M', '三'),
+    (0x1F22B, 'M', '遊'),
+    (0x1F22C, 'M', '左'),
+    (0x1F22D, 'M', '中'),
+    (0x1F22E, 'M', '右'),
+    (0x1F22F, 'M', '指'),
+    (0x1F230, 'M', '走'),
+    (0x1F231, 'M', '打'),
+    (0x1F232, 'M', '禁'),
+    (0x1F233, 'M', '空'),
+    (0x1F234, 'M', '合'),
+    (0x1F235, 'M', '満'),
+    (0x1F236, 'M', '有'),
+    (0x1F237, 'M', '月'),
+    (0x1F238, 'M', '申'),
+    (0x1F239, 'M', '割'),
+    (0x1F23A, 'M', '営'),
+    (0x1F23B, 'M', '配'),
     (0x1F23C, 'X'),
-    (0x1F240, 'M', u'〔本〕'),
-    (0x1F241, 'M', u'〔三〕'),
-    (0x1F242, 'M', u'〔二〕'),
-    (0x1F243, 'M', u'〔安〕'),
-    (0x1F244, 'M', u'〔点〕'),
-    (0x1F245, 'M', u'〔打〕'),
-    (0x1F246, 'M', u'〔盗〕'),
-    (0x1F247, 'M', u'〔勝〕'),
-    (0x1F248, 'M', u'〔敗〕'),
+    (0x1F240, 'M', '〔本〕'),
+    (0x1F241, 'M', '〔三〕'),
+    (0x1F242, 'M', '〔二〕'),
+    (0x1F243, 'M', '〔安〕'),
+    (0x1F244, 'M', '〔点〕'),
+    (0x1F245, 'M', '〔打〕'),
+    (0x1F246, 'M', '〔盗〕'),
+    (0x1F247, 'M', '〔勝〕'),
+    (0x1F248, 'M', '〔敗〕'),
     (0x1F249, 'X'),
-    (0x1F250, 'M', u'得'),
-    (0x1F251, 'M', u'可'),
+    (0x1F250, 'M', '得'),
+    (0x1F251, 'M', '可'),
     (0x1F252, 'X'),
     (0x1F260, 'V'),
     (0x1F266, 'X'),
     (0x1F300, 'V'),
     (0x1F6D8, 'X'),
-    (0x1F6E0, 'V'),
+    (0x1F6DD, 'V'),
     (0x1F6ED, 'X'),
     (0x1F6F0, 'V'),
     (0x1F6FD, 'X'),
@@ -7651,7 +7803,13 @@ def _seg_73():
     (0x1F7D9, 'X'),
     (0x1F7E0, 'V'),
     (0x1F7EC, 'X'),
+    (0x1F7F0, 'V'),
+    (0x1F7F1, 'X'),
     (0x1F800, 'V'),
+    ]
+
+def _seg_75() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
+    return [
     (0x1F80C, 'X'),
     (0x1F810, 'V'),
     (0x1F848, 'X'),
@@ -7664,608 +7822,604 @@ def _seg_73():
     (0x1F8B0, 'V'),
     (0x1F8B2, 'X'),
     (0x1F900, 'V'),
-    (0x1F979, 'X'),
-    (0x1F97A, 'V'),
-    (0x1F9CC, 'X'),
-    (0x1F9CD, 'V'),
     (0x1FA54, 'X'),
     (0x1FA60, 'V'),
     (0x1FA6E, 'X'),
     (0x1FA70, 'V'),
     (0x1FA75, 'X'),
     (0x1FA78, 'V'),
-    (0x1FA7B, 'X'),
+    (0x1FA7D, 'X'),
     (0x1FA80, 'V'),
     (0x1FA87, 'X'),
     (0x1FA90, 'V'),
-    (0x1FAA9, 'X'),
+    (0x1FAAD, 'X'),
     (0x1FAB0, 'V'),
-    (0x1FAB7, 'X'),
+    (0x1FABB, 'X'),
     (0x1FAC0, 'V'),
-    (0x1FAC3, 'X'),
+    (0x1FAC6, 'X'),
     (0x1FAD0, 'V'),
-    (0x1FAD7, 'X'),
+    (0x1FADA, 'X'),
+    (0x1FAE0, 'V'),
+    (0x1FAE8, 'X'),
+    (0x1FAF0, 'V'),
+    (0x1FAF7, 'X'),
     (0x1FB00, 'V'),
     (0x1FB93, 'X'),
     (0x1FB94, 'V'),
     (0x1FBCB, 'X'),
-    (0x1FBF0, 'M', u'0'),
-    (0x1FBF1, 'M', u'1'),
-    (0x1FBF2, 'M', u'2'),
-    (0x1FBF3, 'M', u'3'),
-    (0x1FBF4, 'M', u'4'),
-    (0x1FBF5, 'M', u'5'),
-    (0x1FBF6, 'M', u'6'),
-    (0x1FBF7, 'M', u'7'),
-    (0x1FBF8, 'M', u'8'),
-    (0x1FBF9, 'M', u'9'),
-    ]
-
-def _seg_74():
-    return [
+    (0x1FBF0, 'M', '0'),
+    (0x1FBF1, 'M', '1'),
+    (0x1FBF2, 'M', '2'),
+    (0x1FBF3, 'M', '3'),
+    (0x1FBF4, 'M', '4'),
+    (0x1FBF5, 'M', '5'),
+    (0x1FBF6, 'M', '6'),
+    (0x1FBF7, 'M', '7'),
+    (0x1FBF8, 'M', '8'),
+    (0x1FBF9, 'M', '9'),
     (0x1FBFA, 'X'),
     (0x20000, 'V'),
-    (0x2A6DE, 'X'),
+    (0x2A6E0, 'X'),
     (0x2A700, 'V'),
-    (0x2B735, 'X'),
+    (0x2B739, 'X'),
     (0x2B740, 'V'),
     (0x2B81E, 'X'),
     (0x2B820, 'V'),
     (0x2CEA2, 'X'),
     (0x2CEB0, 'V'),
     (0x2EBE1, 'X'),
-    (0x2F800, 'M', u'丽'),
-    (0x2F801, 'M', u'丸'),
-    (0x2F802, 'M', u'乁'),
-    (0x2F803, 'M', u'𠄢'),
-    (0x2F804, 'M', u'你'),
-    (0x2F805, 'M', u'侮'),
-    (0x2F806, 'M', u'侻'),
-    (0x2F807, 'M', u'倂'),
-    (0x2F808, 'M', u'偺'),
-    (0x2F809, 'M', u'備'),
-    (0x2F80A, 'M', u'僧'),
-    (0x2F80B, 'M', u'像'),
-    (0x2F80C, 'M', u'㒞'),
-    (0x2F80D, 'M', u'𠘺'),
-    (0x2F80E, 'M', u'免'),
-    (0x2F80F, 'M', u'兔'),
-    (0x2F810, 'M', u'兤'),
-    (0x2F811, 'M', u'具'),
-    (0x2F812, 'M', u'𠔜'),
-    (0x2F813, 'M', u'㒹'),
-    (0x2F814, 'M', u'內'),
-    (0x2F815, 'M', u'再'),
-    (0x2F816, 'M', u'𠕋'),
-    (0x2F817, 'M', u'冗'),
-    (0x2F818, 'M', u'冤'),
-    (0x2F819, 'M', u'仌'),
-    (0x2F81A, 'M', u'冬'),
-    (0x2F81B, 'M', u'况'),
-    (0x2F81C, 'M', u'𩇟'),
-    (0x2F81D, 'M', u'凵'),
-    (0x2F81E, 'M', u'刃'),
-    (0x2F81F, 'M', u'㓟'),
-    (0x2F820, 'M', u'刻'),
-    (0x2F821, 'M', u'剆'),
-    (0x2F822, 'M', u'割'),
-    (0x2F823, 'M', u'剷'),
-    (0x2F824, 'M', u'㔕'),
-    (0x2F825, 'M', u'勇'),
-    (0x2F826, 'M', u'勉'),
-    (0x2F827, 'M', u'勤'),
-    (0x2F828, 'M', u'勺'),
-    (0x2F829, 'M', u'包'),
-    (0x2F82A, 'M', u'匆'),
-    (0x2F82B, 'M', u'北'),
-    (0x2F82C, 'M', u'卉'),
-    (0x2F82D, 'M', u'卑'),
-    (0x2F82E, 'M', u'博'),
-    (0x2F82F, 'M', u'即'),
-    (0x2F830, 'M', u'卽'),
-    (0x2F831, 'M', u'卿'),
-    (0x2F834, 'M', u'𠨬'),
-    (0x2F835, 'M', u'灰'),
-    (0x2F836, 'M', u'及'),
-    (0x2F837, 'M', u'叟'),
-    (0x2F838, 'M', u'𠭣'),
-    (0x2F839, 'M', u'叫'),
-    (0x2F83A, 'M', u'叱'),
-    (0x2F83B, 'M', u'吆'),
-    (0x2F83C, 'M', u'咞'),
-    (0x2F83D, 'M', u'吸'),
-    (0x2F83E, 'M', u'呈'),
-    (0x2F83F, 'M', u'周'),
-    (0x2F840, 'M', u'咢'),
-    (0x2F841, 'M', u'哶'),
-    (0x2F842, 'M', u'唐'),
-    (0x2F843, 'M', u'啓'),
-    (0x2F844, 'M', u'啣'),
-    (0x2F845, 'M', u'善'),
-    (0x2F847, 'M', u'喙'),
-    (0x2F848, 'M', u'喫'),
-    (0x2F849, 'M', u'喳'),
-    (0x2F84A, 'M', u'嗂'),
-    (0x2F84B, 'M', u'圖'),
-    (0x2F84C, 'M', u'嘆'),
-    (0x2F84D, 'M', u'圗'),
-    (0x2F84E, 'M', u'噑'),
-    (0x2F84F, 'M', u'噴'),
-    (0x2F850, 'M', u'切'),
-    (0x2F851, 'M', u'壮'),
-    (0x2F852, 'M', u'城'),
-    (0x2F853, 'M', u'埴'),
-    (0x2F854, 'M', u'堍'),
-    (0x2F855, 'M', u'型'),
-    (0x2F856, 'M', u'堲'),
-    (0x2F857, 'M', u'報'),
-    (0x2F858, 'M', u'墬'),
-    (0x2F859, 'M', u'𡓤'),
-    (0x2F85A, 'M', u'売'),
-    (0x2F85B, 'M', u'壷'),
+    (0x2F800, 'M', '丽'),
+    (0x2F801, 'M', '丸'),
+    (0x2F802, 'M', '乁'),
+    (0x2F803, 'M', '𠄢'),
+    (0x2F804, 'M', '你'),
+    (0x2F805, 'M', '侮'),
+    (0x2F806, 'M', '侻'),
+    (0x2F807, 'M', '倂'),
+    (0x2F808, 'M', '偺'),
+    (0x2F809, 'M', '備'),
+    (0x2F80A, 'M', '僧'),
+    (0x2F80B, 'M', '像'),
+    (0x2F80C, 'M', '㒞'),
+    (0x2F80D, 'M', '𠘺'),
+    (0x2F80E, 'M', '免'),
+    (0x2F80F, 'M', '兔'),
+    (0x2F810, 'M', '兤'),
+    (0x2F811, 'M', '具'),
+    (0x2F812, 'M', '𠔜'),
+    (0x2F813, 'M', '㒹'),
+    (0x2F814, 'M', '內'),
+    (0x2F815, 'M', '再'),
+    (0x2F816, 'M', '𠕋'),
+    (0x2F817, 'M', '冗'),
+    (0x2F818, 'M', '冤'),
+    (0x2F819, 'M', '仌'),
+    (0x2F81A, 'M', '冬'),
+    (0x2F81B, 'M', '况'),
+    (0x2F81C, 'M', '𩇟'),
+    (0x2F81D, 'M', '凵'),
+    (0x2F81E, 'M', '刃'),
+    (0x2F81F, 'M', '㓟'),
+    (0x2F820, 'M', '刻'),
+    (0x2F821, 'M', '剆'),
+    (0x2F822, 'M', '割'),
+    (0x2F823, 'M', '剷'),
+    (0x2F824, 'M', '㔕'),
+    (0x2F825, 'M', '勇'),
+    (0x2F826, 'M', '勉'),
+    (0x2F827, 'M', '勤'),
+    (0x2F828, 'M', '勺'),
+    (0x2F829, 'M', '包'),
     ]
 
-def _seg_75():
+def _seg_76() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F85C, 'M', u'夆'),
-    (0x2F85D, 'M', u'多'),
-    (0x2F85E, 'M', u'夢'),
-    (0x2F85F, 'M', u'奢'),
-    (0x2F860, 'M', u'𡚨'),
-    (0x2F861, 'M', u'𡛪'),
-    (0x2F862, 'M', u'姬'),
-    (0x2F863, 'M', u'娛'),
-    (0x2F864, 'M', u'娧'),
-    (0x2F865, 'M', u'姘'),
-    (0x2F866, 'M', u'婦'),
-    (0x2F867, 'M', u'㛮'),
+    (0x2F82A, 'M', '匆'),
+    (0x2F82B, 'M', '北'),
+    (0x2F82C, 'M', '卉'),
+    (0x2F82D, 'M', '卑'),
+    (0x2F82E, 'M', '博'),
+    (0x2F82F, 'M', '即'),
+    (0x2F830, 'M', '卽'),
+    (0x2F831, 'M', '卿'),
+    (0x2F834, 'M', '𠨬'),
+    (0x2F835, 'M', '灰'),
+    (0x2F836, 'M', '及'),
+    (0x2F837, 'M', '叟'),
+    (0x2F838, 'M', '𠭣'),
+    (0x2F839, 'M', '叫'),
+    (0x2F83A, 'M', '叱'),
+    (0x2F83B, 'M', '吆'),
+    (0x2F83C, 'M', '咞'),
+    (0x2F83D, 'M', '吸'),
+    (0x2F83E, 'M', '呈'),
+    (0x2F83F, 'M', '周'),
+    (0x2F840, 'M', '咢'),
+    (0x2F841, 'M', '哶'),
+    (0x2F842, 'M', '唐'),
+    (0x2F843, 'M', '啓'),
+    (0x2F844, 'M', '啣'),
+    (0x2F845, 'M', '善'),
+    (0x2F847, 'M', '喙'),
+    (0x2F848, 'M', '喫'),
+    (0x2F849, 'M', '喳'),
+    (0x2F84A, 'M', '嗂'),
+    (0x2F84B, 'M', '圖'),
+    (0x2F84C, 'M', '嘆'),
+    (0x2F84D, 'M', '圗'),
+    (0x2F84E, 'M', '噑'),
+    (0x2F84F, 'M', '噴'),
+    (0x2F850, 'M', '切'),
+    (0x2F851, 'M', '壮'),
+    (0x2F852, 'M', '城'),
+    (0x2F853, 'M', '埴'),
+    (0x2F854, 'M', '堍'),
+    (0x2F855, 'M', '型'),
+    (0x2F856, 'M', '堲'),
+    (0x2F857, 'M', '報'),
+    (0x2F858, 'M', '墬'),
+    (0x2F859, 'M', '𡓤'),
+    (0x2F85A, 'M', '売'),
+    (0x2F85B, 'M', '壷'),
+    (0x2F85C, 'M', '夆'),
+    (0x2F85D, 'M', '多'),
+    (0x2F85E, 'M', '夢'),
+    (0x2F85F, 'M', '奢'),
+    (0x2F860, 'M', '𡚨'),
+    (0x2F861, 'M', '𡛪'),
+    (0x2F862, 'M', '姬'),
+    (0x2F863, 'M', '娛'),
+    (0x2F864, 'M', '娧'),
+    (0x2F865, 'M', '姘'),
+    (0x2F866, 'M', '婦'),
+    (0x2F867, 'M', '㛮'),
     (0x2F868, 'X'),
-    (0x2F869, 'M', u'嬈'),
-    (0x2F86A, 'M', u'嬾'),
-    (0x2F86C, 'M', u'𡧈'),
-    (0x2F86D, 'M', u'寃'),
-    (0x2F86E, 'M', u'寘'),
-    (0x2F86F, 'M', u'寧'),
-    (0x2F870, 'M', u'寳'),
-    (0x2F871, 'M', u'𡬘'),
-    (0x2F872, 'M', u'寿'),
-    (0x2F873, 'M', u'将'),
+    (0x2F869, 'M', '嬈'),
+    (0x2F86A, 'M', '嬾'),
+    (0x2F86C, 'M', '𡧈'),
+    (0x2F86D, 'M', '寃'),
+    (0x2F86E, 'M', '寘'),
+    (0x2F86F, 'M', '寧'),
+    (0x2F870, 'M', '寳'),
+    (0x2F871, 'M', '𡬘'),
+    (0x2F872, 'M', '寿'),
+    (0x2F873, 'M', '将'),
     (0x2F874, 'X'),
-    (0x2F875, 'M', u'尢'),
-    (0x2F876, 'M', u'㞁'),
-    (0x2F877, 'M', u'屠'),
-    (0x2F878, 'M', u'屮'),
-    (0x2F879, 'M', u'峀'),
-    (0x2F87A, 'M', u'岍'),
-    (0x2F87B, 'M', u'𡷤'),
-    (0x2F87C, 'M', u'嵃'),
-    (0x2F87D, 'M', u'𡷦'),
-    (0x2F87E, 'M', u'嵮'),
-    (0x2F87F, 'M', u'嵫'),
-    (0x2F880, 'M', u'嵼'),
-    (0x2F881, 'M', u'巡'),
-    (0x2F882, 'M', u'巢'),
-    (0x2F883, 'M', u'㠯'),
-    (0x2F884, 'M', u'巽'),
-    (0x2F885, 'M', u'帨'),
-    (0x2F886, 'M', u'帽'),
-    (0x2F887, 'M', u'幩'),
-    (0x2F888, 'M', u'㡢'),
-    (0x2F889, 'M', u'𢆃'),
-    (0x2F88A, 'M', u'㡼'),
-    (0x2F88B, 'M', u'庰'),
-    (0x2F88C, 'M', u'庳'),
-    (0x2F88D, 'M', u'庶'),
-    (0x2F88E, 'M', u'廊'),
-    (0x2F88F, 'M', u'𪎒'),
-    (0x2F890, 'M', u'廾'),
-    (0x2F891, 'M', u'𢌱'),
-    (0x2F893, 'M', u'舁'),
-    (0x2F894, 'M', u'弢'),
-    (0x2F896, 'M', u'㣇'),
-    (0x2F897, 'M', u'𣊸'),
-    (0x2F898, 'M', u'𦇚'),
-    (0x2F899, 'M', u'形'),
-    (0x2F89A, 'M', u'彫'),
-    (0x2F89B, 'M', u'㣣'),
-    (0x2F89C, 'M', u'徚'),
-    (0x2F89D, 'M', u'忍'),
-    (0x2F89E, 'M', u'志'),
-    (0x2F89F, 'M', u'忹'),
-    (0x2F8A0, 'M', u'悁'),
-    (0x2F8A1, 'M', u'㤺'),
-    (0x2F8A2, 'M', u'㤜'),
-    (0x2F8A3, 'M', u'悔'),
-    (0x2F8A4, 'M', u'𢛔'),
-    (0x2F8A5, 'M', u'惇'),
-    (0x2F8A6, 'M', u'慈'),
-    (0x2F8A7, 'M', u'慌'),
-    (0x2F8A8, 'M', u'慎'),
-    (0x2F8A9, 'M', u'慌'),
-    (0x2F8AA, 'M', u'慺'),
-    (0x2F8AB, 'M', u'憎'),
-    (0x2F8AC, 'M', u'憲'),
-    (0x2F8AD, 'M', u'憤'),
-    (0x2F8AE, 'M', u'憯'),
-    (0x2F8AF, 'M', u'懞'),
-    (0x2F8B0, 'M', u'懲'),
-    (0x2F8B1, 'M', u'懶'),
-    (0x2F8B2, 'M', u'成'),
-    (0x2F8B3, 'M', u'戛'),
-    (0x2F8B4, 'M', u'扝'),
-    (0x2F8B5, 'M', u'抱'),
-    (0x2F8B6, 'M', u'拔'),
-    (0x2F8B7, 'M', u'捐'),
-    (0x2F8B8, 'M', u'𢬌'),
-    (0x2F8B9, 'M', u'挽'),
-    (0x2F8BA, 'M', u'拼'),
-    (0x2F8BB, 'M', u'捨'),
-    (0x2F8BC, 'M', u'掃'),
-    (0x2F8BD, 'M', u'揤'),
-    (0x2F8BE, 'M', u'𢯱'),
-    (0x2F8BF, 'M', u'搢'),
-    (0x2F8C0, 'M', u'揅'),
-    (0x2F8C1, 'M', u'掩'),
-    (0x2F8C2, 'M', u'㨮'),
+    (0x2F875, 'M', '尢'),
+    (0x2F876, 'M', '㞁'),
+    (0x2F877, 'M', '屠'),
+    (0x2F878, 'M', '屮'),
+    (0x2F879, 'M', '峀'),
+    (0x2F87A, 'M', '岍'),
+    (0x2F87B, 'M', '𡷤'),
+    (0x2F87C, 'M', '嵃'),
+    (0x2F87D, 'M', '𡷦'),
+    (0x2F87E, 'M', '嵮'),
+    (0x2F87F, 'M', '嵫'),
+    (0x2F880, 'M', '嵼'),
+    (0x2F881, 'M', '巡'),
+    (0x2F882, 'M', '巢'),
+    (0x2F883, 'M', '㠯'),
+    (0x2F884, 'M', '巽'),
+    (0x2F885, 'M', '帨'),
+    (0x2F886, 'M', '帽'),
+    (0x2F887, 'M', '幩'),
+    (0x2F888, 'M', '㡢'),
+    (0x2F889, 'M', '𢆃'),
+    (0x2F88A, 'M', '㡼'),
+    (0x2F88B, 'M', '庰'),
+    (0x2F88C, 'M', '庳'),
+    (0x2F88D, 'M', '庶'),
+    (0x2F88E, 'M', '廊'),
+    (0x2F88F, 'M', '𪎒'),
+    (0x2F890, 'M', '廾'),
+    (0x2F891, 'M', '𢌱'),
     ]
 
-def _seg_76():
+def _seg_77() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F8C3, 'M', u'摩'),
-    (0x2F8C4, 'M', u'摾'),
-    (0x2F8C5, 'M', u'撝'),
-    (0x2F8C6, 'M', u'摷'),
-    (0x2F8C7, 'M', u'㩬'),
-    (0x2F8C8, 'M', u'敏'),
-    (0x2F8C9, 'M', u'敬'),
-    (0x2F8CA, 'M', u'𣀊'),
-    (0x2F8CB, 'M', u'旣'),
-    (0x2F8CC, 'M', u'書'),
-    (0x2F8CD, 'M', u'晉'),
-    (0x2F8CE, 'M', u'㬙'),
-    (0x2F8CF, 'M', u'暑'),
-    (0x2F8D0, 'M', u'㬈'),
-    (0x2F8D1, 'M', u'㫤'),
-    (0x2F8D2, 'M', u'冒'),
-    (0x2F8D3, 'M', u'冕'),
-    (0x2F8D4, 'M', u'最'),
-    (0x2F8D5, 'M', u'暜'),
-    (0x2F8D6, 'M', u'肭'),
-    (0x2F8D7, 'M', u'䏙'),
-    (0x2F8D8, 'M', u'朗'),
-    (0x2F8D9, 'M', u'望'),
-    (0x2F8DA, 'M', u'朡'),
-    (0x2F8DB, 'M', u'杞'),
-    (0x2F8DC, 'M', u'杓'),
-    (0x2F8DD, 'M', u'𣏃'),
-    (0x2F8DE, 'M', u'㭉'),
-    (0x2F8DF, 'M', u'柺'),
-    (0x2F8E0, 'M', u'枅'),
-    (0x2F8E1, 'M', u'桒'),
-    (0x2F8E2, 'M', u'梅'),
-    (0x2F8E3, 'M', u'𣑭'),
-    (0x2F8E4, 'M', u'梎'),
-    (0x2F8E5, 'M', u'栟'),
-    (0x2F8E6, 'M', u'椔'),
-    (0x2F8E7, 'M', u'㮝'),
-    (0x2F8E8, 'M', u'楂'),
-    (0x2F8E9, 'M', u'榣'),
-    (0x2F8EA, 'M', u'槪'),
-    (0x2F8EB, 'M', u'檨'),
-    (0x2F8EC, 'M', u'𣚣'),
-    (0x2F8ED, 'M', u'櫛'),
-    (0x2F8EE, 'M', u'㰘'),
-    (0x2F8EF, 'M', u'次'),
-    (0x2F8F0, 'M', u'𣢧'),
-    (0x2F8F1, 'M', u'歔'),
-    (0x2F8F2, 'M', u'㱎'),
-    (0x2F8F3, 'M', u'歲'),
-    (0x2F8F4, 'M', u'殟'),
-    (0x2F8F5, 'M', u'殺'),
-    (0x2F8F6, 'M', u'殻'),
-    (0x2F8F7, 'M', u'𣪍'),
-    (0x2F8F8, 'M', u'𡴋'),
-    (0x2F8F9, 'M', u'𣫺'),
-    (0x2F8FA, 'M', u'汎'),
-    (0x2F8FB, 'M', u'𣲼'),
-    (0x2F8FC, 'M', u'沿'),
-    (0x2F8FD, 'M', u'泍'),
-    (0x2F8FE, 'M', u'汧'),
-    (0x2F8FF, 'M', u'洖'),
-    (0x2F900, 'M', u'派'),
-    (0x2F901, 'M', u'海'),
-    (0x2F902, 'M', u'流'),
-    (0x2F903, 'M', u'浩'),
-    (0x2F904, 'M', u'浸'),
-    (0x2F905, 'M', u'涅'),
-    (0x2F906, 'M', u'𣴞'),
-    (0x2F907, 'M', u'洴'),
-    (0x2F908, 'M', u'港'),
-    (0x2F909, 'M', u'湮'),
-    (0x2F90A, 'M', u'㴳'),
-    (0x2F90B, 'M', u'滋'),
-    (0x2F90C, 'M', u'滇'),
-    (0x2F90D, 'M', u'𣻑'),
-    (0x2F90E, 'M', u'淹'),
-    (0x2F90F, 'M', u'潮'),
-    (0x2F910, 'M', u'𣽞'),
-    (0x2F911, 'M', u'𣾎'),
-    (0x2F912, 'M', u'濆'),
-    (0x2F913, 'M', u'瀹'),
-    (0x2F914, 'M', u'瀞'),
-    (0x2F915, 'M', u'瀛'),
-    (0x2F916, 'M', u'㶖'),
-    (0x2F917, 'M', u'灊'),
-    (0x2F918, 'M', u'災'),
-    (0x2F919, 'M', u'灷'),
-    (0x2F91A, 'M', u'炭'),
-    (0x2F91B, 'M', u'𠔥'),
-    (0x2F91C, 'M', u'煅'),
-    (0x2F91D, 'M', u'𤉣'),
-    (0x2F91E, 'M', u'熜'),
-    (0x2F91F, 'X'),
-    (0x2F920, 'M', u'爨'),
-    (0x2F921, 'M', u'爵'),
-    (0x2F922, 'M', u'牐'),
-    (0x2F923, 'M', u'𤘈'),
-    (0x2F924, 'M', u'犀'),
-    (0x2F925, 'M', u'犕'),
-    (0x2F926, 'M', u'𤜵'),
+    (0x2F893, 'M', '舁'),
+    (0x2F894, 'M', '弢'),
+    (0x2F896, 'M', '㣇'),
+    (0x2F897, 'M', '𣊸'),
+    (0x2F898, 'M', '𦇚'),
+    (0x2F899, 'M', '形'),
+    (0x2F89A, 'M', '彫'),
+    (0x2F89B, 'M', '㣣'),
+    (0x2F89C, 'M', '徚'),
+    (0x2F89D, 'M', '忍'),
+    (0x2F89E, 'M', '志'),
+    (0x2F89F, 'M', '忹'),
+    (0x2F8A0, 'M', '悁'),
+    (0x2F8A1, 'M', '㤺'),
+    (0x2F8A2, 'M', '㤜'),
+    (0x2F8A3, 'M', '悔'),
+    (0x2F8A4, 'M', '𢛔'),
+    (0x2F8A5, 'M', '惇'),
+    (0x2F8A6, 'M', '慈'),
+    (0x2F8A7, 'M', '慌'),
+    (0x2F8A8, 'M', '慎'),
+    (0x2F8A9, 'M', '慌'),
+    (0x2F8AA, 'M', '慺'),
+    (0x2F8AB, 'M', '憎'),
+    (0x2F8AC, 'M', '憲'),
+    (0x2F8AD, 'M', '憤'),
+    (0x2F8AE, 'M', '憯'),
+    (0x2F8AF, 'M', '懞'),
+    (0x2F8B0, 'M', '懲'),
+    (0x2F8B1, 'M', '懶'),
+    (0x2F8B2, 'M', '成'),
+    (0x2F8B3, 'M', '戛'),
+    (0x2F8B4, 'M', '扝'),
+    (0x2F8B5, 'M', '抱'),
+    (0x2F8B6, 'M', '拔'),
+    (0x2F8B7, 'M', '捐'),
+    (0x2F8B8, 'M', '𢬌'),
+    (0x2F8B9, 'M', '挽'),
+    (0x2F8BA, 'M', '拼'),
+    (0x2F8BB, 'M', '捨'),
+    (0x2F8BC, 'M', '掃'),
+    (0x2F8BD, 'M', '揤'),
+    (0x2F8BE, 'M', '𢯱'),
+    (0x2F8BF, 'M', '搢'),
+    (0x2F8C0, 'M', '揅'),
+    (0x2F8C1, 'M', '掩'),
+    (0x2F8C2, 'M', '㨮'),
+    (0x2F8C3, 'M', '摩'),
+    (0x2F8C4, 'M', '摾'),
+    (0x2F8C5, 'M', '撝'),
+    (0x2F8C6, 'M', '摷'),
+    (0x2F8C7, 'M', '㩬'),
+    (0x2F8C8, 'M', '敏'),
+    (0x2F8C9, 'M', '敬'),
+    (0x2F8CA, 'M', '𣀊'),
+    (0x2F8CB, 'M', '旣'),
+    (0x2F8CC, 'M', '書'),
+    (0x2F8CD, 'M', '晉'),
+    (0x2F8CE, 'M', '㬙'),
+    (0x2F8CF, 'M', '暑'),
+    (0x2F8D0, 'M', '㬈'),
+    (0x2F8D1, 'M', '㫤'),
+    (0x2F8D2, 'M', '冒'),
+    (0x2F8D3, 'M', '冕'),
+    (0x2F8D4, 'M', '最'),
+    (0x2F8D5, 'M', '暜'),
+    (0x2F8D6, 'M', '肭'),
+    (0x2F8D7, 'M', '䏙'),
+    (0x2F8D8, 'M', '朗'),
+    (0x2F8D9, 'M', '望'),
+    (0x2F8DA, 'M', '朡'),
+    (0x2F8DB, 'M', '杞'),
+    (0x2F8DC, 'M', '杓'),
+    (0x2F8DD, 'M', '𣏃'),
+    (0x2F8DE, 'M', '㭉'),
+    (0x2F8DF, 'M', '柺'),
+    (0x2F8E0, 'M', '枅'),
+    (0x2F8E1, 'M', '桒'),
+    (0x2F8E2, 'M', '梅'),
+    (0x2F8E3, 'M', '𣑭'),
+    (0x2F8E4, 'M', '梎'),
+    (0x2F8E5, 'M', '栟'),
+    (0x2F8E6, 'M', '椔'),
+    (0x2F8E7, 'M', '㮝'),
+    (0x2F8E8, 'M', '楂'),
+    (0x2F8E9, 'M', '榣'),
+    (0x2F8EA, 'M', '槪'),
+    (0x2F8EB, 'M', '檨'),
+    (0x2F8EC, 'M', '𣚣'),
+    (0x2F8ED, 'M', '櫛'),
+    (0x2F8EE, 'M', '㰘'),
+    (0x2F8EF, 'M', '次'),
+    (0x2F8F0, 'M', '𣢧'),
+    (0x2F8F1, 'M', '歔'),
+    (0x2F8F2, 'M', '㱎'),
+    (0x2F8F3, 'M', '歲'),
+    (0x2F8F4, 'M', '殟'),
+    (0x2F8F5, 'M', '殺'),
+    (0x2F8F6, 'M', '殻'),
+    (0x2F8F7, 'M', '𣪍'),
     ]
 
-def _seg_77():
+def _seg_78() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F927, 'M', u'𤠔'),
-    (0x2F928, 'M', u'獺'),
-    (0x2F929, 'M', u'王'),
-    (0x2F92A, 'M', u'㺬'),
-    (0x2F92B, 'M', u'玥'),
-    (0x2F92C, 'M', u'㺸'),
-    (0x2F92E, 'M', u'瑇'),
-    (0x2F92F, 'M', u'瑜'),
-    (0x2F930, 'M', u'瑱'),
-    (0x2F931, 'M', u'璅'),
-    (0x2F932, 'M', u'瓊'),
-    (0x2F933, 'M', u'㼛'),
-    (0x2F934, 'M', u'甤'),
-    (0x2F935, 'M', u'𤰶'),
-    (0x2F936, 'M', u'甾'),
-    (0x2F937, 'M', u'𤲒'),
-    (0x2F938, 'M', u'異'),
-    (0x2F939, 'M', u'𢆟'),
-    (0x2F93A, 'M', u'瘐'),
-    (0x2F93B, 'M', u'𤾡'),
-    (0x2F93C, 'M', u'𤾸'),
-    (0x2F93D, 'M', u'𥁄'),
-    (0x2F93E, 'M', u'㿼'),
-    (0x2F93F, 'M', u'䀈'),
-    (0x2F940, 'M', u'直'),
-    (0x2F941, 'M', u'𥃳'),
-    (0x2F942, 'M', u'𥃲'),
-    (0x2F943, 'M', u'𥄙'),
-    (0x2F944, 'M', u'𥄳'),
-    (0x2F945, 'M', u'眞'),
-    (0x2F946, 'M', u'真'),
-    (0x2F948, 'M', u'睊'),
-    (0x2F949, 'M', u'䀹'),
-    (0x2F94A, 'M', u'瞋'),
-    (0x2F94B, 'M', u'䁆'),
-    (0x2F94C, 'M', u'䂖'),
-    (0x2F94D, 'M', u'𥐝'),
-    (0x2F94E, 'M', u'硎'),
-    (0x2F94F, 'M', u'碌'),
-    (0x2F950, 'M', u'磌'),
-    (0x2F951, 'M', u'䃣'),
-    (0x2F952, 'M', u'𥘦'),
-    (0x2F953, 'M', u'祖'),
-    (0x2F954, 'M', u'𥚚'),
-    (0x2F955, 'M', u'𥛅'),
-    (0x2F956, 'M', u'福'),
-    (0x2F957, 'M', u'秫'),
-    (0x2F958, 'M', u'䄯'),
-    (0x2F959, 'M', u'穀'),
-    (0x2F95A, 'M', u'穊'),
-    (0x2F95B, 'M', u'穏'),
-    (0x2F95C, 'M', u'𥥼'),
-    (0x2F95D, 'M', u'𥪧'),
-    (0x2F95F, 'X'),
-    (0x2F960, 'M', u'䈂'),
-    (0x2F961, 'M', u'𥮫'),
-    (0x2F962, 'M', u'篆'),
-    (0x2F963, 'M', u'築'),
-    (0x2F964, 'M', u'䈧'),
-    (0x2F965, 'M', u'𥲀'),
-    (0x2F966, 'M', u'糒'),
-    (0x2F967, 'M', u'䊠'),
-    (0x2F968, 'M', u'糨'),
-    (0x2F969, 'M', u'糣'),
-    (0x2F96A, 'M', u'紀'),
-    (0x2F96B, 'M', u'𥾆'),
-    (0x2F96C, 'M', u'絣'),
-    (0x2F96D, 'M', u'䌁'),
-    (0x2F96E, 'M', u'緇'),
-    (0x2F96F, 'M', u'縂'),
-    (0x2F970, 'M', u'繅'),
-    (0x2F971, 'M', u'䌴'),
-    (0x2F972, 'M', u'𦈨'),
-    (0x2F973, 'M', u'𦉇'),
-    (0x2F974, 'M', u'䍙'),
-    (0x2F975, 'M', u'𦋙'),
-    (0x2F976, 'M', u'罺'),
-    (0x2F977, 'M', u'𦌾'),
-    (0x2F978, 'M', u'羕'),
-    (0x2F979, 'M', u'翺'),
-    (0x2F97A, 'M', u'者'),
-    (0x2F97B, 'M', u'𦓚'),
-    (0x2F97C, 'M', u'𦔣'),
-    (0x2F97D, 'M', u'聠'),
-    (0x2F97E, 'M', u'𦖨'),
-    (0x2F97F, 'M', u'聰'),
-    (0x2F980, 'M', u'𣍟'),
-    (0x2F981, 'M', u'䏕'),
-    (0x2F982, 'M', u'育'),
-    (0x2F983, 'M', u'脃'),
-    (0x2F984, 'M', u'䐋'),
-    (0x2F985, 'M', u'脾'),
-    (0x2F986, 'M', u'媵'),
-    (0x2F987, 'M', u'𦞧'),
-    (0x2F988, 'M', u'𦞵'),
-    (0x2F989, 'M', u'𣎓'),
-    (0x2F98A, 'M', u'𣎜'),
-    (0x2F98B, 'M', u'舁'),
-    (0x2F98C, 'M', u'舄'),
-    (0x2F98D, 'M', u'辞'),
+    (0x2F8F8, 'M', '𡴋'),
+    (0x2F8F9, 'M', '𣫺'),
+    (0x2F8FA, 'M', '汎'),
+    (0x2F8FB, 'M', '𣲼'),
+    (0x2F8FC, 'M', '沿'),
+    (0x2F8FD, 'M', '泍'),
+    (0x2F8FE, 'M', '汧'),
+    (0x2F8FF, 'M', '洖'),
+    (0x2F900, 'M', '派'),
+    (0x2F901, 'M', '海'),
+    (0x2F902, 'M', '流'),
+    (0x2F903, 'M', '浩'),
+    (0x2F904, 'M', '浸'),
+    (0x2F905, 'M', '涅'),
+    (0x2F906, 'M', '𣴞'),
+    (0x2F907, 'M', '洴'),
+    (0x2F908, 'M', '港'),
+    (0x2F909, 'M', '湮'),
+    (0x2F90A, 'M', '㴳'),
+    (0x2F90B, 'M', '滋'),
+    (0x2F90C, 'M', '滇'),
+    (0x2F90D, 'M', '𣻑'),
+    (0x2F90E, 'M', '淹'),
+    (0x2F90F, 'M', '潮'),
+    (0x2F910, 'M', '𣽞'),
+    (0x2F911, 'M', '𣾎'),
+    (0x2F912, 'M', '濆'),
+    (0x2F913, 'M', '瀹'),
+    (0x2F914, 'M', '瀞'),
+    (0x2F915, 'M', '瀛'),
+    (0x2F916, 'M', '㶖'),
+    (0x2F917, 'M', '灊'),
+    (0x2F918, 'M', '災'),
+    (0x2F919, 'M', '灷'),
+    (0x2F91A, 'M', '炭'),
+    (0x2F91B, 'M', '𠔥'),
+    (0x2F91C, 'M', '煅'),
+    (0x2F91D, 'M', '𤉣'),
+    (0x2F91E, 'M', '熜'),
+    (0x2F91F, 'X'),
+    (0x2F920, 'M', '爨'),
+    (0x2F921, 'M', '爵'),
+    (0x2F922, 'M', '牐'),
+    (0x2F923, 'M', '𤘈'),
+    (0x2F924, 'M', '犀'),
+    (0x2F925, 'M', '犕'),
+    (0x2F926, 'M', '𤜵'),
+    (0x2F927, 'M', '𤠔'),
+    (0x2F928, 'M', '獺'),
+    (0x2F929, 'M', '王'),
+    (0x2F92A, 'M', '㺬'),
+    (0x2F92B, 'M', '玥'),
+    (0x2F92C, 'M', '㺸'),
+    (0x2F92E, 'M', '瑇'),
+    (0x2F92F, 'M', '瑜'),
+    (0x2F930, 'M', '瑱'),
+    (0x2F931, 'M', '璅'),
+    (0x2F932, 'M', '瓊'),
+    (0x2F933, 'M', '㼛'),
+    (0x2F934, 'M', '甤'),
+    (0x2F935, 'M', '𤰶'),
+    (0x2F936, 'M', '甾'),
+    (0x2F937, 'M', '𤲒'),
+    (0x2F938, 'M', '異'),
+    (0x2F939, 'M', '𢆟'),
+    (0x2F93A, 'M', '瘐'),
+    (0x2F93B, 'M', '𤾡'),
+    (0x2F93C, 'M', '𤾸'),
+    (0x2F93D, 'M', '𥁄'),
+    (0x2F93E, 'M', '㿼'),
+    (0x2F93F, 'M', '䀈'),
+    (0x2F940, 'M', '直'),
+    (0x2F941, 'M', '𥃳'),
+    (0x2F942, 'M', '𥃲'),
+    (0x2F943, 'M', '𥄙'),
+    (0x2F944, 'M', '𥄳'),
+    (0x2F945, 'M', '眞'),
+    (0x2F946, 'M', '真'),
+    (0x2F948, 'M', '睊'),
+    (0x2F949, 'M', '䀹'),
+    (0x2F94A, 'M', '瞋'),
+    (0x2F94B, 'M', '䁆'),
+    (0x2F94C, 'M', '䂖'),
+    (0x2F94D, 'M', '𥐝'),
+    (0x2F94E, 'M', '硎'),
+    (0x2F94F, 'M', '碌'),
+    (0x2F950, 'M', '磌'),
+    (0x2F951, 'M', '䃣'),
+    (0x2F952, 'M', '𥘦'),
+    (0x2F953, 'M', '祖'),
+    (0x2F954, 'M', '𥚚'),
+    (0x2F955, 'M', '𥛅'),
+    (0x2F956, 'M', '福'),
+    (0x2F957, 'M', '秫'),
+    (0x2F958, 'M', '䄯'),
+    (0x2F959, 'M', '穀'),
+    (0x2F95A, 'M', '穊'),
+    (0x2F95B, 'M', '穏'),
+    (0x2F95C, 'M', '𥥼'),
+    (0x2F95D, 'M', '𥪧'),
     ]
 
-def _seg_78():
+def _seg_79() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F98E, 'M', u'䑫'),
-    (0x2F98F, 'M', u'芑'),
-    (0x2F990, 'M', u'芋'),
-    (0x2F991, 'M', u'芝'),
-    (0x2F992, 'M', u'劳'),
-    (0x2F993, 'M', u'花'),
-    (0x2F994, 'M', u'芳'),
-    (0x2F995, 'M', u'芽'),
-    (0x2F996, 'M', u'苦'),
-    (0x2F997, 'M', u'𦬼'),
-    (0x2F998, 'M', u'若'),
-    (0x2F999, 'M', u'茝'),
-    (0x2F99A, 'M', u'荣'),
-    (0x2F99B, 'M', u'莭'),
-    (0x2F99C, 'M', u'茣'),
-    (0x2F99D, 'M', u'莽'),
-    (0x2F99E, 'M', u'菧'),
-    (0x2F99F, 'M', u'著'),
-    (0x2F9A0, 'M', u'荓'),
-    (0x2F9A1, 'M', u'菊'),
-    (0x2F9A2, 'M', u'菌'),
-    (0x2F9A3, 'M', u'菜'),
-    (0x2F9A4, 'M', u'𦰶'),
-    (0x2F9A5, 'M', u'𦵫'),
-    (0x2F9A6, 'M', u'𦳕'),
-    (0x2F9A7, 'M', u'䔫'),
-    (0x2F9A8, 'M', u'蓱'),
-    (0x2F9A9, 'M', u'蓳'),
-    (0x2F9AA, 'M', u'蔖'),
-    (0x2F9AB, 'M', u'𧏊'),
-    (0x2F9AC, 'M', u'蕤'),
-    (0x2F9AD, 'M', u'𦼬'),
-    (0x2F9AE, 'M', u'䕝'),
-    (0x2F9AF, 'M', u'䕡'),
-    (0x2F9B0, 'M', u'𦾱'),
-    (0x2F9B1, 'M', u'𧃒'),
-    (0x2F9B2, 'M', u'䕫'),
-    (0x2F9B3, 'M', u'虐'),
-    (0x2F9B4, 'M', u'虜'),
-    (0x2F9B5, 'M', u'虧'),
-    (0x2F9B6, 'M', u'虩'),
-    (0x2F9B7, 'M', u'蚩'),
-    (0x2F9B8, 'M', u'蚈'),
-    (0x2F9B9, 'M', u'蜎'),
-    (0x2F9BA, 'M', u'蛢'),
-    (0x2F9BB, 'M', u'蝹'),
-    (0x2F9BC, 'M', u'蜨'),
-    (0x2F9BD, 'M', u'蝫'),
-    (0x2F9BE, 'M', u'螆'),
+    (0x2F95F, 'X'),
+    (0x2F960, 'M', '䈂'),
+    (0x2F961, 'M', '𥮫'),
+    (0x2F962, 'M', '篆'),
+    (0x2F963, 'M', '築'),
+    (0x2F964, 'M', '䈧'),
+    (0x2F965, 'M', '𥲀'),
+    (0x2F966, 'M', '糒'),
+    (0x2F967, 'M', '䊠'),
+    (0x2F968, 'M', '糨'),
+    (0x2F969, 'M', '糣'),
+    (0x2F96A, 'M', '紀'),
+    (0x2F96B, 'M', '𥾆'),
+    (0x2F96C, 'M', '絣'),
+    (0x2F96D, 'M', '䌁'),
+    (0x2F96E, 'M', '緇'),
+    (0x2F96F, 'M', '縂'),
+    (0x2F970, 'M', '繅'),
+    (0x2F971, 'M', '䌴'),
+    (0x2F972, 'M', '𦈨'),
+    (0x2F973, 'M', '𦉇'),
+    (0x2F974, 'M', '䍙'),
+    (0x2F975, 'M', '𦋙'),
+    (0x2F976, 'M', '罺'),
+    (0x2F977, 'M', '𦌾'),
+    (0x2F978, 'M', '羕'),
+    (0x2F979, 'M', '翺'),
+    (0x2F97A, 'M', '者'),
+    (0x2F97B, 'M', '𦓚'),
+    (0x2F97C, 'M', '𦔣'),
+    (0x2F97D, 'M', '聠'),
+    (0x2F97E, 'M', '𦖨'),
+    (0x2F97F, 'M', '聰'),
+    (0x2F980, 'M', '𣍟'),
+    (0x2F981, 'M', '䏕'),
+    (0x2F982, 'M', '育'),
+    (0x2F983, 'M', '脃'),
+    (0x2F984, 'M', '䐋'),
+    (0x2F985, 'M', '脾'),
+    (0x2F986, 'M', '媵'),
+    (0x2F987, 'M', '𦞧'),
+    (0x2F988, 'M', '𦞵'),
+    (0x2F989, 'M', '𣎓'),
+    (0x2F98A, 'M', '𣎜'),
+    (0x2F98B, 'M', '舁'),
+    (0x2F98C, 'M', '舄'),
+    (0x2F98D, 'M', '辞'),
+    (0x2F98E, 'M', '䑫'),
+    (0x2F98F, 'M', '芑'),
+    (0x2F990, 'M', '芋'),
+    (0x2F991, 'M', '芝'),
+    (0x2F992, 'M', '劳'),
+    (0x2F993, 'M', '花'),
+    (0x2F994, 'M', '芳'),
+    (0x2F995, 'M', '芽'),
+    (0x2F996, 'M', '苦'),
+    (0x2F997, 'M', '𦬼'),
+    (0x2F998, 'M', '若'),
+    (0x2F999, 'M', '茝'),
+    (0x2F99A, 'M', '荣'),
+    (0x2F99B, 'M', '莭'),
+    (0x2F99C, 'M', '茣'),
+    (0x2F99D, 'M', '莽'),
+    (0x2F99E, 'M', '菧'),
+    (0x2F99F, 'M', '著'),
+    (0x2F9A0, 'M', '荓'),
+    (0x2F9A1, 'M', '菊'),
+    (0x2F9A2, 'M', '菌'),
+    (0x2F9A3, 'M', '菜'),
+    (0x2F9A4, 'M', '𦰶'),
+    (0x2F9A5, 'M', '𦵫'),
+    (0x2F9A6, 'M', '𦳕'),
+    (0x2F9A7, 'M', '䔫'),
+    (0x2F9A8, 'M', '蓱'),
+    (0x2F9A9, 'M', '蓳'),
+    (0x2F9AA, 'M', '蔖'),
+    (0x2F9AB, 'M', '𧏊'),
+    (0x2F9AC, 'M', '蕤'),
+    (0x2F9AD, 'M', '𦼬'),
+    (0x2F9AE, 'M', '䕝'),
+    (0x2F9AF, 'M', '䕡'),
+    (0x2F9B0, 'M', '𦾱'),
+    (0x2F9B1, 'M', '𧃒'),
+    (0x2F9B2, 'M', '䕫'),
+    (0x2F9B3, 'M', '虐'),
+    (0x2F9B4, 'M', '虜'),
+    (0x2F9B5, 'M', '虧'),
+    (0x2F9B6, 'M', '虩'),
+    (0x2F9B7, 'M', '蚩'),
+    (0x2F9B8, 'M', '蚈'),
+    (0x2F9B9, 'M', '蜎'),
+    (0x2F9BA, 'M', '蛢'),
+    (0x2F9BB, 'M', '蝹'),
+    (0x2F9BC, 'M', '蜨'),
+    (0x2F9BD, 'M', '蝫'),
+    (0x2F9BE, 'M', '螆'),
     (0x2F9BF, 'X'),
-    (0x2F9C0, 'M', u'蟡'),
-    (0x2F9C1, 'M', u'蠁'),
-    (0x2F9C2, 'M', u'䗹'),
-    (0x2F9C3, 'M', u'衠'),
-    (0x2F9C4, 'M', u'衣'),
-    (0x2F9C5, 'M', u'𧙧'),
-    (0x2F9C6, 'M', u'裗'),
-    (0x2F9C7, 'M', u'裞'),
-    (0x2F9C8, 'M', u'䘵'),
-    (0x2F9C9, 'M', u'裺'),
-    (0x2F9CA, 'M', u'㒻'),
-    (0x2F9CB, 'M', u'𧢮'),
-    (0x2F9CC, 'M', u'𧥦'),
-    (0x2F9CD, 'M', u'䚾'),
-    (0x2F9CE, 'M', u'䛇'),
-    (0x2F9CF, 'M', u'誠'),
-    (0x2F9D0, 'M', u'諭'),
-    (0x2F9D1, 'M', u'變'),
-    (0x2F9D2, 'M', u'豕'),
-    (0x2F9D3, 'M', u'𧲨'),
-    (0x2F9D4, 'M', u'貫'),
-    (0x2F9D5, 'M', u'賁'),
-    (0x2F9D6, 'M', u'贛'),
-    (0x2F9D7, 'M', u'起'),
-    (0x2F9D8, 'M', u'𧼯'),
-    (0x2F9D9, 'M', u'𠠄'),
-    (0x2F9DA, 'M', u'跋'),
-    (0x2F9DB, 'M', u'趼'),
-    (0x2F9DC, 'M', u'跰'),
-    (0x2F9DD, 'M', u'𠣞'),
-    (0x2F9DE, 'M', u'軔'),
-    (0x2F9DF, 'M', u'輸'),
-    (0x2F9E0, 'M', u'𨗒'),
-    (0x2F9E1, 'M', u'𨗭'),
-    (0x2F9E2, 'M', u'邔'),
-    (0x2F9E3, 'M', u'郱'),
-    (0x2F9E4, 'M', u'鄑'),
-    (0x2F9E5, 'M', u'𨜮'),
-    (0x2F9E6, 'M', u'鄛'),
-    (0x2F9E7, 'M', u'鈸'),
-    (0x2F9E8, 'M', u'鋗'),
-    (0x2F9E9, 'M', u'鋘'),
-    (0x2F9EA, 'M', u'鉼'),
-    (0x2F9EB, 'M', u'鏹'),
-    (0x2F9EC, 'M', u'鐕'),
-    (0x2F9ED, 'M', u'𨯺'),
-    (0x2F9EE, 'M', u'開'),
-    (0x2F9EF, 'M', u'䦕'),
-    (0x2F9F0, 'M', u'閷'),
-    (0x2F9F1, 'M', u'𨵷'),
+    (0x2F9C0, 'M', '蟡'),
+    (0x2F9C1, 'M', '蠁'),
+    (0x2F9C2, 'M', '䗹'),
     ]
 
-def _seg_79():
+def _seg_80() -> List[Union[Tuple[int, str], Tuple[int, str, str]]]:
     return [
-    (0x2F9F2, 'M', u'䧦'),
-    (0x2F9F3, 'M', u'雃'),
-    (0x2F9F4, 'M', u'嶲'),
-    (0x2F9F5, 'M', u'霣'),
-    (0x2F9F6, 'M', u'𩅅'),
-    (0x2F9F7, 'M', u'𩈚'),
-    (0x2F9F8, 'M', u'䩮'),
-    (0x2F9F9, 'M', u'䩶'),
-    (0x2F9FA, 'M', u'韠'),
-    (0x2F9FB, 'M', u'𩐊'),
-    (0x2F9FC, 'M', u'䪲'),
-    (0x2F9FD, 'M', u'𩒖'),
-    (0x2F9FE, 'M', u'頋'),
-    (0x2FA00, 'M', u'頩'),
-    (0x2FA01, 'M', u'𩖶'),
-    (0x2FA02, 'M', u'飢'),
-    (0x2FA03, 'M', u'䬳'),
-    (0x2FA04, 'M', u'餩'),
-    (0x2FA05, 'M', u'馧'),
-    (0x2FA06, 'M', u'駂'),
-    (0x2FA07, 'M', u'駾'),
-    (0x2FA08, 'M', u'䯎'),
-    (0x2FA09, 'M', u'𩬰'),
-    (0x2FA0A, 'M', u'鬒'),
-    (0x2FA0B, 'M', u'鱀'),
-    (0x2FA0C, 'M', u'鳽'),
-    (0x2FA0D, 'M', u'䳎'),
-    (0x2FA0E, 'M', u'䳭'),
-    (0x2FA0F, 'M', u'鵧'),
-    (0x2FA10, 'M', u'𪃎'),
-    (0x2FA11, 'M', u'䳸'),
-    (0x2FA12, 'M', u'𪄅'),
-    (0x2FA13, 'M', u'𪈎'),
-    (0x2FA14, 'M', u'𪊑'),
-    (0x2FA15, 'M', u'麻'),
-    (0x2FA16, 'M', u'䵖'),
-    (0x2FA17, 'M', u'黹'),
-    (0x2FA18, 'M', u'黾'),
-    (0x2FA19, 'M', u'鼅'),
-    (0x2FA1A, 'M', u'鼏'),
-    (0x2FA1B, 'M', u'鼖'),
-    (0x2FA1C, 'M', u'鼻'),
-    (0x2FA1D, 'M', u'𪘀'),
+    (0x2F9C3, 'M', '衠'),
+    (0x2F9C4, 'M', '衣'),
+    (0x2F9C5, 'M', '𧙧'),
+    (0x2F9C6, 'M', '裗'),
+    (0x2F9C7, 'M', '裞'),
+    (0x2F9C8, 'M', '䘵'),
+    (0x2F9C9, 'M', '裺'),
+    (0x2F9CA, 'M', '㒻'),
+    (0x2F9CB, 'M', '𧢮'),
+    (0x2F9CC, 'M', '𧥦'),
+    (0x2F9CD, 'M', '䚾'),
+    (0x2F9CE, 'M', '䛇'),
+    (0x2F9CF, 'M', '誠'),
+    (0x2F9D0, 'M', '諭'),
+    (0x2F9D1, 'M', '變'),
+    (0x2F9D2, 'M', '豕'),
+    (0x2F9D3, 'M', '𧲨'),
+    (0x2F9D4, 'M', '貫'),
+    (0x2F9D5, 'M', '賁'),
+    (0x2F9D6, 'M', '贛'),
+    (0x2F9D7, 'M', '起'),
+    (0x2F9D8, 'M', '𧼯'),
+    (0x2F9D9, 'M', '𠠄'),
+    (0x2F9DA, 'M', '跋'),
+    (0x2F9DB, 'M', '趼'),
+    (0x2F9DC, 'M', '跰'),
+    (0x2F9DD, 'M', '𠣞'),
+    (0x2F9DE, 'M', '軔'),
+    (0x2F9DF, 'M', '輸'),
+    (0x2F9E0, 'M', '𨗒'),
+    (0x2F9E1, 'M', '𨗭'),
+    (0x2F9E2, 'M', '邔'),
+    (0x2F9E3, 'M', '郱'),
+    (0x2F9E4, 'M', '鄑'),
+    (0x2F9E5, 'M', '𨜮'),
+    (0x2F9E6, 'M', '鄛'),
+    (0x2F9E7, 'M', '鈸'),
+    (0x2F9E8, 'M', '鋗'),
+    (0x2F9E9, 'M', '鋘'),
+    (0x2F9EA, 'M', '鉼'),
+    (0x2F9EB, 'M', '鏹'),
+    (0x2F9EC, 'M', '鐕'),
+    (0x2F9ED, 'M', '𨯺'),
+    (0x2F9EE, 'M', '開'),
+    (0x2F9EF, 'M', '䦕'),
+    (0x2F9F0, 'M', '閷'),
+    (0x2F9F1, 'M', '𨵷'),
+    (0x2F9F2, 'M', '䧦'),
+    (0x2F9F3, 'M', '雃'),
+    (0x2F9F4, 'M', '嶲'),
+    (0x2F9F5, 'M', '霣'),
+    (0x2F9F6, 'M', '𩅅'),
+    (0x2F9F7, 'M', '𩈚'),
+    (0x2F9F8, 'M', '䩮'),
+    (0x2F9F9, 'M', '䩶'),
+    (0x2F9FA, 'M', '韠'),
+    (0x2F9FB, 'M', '𩐊'),
+    (0x2F9FC, 'M', '䪲'),
+    (0x2F9FD, 'M', '𩒖'),
+    (0x2F9FE, 'M', '頋'),
+    (0x2FA00, 'M', '頩'),
+    (0x2FA01, 'M', '𩖶'),
+    (0x2FA02, 'M', '飢'),
+    (0x2FA03, 'M', '䬳'),
+    (0x2FA04, 'M', '餩'),
+    (0x2FA05, 'M', '馧'),
+    (0x2FA06, 'M', '駂'),
+    (0x2FA07, 'M', '駾'),
+    (0x2FA08, 'M', '䯎'),
+    (0x2FA09, 'M', '𩬰'),
+    (0x2FA0A, 'M', '鬒'),
+    (0x2FA0B, 'M', '鱀'),
+    (0x2FA0C, 'M', '鳽'),
+    (0x2FA0D, 'M', '䳎'),
+    (0x2FA0E, 'M', '䳭'),
+    (0x2FA0F, 'M', '鵧'),
+    (0x2FA10, 'M', '𪃎'),
+    (0x2FA11, 'M', '䳸'),
+    (0x2FA12, 'M', '𪄅'),
+    (0x2FA13, 'M', '𪈎'),
+    (0x2FA14, 'M', '𪊑'),
+    (0x2FA15, 'M', '麻'),
+    (0x2FA16, 'M', '䵖'),
+    (0x2FA17, 'M', '黹'),
+    (0x2FA18, 'M', '黾'),
+    (0x2FA19, 'M', '鼅'),
+    (0x2FA1A, 'M', '鼏'),
+    (0x2FA1B, 'M', '鼖'),
+    (0x2FA1C, 'M', '鼻'),
+    (0x2FA1D, 'M', '𪘀'),
     (0x2FA1E, 'X'),
     (0x30000, 'V'),
     (0x3134B, 'X'),
@@ -8354,4 +8508,5 @@ def _seg_79():
     + _seg_77()
     + _seg_78()
     + _seg_79()
-)
+    + _seg_80()
+)  # type: Tuple[Union[Tuple[int, str], Tuple[int, str, str]], ...]
diff --git a/src/pip/_vendor/msgpack/_version.py b/src/pip/_vendor/msgpack/_version.py
index 9f55cf50dc6..fb878b353de 100644
--- a/src/pip/_vendor/msgpack/_version.py
+++ b/src/pip/_vendor/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 0)
+version = (1, 0, 3)
diff --git a/src/pip/_vendor/msgpack/ext.py b/src/pip/_vendor/msgpack/ext.py
index 8341c68b8ab..4eb9dd65adc 100644
--- a/src/pip/_vendor/msgpack/ext.py
+++ b/src/pip/_vendor/msgpack/ext.py
@@ -178,7 +178,9 @@ def to_datetime(self):
 
         :rtype: datetime.
         """
-        return datetime.datetime.fromtimestamp(self.to_unix(), _utc)
+        return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta(
+            seconds=self.to_unix()
+        )
 
     @staticmethod
     def from_datetime(dt):
diff --git a/src/pip/_vendor/msgpack/fallback.py b/src/pip/_vendor/msgpack/fallback.py
index 9f6665b3eb3..b27acb29515 100644
--- a/src/pip/_vendor/msgpack/fallback.py
+++ b/src/pip/_vendor/msgpack/fallback.py
@@ -1,5 +1,4 @@
 """Fallback pure Python implementation of msgpack"""
-
 from datetime import datetime as _DateTime
 import sys
 import struct
@@ -148,6 +147,38 @@ def _unpack_from(f, b, o=0):
 else:
     _unpack_from = struct.unpack_from
 
+_NO_FORMAT_USED = ""
+_MSGPACK_HEADERS = {
+    0xC4: (1, _NO_FORMAT_USED, TYPE_BIN),
+    0xC5: (2, ">H", TYPE_BIN),
+    0xC6: (4, ">I", TYPE_BIN),
+    0xC7: (2, "Bb", TYPE_EXT),
+    0xC8: (3, ">Hb", TYPE_EXT),
+    0xC9: (5, ">Ib", TYPE_EXT),
+    0xCA: (4, ">f"),
+    0xCB: (8, ">d"),
+    0xCC: (1, _NO_FORMAT_USED),
+    0xCD: (2, ">H"),
+    0xCE: (4, ">I"),
+    0xCF: (8, ">Q"),
+    0xD0: (1, "b"),
+    0xD1: (2, ">h"),
+    0xD2: (4, ">i"),
+    0xD3: (8, ">q"),
+    0xD4: (1, "b1s", TYPE_EXT),
+    0xD5: (2, "b2s", TYPE_EXT),
+    0xD6: (4, "b4s", TYPE_EXT),
+    0xD7: (8, "b8s", TYPE_EXT),
+    0xD8: (16, "b16s", TYPE_EXT),
+    0xD9: (1, _NO_FORMAT_USED, TYPE_RAW),
+    0xDA: (2, ">H", TYPE_RAW),
+    0xDB: (4, ">I", TYPE_RAW),
+    0xDC: (2, ">H", TYPE_ARRAY),
+    0xDD: (4, ">I", TYPE_ARRAY),
+    0xDE: (2, ">H", TYPE_MAP),
+    0xDF: (4, ">I", TYPE_MAP),
+}
+
 
 class Unpacker(object):
     """Streaming unpacker.
@@ -229,7 +260,7 @@ class Unpacker(object):
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(max_buffer_size)
+        unpacker = Unpacker()
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -354,7 +385,7 @@ def feed(self, next_bytes):
         self._buffer.extend(view)
 
     def _consume(self):
-        """ Gets rid of the used parts of the buffer. """
+        """Gets rid of the used parts of the buffer."""
         self._stream_offset += self._buff_i - self._buf_checkpoint
         self._buf_checkpoint = self._buff_i
 
@@ -365,18 +396,19 @@ def _get_extradata(self):
         return self._buffer[self._buff_i :]
 
     def read_bytes(self, n):
-        ret = self._read(n)
+        ret = self._read(n, raise_outofdata=False)
         self._consume()
         return ret
 
-    def _read(self, n):
+    def _read(self, n, raise_outofdata=True):
         # (int) -> bytearray
-        self._reserve(n)
+        self._reserve(n, raise_outofdata=raise_outofdata)
         i = self._buff_i
-        self._buff_i = i + n
-        return self._buffer[i : i + n]
+        ret = self._buffer[i : i + n]
+        self._buff_i = i + len(ret)
+        return ret
 
-    def _reserve(self, n):
+    def _reserve(self, n, raise_outofdata=True):
         remain_bytes = len(self._buffer) - self._buff_i - n
 
         # Fast path: buffer has n bytes already
@@ -404,11 +436,11 @@ def _reserve(self, n):
             self._buffer += read_data
             remain_bytes -= len(read_data)
 
-        if len(self._buffer) < n + self._buff_i:
+        if len(self._buffer) < n + self._buff_i and raise_outofdata:
             self._buff_i = 0  # rollback
             raise OutOfData
 
-    def _read_header(self, execute=EX_CONSTRUCT):
+    def _read_header(self):
         typ = TYPE_IMMEDIATE
         n = 0
         obj = None
@@ -423,205 +455,95 @@ def _read_header(self, execute=EX_CONSTRUCT):
             n = b & 0b00011111
             typ = TYPE_RAW
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
             obj = self._read(n)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise ValueError(
+                    "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
+                )
         elif b & 0b11110000 == 0b10000000:
             n = b & 0b00001111
             typ = TYPE_MAP
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
         elif b == 0xC0:
             obj = None
         elif b == 0xC2:
             obj = False
         elif b == 0xC3:
             obj = True
-        elif b == 0xC4:
-            typ = TYPE_BIN
-            self._reserve(1)
-            n = self._buffer[self._buff_i]
-            self._buff_i += 1
-            if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n)
-        elif b == 0xC5:
-            typ = TYPE_BIN
-            self._reserve(2)
-            n = _unpack_from(">H", self._buffer, self._buff_i)[0]
-            self._buff_i += 2
-            if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n)
-        elif b == 0xC6:
-            typ = TYPE_BIN
-            self._reserve(4)
-            n = _unpack_from(">I", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
+        elif 0xC4 <= b <= 0xC6:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            if len(fmt) > 0:
+                n = _unpack_from(fmt, self._buffer, self._buff_i)[0]
+            else:
+                n = self._buffer[self._buff_i]
+            self._buff_i += size
             if n > self._max_bin_len:
                 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
-        elif b == 0xC7:  # ext 8
-            typ = TYPE_EXT
-            self._reserve(2)
-            L, n = _unpack_from("Bb", self._buffer, self._buff_i)
-            self._buff_i += 2
-            if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L)
-        elif b == 0xC8:  # ext 16
-            typ = TYPE_EXT
-            self._reserve(3)
-            L, n = _unpack_from(">Hb", self._buffer, self._buff_i)
-            self._buff_i += 3
-            if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L)
-        elif b == 0xC9:  # ext 32
-            typ = TYPE_EXT
-            self._reserve(5)
-            L, n = _unpack_from(">Ib", self._buffer, self._buff_i)
-            self._buff_i += 5
+        elif 0xC7 <= b <= 0xC9:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            L, n = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size
             if L > self._max_ext_len:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
-        elif b == 0xCA:
-            self._reserve(4)
-            obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-        elif b == 0xCB:
-            self._reserve(8)
-            obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
-            self._buff_i += 8
-        elif b == 0xCC:
-            self._reserve(1)
-            obj = self._buffer[self._buff_i]
-            self._buff_i += 1
-        elif b == 0xCD:
-            self._reserve(2)
-            obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
-            self._buff_i += 2
-        elif b == 0xCE:
-            self._reserve(4)
-            obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-        elif b == 0xCF:
-            self._reserve(8)
-            obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
-            self._buff_i += 8
-        elif b == 0xD0:
-            self._reserve(1)
-            obj = _unpack_from("b", self._buffer, self._buff_i)[0]
-            self._buff_i += 1
-        elif b == 0xD1:
-            self._reserve(2)
-            obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
-            self._buff_i += 2
-        elif b == 0xD2:
-            self._reserve(4)
-            obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-        elif b == 0xD3:
-            self._reserve(8)
-            obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
-            self._buff_i += 8
-        elif b == 0xD4:  # fixext 1
-            typ = TYPE_EXT
-            if self._max_ext_len < 1:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
-            self._reserve(2)
-            n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
-            self._buff_i += 2
-        elif b == 0xD5:  # fixext 2
-            typ = TYPE_EXT
-            if self._max_ext_len < 2:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
-            self._reserve(3)
-            n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
-            self._buff_i += 3
-        elif b == 0xD6:  # fixext 4
-            typ = TYPE_EXT
-            if self._max_ext_len < 4:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
-            self._reserve(5)
-            n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
-            self._buff_i += 5
-        elif b == 0xD7:  # fixext 8
-            typ = TYPE_EXT
-            if self._max_ext_len < 8:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
-            self._reserve(9)
-            n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
-            self._buff_i += 9
-        elif b == 0xD8:  # fixext 16
-            typ = TYPE_EXT
-            if self._max_ext_len < 16:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
-            self._reserve(17)
-            n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
-            self._buff_i += 17
-        elif b == 0xD9:
-            typ = TYPE_RAW
-            self._reserve(1)
-            n = self._buffer[self._buff_i]
-            self._buff_i += 1
-            if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n)
-        elif b == 0xDA:
-            typ = TYPE_RAW
-            self._reserve(2)
-            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
-            self._buff_i += 2
-            if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n)
-        elif b == 0xDB:
-            typ = TYPE_RAW
-            self._reserve(4)
-            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
-            self._buff_i += 4
+        elif 0xCA <= b <= 0xD3:
+            size, fmt = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            if len(fmt) > 0:
+                obj = _unpack_from(fmt, self._buffer, self._buff_i)[0]
+            else:
+                obj = self._buffer[self._buff_i]
+            self._buff_i += size
+        elif 0xD4 <= b <= 0xD8:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            if self._max_ext_len < size:
+                raise ValueError(
+                    "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len)
+                )
+            self._reserve(size + 1)
+            n, obj = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size + 1
+        elif 0xD9 <= b <= 0xDB:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            if len(fmt) > 0:
+                (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            else:
+                n = self._buffer[self._buff_i]
+            self._buff_i += size
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
             obj = self._read(n)
-        elif b == 0xDC:
-            typ = TYPE_ARRAY
-            self._reserve(2)
-            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
-            self._buff_i += 2
-            if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-        elif b == 0xDD:
-            typ = TYPE_ARRAY
-            self._reserve(4)
-            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
-            self._buff_i += 4
+        elif 0xDC <= b <= 0xDD:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-        elif b == 0xDE:
-            self._reserve(2)
-            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
-            self._buff_i += 2
-            if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
-            typ = TYPE_MAP
-        elif b == 0xDF:
-            self._reserve(4)
-            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
-            self._buff_i += 4
+                raise ValueError(
+                    "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
+                )
+        elif 0xDE <= b <= 0xDF:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
-            typ = TYPE_MAP
+                raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
         else:
             raise FormatError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
     def _unpack(self, execute=EX_CONSTRUCT):
-        typ, n, obj = self._read_header(execute)
+        typ, n, obj = self._read_header()
 
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:
@@ -743,7 +665,7 @@ class Packer(object):
     """
     MessagePack Packer
 
-    Usage:
+    Usage::
 
         packer = Packer()
         astream.write(packer.pack(a))
@@ -783,6 +705,29 @@ class Packer(object):
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
+
+    Example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            process(o)
+
+    Example of streaming deserialize from socket::
+
+        unpacker = Unpacker()
+        while True:
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                process(o)
+
+    Raises ``ExtraData`` when *packed* contains extra bytes.
+    Raises ``OutOfData`` when *packed* is incomplete.
+    Raises ``FormatError`` when *packed* is not valid msgpack.
+    Raises ``StackError`` when *packed* contains too nested.
+    Other exceptions can be raised during unpacking.
     """
 
     def __init__(
@@ -920,7 +865,7 @@ def _pack(
                     len(obj), dict_iteritems(obj), nest_limit - 1
                 )
 
-            if self._datetime and check(obj, _DateTime):
+            if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None:
                 obj = Timestamp.from_datetime(obj)
                 default_used = 1
                 continue
@@ -929,6 +874,10 @@ def _pack(
                 obj = self._default(obj)
                 default_used = 1
                 continue
+
+            if self._datetime and check(obj, _DateTime):
+                raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,))
+
             raise TypeError("Cannot serialize %r" % (obj,))
 
     def pack(self, obj):
diff --git a/src/pip/_vendor/packaging.pyi b/src/pip/_vendor/packaging.pyi
deleted file mode 100644
index 3458a3d6374..00000000000
--- a/src/pip/_vendor/packaging.pyi
+++ /dev/null
@@ -1 +0,0 @@
-from packaging import *
\ No newline at end of file
diff --git a/src/pip/_vendor/packaging/__about__.py b/src/pip/_vendor/packaging/__about__.py
index 2d39193b051..3551bc2d298 100644
--- a/src/pip/_vendor/packaging/__about__.py
+++ b/src/pip/_vendor/packaging/__about__.py
@@ -1,7 +1,6 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 __all__ = [
     "__title__",
@@ -18,7 +17,7 @@
 __summary__ = "Core utilities for Python packages"
 __uri__ = "https://github.com/pypa/packaging"
 
-__version__ = "20.8"
+__version__ = "21.3"
 
 __author__ = "Donald Stufft and individual contributors"
 __email__ = "donald@stufft.io"
diff --git a/src/pip/_vendor/packaging/__init__.py b/src/pip/_vendor/packaging/__init__.py
index a0cf67df524..3c50c5dcfee 100644
--- a/src/pip/_vendor/packaging/__init__.py
+++ b/src/pip/_vendor/packaging/__init__.py
@@ -1,7 +1,6 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 from .__about__ import (
     __author__,
diff --git a/src/pip/_vendor/packaging/_compat.py b/src/pip/_vendor/packaging/_compat.py
deleted file mode 100644
index e54bd4ede87..00000000000
--- a/src/pip/_vendor/packaging/_compat.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# This file is dual licensed under the terms of the Apache License, Version
-# 2.0, and the BSD License. See the LICENSE file in the root of this repository
-# for complete details.
-from __future__ import absolute_import, division, print_function
-
-import sys
-
-from ._typing import TYPE_CHECKING
-
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import Any, Dict, Tuple, Type
-
-
-PY2 = sys.version_info[0] == 2
-PY3 = sys.version_info[0] == 3
-
-# flake8: noqa
-
-if PY3:
-    string_types = (str,)
-else:
-    string_types = (basestring,)
-
-
-def with_metaclass(meta, *bases):
-    # type: (Type[Any], Tuple[Type[Any], ...]) -> Any
-    """
-    Create a base class with a metaclass.
-    """
-    # This requires a bit of explanation: the basic idea is to make a dummy
-    # metaclass for one level of class instantiation that replaces itself with
-    # the actual metaclass.
-    class metaclass(meta):  # type: ignore
-        def __new__(cls, name, this_bases, d):
-            # type: (Type[Any], str, Tuple[Any], Dict[Any, Any]) -> Any
-            return meta(name, bases, d)
-
-    return type.__new__(metaclass, "temporary_class", (), {})
diff --git a/src/pip/_vendor/packaging/_manylinux.py b/src/pip/_vendor/packaging/_manylinux.py
new file mode 100644
index 00000000000..4c379aa6f69
--- /dev/null
+++ b/src/pip/_vendor/packaging/_manylinux.py
@@ -0,0 +1,301 @@
+import collections
+import functools
+import os
+import re
+import struct
+import sys
+import warnings
+from typing import IO, Dict, Iterator, NamedTuple, Optional, Tuple
+
+
+# Python does not provide platform information at sufficient granularity to
+# identify the architecture of the running executable in some cases, so we
+# determine it dynamically by reading the information from the running
+# process. This only applies on Linux, which uses the ELF format.
+class _ELFFileHeader:
+    # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header
+    class _InvalidELFFileHeader(ValueError):
+        """
+        An invalid ELF file header was found.
+        """
+
+    ELF_MAGIC_NUMBER = 0x7F454C46
+    ELFCLASS32 = 1
+    ELFCLASS64 = 2
+    ELFDATA2LSB = 1
+    ELFDATA2MSB = 2
+    EM_386 = 3
+    EM_S390 = 22
+    EM_ARM = 40
+    EM_X86_64 = 62
+    EF_ARM_ABIMASK = 0xFF000000
+    EF_ARM_ABI_VER5 = 0x05000000
+    EF_ARM_ABI_FLOAT_HARD = 0x00000400
+
+    def __init__(self, file: IO[bytes]) -> None:
+        def unpack(fmt: str) -> int:
+            try:
+                data = file.read(struct.calcsize(fmt))
+                result: Tuple[int, ...] = struct.unpack(fmt, data)
+            except struct.error:
+                raise _ELFFileHeader._InvalidELFFileHeader()
+            return result[0]
+
+        self.e_ident_magic = unpack(">I")
+        if self.e_ident_magic != self.ELF_MAGIC_NUMBER:
+            raise _ELFFileHeader._InvalidELFFileHeader()
+        self.e_ident_class = unpack("B")
+        if self.e_ident_class not in {self.ELFCLASS32, self.ELFCLASS64}:
+            raise _ELFFileHeader._InvalidELFFileHeader()
+        self.e_ident_data = unpack("B")
+        if self.e_ident_data not in {self.ELFDATA2LSB, self.ELFDATA2MSB}:
+            raise _ELFFileHeader._InvalidELFFileHeader()
+        self.e_ident_version = unpack("B")
+        self.e_ident_osabi = unpack("B")
+        self.e_ident_abiversion = unpack("B")
+        self.e_ident_pad = file.read(7)
+        format_h = "H"
+        format_i = "I"
+        format_q = "Q"
+        format_p = format_i if self.e_ident_class == self.ELFCLASS32 else format_q
+        self.e_type = unpack(format_h)
+        self.e_machine = unpack(format_h)
+        self.e_version = unpack(format_i)
+        self.e_entry = unpack(format_p)
+        self.e_phoff = unpack(format_p)
+        self.e_shoff = unpack(format_p)
+        self.e_flags = unpack(format_i)
+        self.e_ehsize = unpack(format_h)
+        self.e_phentsize = unpack(format_h)
+        self.e_phnum = unpack(format_h)
+        self.e_shentsize = unpack(format_h)
+        self.e_shnum = unpack(format_h)
+        self.e_shstrndx = unpack(format_h)
+
+
+def _get_elf_header() -> Optional[_ELFFileHeader]:
+    try:
+        with open(sys.executable, "rb") as f:
+            elf_header = _ELFFileHeader(f)
+    except (OSError, TypeError, _ELFFileHeader._InvalidELFFileHeader):
+        return None
+    return elf_header
+
+
+def _is_linux_armhf() -> bool:
+    # hard-float ABI can be detected from the ELF header of the running
+    # process
+    # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf
+    elf_header = _get_elf_header()
+    if elf_header is None:
+        return False
+    result = elf_header.e_ident_class == elf_header.ELFCLASS32
+    result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB
+    result &= elf_header.e_machine == elf_header.EM_ARM
+    result &= (
+        elf_header.e_flags & elf_header.EF_ARM_ABIMASK
+    ) == elf_header.EF_ARM_ABI_VER5
+    result &= (
+        elf_header.e_flags & elf_header.EF_ARM_ABI_FLOAT_HARD
+    ) == elf_header.EF_ARM_ABI_FLOAT_HARD
+    return result
+
+
+def _is_linux_i686() -> bool:
+    elf_header = _get_elf_header()
+    if elf_header is None:
+        return False
+    result = elf_header.e_ident_class == elf_header.ELFCLASS32
+    result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB
+    result &= elf_header.e_machine == elf_header.EM_386
+    return result
+
+
+def _have_compatible_abi(arch: str) -> bool:
+    if arch == "armv7l":
+        return _is_linux_armhf()
+    if arch == "i686":
+        return _is_linux_i686()
+    return arch in {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x"}
+
+
+# If glibc ever changes its major version, we need to know what the last
+# minor version was, so we can build the complete list of all versions.
+# For now, guess what the highest minor version might be, assume it will
+# be 50 for testing. Once this actually happens, update the dictionary
+# with the actual value.
+_LAST_GLIBC_MINOR: Dict[int, int] = collections.defaultdict(lambda: 50)
+
+
+class _GLibCVersion(NamedTuple):
+    major: int
+    minor: int
+
+
+def _glibc_version_string_confstr() -> Optional[str]:
+    """
+    Primary implementation of glibc_version_string using os.confstr.
+    """
+    # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely
+    # to be broken or missing. This strategy is used in the standard library
+    # platform module.
+    # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183
+    try:
+        # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17".
+        version_string = os.confstr("CS_GNU_LIBC_VERSION")
+        assert version_string is not None
+        _, version = version_string.split()
+    except (AssertionError, AttributeError, OSError, ValueError):
+        # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)...
+        return None
+    return version
+
+
+def _glibc_version_string_ctypes() -> Optional[str]:
+    """
+    Fallback implementation of glibc_version_string using ctypes.
+    """
+    try:
+        import ctypes
+    except ImportError:
+        return None
+
+    # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
+    # manpage says, "If filename is NULL, then the returned handle is for the
+    # main program". This way we can let the linker do the work to figure out
+    # which libc our process is actually using.
+    #
+    # We must also handle the special case where the executable is not a
+    # dynamically linked executable. This can occur when using musl libc,
+    # for example. In this situation, dlopen() will error, leading to an
+    # OSError. Interestingly, at least in the case of musl, there is no
+    # errno set on the OSError. The single string argument used to construct
+    # OSError comes from libc itself and is therefore not portable to
+    # hard code here. In any case, failure to call dlopen() means we
+    # can proceed, so we bail on our attempt.
+    try:
+        process_namespace = ctypes.CDLL(None)
+    except OSError:
+        return None
+
+    try:
+        gnu_get_libc_version = process_namespace.gnu_get_libc_version
+    except AttributeError:
+        # Symbol doesn't exist -> therefore, we are not linked to
+        # glibc.
+        return None
+
+    # Call gnu_get_libc_version, which returns a string like "2.5"
+    gnu_get_libc_version.restype = ctypes.c_char_p
+    version_str: str = gnu_get_libc_version()
+    # py2 / py3 compatibility:
+    if not isinstance(version_str, str):
+        version_str = version_str.decode("ascii")
+
+    return version_str
+
+
+def _glibc_version_string() -> Optional[str]:
+    """Returns glibc version string, or None if not using glibc."""
+    return _glibc_version_string_confstr() or _glibc_version_string_ctypes()
+
+
+def _parse_glibc_version(version_str: str) -> Tuple[int, int]:
+    """Parse glibc version.
+
+    We use a regexp instead of str.split because we want to discard any
+    random junk that might come after the minor version -- this might happen
+    in patched/forked versions of glibc (e.g. Linaro's version of glibc
+    uses version strings like "2.20-2014.11"). See gh-3588.
+    """
+    m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str)
+    if not m:
+        warnings.warn(
+            "Expected glibc version with 2 components major.minor,"
+            " got: %s" % version_str,
+            RuntimeWarning,
+        )
+        return -1, -1
+    return int(m.group("major")), int(m.group("minor"))
+
+
+@functools.lru_cache()
+def _get_glibc_version() -> Tuple[int, int]:
+    version_str = _glibc_version_string()
+    if version_str is None:
+        return (-1, -1)
+    return _parse_glibc_version(version_str)
+
+
+# From PEP 513, PEP 600
+def _is_compatible(name: str, arch: str, version: _GLibCVersion) -> bool:
+    sys_glibc = _get_glibc_version()
+    if sys_glibc < version:
+        return False
+    # Check for presence of _manylinux module.
+    try:
+        import _manylinux  # noqa
+    except ImportError:
+        return True
+    if hasattr(_manylinux, "manylinux_compatible"):
+        result = _manylinux.manylinux_compatible(version[0], version[1], arch)
+        if result is not None:
+            return bool(result)
+        return True
+    if version == _GLibCVersion(2, 5):
+        if hasattr(_manylinux, "manylinux1_compatible"):
+            return bool(_manylinux.manylinux1_compatible)
+    if version == _GLibCVersion(2, 12):
+        if hasattr(_manylinux, "manylinux2010_compatible"):
+            return bool(_manylinux.manylinux2010_compatible)
+    if version == _GLibCVersion(2, 17):
+        if hasattr(_manylinux, "manylinux2014_compatible"):
+            return bool(_manylinux.manylinux2014_compatible)
+    return True
+
+
+_LEGACY_MANYLINUX_MAP = {
+    # CentOS 7 w/ glibc 2.17 (PEP 599)
+    (2, 17): "manylinux2014",
+    # CentOS 6 w/ glibc 2.12 (PEP 571)
+    (2, 12): "manylinux2010",
+    # CentOS 5 w/ glibc 2.5 (PEP 513)
+    (2, 5): "manylinux1",
+}
+
+
+def platform_tags(linux: str, arch: str) -> Iterator[str]:
+    if not _have_compatible_abi(arch):
+        return
+    # Oldest glibc to be supported regardless of architecture is (2, 17).
+    too_old_glibc2 = _GLibCVersion(2, 16)
+    if arch in {"x86_64", "i686"}:
+        # On x86/i686 also oldest glibc to be supported is (2, 5).
+        too_old_glibc2 = _GLibCVersion(2, 4)
+    current_glibc = _GLibCVersion(*_get_glibc_version())
+    glibc_max_list = [current_glibc]
+    # We can assume compatibility across glibc major versions.
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=24636
+    #
+    # Build a list of maximum glibc versions so that we can
+    # output the canonical list of all glibc from current_glibc
+    # down to too_old_glibc2, including all intermediary versions.
+    for glibc_major in range(current_glibc.major - 1, 1, -1):
+        glibc_minor = _LAST_GLIBC_MINOR[glibc_major]
+        glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor))
+    for glibc_max in glibc_max_list:
+        if glibc_max.major == too_old_glibc2.major:
+            min_minor = too_old_glibc2.minor
+        else:
+            # For other glibc major versions oldest supported is (x, 0).
+            min_minor = -1
+        for glibc_minor in range(glibc_max.minor, min_minor, -1):
+            glibc_version = _GLibCVersion(glibc_max.major, glibc_minor)
+            tag = "manylinux_{}_{}".format(*glibc_version)
+            if _is_compatible(tag, arch, glibc_version):
+                yield linux.replace("linux", tag)
+            # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags.
+            if glibc_version in _LEGACY_MANYLINUX_MAP:
+                legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version]
+                if _is_compatible(legacy_tag, arch, glibc_version):
+                    yield linux.replace("linux", legacy_tag)
diff --git a/src/pip/_vendor/packaging/_musllinux.py b/src/pip/_vendor/packaging/_musllinux.py
new file mode 100644
index 00000000000..8ac3059ba3c
--- /dev/null
+++ b/src/pip/_vendor/packaging/_musllinux.py
@@ -0,0 +1,136 @@
+"""PEP 656 support.
+
+This module implements logic to detect if the currently running Python is
+linked against musl, and what musl version is used.
+"""
+
+import contextlib
+import functools
+import operator
+import os
+import re
+import struct
+import subprocess
+import sys
+from typing import IO, Iterator, NamedTuple, Optional, Tuple
+
+
+def _read_unpacked(f: IO[bytes], fmt: str) -> Tuple[int, ...]:
+    return struct.unpack(fmt, f.read(struct.calcsize(fmt)))
+
+
+def _parse_ld_musl_from_elf(f: IO[bytes]) -> Optional[str]:
+    """Detect musl libc location by parsing the Python executable.
+
+    Based on: https://gist.github.com/lyssdod/f51579ae8d93c8657a5564aefc2ffbca
+    ELF header: https://refspecs.linuxfoundation.org/elf/gabi4+/ch4.eheader.html
+    """
+    f.seek(0)
+    try:
+        ident = _read_unpacked(f, "16B")
+    except struct.error:
+        return None
+    if ident[:4] != tuple(b"\x7fELF"):  # Invalid magic, not ELF.
+        return None
+    f.seek(struct.calcsize("HHI"), 1)  # Skip file type, machine, and version.
+
+    try:
+        # e_fmt: Format for program header.
+        # p_fmt: Format for section header.
+        # p_idx: Indexes to find p_type, p_offset, and p_filesz.
+        e_fmt, p_fmt, p_idx = {
+            1: ("IIIIHHH", "IIIIIIII", (0, 1, 4)),  # 32-bit.
+            2: ("QQQIHHH", "IIQQQQQQ", (0, 2, 5)),  # 64-bit.
+        }[ident[4]]
+    except KeyError:
+        return None
+    else:
+        p_get = operator.itemgetter(*p_idx)
+
+    # Find the interpreter section and return its content.
+    try:
+        _, e_phoff, _, _, _, e_phentsize, e_phnum = _read_unpacked(f, e_fmt)
+    except struct.error:
+        return None
+    for i in range(e_phnum + 1):
+        f.seek(e_phoff + e_phentsize * i)
+        try:
+            p_type, p_offset, p_filesz = p_get(_read_unpacked(f, p_fmt))
+        except struct.error:
+            return None
+        if p_type != 3:  # Not PT_INTERP.
+            continue
+        f.seek(p_offset)
+        interpreter = os.fsdecode(f.read(p_filesz)).strip("\0")
+        if "musl" not in interpreter:
+            return None
+        return interpreter
+    return None
+
+
+class _MuslVersion(NamedTuple):
+    major: int
+    minor: int
+
+
+def _parse_musl_version(output: str) -> Optional[_MuslVersion]:
+    lines = [n for n in (n.strip() for n in output.splitlines()) if n]
+    if len(lines) < 2 or lines[0][:4] != "musl":
+        return None
+    m = re.match(r"Version (\d+)\.(\d+)", lines[1])
+    if not m:
+        return None
+    return _MuslVersion(major=int(m.group(1)), minor=int(m.group(2)))
+
+
+@functools.lru_cache()
+def _get_musl_version(executable: str) -> Optional[_MuslVersion]:
+    """Detect currently-running musl runtime version.
+
+    This is done by checking the specified executable's dynamic linking
+    information, and invoking the loader to parse its output for a version
+    string. If the loader is musl, the output would be something like::
+
+        musl libc (x86_64)
+        Version 1.2.2
+        Dynamic Program Loader
+    """
+    with contextlib.ExitStack() as stack:
+        try:
+            f = stack.enter_context(open(executable, "rb"))
+        except OSError:
+            return None
+        ld = _parse_ld_musl_from_elf(f)
+    if not ld:
+        return None
+    proc = subprocess.run([ld], stderr=subprocess.PIPE, universal_newlines=True)
+    return _parse_musl_version(proc.stderr)
+
+
+def platform_tags(arch: str) -> Iterator[str]:
+    """Generate musllinux tags compatible to the current platform.
+
+    :param arch: Should be the part of platform tag after the ``linux_``
+        prefix, e.g. ``x86_64``. The ``linux_`` prefix is assumed as a
+        prerequisite for the current platform to be musllinux-compatible.
+
+    :returns: An iterator of compatible musllinux tags.
+    """
+    sys_musl = _get_musl_version(sys.executable)
+    if sys_musl is None:  # Python not dynamically linked against musl.
+        return
+    for minor in range(sys_musl.minor, -1, -1):
+        yield f"musllinux_{sys_musl.major}_{minor}_{arch}"
+
+
+if __name__ == "__main__":  # pragma: no cover
+    import sysconfig
+
+    plat = sysconfig.get_platform()
+    assert plat.startswith("linux-"), "not linux"
+
+    print("plat:", plat)
+    print("musl:", _get_musl_version(sys.executable))
+    print("tags:", end=" ")
+    for t in platform_tags(re.sub(r"[.-]", "_", plat.split("-", 1)[-1])):
+        print(t, end="\n      ")
diff --git a/src/pip/_vendor/packaging/_structures.py b/src/pip/_vendor/packaging/_structures.py
index 800d5c5588c..90a6465f968 100644
--- a/src/pip/_vendor/packaging/_structures.py
+++ b/src/pip/_vendor/packaging/_structures.py
@@ -1,85 +1,60 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 
-class InfinityType(object):
-    def __repr__(self):
-        # type: () -> str
+class InfinityType:
+    def __repr__(self) -> str:
         return "Infinity"
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(repr(self))
 
-    def __lt__(self, other):
-        # type: (object) -> bool
+    def __lt__(self, other: object) -> bool:
         return False
 
-    def __le__(self, other):
-        # type: (object) -> bool
+    def __le__(self, other: object) -> bool:
         return False
 
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         return isinstance(other, self.__class__)
 
-    def __ne__(self, other):
-        # type: (object) -> bool
-        return not isinstance(other, self.__class__)
-
-    def __gt__(self, other):
-        # type: (object) -> bool
+    def __gt__(self, other: object) -> bool:
         return True
 
-    def __ge__(self, other):
-        # type: (object) -> bool
+    def __ge__(self, other: object) -> bool:
         return True
 
-    def __neg__(self):
-        # type: (object) -> NegativeInfinityType
+    def __neg__(self: object) -> "NegativeInfinityType":
         return NegativeInfinity
 
 
 Infinity = InfinityType()
 
 
-class NegativeInfinityType(object):
-    def __repr__(self):
-        # type: () -> str
+class NegativeInfinityType:
+    def __repr__(self) -> str:
         return "-Infinity"
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(repr(self))
 
-    def __lt__(self, other):
-        # type: (object) -> bool
+    def __lt__(self, other: object) -> bool:
         return True
 
-    def __le__(self, other):
-        # type: (object) -> bool
+    def __le__(self, other: object) -> bool:
         return True
 
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         return isinstance(other, self.__class__)
 
-    def __ne__(self, other):
-        # type: (object) -> bool
-        return not isinstance(other, self.__class__)
-
-    def __gt__(self, other):
-        # type: (object) -> bool
+    def __gt__(self, other: object) -> bool:
         return False
 
-    def __ge__(self, other):
-        # type: (object) -> bool
+    def __ge__(self, other: object) -> bool:
         return False
 
-    def __neg__(self):
-        # type: (object) -> InfinityType
+    def __neg__(self: object) -> InfinityType:
         return Infinity
 
 
diff --git a/src/pip/_vendor/packaging/_typing.py b/src/pip/_vendor/packaging/_typing.py
deleted file mode 100644
index 2846133bd8d..00000000000
--- a/src/pip/_vendor/packaging/_typing.py
+++ /dev/null
@@ -1,48 +0,0 @@
-"""For neatly implementing static typing in packaging.
-
-`mypy` - the static type analysis tool we use - uses the `typing` module, which
-provides core functionality fundamental to mypy's functioning.
-
-Generally, `typing` would be imported at runtime and used in that fashion -
-it acts as a no-op at runtime and does not have any run-time overhead by
-design.
-
-As it turns out, `typing` is not vendorable - it uses separate sources for
-Python 2/Python 3. Thus, this codebase can not expect it to be present.
-To work around this, mypy allows the typing import to be behind a False-y
-optional to prevent it from running at runtime and type-comments can be used
-to remove the need for the types to be accessible directly during runtime.
-
-This module provides the False-y guard in a nicely named fashion so that a
-curious maintainer can reach here to read this.
-
-In packaging, all static-typing related imports should be guarded as follows:
-
-    from pip._vendor.packaging._typing import TYPE_CHECKING
-
-    if TYPE_CHECKING:
-        from typing import ...
-
-Ref: https://github.com/python/mypy/issues/3216
-"""
-
-__all__ = ["TYPE_CHECKING", "cast"]
-
-# The TYPE_CHECKING constant defined by the typing module is False at runtime
-# but True while type checking.
-if False:  # pragma: no cover
-    from typing import TYPE_CHECKING
-else:
-    TYPE_CHECKING = False
-
-# typing's cast syntax requires calling typing.cast at runtime, but we don't
-# want to import typing at runtime. Here, we inform the type checkers that
-# we're importing `typing.cast` as `cast` and re-implement typing.cast's
-# runtime behavior in a block that is ignored by type checkers.
-if TYPE_CHECKING:  # pragma: no cover
-    # not executed at runtime
-    from typing import cast
-else:
-    # executed at runtime
-    def cast(type_, value):  # noqa
-        return value
diff --git a/src/pip/_vendor/packaging/markers.py b/src/pip/_vendor/packaging/markers.py
index ed642b01fcc..540e7a4dc79 100644
--- a/src/pip/_vendor/packaging/markers.py
+++ b/src/pip/_vendor/packaging/markers.py
@@ -1,26 +1,26 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 import operator
 import os
 import platform
 import sys
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from pip._vendor.pyparsing import (  # noqa: N817
+    Forward,
+    Group,
+    Literal as L,
+    ParseException,
+    ParseResults,
+    QuotedString,
+    ZeroOrMore,
+    stringEnd,
+    stringStart,
+)
 
-from pip._vendor.pyparsing import ParseException, ParseResults, stringStart, stringEnd
-from pip._vendor.pyparsing import ZeroOrMore, Group, Forward, QuotedString
-from pip._vendor.pyparsing import Literal as L  # noqa
-
-from ._compat import string_types
-from ._typing import TYPE_CHECKING
-from .specifiers import Specifier, InvalidSpecifier
-
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import Any, Callable, Dict, List, Optional, Tuple, Union
-
-    Operator = Callable[[str, str], bool]
-
+from .specifiers import InvalidSpecifier, Specifier
 
 __all__ = [
     "InvalidMarker",
@@ -30,6 +30,8 @@
     "default_environment",
 ]
 
+Operator = Callable[[str, str], bool]
+
 
 class InvalidMarker(ValueError):
     """
@@ -50,39 +52,32 @@ class UndefinedEnvironmentName(ValueError):
     """
 
 
-class Node(object):
-    def __init__(self, value):
-        # type: (Any) -> None
+class Node:
+    def __init__(self, value: Any) -> None:
         self.value = value
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return str(self.value)
 
-    def __repr__(self):
-        # type: () -> str
-        return "<{0}({1!r})>".format(self.__class__.__name__, str(self))
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}('{self}')>"
 
-    def serialize(self):
-        # type: () -> str
+    def serialize(self) -> str:
         raise NotImplementedError
 
 
 class Variable(Node):
-    def serialize(self):
-        # type: () -> str
+    def serialize(self) -> str:
         return str(self)
 
 
 class Value(Node):
-    def serialize(self):
-        # type: () -> str
-        return '"{0}"'.format(self)
+    def serialize(self) -> str:
+        return f'"{self}"'
 
 
 class Op(Node):
-    def serialize(self):
-        # type: () -> str
+    def serialize(self) -> str:
         return str(self)
 
 
@@ -143,18 +138,18 @@ def serialize(self):
 MARKER = stringStart + MARKER_EXPR + stringEnd
 
 
-def _coerce_parse_result(results):
-    # type: (Union[ParseResults, List[Any]]) -> List[Any]
+def _coerce_parse_result(results: Union[ParseResults, List[Any]]) -> List[Any]:
     if isinstance(results, ParseResults):
         return [_coerce_parse_result(i) for i in results]
     else:
         return results
 
 
-def _format_marker(marker, first=True):
-    # type: (Union[List[str], Tuple[Node, ...], str], Optional[bool]) -> str
+def _format_marker(
+    marker: Union[List[str], Tuple[Node, ...], str], first: Optional[bool] = True
+) -> str:
 
-    assert isinstance(marker, (list, tuple, string_types))
+    assert isinstance(marker, (list, tuple, str))
 
     # Sometimes we have a structure like [[...]] which is a single item list
     # where the single item is itself it's own list. In that case we want skip
@@ -179,7 +174,7 @@ def _format_marker(marker, first=True):
         return marker
 
 
-_operators = {
+_operators: Dict[str, Operator] = {
     "in": lambda lhs, rhs: lhs in rhs,
     "not in": lambda lhs, rhs: lhs not in rhs,
     "<": operator.lt,
@@ -188,11 +183,10 @@ def _format_marker(marker, first=True):
     "!=": operator.ne,
     ">=": operator.ge,
     ">": operator.gt,
-}  # type: Dict[str, Operator]
+}
 
 
-def _eval_op(lhs, op, rhs):
-    # type: (str, Op, str) -> bool
+def _eval_op(lhs: str, op: Op, rhs: str) -> bool:
     try:
         spec = Specifier("".join([op.serialize(), rhs]))
     except InvalidSpecifier:
@@ -200,40 +194,36 @@ def _eval_op(lhs, op, rhs):
     else:
         return spec.contains(lhs)
 
-    oper = _operators.get(op.serialize())  # type: Optional[Operator]
+    oper: Optional[Operator] = _operators.get(op.serialize())
     if oper is None:
-        raise UndefinedComparison(
-            "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs)
-        )
+        raise UndefinedComparison(f"Undefined {op!r} on {lhs!r} and {rhs!r}.")
 
     return oper(lhs, rhs)
 
 
-class Undefined(object):
+class Undefined:
     pass
 
 
 _undefined = Undefined()
 
 
-def _get_env(environment, name):
-    # type: (Dict[str, str], str) -> str
-    value = environment.get(name, _undefined)  # type: Union[str, Undefined]
+def _get_env(environment: Dict[str, str], name: str) -> str:
+    value: Union[str, Undefined] = environment.get(name, _undefined)
 
     if isinstance(value, Undefined):
         raise UndefinedEnvironmentName(
-            "{0!r} does not exist in evaluation environment.".format(name)
+            f"{name!r} does not exist in evaluation environment."
         )
 
     return value
 
 
-def _evaluate_markers(markers, environment):
-    # type: (List[Any], Dict[str, str]) -> bool
-    groups = [[]]  # type: List[List[bool]]
+def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool:
+    groups: List[List[bool]] = [[]]
 
     for marker in markers:
-        assert isinstance(marker, (list, tuple, string_types))
+        assert isinstance(marker, (list, tuple, str))
 
         if isinstance(marker, list):
             groups[-1].append(_evaluate_markers(marker, environment))
@@ -256,8 +246,7 @@ def _evaluate_markers(markers, environment):
     return any(all(item) for item in groups)
 
 
-def format_full_version(info):
-    # type: (sys._version_info) -> str
+def format_full_version(info: "sys._version_info") -> str:
     version = "{0.major}.{0.minor}.{0.micro}".format(info)
     kind = info.releaselevel
     if kind != "final":
@@ -265,18 +254,9 @@ def format_full_version(info):
     return version
 
 
-def default_environment():
-    # type: () -> Dict[str, str]
-    if hasattr(sys, "implementation"):
-        # Ignoring the `sys.implementation` reference for type checking due to
-        # mypy not liking that the attribute doesn't exist in Python 2.7 when
-        # run with the `--py27` flag.
-        iver = format_full_version(sys.implementation.version)  # type: ignore
-        implementation_name = sys.implementation.name  # type: ignore
-    else:
-        iver = "0"
-        implementation_name = ""
-
+def default_environment() -> Dict[str, str]:
+    iver = format_full_version(sys.implementation.version)
+    implementation_name = sys.implementation.name
     return {
         "implementation_name": implementation_name,
         "implementation_version": iver,
@@ -292,27 +272,23 @@ def default_environment():
     }
 
 
-class Marker(object):
-    def __init__(self, marker):
-        # type: (str) -> None
+class Marker:
+    def __init__(self, marker: str) -> None:
         try:
             self._markers = _coerce_parse_result(MARKER.parseString(marker))
         except ParseException as e:
-            err_str = "Invalid marker: {0!r}, parse error at {1!r}".format(
-                marker, marker[e.loc : e.loc + 8]
+            raise InvalidMarker(
+                f"Invalid marker: {marker!r}, parse error at "
+                f"{marker[e.loc : e.loc + 8]!r}"
             )
-            raise InvalidMarker(err_str)
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return _format_marker(self._markers)
 
-    def __repr__(self):
-        # type: () -> str
-        return "".format(str(self))
+    def __repr__(self) -> str:
+        return f""
 
-    def evaluate(self, environment=None):
-        # type: (Optional[Dict[str, str]]) -> bool
+    def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool:
         """Evaluate a marker.
 
         Return the boolean from evaluating the given marker against the
diff --git a/src/pip/_vendor/packaging/requirements.py b/src/pip/_vendor/packaging/requirements.py
index df7f41d2c08..1eab7dd66d9 100644
--- a/src/pip/_vendor/packaging/requirements.py
+++ b/src/pip/_vendor/packaging/requirements.py
@@ -1,29 +1,28 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
-import string
 import re
-import sys
-
-from pip._vendor.pyparsing import stringStart, stringEnd, originalTextFor, ParseException
-from pip._vendor.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine
-from pip._vendor.pyparsing import Literal as L  # noqa
+import string
+import urllib.parse
+from typing import List, Optional as TOptional, Set
+
+from pip._vendor.pyparsing import (  # noqa
+    Combine,
+    Literal as L,
+    Optional,
+    ParseException,
+    Regex,
+    Word,
+    ZeroOrMore,
+    originalTextFor,
+    stringEnd,
+    stringStart,
+)
 
-from ._typing import TYPE_CHECKING
 from .markers import MARKER_EXPR, Marker
 from .specifiers import LegacySpecifier, Specifier, SpecifierSet
 
-if sys.version_info[0] >= 3:
-    from urllib import parse as urlparse  # pragma: no cover
-else:  # pragma: no cover
-    import urlparse
-
-
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import List, Optional as TOptional, Set
-
 
 class InvalidRequirement(ValueError):
     """
@@ -61,7 +60,7 @@ class InvalidRequirement(ValueError):
 VERSION_MANY = Combine(
     VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False
 )("_raw_spec")
-_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY))
+_VERSION_SPEC = Optional((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)
 _VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "")
 
 VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier")
@@ -85,7 +84,7 @@ class InvalidRequirement(ValueError):
 REQUIREMENT.parseString("x[]")
 
 
-class Requirement(object):
+class Requirement:
     """Parse a requirement.
 
     Parse a given requirement string into its parts, such as name, specifier,
@@ -98,54 +97,50 @@ class Requirement(object):
     #       the thing as well as the version? What about the markers?
     # TODO: Can we normalize the name and extra name?
 
-    def __init__(self, requirement_string):
-        # type: (str) -> None
+    def __init__(self, requirement_string: str) -> None:
         try:
             req = REQUIREMENT.parseString(requirement_string)
         except ParseException as e:
             raise InvalidRequirement(
-                'Parse error at "{0!r}": {1}'.format(
-                    requirement_string[e.loc : e.loc + 8], e.msg
-                )
+                f'Parse error at "{ requirement_string[e.loc : e.loc + 8]!r}": {e.msg}'
             )
 
-        self.name = req.name  # type: str
+        self.name: str = req.name
         if req.url:
-            parsed_url = urlparse.urlparse(req.url)
+            parsed_url = urllib.parse.urlparse(req.url)
             if parsed_url.scheme == "file":
-                if urlparse.urlunparse(parsed_url) != req.url:
+                if urllib.parse.urlunparse(parsed_url) != req.url:
                     raise InvalidRequirement("Invalid URL given")
             elif not (parsed_url.scheme and parsed_url.netloc) or (
                 not parsed_url.scheme and not parsed_url.netloc
             ):
-                raise InvalidRequirement("Invalid URL: {0}".format(req.url))
-            self.url = req.url  # type: TOptional[str]
+                raise InvalidRequirement(f"Invalid URL: {req.url}")
+            self.url: TOptional[str] = req.url
         else:
             self.url = None
-        self.extras = set(req.extras.asList() if req.extras else [])  # type: Set[str]
-        self.specifier = SpecifierSet(req.specifier)  # type: SpecifierSet
-        self.marker = req.marker if req.marker else None  # type: TOptional[Marker]
+        self.extras: Set[str] = set(req.extras.asList() if req.extras else [])
+        self.specifier: SpecifierSet = SpecifierSet(req.specifier)
+        self.marker: TOptional[Marker] = req.marker if req.marker else None
 
-    def __str__(self):
-        # type: () -> str
-        parts = [self.name]  # type: List[str]
+    def __str__(self) -> str:
+        parts: List[str] = [self.name]
 
         if self.extras:
-            parts.append("[{0}]".format(",".join(sorted(self.extras))))
+            formatted_extras = ",".join(sorted(self.extras))
+            parts.append(f"[{formatted_extras}]")
 
         if self.specifier:
             parts.append(str(self.specifier))
 
         if self.url:
-            parts.append("@ {0}".format(self.url))
+            parts.append(f"@ {self.url}")
             if self.marker:
                 parts.append(" ")
 
         if self.marker:
-            parts.append("; {0}".format(self.marker))
+            parts.append(f"; {self.marker}")
 
         return "".join(parts)
 
-    def __repr__(self):
-        # type: () -> str
-        return "".format(str(self))
+    def __repr__(self) -> str:
+        return f""
diff --git a/src/pip/_vendor/packaging/specifiers.py b/src/pip/_vendor/packaging/specifiers.py
index a42cbfef332..0e218a6f9f7 100644
--- a/src/pip/_vendor/packaging/specifiers.py
+++ b/src/pip/_vendor/packaging/specifiers.py
@@ -1,25 +1,33 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 import abc
 import functools
 import itertools
 import re
 import warnings
+from typing import (
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Pattern,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+)
 
-from ._compat import string_types, with_metaclass
-from ._typing import TYPE_CHECKING
 from .utils import canonicalize_version
-from .version import Version, LegacyVersion, parse
+from .version import LegacyVersion, Version, parse
 
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import List, Dict, Union, Iterable, Iterator, Optional, Callable, Tuple
-
-    ParsedVersion = Union[Version, LegacyVersion]
-    UnparsedVersion = Union[Version, LegacyVersion, str]
-    CallableOperator = Callable[[ParsedVersion, str], bool]
+ParsedVersion = Union[Version, LegacyVersion]
+UnparsedVersion = Union[Version, LegacyVersion, str]
+VersionTypeVar = TypeVar("VersionTypeVar", bound=UnparsedVersion)
+CallableOperator = Callable[[ParsedVersion, str], bool]
 
 
 class InvalidSpecifier(ValueError):
@@ -28,64 +36,51 @@ class InvalidSpecifier(ValueError):
     """
 
 
-class BaseSpecifier(with_metaclass(abc.ABCMeta, object)):  # type: ignore
+class BaseSpecifier(metaclass=abc.ABCMeta):
     @abc.abstractmethod
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         """
         Returns the str representation of this Specifier like object. This
         should be representative of the Specifier itself.
         """
 
     @abc.abstractmethod
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         """
         Returns a hash value for this Specifier like object.
         """
 
     @abc.abstractmethod
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         """
         Returns a boolean representing whether or not the two Specifier like
         objects are equal.
         """
 
-    @abc.abstractmethod
-    def __ne__(self, other):
-        # type: (object) -> bool
-        """
-        Returns a boolean representing whether or not the two Specifier like
-        objects are not equal.
-        """
-
     @abc.abstractproperty
-    def prereleases(self):
-        # type: () -> Optional[bool]
+    def prereleases(self) -> Optional[bool]:
         """
         Returns whether or not pre-releases as a whole are allowed by this
         specifier.
         """
 
     @prereleases.setter
-    def prereleases(self, value):
-        # type: (bool) -> None
+    def prereleases(self, value: bool) -> None:
         """
         Sets whether or not pre-releases as a whole are allowed by this
         specifier.
         """
 
     @abc.abstractmethod
-    def contains(self, item, prereleases=None):
-        # type: (str, Optional[bool]) -> bool
+    def contains(self, item: str, prereleases: Optional[bool] = None) -> bool:
         """
         Determines if the given item is contained within this specifier.
         """
 
     @abc.abstractmethod
-    def filter(self, iterable, prereleases=None):
-        # type: (Iterable[UnparsedVersion], Optional[bool]) -> Iterable[UnparsedVersion]
+    def filter(
+        self, iterable: Iterable[VersionTypeVar], prereleases: Optional[bool] = None
+    ) -> Iterable[VersionTypeVar]:
         """
         Takes an iterable of items and filters them so that only items which
         are contained within this specifier are allowed in it.
@@ -94,48 +89,43 @@ def filter(self, iterable, prereleases=None):
 
 class _IndividualSpecifier(BaseSpecifier):
 
-    _operators = {}  # type: Dict[str, str]
+    _operators: Dict[str, str] = {}
+    _regex: Pattern[str]
 
-    def __init__(self, spec="", prereleases=None):
-        # type: (str, Optional[bool]) -> None
+    def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None:
         match = self._regex.search(spec)
         if not match:
-            raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec))
+            raise InvalidSpecifier(f"Invalid specifier: '{spec}'")
 
-        self._spec = (
+        self._spec: Tuple[str, str] = (
             match.group("operator").strip(),
             match.group("version").strip(),
-        )  # type: Tuple[str, str]
+        )
 
         # Store whether or not this Specifier should accept prereleases
         self._prereleases = prereleases
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         pre = (
-            ", prereleases={0!r}".format(self.prereleases)
+            f", prereleases={self.prereleases!r}"
             if self._prereleases is not None
             else ""
         )
 
-        return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre)
+        return f"<{self.__class__.__name__}({str(self)!r}{pre})>"
 
-    def __str__(self):
-        # type: () -> str
-        return "{0}{1}".format(*self._spec)
+    def __str__(self) -> str:
+        return "{}{}".format(*self._spec)
 
     @property
-    def _canonical_spec(self):
-        # type: () -> Tuple[str, Union[Version, str]]
+    def _canonical_spec(self) -> Tuple[str, str]:
         return self._spec[0], canonicalize_version(self._spec[1])
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(self._canonical_spec)
 
-    def __eq__(self, other):
-        # type: (object) -> bool
-        if isinstance(other, string_types):
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, str):
             try:
                 other = self.__class__(str(other))
             except InvalidSpecifier:
@@ -145,57 +135,39 @@ def __eq__(self, other):
 
         return self._canonical_spec == other._canonical_spec
 
-    def __ne__(self, other):
-        # type: (object) -> bool
-        if isinstance(other, string_types):
-            try:
-                other = self.__class__(str(other))
-            except InvalidSpecifier:
-                return NotImplemented
-        elif not isinstance(other, self.__class__):
-            return NotImplemented
-
-        return self._spec != other._spec
-
-    def _get_operator(self, op):
-        # type: (str) -> CallableOperator
-        operator_callable = getattr(
-            self, "_compare_{0}".format(self._operators[op])
-        )  # type: CallableOperator
+    def _get_operator(self, op: str) -> CallableOperator:
+        operator_callable: CallableOperator = getattr(
+            self, f"_compare_{self._operators[op]}"
+        )
         return operator_callable
 
-    def _coerce_version(self, version):
-        # type: (UnparsedVersion) -> ParsedVersion
+    def _coerce_version(self, version: UnparsedVersion) -> ParsedVersion:
         if not isinstance(version, (LegacyVersion, Version)):
             version = parse(version)
         return version
 
     @property
-    def operator(self):
-        # type: () -> str
+    def operator(self) -> str:
         return self._spec[0]
 
     @property
-    def version(self):
-        # type: () -> str
+    def version(self) -> str:
         return self._spec[1]
 
     @property
-    def prereleases(self):
-        # type: () -> Optional[bool]
+    def prereleases(self) -> Optional[bool]:
         return self._prereleases
 
     @prereleases.setter
-    def prereleases(self, value):
-        # type: (bool) -> None
+    def prereleases(self, value: bool) -> None:
         self._prereleases = value
 
-    def __contains__(self, item):
-        # type: (str) -> bool
+    def __contains__(self, item: str) -> bool:
         return self.contains(item)
 
-    def contains(self, item, prereleases=None):
-        # type: (UnparsedVersion, Optional[bool]) -> bool
+    def contains(
+        self, item: UnparsedVersion, prereleases: Optional[bool] = None
+    ) -> bool:
 
         # Determine if prereleases are to be allowed or not.
         if prereleases is None:
@@ -213,11 +185,12 @@ def contains(self, item, prereleases=None):
 
         # Actually do the comparison to determine if this item is contained
         # within this Specifier or not.
-        operator_callable = self._get_operator(self.operator)  # type: CallableOperator
+        operator_callable: CallableOperator = self._get_operator(self.operator)
         return operator_callable(normalized_item, self.version)
 
-    def filter(self, iterable, prereleases=None):
-        # type: (Iterable[UnparsedVersion], Optional[bool]) -> Iterable[UnparsedVersion]
+    def filter(
+        self, iterable: Iterable[VersionTypeVar], prereleases: Optional[bool] = None
+    ) -> Iterable[VersionTypeVar]:
 
         yielded = False
         found_prereleases = []
@@ -231,7 +204,7 @@ def filter(self, iterable, prereleases=None):
 
             if self.contains(parsed_version, **kw):
                 # If our version is a prerelease, and we were not set to allow
-                # prereleases, then we'll store it for later incase nothing
+                # prereleases, then we'll store it for later in case nothing
                 # else matches this specifier.
                 if parsed_version.is_prerelease and not (
                     prereleases or self.prereleases
@@ -276,9 +249,8 @@ class LegacySpecifier(_IndividualSpecifier):
         ">": "greater_than",
     }
 
-    def __init__(self, spec="", prereleases=None):
-        # type: (str, Optional[bool]) -> None
-        super(LegacySpecifier, self).__init__(spec, prereleases)
+    def __init__(self, spec: str = "", prereleases: Optional[bool] = None) -> None:
+        super().__init__(spec, prereleases)
 
         warnings.warn(
             "Creating a LegacyVersion has been deprecated and will be "
@@ -286,44 +258,37 @@ def __init__(self, spec="", prereleases=None):
             DeprecationWarning,
         )
 
-    def _coerce_version(self, version):
-        # type: (Union[ParsedVersion, str]) -> LegacyVersion
+    def _coerce_version(self, version: UnparsedVersion) -> LegacyVersion:
         if not isinstance(version, LegacyVersion):
             version = LegacyVersion(str(version))
         return version
 
-    def _compare_equal(self, prospective, spec):
-        # type: (LegacyVersion, str) -> bool
+    def _compare_equal(self, prospective: LegacyVersion, spec: str) -> bool:
         return prospective == self._coerce_version(spec)
 
-    def _compare_not_equal(self, prospective, spec):
-        # type: (LegacyVersion, str) -> bool
+    def _compare_not_equal(self, prospective: LegacyVersion, spec: str) -> bool:
         return prospective != self._coerce_version(spec)
 
-    def _compare_less_than_equal(self, prospective, spec):
-        # type: (LegacyVersion, str) -> bool
+    def _compare_less_than_equal(self, prospective: LegacyVersion, spec: str) -> bool:
         return prospective <= self._coerce_version(spec)
 
-    def _compare_greater_than_equal(self, prospective, spec):
-        # type: (LegacyVersion, str) -> bool
+    def _compare_greater_than_equal(
+        self, prospective: LegacyVersion, spec: str
+    ) -> bool:
         return prospective >= self._coerce_version(spec)
 
-    def _compare_less_than(self, prospective, spec):
-        # type: (LegacyVersion, str) -> bool
+    def _compare_less_than(self, prospective: LegacyVersion, spec: str) -> bool:
         return prospective < self._coerce_version(spec)
 
-    def _compare_greater_than(self, prospective, spec):
-        # type: (LegacyVersion, str) -> bool
+    def _compare_greater_than(self, prospective: LegacyVersion, spec: str) -> bool:
         return prospective > self._coerce_version(spec)
 
 
 def _require_version_compare(
-    fn,  # type: (Callable[[Specifier, ParsedVersion, str], bool])
-):
-    # type: (...) -> Callable[[Specifier, ParsedVersion, str], bool]
+    fn: Callable[["Specifier", ParsedVersion, str], bool]
+) -> Callable[["Specifier", ParsedVersion, str], bool]:
     @functools.wraps(fn)
-    def wrapped(self, prospective, spec):
-        # type: (Specifier, ParsedVersion, str) -> bool
+    def wrapped(self: "Specifier", prospective: ParsedVersion, spec: str) -> bool:
         if not isinstance(prospective, Version):
             return False
         return fn(self, prospective, spec)
@@ -440,8 +405,7 @@ class Specifier(_IndividualSpecifier):
     }
 
     @_require_version_compare
-    def _compare_compatible(self, prospective, spec):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_compatible(self, prospective: ParsedVersion, spec: str) -> bool:
 
         # Compatible releases have an equivalent combination of >= and ==. That
         # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to
@@ -450,15 +414,9 @@ def _compare_compatible(self, prospective, spec):
         # the other specifiers.
 
         # We want everything but the last item in the version, but we want to
-        # ignore post and dev releases and we want to treat the pre-release as
-        # it's own separate segment.
+        # ignore suffix segments.
         prefix = ".".join(
-            list(
-                itertools.takewhile(
-                    lambda x: (not x.startswith("post") and not x.startswith("dev")),
-                    _version_split(spec),
-                )
-            )[:-1]
+            list(itertools.takewhile(_is_not_suffix, _version_split(spec)))[:-1]
         )
 
         # Add the prefix notation to the end of our string
@@ -469,8 +427,7 @@ def _compare_compatible(self, prospective, spec):
         )
 
     @_require_version_compare
-    def _compare_equal(self, prospective, spec):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_equal(self, prospective: ParsedVersion, spec: str) -> bool:
 
         # We need special logic to handle prefix matching
         if spec.endswith(".*"):
@@ -510,13 +467,11 @@ def _compare_equal(self, prospective, spec):
             return prospective == spec_version
 
     @_require_version_compare
-    def _compare_not_equal(self, prospective, spec):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_not_equal(self, prospective: ParsedVersion, spec: str) -> bool:
         return not self._compare_equal(prospective, spec)
 
     @_require_version_compare
-    def _compare_less_than_equal(self, prospective, spec):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_less_than_equal(self, prospective: ParsedVersion, spec: str) -> bool:
 
         # NB: Local version identifiers are NOT permitted in the version
         # specifier, so local version labels can be universally removed from
@@ -524,8 +479,9 @@ def _compare_less_than_equal(self, prospective, spec):
         return Version(prospective.public) <= Version(spec)
 
     @_require_version_compare
-    def _compare_greater_than_equal(self, prospective, spec):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_greater_than_equal(
+        self, prospective: ParsedVersion, spec: str
+    ) -> bool:
 
         # NB: Local version identifiers are NOT permitted in the version
         # specifier, so local version labels can be universally removed from
@@ -533,8 +489,7 @@ def _compare_greater_than_equal(self, prospective, spec):
         return Version(prospective.public) >= Version(spec)
 
     @_require_version_compare
-    def _compare_less_than(self, prospective, spec_str):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_less_than(self, prospective: ParsedVersion, spec_str: str) -> bool:
 
         # Convert our spec to a Version instance, since we'll want to work with
         # it as a version.
@@ -560,8 +515,7 @@ def _compare_less_than(self, prospective, spec_str):
         return True
 
     @_require_version_compare
-    def _compare_greater_than(self, prospective, spec_str):
-        # type: (ParsedVersion, str) -> bool
+    def _compare_greater_than(self, prospective: ParsedVersion, spec_str: str) -> bool:
 
         # Convert our spec to a Version instance, since we'll want to work with
         # it as a version.
@@ -592,13 +546,11 @@ def _compare_greater_than(self, prospective, spec_str):
         # same version in the spec.
         return True
 
-    def _compare_arbitrary(self, prospective, spec):
-        # type: (Version, str) -> bool
+    def _compare_arbitrary(self, prospective: Version, spec: str) -> bool:
         return str(prospective).lower() == str(spec).lower()
 
     @property
-    def prereleases(self):
-        # type: () -> bool
+    def prereleases(self) -> bool:
 
         # If there is an explicit prereleases set for this, then we'll just
         # blindly use that.
@@ -623,17 +575,15 @@ def prereleases(self):
         return False
 
     @prereleases.setter
-    def prereleases(self, value):
-        # type: (bool) -> None
+    def prereleases(self, value: bool) -> None:
         self._prereleases = value
 
 
 _prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$")
 
 
-def _version_split(version):
-    # type: (str) -> List[str]
-    result = []  # type: List[str]
+def _version_split(version: str) -> List[str]:
+    result: List[str] = []
     for item in version.split("."):
         match = _prefix_regex.search(item)
         if match:
@@ -643,8 +593,13 @@ def _version_split(version):
     return result
 
 
-def _pad_version(left, right):
-    # type: (List[str], List[str]) -> Tuple[List[str], List[str]]
+def _is_not_suffix(segment: str) -> bool:
+    return not any(
+        segment.startswith(prefix) for prefix in ("dev", "a", "b", "rc", "post")
+    )
+
+
+def _pad_version(left: List[str], right: List[str]) -> Tuple[List[str], List[str]]:
     left_split, right_split = [], []
 
     # Get the release segment of our versions
@@ -663,8 +618,9 @@ def _pad_version(left, right):
 
 
 class SpecifierSet(BaseSpecifier):
-    def __init__(self, specifiers="", prereleases=None):
-        # type: (str, Optional[bool]) -> None
+    def __init__(
+        self, specifiers: str = "", prereleases: Optional[bool] = None
+    ) -> None:
 
         # Split on , to break each individual specifier into it's own item, and
         # strip each item to remove leading/trailing whitespace.
@@ -672,7 +628,7 @@ def __init__(self, specifiers="", prereleases=None):
 
         # Parsed each individual specifier, attempting first to make it a
         # Specifier and falling back to a LegacySpecifier.
-        parsed = set()
+        parsed: Set[_IndividualSpecifier] = set()
         for specifier in split_specifiers:
             try:
                 parsed.add(Specifier(specifier))
@@ -686,27 +642,23 @@ def __init__(self, specifiers="", prereleases=None):
         # we accept prereleases or not.
         self._prereleases = prereleases
 
-    def __repr__(self):
-        # type: () -> str
+    def __repr__(self) -> str:
         pre = (
-            ", prereleases={0!r}".format(self.prereleases)
+            f", prereleases={self.prereleases!r}"
             if self._prereleases is not None
             else ""
         )
 
-        return "".format(str(self), pre)
+        return f""
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return ",".join(sorted(str(s) for s in self._specs))
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(self._specs)
 
-    def __and__(self, other):
-        # type: (Union[SpecifierSet, str]) -> SpecifierSet
-        if isinstance(other, string_types):
+    def __and__(self, other: Union["SpecifierSet", str]) -> "SpecifierSet":
+        if isinstance(other, str):
             other = SpecifierSet(other)
         elif not isinstance(other, SpecifierSet):
             return NotImplemented
@@ -728,35 +680,22 @@ def __and__(self, other):
 
         return specifier
 
-    def __eq__(self, other):
-        # type: (object) -> bool
-        if isinstance(other, (string_types, _IndividualSpecifier)):
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, (str, _IndividualSpecifier)):
             other = SpecifierSet(str(other))
         elif not isinstance(other, SpecifierSet):
             return NotImplemented
 
         return self._specs == other._specs
 
-    def __ne__(self, other):
-        # type: (object) -> bool
-        if isinstance(other, (string_types, _IndividualSpecifier)):
-            other = SpecifierSet(str(other))
-        elif not isinstance(other, SpecifierSet):
-            return NotImplemented
-
-        return self._specs != other._specs
-
-    def __len__(self):
-        # type: () -> int
+    def __len__(self) -> int:
         return len(self._specs)
 
-    def __iter__(self):
-        # type: () -> Iterator[_IndividualSpecifier]
+    def __iter__(self) -> Iterator[_IndividualSpecifier]:
         return iter(self._specs)
 
     @property
-    def prereleases(self):
-        # type: () -> Optional[bool]
+    def prereleases(self) -> Optional[bool]:
 
         # If we have been given an explicit prerelease modifier, then we'll
         # pass that through here.
@@ -774,16 +713,15 @@ def prereleases(self):
         return any(s.prereleases for s in self._specs)
 
     @prereleases.setter
-    def prereleases(self, value):
-        # type: (bool) -> None
+    def prereleases(self, value: bool) -> None:
         self._prereleases = value
 
-    def __contains__(self, item):
-        # type: (Union[ParsedVersion, str]) -> bool
+    def __contains__(self, item: UnparsedVersion) -> bool:
         return self.contains(item)
 
-    def contains(self, item, prereleases=None):
-        # type: (Union[ParsedVersion, str], Optional[bool]) -> bool
+    def contains(
+        self, item: UnparsedVersion, prereleases: Optional[bool] = None
+    ) -> bool:
 
         # Ensure that our item is a Version or LegacyVersion instance.
         if not isinstance(item, (LegacyVersion, Version)):
@@ -811,11 +749,8 @@ def contains(self, item, prereleases=None):
         return all(s.contains(item, prereleases=prereleases) for s in self._specs)
 
     def filter(
-        self,
-        iterable,  # type: Iterable[Union[ParsedVersion, str]]
-        prereleases=None,  # type: Optional[bool]
-    ):
-        # type: (...) -> Iterable[Union[ParsedVersion, str]]
+        self, iterable: Iterable[VersionTypeVar], prereleases: Optional[bool] = None
+    ) -> Iterable[VersionTypeVar]:
 
         # Determine if we're forcing a prerelease or not, if we're not forcing
         # one for this particular filter call, then we'll use whatever the
@@ -834,8 +769,11 @@ def filter(
         # which will filter out any pre-releases, unless there are no final
         # releases, and which will filter out LegacyVersion in general.
         else:
-            filtered = []  # type: List[Union[ParsedVersion, str]]
-            found_prereleases = []  # type: List[Union[ParsedVersion, str]]
+            filtered: List[VersionTypeVar] = []
+            found_prereleases: List[VersionTypeVar] = []
+
+            item: UnparsedVersion
+            parsed_version: Union[Version, LegacyVersion]
 
             for item in iterable:
                 # Ensure that we some kind of Version class for this item.
diff --git a/src/pip/_vendor/packaging/tags.py b/src/pip/_vendor/packaging/tags.py
index 13798e38bce..9a3d25a71c7 100644
--- a/src/pip/_vendor/packaging/tags.py
+++ b/src/pip/_vendor/packaging/tags.py
@@ -2,81 +2,44 @@
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
 
-from __future__ import absolute_import
-
-import distutils.util
-
-try:
-    from importlib.machinery import EXTENSION_SUFFIXES
-except ImportError:  # pragma: no cover
-    import imp
-
-    EXTENSION_SUFFIXES = [x[0] for x in imp.get_suffixes()]
-    del imp
-import collections
 import logging
-import os
 import platform
-import re
-import struct
 import sys
 import sysconfig
-import warnings
-
-from ._typing import TYPE_CHECKING, cast
-
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import (
-        Dict,
-        FrozenSet,
-        IO,
-        Iterable,
-        Iterator,
-        List,
-        Optional,
-        Sequence,
-        Tuple,
-        Union,
-    )
-
-    PythonVersion = Sequence[int]
-    MacVersion = Tuple[int, int]
-    GlibcVersion = Tuple[int, int]
-
+from importlib.machinery import EXTENSION_SUFFIXES
+from typing import (
+    Dict,
+    FrozenSet,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
+
+from . import _manylinux, _musllinux
 
 logger = logging.getLogger(__name__)
 
-INTERPRETER_SHORT_NAMES = {
+PythonVersion = Sequence[int]
+MacVersion = Tuple[int, int]
+
+INTERPRETER_SHORT_NAMES: Dict[str, str] = {
     "python": "py",  # Generic.
     "cpython": "cp",
     "pypy": "pp",
     "ironpython": "ip",
     "jython": "jy",
-}  # type: Dict[str, str]
+}
 
 
 _32_BIT_INTERPRETER = sys.maxsize <= 2 ** 32
 
 
-_LEGACY_MANYLINUX_MAP = {
-    # CentOS 7 w/ glibc 2.17 (PEP 599)
-    (2, 17): "manylinux2014",
-    # CentOS 6 w/ glibc 2.12 (PEP 571)
-    (2, 12): "manylinux2010",
-    # CentOS 5 w/ glibc 2.5 (PEP 513)
-    (2, 5): "manylinux1",
-}
-
-# If glibc ever changes its major version, we need to know what the last
-# minor version was, so we can build the complete list of all versions.
-# For now, guess what the highest minor version might be, assume it will
-# be 50 for testing. Once this actually happens, update the dictionary
-# with the actual value.
-_LAST_GLIBC_MINOR = collections.defaultdict(lambda: 50)  # type: Dict[int, int]
-glibcVersion = collections.namedtuple("Version", ["major", "minor"])
-
-
-class Tag(object):
+class Tag:
     """
     A representation of the tag triple for a wheel.
 
@@ -86,8 +49,7 @@ class Tag(object):
 
     __slots__ = ["_interpreter", "_abi", "_platform", "_hash"]
 
-    def __init__(self, interpreter, abi, platform):
-        # type: (str, str, str) -> None
+    def __init__(self, interpreter: str, abi: str, platform: str) -> None:
         self._interpreter = interpreter.lower()
         self._abi = abi.lower()
         self._platform = platform.lower()
@@ -99,46 +61,39 @@ def __init__(self, interpreter, abi, platform):
         self._hash = hash((self._interpreter, self._abi, self._platform))
 
     @property
-    def interpreter(self):
-        # type: () -> str
+    def interpreter(self) -> str:
         return self._interpreter
 
     @property
-    def abi(self):
-        # type: () -> str
+    def abi(self) -> str:
         return self._abi
 
     @property
-    def platform(self):
-        # type: () -> str
+    def platform(self) -> str:
         return self._platform
 
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, Tag):
             return NotImplemented
 
         return (
-            (self.platform == other.platform)
-            and (self.abi == other.abi)
-            and (self.interpreter == other.interpreter)
+            (self._hash == other._hash)  # Short-circuit ASAP for perf reasons.
+            and (self._platform == other._platform)
+            and (self._abi == other._abi)
+            and (self._interpreter == other._interpreter)
         )
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return self._hash
 
-    def __str__(self):
-        # type: () -> str
-        return "{}-{}-{}".format(self._interpreter, self._abi, self._platform)
+    def __str__(self) -> str:
+        return f"{self._interpreter}-{self._abi}-{self._platform}"
 
-    def __repr__(self):
-        # type: () -> str
-        return "<{self} @ {self_id}>".format(self=self, self_id=id(self))
+    def __repr__(self) -> str:
+        return f"<{self} @ {id(self)}>"
 
 
-def parse_tag(tag):
-    # type: (str) -> FrozenSet[Tag]
+def parse_tag(tag: str) -> FrozenSet[Tag]:
     """
     Parses the provided tag (e.g. `py3-none-any`) into a frozenset of Tag instances.
 
@@ -154,24 +109,7 @@ def parse_tag(tag):
     return frozenset(tags)
 
 
-def _warn_keyword_parameter(func_name, kwargs):
-    # type: (str, Dict[str, bool]) -> bool
-    """
-    Backwards-compatibility with Python 2.7 to allow treating 'warn' as keyword-only.
-    """
-    if not kwargs:
-        return False
-    elif len(kwargs) > 1 or "warn" not in kwargs:
-        kwargs.pop("warn", None)
-        arg = next(iter(kwargs.keys()))
-        raise TypeError(
-            "{}() got an unexpected keyword argument {!r}".format(func_name, arg)
-        )
-    return kwargs["warn"]
-
-
-def _get_config_var(name, warn=False):
-    # type: (str, bool) -> Union[int, str, None]
+def _get_config_var(name: str, warn: bool = False) -> Union[int, str, None]:
     value = sysconfig.get_config_var(name)
     if value is None and warn:
         logger.debug(
@@ -180,13 +118,11 @@ def _get_config_var(name, warn=False):
     return value
 
 
-def _normalize_string(string):
-    # type: (str) -> str
+def _normalize_string(string: str) -> str:
     return string.replace(".", "_").replace("-", "_")
 
 
-def _abi3_applies(python_version):
-    # type: (PythonVersion) -> bool
+def _abi3_applies(python_version: PythonVersion) -> bool:
     """
     Determine if the Python version supports abi3.
 
@@ -195,8 +131,7 @@ def _abi3_applies(python_version):
     return len(python_version) > 1 and tuple(python_version) >= (3, 2)
 
 
-def _cpython_abis(py_version, warn=False):
-    # type: (PythonVersion, bool) -> List[str]
+def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> List[str]:
     py_version = tuple(py_version)  # To allow for version comparison.
     abis = []
     version = _version_nodot(py_version[:2])
@@ -222,7 +157,7 @@ def _cpython_abis(py_version, warn=False):
     elif debug:
         # Debug builds can also load "normal" extension modules.
         # We can also assume no UCS-4 or pymalloc requirement.
-        abis.append("cp{version}".format(version=version))
+        abis.append(f"cp{version}")
     abis.insert(
         0,
         "cp{version}{debug}{pymalloc}{ucs4}".format(
@@ -233,12 +168,12 @@ def _cpython_abis(py_version, warn=False):
 
 
 def cpython_tags(
-    python_version=None,  # type: Optional[PythonVersion]
-    abis=None,  # type: Optional[Iterable[str]]
-    platforms=None,  # type: Optional[Iterable[str]]
-    **kwargs  # type: bool
-):
-    # type: (...) -> Iterator[Tag]
+    python_version: Optional[PythonVersion] = None,
+    abis: Optional[Iterable[str]] = None,
+    platforms: Optional[Iterable[str]] = None,
+    *,
+    warn: bool = False,
+) -> Iterator[Tag]:
     """
     Yields the tags for a CPython interpreter.
 
@@ -254,11 +189,10 @@ def cpython_tags(
     If 'abi3' or 'none' are specified in 'abis' then they will be yielded at
     their normal position and not at the beginning.
     """
-    warn = _warn_keyword_parameter("cpython_tags", kwargs)
     if not python_version:
         python_version = sys.version_info[:2]
 
-    interpreter = "cp{}".format(_version_nodot(python_version[:2]))
+    interpreter = f"cp{_version_nodot(python_version[:2])}"
 
     if abis is None:
         if len(python_version) > 1:
@@ -273,15 +207,13 @@ def cpython_tags(
         except ValueError:
             pass
 
-    platforms = list(platforms or _platform_tags())
+    platforms = list(platforms or platform_tags())
     for abi in abis:
         for platform_ in platforms:
             yield Tag(interpreter, abi, platform_)
     if _abi3_applies(python_version):
-        for tag in (Tag(interpreter, "abi3", platform_) for platform_ in platforms):
-            yield tag
-    for tag in (Tag(interpreter, "none", platform_) for platform_ in platforms):
-        yield tag
+        yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms)
+    yield from (Tag(interpreter, "none", platform_) for platform_ in platforms)
 
     if _abi3_applies(python_version):
         for minor_version in range(python_version[1] - 1, 1, -1):
@@ -292,20 +224,19 @@ def cpython_tags(
                 yield Tag(interpreter, "abi3", platform_)
 
 
-def _generic_abi():
-    # type: () -> Iterator[str]
+def _generic_abi() -> Iterator[str]:
     abi = sysconfig.get_config_var("SOABI")
     if abi:
         yield _normalize_string(abi)
 
 
 def generic_tags(
-    interpreter=None,  # type: Optional[str]
-    abis=None,  # type: Optional[Iterable[str]]
-    platforms=None,  # type: Optional[Iterable[str]]
-    **kwargs  # type: bool
-):
-    # type: (...) -> Iterator[Tag]
+    interpreter: Optional[str] = None,
+    abis: Optional[Iterable[str]] = None,
+    platforms: Optional[Iterable[str]] = None,
+    *,
+    warn: bool = False,
+) -> Iterator[Tag]:
     """
     Yields the tags for a generic interpreter.
 
@@ -314,14 +245,13 @@ def generic_tags(
 
     The "none" ABI will be added if it was not explicitly provided.
     """
-    warn = _warn_keyword_parameter("generic_tags", kwargs)
     if not interpreter:
         interp_name = interpreter_name()
         interp_version = interpreter_version(warn=warn)
         interpreter = "".join([interp_name, interp_version])
     if abis is None:
         abis = _generic_abi()
-    platforms = list(platforms or _platform_tags())
+    platforms = list(platforms or platform_tags())
     abis = list(abis)
     if "none" not in abis:
         abis.append("none")
@@ -330,8 +260,7 @@ def generic_tags(
             yield Tag(interpreter, abi, platform_)
 
 
-def _py_interpreter_range(py_version):
-    # type: (PythonVersion) -> Iterator[str]
+def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]:
     """
     Yields Python versions in descending order.
 
@@ -339,19 +268,18 @@ def _py_interpreter_range(py_version):
     all previous versions of that major version.
     """
     if len(py_version) > 1:
-        yield "py{version}".format(version=_version_nodot(py_version[:2]))
-    yield "py{major}".format(major=py_version[0])
+        yield f"py{_version_nodot(py_version[:2])}"
+    yield f"py{py_version[0]}"
     if len(py_version) > 1:
         for minor in range(py_version[1] - 1, -1, -1):
-            yield "py{version}".format(version=_version_nodot((py_version[0], minor)))
+            yield f"py{_version_nodot((py_version[0], minor))}"
 
 
 def compatible_tags(
-    python_version=None,  # type: Optional[PythonVersion]
-    interpreter=None,  # type: Optional[str]
-    platforms=None,  # type: Optional[Iterable[str]]
-):
-    # type: (...) -> Iterator[Tag]
+    python_version: Optional[PythonVersion] = None,
+    interpreter: Optional[str] = None,
+    platforms: Optional[Iterable[str]] = None,
+) -> Iterator[Tag]:
     """
     Yields the sequence of tags that are compatible with a specific version of Python.
 
@@ -362,7 +290,7 @@ def compatible_tags(
     """
     if not python_version:
         python_version = sys.version_info[:2]
-    platforms = list(platforms or _platform_tags())
+    platforms = list(platforms or platform_tags())
     for version in _py_interpreter_range(python_version):
         for platform_ in platforms:
             yield Tag(version, "none", platform_)
@@ -372,8 +300,7 @@ def compatible_tags(
         yield Tag(version, "none", "any")
 
 
-def _mac_arch(arch, is_32bit=_32_BIT_INTERPRETER):
-    # type: (str, bool) -> str
+def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str:
     if not is_32bit:
         return arch
 
@@ -383,8 +310,7 @@ def _mac_arch(arch, is_32bit=_32_BIT_INTERPRETER):
     return "i386"
 
 
-def _mac_binary_formats(version, cpu_arch):
-    # type: (MacVersion, str) -> List[str]
+def _mac_binary_formats(version: MacVersion, cpu_arch: str) -> List[str]:
     formats = [cpu_arch]
     if cpu_arch == "x86_64":
         if version < (10, 4):
@@ -416,8 +342,9 @@ def _mac_binary_formats(version, cpu_arch):
     return formats
 
 
-def mac_platforms(version=None, arch=None):
-    # type: (Optional[MacVersion], Optional[str]) -> Iterator[str]
+def mac_platforms(
+    version: Optional[MacVersion] = None, arch: Optional[str] = None
+) -> Iterator[str]:
     """
     Yields the platform tags for a macOS system.
 
@@ -426,7 +353,7 @@ def mac_platforms(version=None, arch=None):
     generate platform tags for. Both parameters default to the appropriate value
     for the current system.
     """
-    version_str, _, cpu_arch = platform.mac_ver()  # type: ignore
+    version_str, _, cpu_arch = platform.mac_ver()
     if version is None:
         version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2])))
     else:
@@ -458,14 +385,28 @@ def mac_platforms(version=None, arch=None):
                     major=major_version, minor=0, binary_format=binary_format
                 )
 
-    if version >= (11, 0) and arch == "x86_64":
+    if version >= (11, 0):
         # Mac OS 11 on x86_64 is compatible with binaries from previous releases.
         # Arm64 support was introduced in 11.0, so no Arm binaries from previous
         # releases exist.
-        for minor_version in range(16, 3, -1):
-            compat_version = 10, minor_version
-            binary_formats = _mac_binary_formats(compat_version, arch)
-            for binary_format in binary_formats:
+        #
+        # However, the "universal2" binary format can have a
+        # macOS version earlier than 11.0 when the x86_64 part of the binary supports
+        # that version of macOS.
+        if arch == "x86_64":
+            for minor_version in range(16, 3, -1):
+                compat_version = 10, minor_version
+                binary_formats = _mac_binary_formats(compat_version, arch)
+                for binary_format in binary_formats:
+                    yield "macosx_{major}_{minor}_{binary_format}".format(
+                        major=compat_version[0],
+                        minor=compat_version[1],
+                        binary_format=binary_format,
+                    )
+        else:
+            for minor_version in range(16, 3, -1):
+                compat_version = 10, minor_version
+                binary_format = "universal2"
                 yield "macosx_{major}_{minor}_{binary_format}".format(
                     major=compat_version[0],
                     minor=compat_version[1],
@@ -473,320 +414,24 @@ def mac_platforms(version=None, arch=None):
                 )
 
 
-# From PEP 513, PEP 600
-def _is_manylinux_compatible(name, arch, glibc_version):
-    # type: (str, str, GlibcVersion) -> bool
-    sys_glibc = _get_glibc_version()
-    if sys_glibc < glibc_version:
-        return False
-    # Check for presence of _manylinux module.
-    try:
-        import _manylinux  # noqa
-    except ImportError:
-        pass
-    else:
-        if hasattr(_manylinux, "manylinux_compatible"):
-            result = _manylinux.manylinux_compatible(
-                glibc_version[0], glibc_version[1], arch
-            )
-            if result is not None:
-                return bool(result)
-        else:
-            if glibc_version == (2, 5):
-                if hasattr(_manylinux, "manylinux1_compatible"):
-                    return bool(_manylinux.manylinux1_compatible)
-            if glibc_version == (2, 12):
-                if hasattr(_manylinux, "manylinux2010_compatible"):
-                    return bool(_manylinux.manylinux2010_compatible)
-            if glibc_version == (2, 17):
-                if hasattr(_manylinux, "manylinux2014_compatible"):
-                    return bool(_manylinux.manylinux2014_compatible)
-    return True
-
-
-def _glibc_version_string():
-    # type: () -> Optional[str]
-    # Returns glibc version string, or None if not using glibc.
-    return _glibc_version_string_confstr() or _glibc_version_string_ctypes()
-
-
-def _glibc_version_string_confstr():
-    # type: () -> Optional[str]
-    """
-    Primary implementation of glibc_version_string using os.confstr.
-    """
-    # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely
-    # to be broken or missing. This strategy is used in the standard library
-    # platform module.
-    # https://github.com/python/cpython/blob/fcf1d003bf4f0100c9d0921ff3d70e1127ca1b71/Lib/platform.py#L175-L183
-    try:
-        # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17".
-        version_string = os.confstr(  # type: ignore[attr-defined] # noqa: F821
-            "CS_GNU_LIBC_VERSION"
-        )
-        assert version_string is not None
-        _, version = version_string.split()  # type: Tuple[str, str]
-    except (AssertionError, AttributeError, OSError, ValueError):
-        # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)...
-        return None
-    return version
-
-
-def _glibc_version_string_ctypes():
-    # type: () -> Optional[str]
-    """
-    Fallback implementation of glibc_version_string using ctypes.
-    """
-    try:
-        import ctypes
-    except ImportError:
-        return None
-
-    # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
-    # manpage says, "If filename is NULL, then the returned handle is for the
-    # main program". This way we can let the linker do the work to figure out
-    # which libc our process is actually using.
-    #
-    # We must also handle the special case where the executable is not a
-    # dynamically linked executable. This can occur when using musl libc,
-    # for example. In this situation, dlopen() will error, leading to an
-    # OSError. Interestingly, at least in the case of musl, there is no
-    # errno set on the OSError. The single string argument used to construct
-    # OSError comes from libc itself and is therefore not portable to
-    # hard code here. In any case, failure to call dlopen() means we
-    # can proceed, so we bail on our attempt.
-    try:
-        # Note: typeshed is wrong here so we are ignoring this line.
-        process_namespace = ctypes.CDLL(None)  # type: ignore
-    except OSError:
-        return None
-
-    try:
-        gnu_get_libc_version = process_namespace.gnu_get_libc_version
-    except AttributeError:
-        # Symbol doesn't exist -> therefore, we are not linked to
-        # glibc.
-        return None
-
-    # Call gnu_get_libc_version, which returns a string like "2.5"
-    gnu_get_libc_version.restype = ctypes.c_char_p
-    version_str = gnu_get_libc_version()  # type: str
-    # py2 / py3 compatibility:
-    if not isinstance(version_str, str):
-        version_str = version_str.decode("ascii")
-
-    return version_str
-
-
-def _parse_glibc_version(version_str):
-    # type: (str) -> Tuple[int, int]
-    # Parse glibc version.
-    #
-    # We use a regexp instead of str.split because we want to discard any
-    # random junk that might come after the minor version -- this might happen
-    # in patched/forked versions of glibc (e.g. Linaro's version of glibc
-    # uses version strings like "2.20-2014.11"). See gh-3588.
-    m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str)
-    if not m:
-        warnings.warn(
-            "Expected glibc version with 2 components major.minor,"
-            " got: %s" % version_str,
-            RuntimeWarning,
-        )
-        return -1, -1
-    return (int(m.group("major")), int(m.group("minor")))
-
-
-_glibc_version = []  #  type: List[Tuple[int, int]]
-
-
-def _get_glibc_version():
-    # type: () -> Tuple[int, int]
-    if _glibc_version:
-        return _glibc_version[0]
-    version_str = _glibc_version_string()
-    if version_str is None:
-        _glibc_version.append((-1, -1))
-    else:
-        _glibc_version.append(_parse_glibc_version(version_str))
-    return _glibc_version[0]
-
-
-# Python does not provide platform information at sufficient granularity to
-# identify the architecture of the running executable in some cases, so we
-# determine it dynamically by reading the information from the running
-# process. This only applies on Linux, which uses the ELF format.
-class _ELFFileHeader(object):
-    # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header
-    class _InvalidELFFileHeader(ValueError):
-        """
-        An invalid ELF file header was found.
-        """
-
-    ELF_MAGIC_NUMBER = 0x7F454C46
-    ELFCLASS32 = 1
-    ELFCLASS64 = 2
-    ELFDATA2LSB = 1
-    ELFDATA2MSB = 2
-    EM_386 = 3
-    EM_S390 = 22
-    EM_ARM = 40
-    EM_X86_64 = 62
-    EF_ARM_ABIMASK = 0xFF000000
-    EF_ARM_ABI_VER5 = 0x05000000
-    EF_ARM_ABI_FLOAT_HARD = 0x00000400
-
-    def __init__(self, file):
-        # type: (IO[bytes]) -> None
-        def unpack(fmt):
-            # type: (str) -> int
-            try:
-                (result,) = struct.unpack(
-                    fmt, file.read(struct.calcsize(fmt))
-                )  # type: (int, )
-            except struct.error:
-                raise _ELFFileHeader._InvalidELFFileHeader()
-            return result
-
-        self.e_ident_magic = unpack(">I")
-        if self.e_ident_magic != self.ELF_MAGIC_NUMBER:
-            raise _ELFFileHeader._InvalidELFFileHeader()
-        self.e_ident_class = unpack("B")
-        if self.e_ident_class not in {self.ELFCLASS32, self.ELFCLASS64}:
-            raise _ELFFileHeader._InvalidELFFileHeader()
-        self.e_ident_data = unpack("B")
-        if self.e_ident_data not in {self.ELFDATA2LSB, self.ELFDATA2MSB}:
-            raise _ELFFileHeader._InvalidELFFileHeader()
-        self.e_ident_version = unpack("B")
-        self.e_ident_osabi = unpack("B")
-        self.e_ident_abiversion = unpack("B")
-        self.e_ident_pad = file.read(7)
-        format_h = "H"
-        format_i = "I"
-        format_q = "Q"
-        format_p = format_i if self.e_ident_class == self.ELFCLASS32 else format_q
-        self.e_type = unpack(format_h)
-        self.e_machine = unpack(format_h)
-        self.e_version = unpack(format_i)
-        self.e_entry = unpack(format_p)
-        self.e_phoff = unpack(format_p)
-        self.e_shoff = unpack(format_p)
-        self.e_flags = unpack(format_i)
-        self.e_ehsize = unpack(format_h)
-        self.e_phentsize = unpack(format_h)
-        self.e_phnum = unpack(format_h)
-        self.e_shentsize = unpack(format_h)
-        self.e_shnum = unpack(format_h)
-        self.e_shstrndx = unpack(format_h)
-
-
-def _get_elf_header():
-    # type: () -> Optional[_ELFFileHeader]
-    try:
-        with open(sys.executable, "rb") as f:
-            elf_header = _ELFFileHeader(f)
-    except (IOError, OSError, TypeError, _ELFFileHeader._InvalidELFFileHeader):
-        return None
-    return elf_header
-
-
-def _is_linux_armhf():
-    # type: () -> bool
-    # hard-float ABI can be detected from the ELF header of the running
-    # process
-    # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf
-    elf_header = _get_elf_header()
-    if elf_header is None:
-        return False
-    result = elf_header.e_ident_class == elf_header.ELFCLASS32
-    result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB
-    result &= elf_header.e_machine == elf_header.EM_ARM
-    result &= (
-        elf_header.e_flags & elf_header.EF_ARM_ABIMASK
-    ) == elf_header.EF_ARM_ABI_VER5
-    result &= (
-        elf_header.e_flags & elf_header.EF_ARM_ABI_FLOAT_HARD
-    ) == elf_header.EF_ARM_ABI_FLOAT_HARD
-    return result
-
-
-def _is_linux_i686():
-    # type: () -> bool
-    elf_header = _get_elf_header()
-    if elf_header is None:
-        return False
-    result = elf_header.e_ident_class == elf_header.ELFCLASS32
-    result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB
-    result &= elf_header.e_machine == elf_header.EM_386
-    return result
-
-
-def _have_compatible_manylinux_abi(arch):
-    # type: (str) -> bool
-    if arch == "armv7l":
-        return _is_linux_armhf()
-    if arch == "i686":
-        return _is_linux_i686()
-    return arch in {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x"}
-
-
-def _manylinux_tags(linux, arch):
-    # type: (str, str) -> Iterator[str]
-    # Oldest glibc to be supported regardless of architecture is (2, 17).
-    too_old_glibc2 = glibcVersion(2, 16)
-    if arch in {"x86_64", "i686"}:
-        # On x86/i686 also oldest glibc to be supported is (2, 5).
-        too_old_glibc2 = glibcVersion(2, 4)
-    current_glibc = glibcVersion(*_get_glibc_version())
-    glibc_max_list = [current_glibc]
-    # We can assume compatibility across glibc major versions.
-    # https://sourceware.org/bugzilla/show_bug.cgi?id=24636
-    #
-    # Build a list of maximum glibc versions so that we can
-    # output the canonical list of all glibc from current_glibc
-    # down to too_old_glibc2, including all intermediary versions.
-    for glibc_major in range(current_glibc.major - 1, 1, -1):
-        glibc_max_list.append(glibcVersion(glibc_major, _LAST_GLIBC_MINOR[glibc_major]))
-    for glibc_max in glibc_max_list:
-        if glibc_max.major == too_old_glibc2.major:
-            min_minor = too_old_glibc2.minor
-        else:
-            # For other glibc major versions oldest supported is (x, 0).
-            min_minor = -1
-        for glibc_minor in range(glibc_max.minor, min_minor, -1):
-            glibc_version = (glibc_max.major, glibc_minor)
-            tag = "manylinux_{}_{}".format(*glibc_version)
-            if _is_manylinux_compatible(tag, arch, glibc_version):
-                yield linux.replace("linux", tag)
-            # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags.
-            if glibc_version in _LEGACY_MANYLINUX_MAP:
-                legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version]
-                if _is_manylinux_compatible(legacy_tag, arch, glibc_version):
-                    yield linux.replace("linux", legacy_tag)
-
-
-def _linux_platforms(is_32bit=_32_BIT_INTERPRETER):
-    # type: (bool) -> Iterator[str]
-    linux = _normalize_string(distutils.util.get_platform())
+def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]:
+    linux = _normalize_string(sysconfig.get_platform())
     if is_32bit:
         if linux == "linux_x86_64":
             linux = "linux_i686"
         elif linux == "linux_aarch64":
             linux = "linux_armv7l"
     _, arch = linux.split("_", 1)
-    if _have_compatible_manylinux_abi(arch):
-        for tag in _manylinux_tags(linux, arch):
-            yield tag
+    yield from _manylinux.platform_tags(linux, arch)
+    yield from _musllinux.platform_tags(arch)
     yield linux
 
 
-def _generic_platforms():
-    # type: () -> Iterator[str]
-    yield _normalize_string(distutils.util.get_platform())
+def _generic_platforms() -> Iterator[str]:
+    yield _normalize_string(sysconfig.get_platform())
 
 
-def _platform_tags():
-    # type: () -> Iterator[str]
+def platform_tags() -> Iterator[str]:
     """
     Provides the platform tags for this installation.
     """
@@ -798,25 +443,18 @@ def _platform_tags():
         return _generic_platforms()
 
 
-def interpreter_name():
-    # type: () -> str
+def interpreter_name() -> str:
     """
     Returns the name of the running interpreter.
     """
-    try:
-        name = sys.implementation.name  # type: ignore
-    except AttributeError:  # pragma: no cover
-        # Python 2.7 compatibility.
-        name = platform.python_implementation().lower()
+    name = sys.implementation.name
     return INTERPRETER_SHORT_NAMES.get(name) or name
 
 
-def interpreter_version(**kwargs):
-    # type: (bool) -> str
+def interpreter_version(*, warn: bool = False) -> str:
     """
     Returns the version of the running interpreter.
     """
-    warn = _warn_keyword_parameter("interpreter_version", kwargs)
     version = _get_config_var("py_version_nodot", warn=warn)
     if version:
         version = str(version)
@@ -825,28 +463,25 @@ def interpreter_version(**kwargs):
     return version
 
 
-def _version_nodot(version):
-    # type: (PythonVersion) -> str
+def _version_nodot(version: PythonVersion) -> str:
     return "".join(map(str, version))
 
 
-def sys_tags(**kwargs):
-    # type: (bool) -> Iterator[Tag]
+def sys_tags(*, warn: bool = False) -> Iterator[Tag]:
     """
     Returns the sequence of tag triples for the running interpreter.
 
     The order of the sequence corresponds to priority order for the
     interpreter, from most to least important.
     """
-    warn = _warn_keyword_parameter("sys_tags", kwargs)
 
     interp_name = interpreter_name()
     if interp_name == "cp":
-        for tag in cpython_tags(warn=warn):
-            yield tag
+        yield from cpython_tags(warn=warn)
     else:
-        for tag in generic_tags():
-            yield tag
+        yield from generic_tags()
 
-    for tag in compatible_tags():
-        yield tag
+    if interp_name == "pp":
+        yield from compatible_tags(interpreter="pp3")
+    else:
+        yield from compatible_tags()
diff --git a/src/pip/_vendor/packaging/utils.py b/src/pip/_vendor/packaging/utils.py
index 92c7b00b778..bab11b80c60 100644
--- a/src/pip/_vendor/packaging/utils.py
+++ b/src/pip/_vendor/packaging/utils.py
@@ -1,67 +1,136 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 import re
+from typing import FrozenSet, NewType, Tuple, Union, cast
 
-from ._typing import TYPE_CHECKING, cast
+from .tags import Tag, parse_tag
 from .version import InvalidVersion, Version
 
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import NewType, Union
+BuildTag = Union[Tuple[()], Tuple[int, str]]
+NormalizedName = NewType("NormalizedName", str)
+
+
+class InvalidWheelFilename(ValueError):
+    """
+    An invalid wheel filename was found, users should refer to PEP 427.
+    """
+
+
+class InvalidSdistFilename(ValueError):
+    """
+    An invalid sdist filename was found, users should refer to the packaging user guide.
+    """
 
-    NormalizedName = NewType("NormalizedName", str)
-else:
-    NormalizedName = str
 
 _canonicalize_regex = re.compile(r"[-_.]+")
+# PEP 427: The build number must start with a digit.
+_build_tag_regex = re.compile(r"(\d+)(.*)")
 
 
-def canonicalize_name(name):
-    # type: (str) -> NormalizedName
+def canonicalize_name(name: str) -> NormalizedName:
     # This is taken from PEP 503.
     value = _canonicalize_regex.sub("-", name).lower()
-    return cast("NormalizedName", value)
+    return cast(NormalizedName, value)
 
 
-def canonicalize_version(version):
-    # type: (Union[Version, str]) -> Union[Version, str]
+def canonicalize_version(version: Union[Version, str]) -> str:
     """
     This is very similar to Version.__str__, but has one subtle difference
     with the way it handles the release segment.
     """
-    if not isinstance(version, Version):
+    if isinstance(version, str):
         try:
-            version = Version(version)
+            parsed = Version(version)
         except InvalidVersion:
             # Legacy versions cannot be normalized
             return version
+    else:
+        parsed = version
 
     parts = []
 
     # Epoch
-    if version.epoch != 0:
-        parts.append("{0}!".format(version.epoch))
+    if parsed.epoch != 0:
+        parts.append(f"{parsed.epoch}!")
 
     # Release segment
     # NB: This strips trailing '.0's to normalize
-    parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release)))
+    parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in parsed.release)))
 
     # Pre-release
-    if version.pre is not None:
-        parts.append("".join(str(x) for x in version.pre))
+    if parsed.pre is not None:
+        parts.append("".join(str(x) for x in parsed.pre))
 
     # Post-release
-    if version.post is not None:
-        parts.append(".post{0}".format(version.post))
+    if parsed.post is not None:
+        parts.append(f".post{parsed.post}")
 
     # Development release
-    if version.dev is not None:
-        parts.append(".dev{0}".format(version.dev))
+    if parsed.dev is not None:
+        parts.append(f".dev{parsed.dev}")
 
     # Local version segment
-    if version.local is not None:
-        parts.append("+{0}".format(version.local))
+    if parsed.local is not None:
+        parts.append(f"+{parsed.local}")
 
     return "".join(parts)
+
+
+def parse_wheel_filename(
+    filename: str,
+) -> Tuple[NormalizedName, Version, BuildTag, FrozenSet[Tag]]:
+    if not filename.endswith(".whl"):
+        raise InvalidWheelFilename(
+            f"Invalid wheel filename (extension must be '.whl'): {filename}"
+        )
+
+    filename = filename[:-4]
+    dashes = filename.count("-")
+    if dashes not in (4, 5):
+        raise InvalidWheelFilename(
+            f"Invalid wheel filename (wrong number of parts): {filename}"
+        )
+
+    parts = filename.split("-", dashes - 2)
+    name_part = parts[0]
+    # See PEP 427 for the rules on escaping the project name
+    if "__" in name_part or re.match(r"^[\w\d._]*$", name_part, re.UNICODE) is None:
+        raise InvalidWheelFilename(f"Invalid project name: {filename}")
+    name = canonicalize_name(name_part)
+    version = Version(parts[1])
+    if dashes == 5:
+        build_part = parts[2]
+        build_match = _build_tag_regex.match(build_part)
+        if build_match is None:
+            raise InvalidWheelFilename(
+                f"Invalid build number: {build_part} in '{filename}'"
+            )
+        build = cast(BuildTag, (int(build_match.group(1)), build_match.group(2)))
+    else:
+        build = ()
+    tags = parse_tag(parts[-1])
+    return (name, version, build, tags)
+
+
+def parse_sdist_filename(filename: str) -> Tuple[NormalizedName, Version]:
+    if filename.endswith(".tar.gz"):
+        file_stem = filename[: -len(".tar.gz")]
+    elif filename.endswith(".zip"):
+        file_stem = filename[: -len(".zip")]
+    else:
+        raise InvalidSdistFilename(
+            f"Invalid sdist filename (extension must be '.tar.gz' or '.zip'):"
+            f" {filename}"
+        )
+
+    # We are requiring a PEP 440 version, which cannot contain dashes,
+    # so we split on the last dash.
+    name_part, sep, version_part = file_stem.rpartition("-")
+    if not sep:
+        raise InvalidSdistFilename(f"Invalid sdist filename: {filename}")
+
+    name = canonicalize_name(name_part)
+    version = Version(version_part)
+    return (name, version)
diff --git a/src/pip/_vendor/packaging/version.py b/src/pip/_vendor/packaging/version.py
index 517d91f2485..de9a09a4ed3 100644
--- a/src/pip/_vendor/packaging/version.py
+++ b/src/pip/_vendor/packaging/version.py
@@ -1,53 +1,45 @@
 # This file is dual licensed under the terms of the Apache License, Version
 # 2.0, and the BSD License. See the LICENSE file in the root of this repository
 # for complete details.
-from __future__ import absolute_import, division, print_function
 
 import collections
 import itertools
 import re
 import warnings
+from typing import Callable, Iterator, List, Optional, SupportsInt, Tuple, Union
 
-from ._structures import Infinity, NegativeInfinity
-from ._typing import TYPE_CHECKING
-
-if TYPE_CHECKING:  # pragma: no cover
-    from typing import Callable, Iterator, List, Optional, SupportsInt, Tuple, Union
-
-    from ._structures import InfinityType, NegativeInfinityType
-
-    InfiniteTypes = Union[InfinityType, NegativeInfinityType]
-    PrePostDevType = Union[InfiniteTypes, Tuple[str, int]]
-    SubLocalType = Union[InfiniteTypes, int, str]
-    LocalType = Union[
-        NegativeInfinityType,
-        Tuple[
-            Union[
-                SubLocalType,
-                Tuple[SubLocalType, str],
-                Tuple[NegativeInfinityType, SubLocalType],
-            ],
-            ...,
-        ],
-    ]
-    CmpKey = Tuple[
-        int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType
-    ]
-    LegacyCmpKey = Tuple[int, Tuple[str, ...]]
-    VersionComparisonMethod = Callable[
-        [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool
-    ]
+from ._structures import Infinity, InfinityType, NegativeInfinity, NegativeInfinityType
 
 __all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"]
 
+InfiniteTypes = Union[InfinityType, NegativeInfinityType]
+PrePostDevType = Union[InfiniteTypes, Tuple[str, int]]
+SubLocalType = Union[InfiniteTypes, int, str]
+LocalType = Union[
+    NegativeInfinityType,
+    Tuple[
+        Union[
+            SubLocalType,
+            Tuple[SubLocalType, str],
+            Tuple[NegativeInfinityType, SubLocalType],
+        ],
+        ...,
+    ],
+]
+CmpKey = Tuple[
+    int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType
+]
+LegacyCmpKey = Tuple[int, Tuple[str, ...]]
+VersionComparisonMethod = Callable[
+    [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool
+]
 
 _Version = collections.namedtuple(
     "_Version", ["epoch", "release", "dev", "pre", "post", "local"]
 )
 
 
-def parse(version):
-    # type: (str) -> Union[LegacyVersion, Version]
+def parse(version: str) -> Union["LegacyVersion", "Version"]:
     """
     Parse the given version string and return either a :class:`Version` object
     or a :class:`LegacyVersion` object depending on if the given version is
@@ -65,53 +57,46 @@ class InvalidVersion(ValueError):
     """
 
 
-class _BaseVersion(object):
-    _key = None  # type: Union[CmpKey, LegacyCmpKey]
+class _BaseVersion:
+    _key: Union[CmpKey, LegacyCmpKey]
 
-    def __hash__(self):
-        # type: () -> int
+    def __hash__(self) -> int:
         return hash(self._key)
 
     # Please keep the duplicated `isinstance` check
     # in the six comparisons hereunder
     # unless you find a way to avoid adding overhead function calls.
-    def __lt__(self, other):
-        # type: (_BaseVersion) -> bool
+    def __lt__(self, other: "_BaseVersion") -> bool:
         if not isinstance(other, _BaseVersion):
             return NotImplemented
 
         return self._key < other._key
 
-    def __le__(self, other):
-        # type: (_BaseVersion) -> bool
+    def __le__(self, other: "_BaseVersion") -> bool:
         if not isinstance(other, _BaseVersion):
             return NotImplemented
 
         return self._key <= other._key
 
-    def __eq__(self, other):
-        # type: (object) -> bool
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, _BaseVersion):
             return NotImplemented
 
         return self._key == other._key
 
-    def __ge__(self, other):
-        # type: (_BaseVersion) -> bool
+    def __ge__(self, other: "_BaseVersion") -> bool:
         if not isinstance(other, _BaseVersion):
             return NotImplemented
 
         return self._key >= other._key
 
-    def __gt__(self, other):
-        # type: (_BaseVersion) -> bool
+    def __gt__(self, other: "_BaseVersion") -> bool:
         if not isinstance(other, _BaseVersion):
             return NotImplemented
 
         return self._key > other._key
 
-    def __ne__(self, other):
-        # type: (object) -> bool
+    def __ne__(self, other: object) -> bool:
         if not isinstance(other, _BaseVersion):
             return NotImplemented
 
@@ -119,8 +104,7 @@ def __ne__(self, other):
 
 
 class LegacyVersion(_BaseVersion):
-    def __init__(self, version):
-        # type: (str) -> None
+    def __init__(self, version: str) -> None:
         self._version = str(version)
         self._key = _legacy_cmpkey(self._version)
 
@@ -130,67 +114,54 @@ def __init__(self, version):
             DeprecationWarning,
         )
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         return self._version
 
-    def __repr__(self):
-        # type: () -> str
-        return "".format(repr(str(self)))
+    def __repr__(self) -> str:
+        return f""
 
     @property
-    def public(self):
-        # type: () -> str
+    def public(self) -> str:
         return self._version
 
     @property
-    def base_version(self):
-        # type: () -> str
+    def base_version(self) -> str:
         return self._version
 
     @property
-    def epoch(self):
-        # type: () -> int
+    def epoch(self) -> int:
         return -1
 
     @property
-    def release(self):
-        # type: () -> None
+    def release(self) -> None:
         return None
 
     @property
-    def pre(self):
-        # type: () -> None
+    def pre(self) -> None:
         return None
 
     @property
-    def post(self):
-        # type: () -> None
+    def post(self) -> None:
         return None
 
     @property
-    def dev(self):
-        # type: () -> None
+    def dev(self) -> None:
         return None
 
     @property
-    def local(self):
-        # type: () -> None
+    def local(self) -> None:
         return None
 
     @property
-    def is_prerelease(self):
-        # type: () -> bool
+    def is_prerelease(self) -> bool:
         return False
 
     @property
-    def is_postrelease(self):
-        # type: () -> bool
+    def is_postrelease(self) -> bool:
         return False
 
     @property
-    def is_devrelease(self):
-        # type: () -> bool
+    def is_devrelease(self) -> bool:
         return False
 
 
@@ -205,8 +176,7 @@ def is_devrelease(self):
 }
 
 
-def _parse_version_parts(s):
-    # type: (str) -> Iterator[str]
+def _parse_version_parts(s: str) -> Iterator[str]:
     for part in _legacy_version_component_re.split(s):
         part = _legacy_version_replacement_map.get(part, part)
 
@@ -223,8 +193,7 @@ def _parse_version_parts(s):
     yield "*final"
 
 
-def _legacy_cmpkey(version):
-    # type: (str) -> LegacyCmpKey
+def _legacy_cmpkey(version: str) -> LegacyCmpKey:
 
     # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch
     # greater than or equal to 0. This will effectively put the LegacyVersion,
@@ -234,7 +203,7 @@ def _legacy_cmpkey(version):
 
     # This scheme is taken from pkg_resources.parse_version setuptools prior to
     # it's adoption of the packaging library.
-    parts = []  # type: List[str]
+    parts: List[str] = []
     for part in _parse_version_parts(version.lower()):
         if part.startswith("*"):
             # remove "-" before a prerelease tag
@@ -289,13 +258,12 @@ class Version(_BaseVersion):
 
     _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
 
-    def __init__(self, version):
-        # type: (str) -> None
+    def __init__(self, version: str) -> None:
 
         # Validate the version and parse it into pieces
         match = self._regex.search(version)
         if not match:
-            raise InvalidVersion("Invalid version: '{0}'".format(version))
+            raise InvalidVersion(f"Invalid version: '{version}'")
 
         # Store the parsed out pieces of the version
         self._version = _Version(
@@ -319,17 +287,15 @@ def __init__(self, version):
             self._version.local,
         )
 
-    def __repr__(self):
-        # type: () -> str
-        return "".format(repr(str(self)))
+    def __repr__(self) -> str:
+        return f""
 
-    def __str__(self):
-        # type: () -> str
+    def __str__(self) -> str:
         parts = []
 
         # Epoch
         if self.epoch != 0:
-            parts.append("{0}!".format(self.epoch))
+            parts.append(f"{self.epoch}!")
 
         # Release segment
         parts.append(".".join(str(x) for x in self.release))
@@ -340,67 +306,59 @@ def __str__(self):
 
         # Post-release
         if self.post is not None:
-            parts.append(".post{0}".format(self.post))
+            parts.append(f".post{self.post}")
 
         # Development release
         if self.dev is not None:
-            parts.append(".dev{0}".format(self.dev))
+            parts.append(f".dev{self.dev}")
 
         # Local version segment
         if self.local is not None:
-            parts.append("+{0}".format(self.local))
+            parts.append(f"+{self.local}")
 
         return "".join(parts)
 
     @property
-    def epoch(self):
-        # type: () -> int
-        _epoch = self._version.epoch  # type: int
+    def epoch(self) -> int:
+        _epoch: int = self._version.epoch
         return _epoch
 
     @property
-    def release(self):
-        # type: () -> Tuple[int, ...]
-        _release = self._version.release  # type: Tuple[int, ...]
+    def release(self) -> Tuple[int, ...]:
+        _release: Tuple[int, ...] = self._version.release
         return _release
 
     @property
-    def pre(self):
-        # type: () -> Optional[Tuple[str, int]]
-        _pre = self._version.pre  # type: Optional[Tuple[str, int]]
+    def pre(self) -> Optional[Tuple[str, int]]:
+        _pre: Optional[Tuple[str, int]] = self._version.pre
         return _pre
 
     @property
-    def post(self):
-        # type: () -> Optional[Tuple[str, int]]
+    def post(self) -> Optional[int]:
         return self._version.post[1] if self._version.post else None
 
     @property
-    def dev(self):
-        # type: () -> Optional[Tuple[str, int]]
+    def dev(self) -> Optional[int]:
         return self._version.dev[1] if self._version.dev else None
 
     @property
-    def local(self):
-        # type: () -> Optional[str]
+    def local(self) -> Optional[str]:
         if self._version.local:
             return ".".join(str(x) for x in self._version.local)
         else:
             return None
 
     @property
-    def public(self):
-        # type: () -> str
+    def public(self) -> str:
         return str(self).split("+", 1)[0]
 
     @property
-    def base_version(self):
-        # type: () -> str
+    def base_version(self) -> str:
         parts = []
 
         # Epoch
         if self.epoch != 0:
-            parts.append("{0}!".format(self.epoch))
+            parts.append(f"{self.epoch}!")
 
         # Release segment
         parts.append(".".join(str(x) for x in self.release))
@@ -408,41 +366,33 @@ def base_version(self):
         return "".join(parts)
 
     @property
-    def is_prerelease(self):
-        # type: () -> bool
+    def is_prerelease(self) -> bool:
         return self.dev is not None or self.pre is not None
 
     @property
-    def is_postrelease(self):
-        # type: () -> bool
+    def is_postrelease(self) -> bool:
         return self.post is not None
 
     @property
-    def is_devrelease(self):
-        # type: () -> bool
+    def is_devrelease(self) -> bool:
         return self.dev is not None
 
     @property
-    def major(self):
-        # type: () -> int
+    def major(self) -> int:
         return self.release[0] if len(self.release) >= 1 else 0
 
     @property
-    def minor(self):
-        # type: () -> int
+    def minor(self) -> int:
         return self.release[1] if len(self.release) >= 2 else 0
 
     @property
-    def micro(self):
-        # type: () -> int
+    def micro(self) -> int:
         return self.release[2] if len(self.release) >= 3 else 0
 
 
 def _parse_letter_version(
-    letter,  # type: str
-    number,  # type: Union[str, bytes, SupportsInt]
-):
-    # type: (...) -> Optional[Tuple[str, int]]
+    letter: str, number: Union[str, bytes, SupportsInt]
+) -> Optional[Tuple[str, int]]:
 
     if letter:
         # We consider there to be an implicit 0 in a pre-release if there is
@@ -479,8 +429,7 @@ def _parse_letter_version(
 _local_version_separators = re.compile(r"[\._-]")
 
 
-def _parse_local_version(local):
-    # type: (str) -> Optional[LocalType]
+def _parse_local_version(local: str) -> Optional[LocalType]:
     """
     Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
     """
@@ -493,14 +442,13 @@ def _parse_local_version(local):
 
 
 def _cmpkey(
-    epoch,  # type: int
-    release,  # type: Tuple[int, ...]
-    pre,  # type: Optional[Tuple[str, int]]
-    post,  # type: Optional[Tuple[str, int]]
-    dev,  # type: Optional[Tuple[str, int]]
-    local,  # type: Optional[Tuple[SubLocalType]]
-):
-    # type: (...) -> CmpKey
+    epoch: int,
+    release: Tuple[int, ...],
+    pre: Optional[Tuple[str, int]],
+    post: Optional[Tuple[str, int]],
+    dev: Optional[Tuple[str, int]],
+    local: Optional[Tuple[SubLocalType]],
+) -> CmpKey:
 
     # When we compare a release version, we want to compare it with all of the
     # trailing zeros removed. So we'll use a reverse the list, drop all the now
@@ -516,7 +464,7 @@ def _cmpkey(
     # if there is not a pre or a post segment. If we have one of those then
     # the normal sorting rules will handle this case correctly.
     if pre is None and post is None and dev is not None:
-        _pre = NegativeInfinity  # type: PrePostDevType
+        _pre: PrePostDevType = NegativeInfinity
     # Versions without a pre-release (except as noted above) should sort after
     # those with one.
     elif pre is None:
@@ -526,21 +474,21 @@ def _cmpkey(
 
     # Versions without a post segment should sort before those with one.
     if post is None:
-        _post = NegativeInfinity  # type: PrePostDevType
+        _post: PrePostDevType = NegativeInfinity
 
     else:
         _post = post
 
     # Versions without a development segment should sort after those with one.
     if dev is None:
-        _dev = Infinity  # type: PrePostDevType
+        _dev: PrePostDevType = Infinity
 
     else:
         _dev = dev
 
     if local is None:
         # Versions without a local segment should sort before those with one.
-        _local = NegativeInfinity  # type: LocalType
+        _local: LocalType = NegativeInfinity
     else:
         # Versions with a local segment need that segment parsed to implement
         # the sorting rules in PEP440.
diff --git a/src/pip/_vendor/pep517/__init__.py b/src/pip/_vendor/pep517/__init__.py
index 10687486e2b..2b6b8856790 100644
--- a/src/pip/_vendor/pep517/__init__.py
+++ b/src/pip/_vendor/pep517/__init__.py
@@ -1,6 +1,6 @@
 """Wrappers to build Python packages using PEP 517 hooks
 """
 
-__version__ = '0.9.1'
+__version__ = '0.12.0'
 
 from .wrappers import *  # noqa: F401, F403
diff --git a/src/pip/_vendor/pep517/build.py b/src/pip/_vendor/pep517/build.py
index 264301447e2..bc463b2ba6d 100644
--- a/src/pip/_vendor/pep517/build.py
+++ b/src/pip/_vendor/pep517/build.py
@@ -1,15 +1,15 @@
 """Build a project using PEP 517 hooks.
 """
 import argparse
+import io
 import logging
 import os
-from pip._vendor import toml
 import shutil
 
 from .envbuild import BuildEnvironment
 from .wrappers import Pep517HookCaller
 from .dirtools import tempdir, mkdir_p
-from .compat import FileNotFoundError
+from .compat import FileNotFoundError, toml_load
 
 log = logging.getLogger(__name__)
 
@@ -31,8 +31,8 @@ def load_system(source_dir):
     Load the build system from a source dir (pyproject.toml).
     """
     pyproject = os.path.join(source_dir, 'pyproject.toml')
-    with open(pyproject) as f:
-        pyproject_data = toml.load(f)
+    with io.open(pyproject, 'rb') as f:
+        pyproject_data = toml_load(f)
     return pyproject_data['build-system']
 
 
@@ -110,6 +110,9 @@ def build(source_dir, dist, dest=None, system=None):
 
 
 def main(args):
+    log.warning('pep517.build is deprecated. '
+                'Consider switching to https://pypi.org/project/build/')
+
     # determine which dists to build
     dists = list(filter(None, (
         'sdist' if args.source or not args.binary else None,
diff --git a/src/pip/_vendor/pep517/check.py b/src/pip/_vendor/pep517/check.py
index 13e722a3748..bf3c722641e 100644
--- a/src/pip/_vendor/pep517/check.py
+++ b/src/pip/_vendor/pep517/check.py
@@ -1,10 +1,10 @@
 """Check a project and backend by attempting to build using PEP 517 hooks.
 """
 import argparse
+import io
 import logging
 import os
 from os.path import isfile, join as pjoin
-from pip._vendor.toml import TomlDecodeError, load as toml_load
 import shutil
 from subprocess import CalledProcessError
 import sys
@@ -13,6 +13,7 @@
 import zipfile
 
 from .colorlog import enable_colourful_output
+from .compat import TOMLDecodeError, toml_load
 from .envbuild import BuildEnvironment
 from .wrappers import Pep517HookCaller
 
@@ -141,7 +142,7 @@ def check(source_dir):
         return False
 
     try:
-        with open(pyproject) as f:
+        with io.open(pyproject, 'rb') as f:
             pyproject_data = toml_load(f)
         # Ensure the mandatory data can be loaded
         buildsys = pyproject_data['build-system']
@@ -149,7 +150,7 @@ def check(source_dir):
         backend = buildsys['build-backend']
         backend_path = buildsys.get('backend-path')
         log.info('Loaded pyproject.toml')
-    except (TomlDecodeError, KeyError):
+    except (TOMLDecodeError, KeyError):
         log.error("Invalid pyproject.toml", exc_info=True)
         return False
 
@@ -167,6 +168,9 @@ def check(source_dir):
 
 
 def main(argv=None):
+    log.warning('pep517.check is deprecated. '
+                'Consider switching to https://pypi.org/project/build/')
+
     ap = argparse.ArgumentParser()
     ap.add_argument(
         'source_dir',
diff --git a/src/pip/_vendor/pep517/compat.py b/src/pip/_vendor/pep517/compat.py
index 8432acb7324..730ef5ffaa1 100644
--- a/src/pip/_vendor/pep517/compat.py
+++ b/src/pip/_vendor/pep517/compat.py
@@ -1,4 +1,5 @@
 """Python 2/3 compatibility"""
+import io
 import json
 import sys
 
@@ -32,3 +33,19 @@ def read_json(path):
     FileNotFoundError = FileNotFoundError
 except NameError:
     FileNotFoundError = IOError
+
+
+if sys.version_info < (3, 6):
+    from toml import load as _toml_load  # noqa: F401
+
+    def toml_load(f):
+        w = io.TextIOWrapper(f, encoding="utf8", newline="")
+        try:
+            return _toml_load(w)
+        finally:
+            w.detach()
+
+    from toml import TomlDecodeError as TOMLDecodeError  # noqa: F401
+else:
+    from pip._vendor.tomli import load as toml_load  # noqa: F401
+    from pip._vendor.tomli import TOMLDecodeError  # noqa: F401
diff --git a/src/pip/_vendor/pep517/envbuild.py b/src/pip/_vendor/pep517/envbuild.py
index 4088dcdb40a..fe8873c64a9 100644
--- a/src/pip/_vendor/pep517/envbuild.py
+++ b/src/pip/_vendor/pep517/envbuild.py
@@ -1,23 +1,27 @@
 """Build wheels/sdists by installing build deps to a temporary environment.
 """
 
+import io
 import os
 import logging
-from pip._vendor import toml
 import shutil
 from subprocess import check_call
 import sys
 from sysconfig import get_paths
 from tempfile import mkdtemp
 
+from .compat import toml_load
 from .wrappers import Pep517HookCaller, LoggerWrapper
 
 log = logging.getLogger(__name__)
 
 
 def _load_pyproject(source_dir):
-    with open(os.path.join(source_dir, 'pyproject.toml')) as f:
-        pyproject_data = toml.load(f)
+    with io.open(
+            os.path.join(source_dir, 'pyproject.toml'),
+            'rb',
+            ) as f:
+        pyproject_data = toml_load(f)
     buildsys = pyproject_data['build-system']
     return (
         buildsys['requires'],
diff --git a/src/pip/_vendor/pep517/in_process/__init__.py b/src/pip/_vendor/pep517/in_process/__init__.py
new file mode 100644
index 00000000000..c932313b328
--- /dev/null
+++ b/src/pip/_vendor/pep517/in_process/__init__.py
@@ -0,0 +1,17 @@
+"""This is a subpackage because the directory is on sys.path for _in_process.py
+
+The subpackage should stay as empty as possible to avoid shadowing modules that
+the backend might import.
+"""
+from os.path import dirname, abspath, join as pjoin
+from contextlib import contextmanager
+
+try:
+    import importlib.resources as resources
+
+    def _in_proc_script_path():
+        return resources.path(__package__, '_in_process.py')
+except ImportError:
+    @contextmanager
+    def _in_proc_script_path():
+        yield pjoin(dirname(abspath(__file__)), '_in_process.py')
diff --git a/src/pip/_vendor/pep517/_in_process.py b/src/pip/_vendor/pep517/in_process/_in_process.py
similarity index 72%
rename from src/pip/_vendor/pep517/_in_process.py
rename to src/pip/_vendor/pep517/in_process/_in_process.py
index a536b03e6bb..954a4ab05e9 100644
--- a/src/pip/_vendor/pep517/_in_process.py
+++ b/src/pip/_vendor/pep517/in_process/_in_process.py
@@ -63,6 +63,9 @@ def __init__(self, message):
 
 class HookMissing(Exception):
     """Raised if a hook is missing and we are not executing the fallback"""
+    def __init__(self, hook_name=None):
+        super(HookMissing, self).__init__(hook_name)
+        self.hook_name = hook_name
 
 
 def contained_in(filename, directory):
@@ -100,6 +103,19 @@ def _build_backend():
     return obj
 
 
+def _supported_features():
+    """Return the list of options features supported by the backend.
+
+    Returns a list of strings.
+    The only possible value is 'build_editable'.
+    """
+    backend = _build_backend()
+    features = []
+    if hasattr(backend, "build_editable"):
+        features.append("build_editable")
+    return features
+
+
 def get_requires_for_build_wheel(config_settings):
     """Invoke the optional get_requires_for_build_wheel hook
 
@@ -114,6 +130,20 @@ def get_requires_for_build_wheel(config_settings):
         return hook(config_settings)
 
 
+def get_requires_for_build_editable(config_settings):
+    """Invoke the optional get_requires_for_build_editable hook
+
+    Returns [] if the hook is not defined.
+    """
+    backend = _build_backend()
+    try:
+        hook = backend.get_requires_for_build_editable
+    except AttributeError:
+        return []
+    else:
+        return hook(config_settings)
+
+
 def prepare_metadata_for_build_wheel(
         metadata_directory, config_settings, _allow_fallback):
     """Invoke optional prepare_metadata_for_build_wheel
@@ -127,12 +157,40 @@ def prepare_metadata_for_build_wheel(
     except AttributeError:
         if not _allow_fallback:
             raise HookMissing()
-        return _get_wheel_metadata_from_wheel(backend, metadata_directory,
+        whl_basename = backend.build_wheel(metadata_directory, config_settings)
+        return _get_wheel_metadata_from_wheel(whl_basename, metadata_directory,
                                               config_settings)
     else:
         return hook(metadata_directory, config_settings)
 
 
+def prepare_metadata_for_build_editable(
+        metadata_directory, config_settings, _allow_fallback):
+    """Invoke optional prepare_metadata_for_build_editable
+
+    Implements a fallback by building an editable wheel if the hook isn't
+    defined, unless _allow_fallback is False in which case HookMissing is
+    raised.
+    """
+    backend = _build_backend()
+    try:
+        hook = backend.prepare_metadata_for_build_editable
+    except AttributeError:
+        if not _allow_fallback:
+            raise HookMissing()
+        try:
+            build_hook = backend.build_editable
+        except AttributeError:
+            raise HookMissing(hook_name='build_editable')
+        else:
+            whl_basename = build_hook(metadata_directory, config_settings)
+            return _get_wheel_metadata_from_wheel(whl_basename,
+                                                  metadata_directory,
+                                                  config_settings)
+    else:
+        return hook(metadata_directory, config_settings)
+
+
 WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL'
 
 
@@ -149,14 +207,13 @@ def _dist_info_files(whl_zip):
 
 
 def _get_wheel_metadata_from_wheel(
-        backend, metadata_directory, config_settings):
-    """Build a wheel and extract the metadata from it.
+        whl_basename, metadata_directory, config_settings):
+    """Extract the metadata from a wheel.
 
     Fallback for when the build backend does not
     define the 'get_wheel_metadata' hook.
     """
     from zipfile import ZipFile
-    whl_basename = backend.build_wheel(metadata_directory, config_settings)
     with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'):
         pass  # Touch marker file
 
@@ -205,6 +262,27 @@ def build_wheel(wheel_directory, config_settings, metadata_directory=None):
                                         metadata_directory)
 
 
+def build_editable(wheel_directory, config_settings, metadata_directory=None):
+    """Invoke the optional build_editable hook.
+
+    If a wheel was already built in the
+    prepare_metadata_for_build_editable fallback, this
+    will copy it rather than rebuilding the wheel.
+    """
+    backend = _build_backend()
+    try:
+        hook = backend.build_editable
+    except AttributeError:
+        raise HookMissing()
+    else:
+        prebuilt_whl = _find_already_built_wheel(metadata_directory)
+        if prebuilt_whl:
+            shutil.copy2(prebuilt_whl, wheel_directory)
+            return os.path.basename(prebuilt_whl)
+
+        return hook(wheel_directory, config_settings, metadata_directory)
+
+
 def get_requires_for_build_sdist(config_settings):
     """Invoke the optional get_requires_for_build_wheel hook
 
@@ -242,8 +320,12 @@ def build_sdist(sdist_directory, config_settings):
     'get_requires_for_build_wheel',
     'prepare_metadata_for_build_wheel',
     'build_wheel',
+    'get_requires_for_build_editable',
+    'prepare_metadata_for_build_editable',
+    'build_editable',
     'get_requires_for_build_sdist',
     'build_sdist',
+    '_supported_features',
 }
 
 
@@ -270,8 +352,9 @@ def main():
     except GotUnsupportedOperation as e:
         json_out['unsupported'] = True
         json_out['traceback'] = e.traceback
-    except HookMissing:
+    except HookMissing as e:
         json_out['hook_missing'] = True
+        json_out['missing_hook_name'] = e.hook_name or hook_name
 
     write_json(json_out, pjoin(control_dir, 'output.json'), indent=2)
 
diff --git a/src/pip/_vendor/pep517/wrappers.py b/src/pip/_vendor/pep517/wrappers.py
index d6338ea5201..e031ed70875 100644
--- a/src/pip/_vendor/pep517/wrappers.py
+++ b/src/pip/_vendor/pep517/wrappers.py
@@ -1,13 +1,14 @@
 import threading
 from contextlib import contextmanager
 import os
-from os.path import dirname, abspath, join as pjoin
+from os.path import abspath, join as pjoin
 import shutil
 from subprocess import check_call, check_output, STDOUT
 import sys
 from tempfile import mkdtemp
 
 from . import compat
+from .in_process import _in_proc_script_path
 
 __all__ = [
     'BackendUnavailable',
@@ -19,16 +20,6 @@
     'Pep517HookCaller',
 ]
 
-try:
-    import importlib.resources as resources
-
-    def _in_proc_script_path():
-        return resources.path(__package__, '_in_process.py')
-except ImportError:
-    @contextmanager
-    def _in_proc_script_path():
-        yield pjoin(dirname(abspath(__file__)), '_in_process.py')
-
 
 @contextmanager
 def tempdir():
@@ -163,6 +154,10 @@ def subprocess_runner(self, runner):
         finally:
             self._subprocess_runner = prev
 
+    def _supported_features(self):
+        """Return the list of optional features supported by the backend."""
+        return self._call_hook('_supported_features', {})
+
     def get_requires_for_build_wheel(self, config_settings=None):
         """Identify packages required for building a wheel
 
@@ -216,6 +211,59 @@ def build_wheel(
             'metadata_directory': metadata_directory,
         })
 
+    def get_requires_for_build_editable(self, config_settings=None):
+        """Identify packages required for building an editable wheel
+
+        Returns a list of dependency specifications, e.g.::
+
+            ["wheel >= 0.25", "setuptools"]
+
+        This does not include requirements specified in pyproject.toml.
+        It returns the result of calling the equivalently named hook in a
+        subprocess.
+        """
+        return self._call_hook('get_requires_for_build_editable', {
+            'config_settings': config_settings
+        })
+
+    def prepare_metadata_for_build_editable(
+            self, metadata_directory, config_settings=None,
+            _allow_fallback=True):
+        """Prepare a ``*.dist-info`` folder with metadata for this project.
+
+        Returns the name of the newly created folder.
+
+        If the build backend defines a hook with this name, it will be called
+        in a subprocess. If not, the backend will be asked to build an editable
+        wheel, and the dist-info extracted from that (unless _allow_fallback is
+        False).
+        """
+        return self._call_hook('prepare_metadata_for_build_editable', {
+            'metadata_directory': abspath(metadata_directory),
+            'config_settings': config_settings,
+            '_allow_fallback': _allow_fallback,
+        })
+
+    def build_editable(
+            self, wheel_directory, config_settings=None,
+            metadata_directory=None):
+        """Build an editable wheel from this project.
+
+        Returns the name of the newly created file.
+
+        In general, this will call the 'build_editable' hook in the backend.
+        However, if that was previously called by
+        'prepare_metadata_for_build_editable', and the same metadata_directory
+        is used, the previously built wheel will be copied to wheel_directory.
+        """
+        if metadata_directory is not None:
+            metadata_directory = abspath(metadata_directory)
+        return self._call_hook('build_editable', {
+            'wheel_directory': abspath(wheel_directory),
+            'config_settings': config_settings,
+            'metadata_directory': metadata_directory,
+        })
+
     def get_requires_for_build_sdist(self, config_settings=None):
         """Identify packages required for building a wheel
 
@@ -289,7 +337,7 @@ def _call_hook(self, hook_name, kwargs):
                     message=data.get('backend_error', '')
                 )
             if data.get('hook_missing'):
-                raise HookMissing(hook_name)
+                raise HookMissing(data.get('missing_hook_name') or hook_name)
             return data['return_val']
 
 
diff --git a/src/pip/_vendor/pkg_resources/__init__.py b/src/pip/_vendor/pkg_resources/__init__.py
index a457ff27ef0..4cd562cf94c 100644
--- a/src/pip/_vendor/pkg_resources/__init__.py
+++ b/src/pip/_vendor/pkg_resources/__init__.py
@@ -77,7 +77,7 @@
     importlib_machinery = None
 
 from . import py31compat
-from pip._vendor import appdirs
+from pip._vendor import platformdirs
 from pip._vendor import packaging
 __import__('pip._vendor.packaging.version')
 __import__('pip._vendor.packaging.specifiers')
@@ -1310,7 +1310,7 @@ def get_default_cache():
     """
     return (
         os.environ.get('PYTHON_EGG_CACHE')
-        or appdirs.user_cache_dir(appname='Python-Eggs')
+        or platformdirs.user_cache_dir(appname='Python-Eggs')
     )
 
 
diff --git a/src/pip/_vendor/appdirs.LICENSE.txt b/src/pip/_vendor/platformdirs/LICENSE.txt
similarity index 99%
rename from src/pip/_vendor/appdirs.LICENSE.txt
rename to src/pip/_vendor/platformdirs/LICENSE.txt
index 107c61405e3..f0bbd69f0c8 100644
--- a/src/pip/_vendor/appdirs.LICENSE.txt
+++ b/src/pip/_vendor/platformdirs/LICENSE.txt
@@ -20,4 +20,3 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
diff --git a/src/pip/_vendor/platformdirs/__init__.py b/src/pip/_vendor/platformdirs/__init__.py
new file mode 100644
index 00000000000..089b5157436
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/__init__.py
@@ -0,0 +1,331 @@
+"""
+Utilities for determining application-specific dirs. See  for details and
+usage.
+"""
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pip._vendor.typing_extensions import Literal  # pragma: no cover
+
+from .api import PlatformDirsABC
+from .version import __version__, __version_info__
+
+
+def _set_platform_dir_class() -> type[PlatformDirsABC]:
+    if os.getenv("ANDROID_DATA") == "/data" and os.getenv("ANDROID_ROOT") == "/system":
+        module, name = "pip._vendor.platformdirs.android", "Android"
+    elif sys.platform == "win32":
+        module, name = "pip._vendor.platformdirs.windows", "Windows"
+    elif sys.platform == "darwin":
+        module, name = "pip._vendor.platformdirs.macos", "MacOS"
+    else:
+        module, name = "pip._vendor.platformdirs.unix", "Unix"
+    result: type[PlatformDirsABC] = getattr(importlib.import_module(module), name)
+    return result
+
+
+PlatformDirs = _set_platform_dir_class()  #: Currently active platform
+AppDirs = PlatformDirs  #: Backwards compatibility with appdirs
+
+
+def user_data_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    roaming: bool = False,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param roaming: See `roaming `.
+    :returns: data directory tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_data_dir
+
+
+def site_data_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    multipath: bool = False,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param multipath: See `roaming `.
+    :returns: data directory shared by users
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_data_dir
+
+
+def user_config_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    roaming: bool = False,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param roaming: See `roaming `.
+    :returns: config directory tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_config_dir
+
+
+def site_config_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    multipath: bool = False,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param multipath: See `roaming `.
+    :returns: config directory shared by the users
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_config_dir
+
+
+def user_cache_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    opinion: bool = True,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param opinion: See `roaming `.
+    :returns: cache directory tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_cache_dir
+
+
+def user_state_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    roaming: bool = False,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param roaming: See `roaming `.
+    :returns: state directory tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_state_dir
+
+
+def user_log_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    opinion: bool = True,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param opinion: See `roaming `.
+    :returns: log directory tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_log_dir
+
+
+def user_documents_dir() -> str:
+    """
+    :returns: documents directory tied to the user
+    """
+    return PlatformDirs().user_documents_dir
+
+
+def user_runtime_dir(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    opinion: bool = True,
+) -> str:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param opinion: See `opinion `.
+    :returns: runtime directory tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_runtime_dir
+
+
+def user_data_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    roaming: bool = False,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param roaming: See `roaming `.
+    :returns: data path tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_data_path
+
+
+def site_data_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    multipath: bool = False,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param multipath: See `multipath `.
+    :returns: data path shared by users
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_data_path
+
+
+def user_config_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    roaming: bool = False,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param roaming: See `roaming `.
+    :returns: config path tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_config_path
+
+
+def site_config_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    multipath: bool = False,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param multipath: See `roaming `.
+    :returns: config path shared by the users
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, multipath=multipath).site_config_path
+
+
+def user_cache_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    opinion: bool = True,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param opinion: See `roaming `.
+    :returns: cache path tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_cache_path
+
+
+def user_state_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    roaming: bool = False,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param roaming: See `roaming `.
+    :returns: state path tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, roaming=roaming).user_state_path
+
+
+def user_log_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    opinion: bool = True,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param opinion: See `roaming `.
+    :returns: log path tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_log_path
+
+
+def user_documents_path() -> Path:
+    """
+    :returns: documents path tied to the user
+    """
+    return PlatformDirs().user_documents_path
+
+
+def user_runtime_path(
+    appname: str | None = None,
+    appauthor: str | None | Literal[False] = None,
+    version: str | None = None,
+    opinion: bool = True,
+) -> Path:
+    """
+    :param appname: See `appname `.
+    :param appauthor: See `appauthor `.
+    :param version: See `version `.
+    :param opinion: See `opinion `.
+    :returns: runtime path tied to the user
+    """
+    return PlatformDirs(appname=appname, appauthor=appauthor, version=version, opinion=opinion).user_runtime_path
+
+
+__all__ = [
+    "__version__",
+    "__version_info__",
+    "PlatformDirs",
+    "AppDirs",
+    "PlatformDirsABC",
+    "user_data_dir",
+    "user_config_dir",
+    "user_cache_dir",
+    "user_state_dir",
+    "user_log_dir",
+    "user_documents_dir",
+    "user_runtime_dir",
+    "site_data_dir",
+    "site_config_dir",
+    "user_data_path",
+    "user_config_path",
+    "user_cache_path",
+    "user_state_path",
+    "user_log_path",
+    "user_documents_path",
+    "user_runtime_path",
+    "site_data_path",
+    "site_config_path",
+]
diff --git a/src/pip/_vendor/platformdirs/__main__.py b/src/pip/_vendor/platformdirs/__main__.py
new file mode 100644
index 00000000000..9c54bfb438d
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/__main__.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from pip._vendor.platformdirs import PlatformDirs, __version__
+
+PROPS = (
+    "user_data_dir",
+    "user_config_dir",
+    "user_cache_dir",
+    "user_state_dir",
+    "user_log_dir",
+    "user_documents_dir",
+    "user_runtime_dir",
+    "site_data_dir",
+    "site_config_dir",
+)
+
+
+def main() -> None:
+    app_name = "MyApp"
+    app_author = "MyCompany"
+
+    print(f"-- platformdirs {__version__} --")
+
+    print("-- app dirs (with optional 'version')")
+    dirs = PlatformDirs(app_name, app_author, version="1.0")
+    for prop in PROPS:
+        print(f"{prop}: {getattr(dirs, prop)}")
+
+    print("\n-- app dirs (without optional 'version')")
+    dirs = PlatformDirs(app_name, app_author)
+    for prop in PROPS:
+        print(f"{prop}: {getattr(dirs, prop)}")
+
+    print("\n-- app dirs (without optional 'appauthor')")
+    dirs = PlatformDirs(app_name)
+    for prop in PROPS:
+        print(f"{prop}: {getattr(dirs, prop)}")
+
+    print("\n-- app dirs (with disabled 'appauthor')")
+    dirs = PlatformDirs(app_name, appauthor=False)
+    for prop in PROPS:
+        print(f"{prop}: {getattr(dirs, prop)}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/pip/_vendor/platformdirs/android.py b/src/pip/_vendor/platformdirs/android.py
new file mode 100644
index 00000000000..a68405871f2
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/android.py
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import os
+import re
+import sys
+from functools import lru_cache
+
+from .api import PlatformDirsABC
+
+
+class Android(PlatformDirsABC):
+    """
+    Follows the guidance `from here `_. Makes use of the
+    `appname ` and
+    `version `.
+    """
+
+    @property
+    def user_data_dir(self) -> str:
+        """:return: data directory tied to the user, e.g. ``/data/user///files/``"""
+        return self._append_app_name_and_version(_android_folder(), "files")
+
+    @property
+    def site_data_dir(self) -> str:
+        """:return: data directory shared by users, same as `user_data_dir`"""
+        return self.user_data_dir
+
+    @property
+    def user_config_dir(self) -> str:
+        """
+        :return: config directory tied to the user, e.g. ``/data/user///shared_prefs/``
+        """
+        return self._append_app_name_and_version(_android_folder(), "shared_prefs")
+
+    @property
+    def site_config_dir(self) -> str:
+        """:return: config directory shared by the users, same as `user_config_dir`"""
+        return self.user_config_dir
+
+    @property
+    def user_cache_dir(self) -> str:
+        """:return: cache directory tied to the user, e.g. e.g. ``/data/user///cache/``"""
+        return self._append_app_name_and_version(_android_folder(), "cache")
+
+    @property
+    def user_state_dir(self) -> str:
+        """:return: state directory tied to the user, same as `user_data_dir`"""
+        return self.user_data_dir
+
+    @property
+    def user_log_dir(self) -> str:
+        """
+        :return: log directory tied to the user, same as `user_cache_dir` if not opinionated else ``log`` in it,
+          e.g. ``/data/user///cache//log``
+        """
+        path = self.user_cache_dir
+        if self.opinion:
+            path = os.path.join(path, "log")
+        return path
+
+    @property
+    def user_documents_dir(self) -> str:
+        """
+        :return: documents directory tied to the user e.g. ``/storage/emulated/0/Documents``
+        """
+        return _android_documents_folder()
+
+    @property
+    def user_runtime_dir(self) -> str:
+        """
+        :return: runtime directory tied to the user, same as `user_cache_dir` if not opinionated else ``tmp`` in it,
+          e.g. ``/data/user///cache//tmp``
+        """
+        path = self.user_cache_dir
+        if self.opinion:
+            path = os.path.join(path, "tmp")
+        return path
+
+
+@lru_cache(maxsize=1)
+def _android_folder() -> str:
+    """:return: base folder for the Android OS"""
+    try:
+        # First try to get path to android app via pyjnius
+        from jnius import autoclass
+
+        Context = autoclass("android.content.Context")  # noqa: N806
+        result: str = Context.getFilesDir().getParentFile().getAbsolutePath()
+    except Exception:
+        # if fails find an android folder looking path on the sys.path
+        pattern = re.compile(r"/data/(data|user/\d+)/(.+)/files")
+        for path in sys.path:
+            if pattern.match(path):
+                result = path.split("/files")[0]
+                break
+        else:
+            raise OSError("Cannot find path to android app folder")
+    return result
+
+
+@lru_cache(maxsize=1)
+def _android_documents_folder() -> str:
+    """:return: documents folder for the Android OS"""
+    # Get directories with pyjnius
+    try:
+        from jnius import autoclass
+
+        Context = autoclass("android.content.Context")  # noqa: N806
+        Environment = autoclass("android.os.Environment")  # noqa: N806
+        documents_dir: str = Context.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath()
+    except Exception:
+        documents_dir = "/storage/emulated/0/Documents"
+
+    return documents_dir
+
+
+__all__ = [
+    "Android",
+]
diff --git a/src/pip/_vendor/platformdirs/api.py b/src/pip/_vendor/platformdirs/api.py
new file mode 100644
index 00000000000..6f6e2c2c69d
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/api.py
@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import os
+import sys
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+if sys.version_info >= (3, 8):  # pragma: no branch
+    from typing import Literal  # pragma: no cover
+
+
+class PlatformDirsABC(ABC):
+    """
+    Abstract base class for platform directories.
+    """
+
+    def __init__(
+        self,
+        appname: str | None = None,
+        appauthor: str | None | Literal[False] = None,
+        version: str | None = None,
+        roaming: bool = False,
+        multipath: bool = False,
+        opinion: bool = True,
+    ):
+        """
+        Create a new platform directory.
+
+        :param appname: See `appname`.
+        :param appauthor: See `appauthor`.
+        :param version: See `version`.
+        :param roaming: See `roaming`.
+        :param multipath: See `multipath`.
+        :param opinion: See `opinion`.
+        """
+        self.appname = appname  #: The name of application.
+        self.appauthor = appauthor
+        """
+        The name of the app author or distributing body for this application. Typically, it is the owning company name.
+        Defaults to `appname`. You may pass ``False`` to disable it.
+        """
+        self.version = version
+        """
+        An optional version path element to append to the path. You might want to use this if you want multiple versions
+        of your app to be able to run independently. If used, this would typically be ``.``.
+        """
+        self.roaming = roaming
+        """
+        Whether to use the roaming appdata directory on Windows. That means that for users on a Windows network setup
+        for roaming profiles, this user data will be synced on login (see
+        `here `_).
+        """
+        self.multipath = multipath
+        """
+        An optional parameter only applicable to Unix/Linux which indicates that the entire list of data dirs should be
+        returned. By default, the first item would only be returned.
+        """
+        self.opinion = opinion  #: A flag to indicating to use opinionated values.
+
+    def _append_app_name_and_version(self, *base: str) -> str:
+        params = list(base[1:])
+        if self.appname:
+            params.append(self.appname)
+            if self.version:
+                params.append(self.version)
+        return os.path.join(base[0], *params)
+
+    @property
+    @abstractmethod
+    def user_data_dir(self) -> str:
+        """:return: data directory tied to the user"""
+
+    @property
+    @abstractmethod
+    def site_data_dir(self) -> str:
+        """:return: data directory shared by users"""
+
+    @property
+    @abstractmethod
+    def user_config_dir(self) -> str:
+        """:return: config directory tied to the user"""
+
+    @property
+    @abstractmethod
+    def site_config_dir(self) -> str:
+        """:return: config directory shared by the users"""
+
+    @property
+    @abstractmethod
+    def user_cache_dir(self) -> str:
+        """:return: cache directory tied to the user"""
+
+    @property
+    @abstractmethod
+    def user_state_dir(self) -> str:
+        """:return: state directory tied to the user"""
+
+    @property
+    @abstractmethod
+    def user_log_dir(self) -> str:
+        """:return: log directory tied to the user"""
+
+    @property
+    @abstractmethod
+    def user_documents_dir(self) -> str:
+        """:return: documents directory tied to the user"""
+
+    @property
+    @abstractmethod
+    def user_runtime_dir(self) -> str:
+        """:return: runtime directory tied to the user"""
+
+    @property
+    def user_data_path(self) -> Path:
+        """:return: data path tied to the user"""
+        return Path(self.user_data_dir)
+
+    @property
+    def site_data_path(self) -> Path:
+        """:return: data path shared by users"""
+        return Path(self.site_data_dir)
+
+    @property
+    def user_config_path(self) -> Path:
+        """:return: config path tied to the user"""
+        return Path(self.user_config_dir)
+
+    @property
+    def site_config_path(self) -> Path:
+        """:return: config path shared by the users"""
+        return Path(self.site_config_dir)
+
+    @property
+    def user_cache_path(self) -> Path:
+        """:return: cache path tied to the user"""
+        return Path(self.user_cache_dir)
+
+    @property
+    def user_state_path(self) -> Path:
+        """:return: state path tied to the user"""
+        return Path(self.user_state_dir)
+
+    @property
+    def user_log_path(self) -> Path:
+        """:return: log path tied to the user"""
+        return Path(self.user_log_dir)
+
+    @property
+    def user_documents_path(self) -> Path:
+        """:return: documents path tied to the user"""
+        return Path(self.user_documents_dir)
+
+    @property
+    def user_runtime_path(self) -> Path:
+        """:return: runtime path tied to the user"""
+        return Path(self.user_runtime_dir)
diff --git a/src/pip/_vendor/platformdirs/macos.py b/src/pip/_vendor/platformdirs/macos.py
new file mode 100644
index 00000000000..a01337c7764
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/macos.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import os
+
+from .api import PlatformDirsABC
+
+
+class MacOS(PlatformDirsABC):
+    """
+    Platform directories for the macOS operating system. Follows the guidance from `Apple documentation
+    `_.
+    Makes use of the `appname ` and
+    `version `.
+    """
+
+    @property
+    def user_data_dir(self) -> str:
+        """:return: data directory tied to the user, e.g. ``~/Library/Application Support/$appname/$version``"""
+        return self._append_app_name_and_version(os.path.expanduser("~/Library/Application Support/"))
+
+    @property
+    def site_data_dir(self) -> str:
+        """:return: data directory shared by users, e.g. ``/Library/Application Support/$appname/$version``"""
+        return self._append_app_name_and_version("/Library/Application Support")
+
+    @property
+    def user_config_dir(self) -> str:
+        """:return: config directory tied to the user, e.g. ``~/Library/Preferences/$appname/$version``"""
+        return self._append_app_name_and_version(os.path.expanduser("~/Library/Preferences/"))
+
+    @property
+    def site_config_dir(self) -> str:
+        """:return: config directory shared by the users, e.g. ``/Library/Preferences/$appname``"""
+        return self._append_app_name_and_version("/Library/Preferences")
+
+    @property
+    def user_cache_dir(self) -> str:
+        """:return: cache directory tied to the user, e.g. ``~/Library/Caches/$appname/$version``"""
+        return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches"))
+
+    @property
+    def user_state_dir(self) -> str:
+        """:return: state directory tied to the user, same as `user_data_dir`"""
+        return self.user_data_dir
+
+    @property
+    def user_log_dir(self) -> str:
+        """:return: log directory tied to the user, e.g. ``~/Library/Logs/$appname/$version``"""
+        return self._append_app_name_and_version(os.path.expanduser("~/Library/Logs"))
+
+    @property
+    def user_documents_dir(self) -> str:
+        """:return: documents directory tied to the user, e.g. ``~/Documents``"""
+        return os.path.expanduser("~/Documents")
+
+    @property
+    def user_runtime_dir(self) -> str:
+        """:return: runtime directory tied to the user, e.g. ``~/Library/Caches/TemporaryItems/$appname/$version``"""
+        return self._append_app_name_and_version(os.path.expanduser("~/Library/Caches/TemporaryItems"))
+
+
+__all__ = [
+    "MacOS",
+]
diff --git a/news/205edb4f-502c-4213-b8b8-c9173718e8ab.trivial.rst b/src/pip/_vendor/platformdirs/py.typed
similarity index 100%
rename from news/205edb4f-502c-4213-b8b8-c9173718e8ab.trivial.rst
rename to src/pip/_vendor/platformdirs/py.typed
diff --git a/src/pip/_vendor/platformdirs/unix.py b/src/pip/_vendor/platformdirs/unix.py
new file mode 100644
index 00000000000..2fbd4d4f367
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/unix.py
@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import os
+import sys
+from configparser import ConfigParser
+from pathlib import Path
+
+from .api import PlatformDirsABC
+
+if sys.platform.startswith("linux"):  # pragma: no branch # no op check, only to please the type checker
+    from os import getuid
+else:
+
+    def getuid() -> int:
+        raise RuntimeError("should only be used on Linux")
+
+
+class Unix(PlatformDirsABC):
+    """
+    On Unix/Linux, we follow the
+    `XDG Basedir Spec `_. The spec allows
+    overriding directories with environment variables. The examples show are the default values, alongside the name of
+    the environment variable that overrides them. Makes use of the
+    `appname `,
+    `version `,
+    `multipath `,
+    `opinion `.
+    """
+
+    @property
+    def user_data_dir(self) -> str:
+        """
+        :return: data directory tied to the user, e.g. ``~/.local/share/$appname/$version`` or
+         ``$XDG_DATA_HOME/$appname/$version``
+        """
+        path = os.environ.get("XDG_DATA_HOME", "")
+        if not path.strip():
+            path = os.path.expanduser("~/.local/share")
+        return self._append_app_name_and_version(path)
+
+    @property
+    def site_data_dir(self) -> str:
+        """
+        :return: data directories shared by users (if `multipath ` is
+         enabled and ``XDG_DATA_DIR`` is set and a multi path the response is also a multi path separated by the OS
+         path separator), e.g. ``/usr/local/share/$appname/$version`` or ``/usr/share/$appname/$version``
+        """
+        # XDG default for $XDG_DATA_DIRS; only first, if multipath is False
+        path = os.environ.get("XDG_DATA_DIRS", "")
+        if not path.strip():
+            path = f"/usr/local/share{os.pathsep}/usr/share"
+        return self._with_multi_path(path)
+
+    def _with_multi_path(self, path: str) -> str:
+        path_list = path.split(os.pathsep)
+        if not self.multipath:
+            path_list = path_list[0:1]
+        path_list = [self._append_app_name_and_version(os.path.expanduser(p)) for p in path_list]
+        return os.pathsep.join(path_list)
+
+    @property
+    def user_config_dir(self) -> str:
+        """
+        :return: config directory tied to the user, e.g. ``~/.config/$appname/$version`` or
+         ``$XDG_CONFIG_HOME/$appname/$version``
+        """
+        path = os.environ.get("XDG_CONFIG_HOME", "")
+        if not path.strip():
+            path = os.path.expanduser("~/.config")
+        return self._append_app_name_and_version(path)
+
+    @property
+    def site_config_dir(self) -> str:
+        """
+        :return: config directories shared by users (if `multipath `
+         is enabled and ``XDG_DATA_DIR`` is set and a multi path the response is also a multi path separated by the OS
+         path separator), e.g. ``/etc/xdg/$appname/$version``
+        """
+        # XDG default for $XDG_CONFIG_DIRS only first, if multipath is False
+        path = os.environ.get("XDG_CONFIG_DIRS", "")
+        if not path.strip():
+            path = "/etc/xdg"
+        return self._with_multi_path(path)
+
+    @property
+    def user_cache_dir(self) -> str:
+        """
+        :return: cache directory tied to the user, e.g. ``~/.cache/$appname/$version`` or
+         ``~/$XDG_CACHE_HOME/$appname/$version``
+        """
+        path = os.environ.get("XDG_CACHE_HOME", "")
+        if not path.strip():
+            path = os.path.expanduser("~/.cache")
+        return self._append_app_name_and_version(path)
+
+    @property
+    def user_state_dir(self) -> str:
+        """
+        :return: state directory tied to the user, e.g. ``~/.local/state/$appname/$version`` or
+         ``$XDG_STATE_HOME/$appname/$version``
+        """
+        path = os.environ.get("XDG_STATE_HOME", "")
+        if not path.strip():
+            path = os.path.expanduser("~/.local/state")
+        return self._append_app_name_and_version(path)
+
+    @property
+    def user_log_dir(self) -> str:
+        """
+        :return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``log`` in it
+        """
+        path = self.user_cache_dir
+        if self.opinion:
+            path = os.path.join(path, "log")
+        return path
+
+    @property
+    def user_documents_dir(self) -> str:
+        """
+        :return: documents directory tied to the user, e.g. ``~/Documents``
+        """
+        documents_dir = _get_user_dirs_folder("XDG_DOCUMENTS_DIR")
+        if documents_dir is None:
+            documents_dir = os.environ.get("XDG_DOCUMENTS_DIR", "").strip()
+            if not documents_dir:
+                documents_dir = os.path.expanduser("~/Documents")
+
+        return documents_dir
+
+    @property
+    def user_runtime_dir(self) -> str:
+        """
+        :return: runtime directory tied to the user, e.g. ``/run/user/$(id -u)/$appname/$version`` or
+         ``$XDG_RUNTIME_DIR/$appname/$version``
+        """
+        path = os.environ.get("XDG_RUNTIME_DIR", "")
+        if not path.strip():
+            path = f"/run/user/{getuid()}"
+        return self._append_app_name_and_version(path)
+
+    @property
+    def site_data_path(self) -> Path:
+        """:return: data path shared by users. Only return first item, even if ``multipath`` is set to ``True``"""
+        return self._first_item_as_path_if_multipath(self.site_data_dir)
+
+    @property
+    def site_config_path(self) -> Path:
+        """:return: config path shared by the users. Only return first item, even if ``multipath`` is set to ``True``"""
+        return self._first_item_as_path_if_multipath(self.site_config_dir)
+
+    def _first_item_as_path_if_multipath(self, directory: str) -> Path:
+        if self.multipath:
+            # If multipath is True, the first path is returned.
+            directory = directory.split(os.pathsep)[0]
+        return Path(directory)
+
+
+def _get_user_dirs_folder(key: str) -> str | None:
+    """Return directory from user-dirs.dirs config file. See https://freedesktop.org/wiki/Software/xdg-user-dirs/"""
+    user_dirs_config_path = os.path.join(Unix().user_config_dir, "user-dirs.dirs")
+    if os.path.exists(user_dirs_config_path):
+        parser = ConfigParser()
+
+        with open(user_dirs_config_path) as stream:
+            # Add fake section header, so ConfigParser doesn't complain
+            parser.read_string(f"[top]\n{stream.read()}")
+
+        if key not in parser["top"]:
+            return None
+
+        path = parser["top"][key].strip('"')
+        # Handle relative home paths
+        path = path.replace("$HOME", os.path.expanduser("~"))
+        return path
+
+    return None
+
+
+__all__ = [
+    "Unix",
+]
diff --git a/src/pip/_vendor/platformdirs/version.py b/src/pip/_vendor/platformdirs/version.py
new file mode 100644
index 00000000000..175ded85617
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/version.py
@@ -0,0 +1,4 @@
+""" Version information """
+
+__version__ = "2.4.1"
+__version_info__ = (2, 4, 1)
diff --git a/src/pip/_vendor/platformdirs/windows.py b/src/pip/_vendor/platformdirs/windows.py
new file mode 100644
index 00000000000..ef972bdf29c
--- /dev/null
+++ b/src/pip/_vendor/platformdirs/windows.py
@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import ctypes
+import os
+from functools import lru_cache
+from typing import Callable
+
+from .api import PlatformDirsABC
+
+
+class Windows(PlatformDirsABC):
+    """`MSDN on where to store app data files
+    `_.
+    Makes use of the
+    `appname `,
+    `appauthor `,
+    `version `,
+    `roaming `,
+    `opinion `."""
+
+    @property
+    def user_data_dir(self) -> str:
+        """
+        :return: data directory tied to the user, e.g.
+         ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname`` (not roaming) or
+         ``%USERPROFILE%\\AppData\\Roaming\\$appauthor\\$appname`` (roaming)
+        """
+        const = "CSIDL_APPDATA" if self.roaming else "CSIDL_LOCAL_APPDATA"
+        path = os.path.normpath(get_win_folder(const))
+        return self._append_parts(path)
+
+    def _append_parts(self, path: str, *, opinion_value: str | None = None) -> str:
+        params = []
+        if self.appname:
+            if self.appauthor is not False:
+                author = self.appauthor or self.appname
+                params.append(author)
+            params.append(self.appname)
+            if opinion_value is not None and self.opinion:
+                params.append(opinion_value)
+            if self.version:
+                params.append(self.version)
+        return os.path.join(path, *params)
+
+    @property
+    def site_data_dir(self) -> str:
+        """:return: data directory shared by users, e.g. ``C:\\ProgramData\\$appauthor\\$appname``"""
+        path = os.path.normpath(get_win_folder("CSIDL_COMMON_APPDATA"))
+        return self._append_parts(path)
+
+    @property
+    def user_config_dir(self) -> str:
+        """:return: config directory tied to the user, same as `user_data_dir`"""
+        return self.user_data_dir
+
+    @property
+    def site_config_dir(self) -> str:
+        """:return: config directory shared by the users, same as `site_data_dir`"""
+        return self.site_data_dir
+
+    @property
+    def user_cache_dir(self) -> str:
+        """
+        :return: cache directory tied to the user (if opinionated with ``Cache`` folder within ``$appname``) e.g.
+         ``%USERPROFILE%\\AppData\\Local\\$appauthor\\$appname\\Cache\\$version``
+        """
+        path = os.path.normpath(get_win_folder("CSIDL_LOCAL_APPDATA"))
+        return self._append_parts(path, opinion_value="Cache")
+
+    @property
+    def user_state_dir(self) -> str:
+        """:return: state directory tied to the user, same as `user_data_dir`"""
+        return self.user_data_dir
+
+    @property
+    def user_log_dir(self) -> str:
+        """
+        :return: log directory tied to the user, same as `user_data_dir` if not opinionated else ``Logs`` in it
+        """
+        path = self.user_data_dir
+        if self.opinion:
+            path = os.path.join(path, "Logs")
+        return path
+
+    @property
+    def user_documents_dir(self) -> str:
+        """
+        :return: documents directory tied to the user e.g. ``%USERPROFILE%\\Documents``
+        """
+        return os.path.normpath(get_win_folder("CSIDL_PERSONAL"))
+
+    @property
+    def user_runtime_dir(self) -> str:
+        """
+        :return: runtime directory tied to the user, e.g.
+         ``%USERPROFILE%\\AppData\\Local\\Temp\\$appauthor\\$appname``
+        """
+        path = os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Temp"))
+        return self._append_parts(path)
+
+
+def get_win_folder_from_env_vars(csidl_name: str) -> str:
+    """Get folder from environment variables."""
+    if csidl_name == "CSIDL_PERSONAL":  # does not have an environment name
+        return os.path.join(os.path.normpath(os.environ["USERPROFILE"]), "Documents")
+
+    env_var_name = {
+        "CSIDL_APPDATA": "APPDATA",
+        "CSIDL_COMMON_APPDATA": "ALLUSERSPROFILE",
+        "CSIDL_LOCAL_APPDATA": "LOCALAPPDATA",
+    }.get(csidl_name)
+    if env_var_name is None:
+        raise ValueError(f"Unknown CSIDL name: {csidl_name}")
+    result = os.environ.get(env_var_name)
+    if result is None:
+        raise ValueError(f"Unset environment variable: {env_var_name}")
+    return result
+
+
+def get_win_folder_from_registry(csidl_name: str) -> str:
+    """Get folder from the registry.
+
+    This is a fallback technique at best. I'm not sure if using the
+    registry for this guarantees us the correct answer for all CSIDL_*
+    names.
+    """
+    shell_folder_name = {
+        "CSIDL_APPDATA": "AppData",
+        "CSIDL_COMMON_APPDATA": "Common AppData",
+        "CSIDL_LOCAL_APPDATA": "Local AppData",
+        "CSIDL_PERSONAL": "Personal",
+    }.get(csidl_name)
+    if shell_folder_name is None:
+        raise ValueError(f"Unknown CSIDL name: {csidl_name}")
+
+    import winreg
+
+    key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
+    directory, _ = winreg.QueryValueEx(key, shell_folder_name)
+    return str(directory)
+
+
+def get_win_folder_via_ctypes(csidl_name: str) -> str:
+    """Get folder with ctypes."""
+    csidl_const = {
+        "CSIDL_APPDATA": 26,
+        "CSIDL_COMMON_APPDATA": 35,
+        "CSIDL_LOCAL_APPDATA": 28,
+        "CSIDL_PERSONAL": 5,
+    }.get(csidl_name)
+    if csidl_const is None:
+        raise ValueError(f"Unknown CSIDL name: {csidl_name}")
+
+    buf = ctypes.create_unicode_buffer(1024)
+    windll = getattr(ctypes, "windll")  # noqa: B009 # using getattr to avoid false positive with mypy type checker
+    windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
+
+    # Downgrade to short path name if it has highbit chars.
+    if any(ord(c) > 255 for c in buf):
+        buf2 = ctypes.create_unicode_buffer(1024)
+        if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024):
+            buf = buf2
+
+    return buf.value
+
+
+def _pick_get_win_folder() -> Callable[[str], str]:
+    if hasattr(ctypes, "windll"):
+        return get_win_folder_via_ctypes
+    try:
+        import winreg  # noqa: F401
+    except ImportError:
+        return get_win_folder_from_env_vars
+    else:
+        return get_win_folder_from_registry
+
+
+get_win_folder = lru_cache(maxsize=None)(_pick_get_win_folder())
+
+__all__ = [
+    "Windows",
+]
diff --git a/src/pip/_vendor/progress/LICENSE b/src/pip/_vendor/progress/LICENSE
index 059cc056616..fb3ac14a693 100644
--- a/src/pip/_vendor/progress/LICENSE
+++ b/src/pip/_vendor/progress/LICENSE
@@ -1,4 +1,4 @@
-# Copyright (c) 2012 Giorgos Verigakis 
+# Copyright (c) 2012 Georgios Verigakis 
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/src/pip/_vendor/progress/__init__.py b/src/pip/_vendor/progress/__init__.py
index e434c257fef..b434b300ad7 100644
--- a/src/pip/_vendor/progress/__init__.py
+++ b/src/pip/_vendor/progress/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2012 Giorgos Verigakis 
+# Copyright (c) 2012 Georgios Verigakis 
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -24,7 +24,7 @@
     from time import time as monotonic
 
 
-__version__ = '1.5'
+__version__ = '1.6'
 
 HIDE_CURSOR = '\x1b[?25l'
 SHOW_CURSOR = '\x1b[?25h'
@@ -46,14 +46,19 @@ def __init__(self, message='', **kwargs):
         for key, val in kwargs.items():
             setattr(self, key, val)
 
-        self._width = 0
+        self._max_width = 0
+        self._hidden_cursor = False
         self.message = message
 
         if self.file and self.is_tty():
             if self.hide_cursor:
                 print(HIDE_CURSOR, end='', file=self.file)
-            print(self.message, end='', file=self.file)
-            self.file.flush()
+                self._hidden_cursor = True
+        self.writeln('')
+
+    def __del__(self):
+        if self._hidden_cursor:
+            print(SHOW_CURSOR, end='', file=self.file)
 
     def __getitem__(self, key):
         if key.startswith('_'):
@@ -85,31 +90,30 @@ def update(self):
     def start(self):
         pass
 
-    def clearln(self):
-        if self.file and self.is_tty():
-            print('\r\x1b[K', end='', file=self.file)
-
-    def write(self, s):
-        if self.file and self.is_tty():
-            line = self.message + s.ljust(self._width)
-            print('\r' + line, end='', file=self.file)
-            self._width = max(self._width, len(s))
-            self.file.flush()
-
     def writeln(self, line):
         if self.file and self.is_tty():
-            self.clearln()
-            print(line, end='', file=self.file)
+            width = len(line)
+            if width < self._max_width:
+                # Add padding to cover previous contents
+                line += ' ' * (self._max_width - width)
+            else:
+                self._max_width = width
+            print('\r' + line, end='', file=self.file)
             self.file.flush()
 
     def finish(self):
         if self.file and self.is_tty():
             print(file=self.file)
-            if self.hide_cursor:
+            if self._hidden_cursor:
                 print(SHOW_CURSOR, end='', file=self.file)
+                self._hidden_cursor = False
 
     def is_tty(self):
-        return self.file.isatty() if self.check_tty else True
+        try:
+            return self.file.isatty() if self.check_tty else True
+        except AttributeError:
+            msg = "%s has no attribute 'isatty'. Try setting check_tty=False." % self
+            raise AttributeError(msg)
 
     def next(self, n=1):
         now = monotonic()
@@ -120,10 +124,13 @@ def next(self, n=1):
         self.update()
 
     def iter(self, it):
+        self.iter_value = None
         with self:
             for x in it:
+                self.iter_value = x
                 yield x
                 self.next()
+        del self.iter_value
 
     def __enter__(self):
         self.start()
@@ -152,6 +159,8 @@ def percent(self):
 
     @property
     def progress(self):
+        if self.max == 0:
+            return 0
         return min(1, self.index / self.max)
 
     @property
@@ -171,7 +180,10 @@ def iter(self, it):
         except TypeError:
             pass
 
+        self.iter_value = None
         with self:
             for x in it:
+                self.iter_value = x
                 yield x
                 self.next()
+        del self.iter_value
diff --git a/src/pip/_vendor/progress/bar.py b/src/pip/_vendor/progress/bar.py
index 8819efda65b..df4e8b61f89 100644
--- a/src/pip/_vendor/progress/bar.py
+++ b/src/pip/_vendor/progress/bar.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2012 Giorgos Verigakis 
+# Copyright (c) 2012 Georgios Verigakis 
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -19,6 +19,7 @@
 import sys
 
 from . import Progress
+from .colors import color
 
 
 class Bar(Progress):
@@ -28,13 +29,14 @@ class Bar(Progress):
     bar_suffix = '| '
     empty_fill = ' '
     fill = '#'
+    color = None
 
     def update(self):
         filled_length = int(self.width * self.progress)
         empty_length = self.width - filled_length
 
         message = self.message % self
-        bar = self.fill * filled_length
+        bar = color(self.fill * filled_length, fg=self.color)
         empty = self.empty_fill * empty_length
         suffix = self.suffix % self
         line = ''.join([message, self.bar_prefix, bar, empty, self.bar_suffix,
@@ -74,7 +76,7 @@ def update(self):
         nempty = self.width - nfull                  # Number of empty chars
 
         message = self.message % self
-        bar = self.phases[-1] * nfull
+        bar = color(self.phases[-1] * nfull, fg=self.color)
         current = self.phases[phase] if phase > 0 else ''
         empty = self.empty_fill * max(0, nempty - len(current))
         suffix = self.suffix % self
diff --git a/src/pip/_vendor/progress/colors.py b/src/pip/_vendor/progress/colors.py
new file mode 100644
index 00000000000..4e770f868bf
--- /dev/null
+++ b/src/pip/_vendor/progress/colors.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2020 Georgios Verigakis 
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from functools import partial
+
+
+COLORS = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan',
+          'white')
+STYLES = ('bold', 'faint', 'italic', 'underline', 'blink', 'blink2',
+          'negative', 'concealed', 'crossed')
+
+
+def color(s, fg=None, bg=None, style=None):
+    sgr = []
+
+    if fg:
+        if fg in COLORS:
+            sgr.append(str(30 + COLORS.index(fg)))
+        elif isinstance(fg, int) and 0 <= fg <= 255:
+            sgr.append('38;5;%d' % int(fg))
+        else:
+            raise Exception('Invalid color "%s"' % fg)
+
+    if bg:
+        if bg in COLORS:
+            sgr.append(str(40 + COLORS.index(bg)))
+        elif isinstance(bg, int) and 0 <= bg <= 255:
+            sgr.append('48;5;%d' % bg)
+        else:
+            raise Exception('Invalid color "%s"' % bg)
+
+    if style:
+        for st in style.split('+'):
+            if st in STYLES:
+                sgr.append(str(1 + STYLES.index(st)))
+            else:
+                raise Exception('Invalid style "%s"' % st)
+
+    if sgr:
+        prefix = '\x1b[' + ';'.join(sgr) + 'm'
+        suffix = '\x1b[0m'
+        return prefix + s + suffix
+    else:
+        return s
+
+
+# Foreground shortcuts
+black = partial(color, fg='black')
+red = partial(color, fg='red')
+green = partial(color, fg='green')
+yellow = partial(color, fg='yellow')
+blue = partial(color, fg='blue')
+magenta = partial(color, fg='magenta')
+cyan = partial(color, fg='cyan')
+white = partial(color, fg='white')
+
+# Style shortcuts
+bold = partial(color, style='bold')
+faint = partial(color, style='faint')
+italic = partial(color, style='italic')
+underline = partial(color, style='underline')
+blink = partial(color, style='blink')
+blink2 = partial(color, style='blink2')
+negative = partial(color, style='negative')
+concealed = partial(color, style='concealed')
+crossed = partial(color, style='crossed')
diff --git a/src/pip/_vendor/progress/counter.py b/src/pip/_vendor/progress/counter.py
index d955ca4771b..d0fbe7ef354 100644
--- a/src/pip/_vendor/progress/counter.py
+++ b/src/pip/_vendor/progress/counter.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2012 Giorgos Verigakis 
+# Copyright (c) 2012 Georgios Verigakis 
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -20,12 +20,16 @@
 
 class Counter(Infinite):
     def update(self):
-        self.write(str(self.index))
+        message = self.message % self
+        line = ''.join([message, str(self.index)])
+        self.writeln(line)
 
 
 class Countdown(Progress):
     def update(self):
-        self.write(str(self.remaining))
+        message = self.message % self
+        line = ''.join([message, str(self.remaining)])
+        self.writeln(line)
 
 
 class Stack(Progress):
@@ -34,7 +38,9 @@ class Stack(Progress):
     def update(self):
         nphases = len(self.phases)
         i = min(nphases - 1, int(self.progress * nphases))
-        self.write(self.phases[i])
+        message = self.message % self
+        line = ''.join([message, self.phases[i]])
+        self.writeln(line)
 
 
 class Pie(Stack):
diff --git a/src/pip/_vendor/progress/spinner.py b/src/pip/_vendor/progress/spinner.py
index 4e100cabb9b..d593a203e08 100644
--- a/src/pip/_vendor/progress/spinner.py
+++ b/src/pip/_vendor/progress/spinner.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2012 Giorgos Verigakis 
+# Copyright (c) 2012 Georgios Verigakis 
 #
 # Permission to use, copy, modify, and distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
@@ -24,7 +24,9 @@ class Spinner(Infinite):
 
     def update(self):
         i = self.index % len(self.phases)
-        self.write(self.phases[i])
+        message = self.message % self
+        line = ''.join([message, self.phases[i]])
+        self.writeln(line)
 
 
 class PieSpinner(Spinner):
diff --git a/src/pip/_vendor/pygments.pyi b/src/pip/_vendor/pygments.pyi
new file mode 100644
index 00000000000..566eaff369b
--- /dev/null
+++ b/src/pip/_vendor/pygments.pyi
@@ -0,0 +1 @@
+from pygments import *
\ No newline at end of file
diff --git a/src/pip/_vendor/pygments/LICENSE b/src/pip/_vendor/pygments/LICENSE
new file mode 100644
index 00000000000..e1b15663d95
--- /dev/null
+++ b/src/pip/_vendor/pygments/LICENSE
@@ -0,0 +1,25 @@
+Copyright (c) 2006-2021 by the respective authors (see AUTHORS file).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/pip/_vendor/pygments/__init__.py b/src/pip/_vendor/pygments/__init__.py
new file mode 100644
index 00000000000..22c50b356ad
--- /dev/null
+++ b/src/pip/_vendor/pygments/__init__.py
@@ -0,0 +1,83 @@
+"""
+    Pygments
+    ~~~~~~~~
+
+    Pygments is a syntax highlighting package written in Python.
+
+    It is a generic syntax highlighter for general use in all kinds of software
+    such as forum systems, wikis or other applications that need to prettify
+    source code. Highlights are:
+
+    * a wide range of common languages and markup formats is supported
+    * special attention is paid to details, increasing quality by a fair amount
+    * support for new languages and formats are added easily
+    * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
+      formats that PIL supports, and ANSI sequences
+    * it is usable as a command-line tool and as a library
+    * ... and it highlights even Brainfuck!
+
+    The `Pygments master branch`_ is installable with ``easy_install Pygments==dev``.
+
+    .. _Pygments master branch:
+       https://github.com/pygments/pygments/archive/master.zip#egg=Pygments-dev
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+from io import StringIO, BytesIO
+
+__version__ = '2.11.2'
+__docformat__ = 'restructuredtext'
+
+__all__ = ['lex', 'format', 'highlight']
+
+
+def lex(code, lexer):
+    """
+    Lex ``code`` with ``lexer`` and return an iterable of tokens.
+    """
+    try:
+        return lexer.get_tokens(code)
+    except TypeError as err:
+        if (isinstance(err.args[0], str) and
+            ('unbound method get_tokens' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
+            raise TypeError('lex() argument must be a lexer instance, '
+                            'not a class')
+        raise
+
+
+def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
+    """
+    Format a tokenlist ``tokens`` with the formatter ``formatter``.
+
+    If ``outfile`` is given and a valid file object (an object
+    with a ``write`` method), the result will be written to it, otherwise
+    it is returned as a string.
+    """
+    try:
+        if not outfile:
+            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
+            formatter.format(tokens, realoutfile)
+            return realoutfile.getvalue()
+        else:
+            formatter.format(tokens, outfile)
+    except TypeError as err:
+        if (isinstance(err.args[0], str) and
+            ('unbound method format' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
+            raise TypeError('format() argument must be a formatter instance, '
+                            'not a class')
+        raise
+
+
+def highlight(code, lexer, formatter, outfile=None):
+    """
+    Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
+
+    If ``outfile`` is given and a valid file object (an object
+    with a ``write`` method), the result will be written to it, otherwise
+    it is returned as a string.
+    """
+    return format(lex(code, lexer), formatter, outfile)
+
diff --git a/src/pip/_vendor/pygments/__main__.py b/src/pip/_vendor/pygments/__main__.py
new file mode 100644
index 00000000000..010896b88ff
--- /dev/null
+++ b/src/pip/_vendor/pygments/__main__.py
@@ -0,0 +1,17 @@
+"""
+    pygments.__main__
+    ~~~~~~~~~~~~~~~~~
+
+    Main entry point for ``python -m pygments``.
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+from pip._vendor.pygments.cmdline import main
+
+try:
+    sys.exit(main(sys.argv))
+except KeyboardInterrupt:
+    sys.exit(1)
diff --git a/src/pip/_vendor/pygments/cmdline.py b/src/pip/_vendor/pygments/cmdline.py
new file mode 100644
index 00000000000..908064e2f63
--- /dev/null
+++ b/src/pip/_vendor/pygments/cmdline.py
@@ -0,0 +1,663 @@
+"""
+    pygments.cmdline
+    ~~~~~~~~~~~~~~~~
+
+    Command line interface.
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import sys
+import shutil
+import argparse
+from textwrap import dedent
+
+from pip._vendor.pygments import __version__, highlight
+from pip._vendor.pygments.util import ClassNotFound, OptionError, docstring_headline, \
+    guess_decode, guess_decode_from_terminal, terminal_encoding, \
+    UnclosingTextIOWrapper
+from pip._vendor.pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
+    load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
+from pip._vendor.pygments.lexers.special import TextLexer
+from pip._vendor.pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
+from pip._vendor.pygments.formatters import get_all_formatters, get_formatter_by_name, \
+    load_formatter_from_file, get_formatter_for_filename, find_formatter_class
+from pip._vendor.pygments.formatters.terminal import TerminalFormatter
+from pip._vendor.pygments.formatters.terminal256 import Terminal256Formatter
+from pip._vendor.pygments.filters import get_all_filters, find_filter_class
+from pip._vendor.pygments.styles import get_all_styles, get_style_by_name
+
+
+def _parse_options(o_strs):
+    opts = {}
+    if not o_strs:
+        return opts
+    for o_str in o_strs:
+        if not o_str.strip():
+            continue
+        o_args = o_str.split(',')
+        for o_arg in o_args:
+            o_arg = o_arg.strip()
+            try:
+                o_key, o_val = o_arg.split('=', 1)
+                o_key = o_key.strip()
+                o_val = o_val.strip()
+            except ValueError:
+                opts[o_arg] = True
+            else:
+                opts[o_key] = o_val
+    return opts
+
+
+def _parse_filters(f_strs):
+    filters = []
+    if not f_strs:
+        return filters
+    for f_str in f_strs:
+        if ':' in f_str:
+            fname, fopts = f_str.split(':', 1)
+            filters.append((fname, _parse_options([fopts])))
+        else:
+            filters.append((f_str, {}))
+    return filters
+
+
+def _print_help(what, name):
+    try:
+        if what == 'lexer':
+            cls = get_lexer_by_name(name)
+            print("Help on the %s lexer:" % cls.name)
+            print(dedent(cls.__doc__))
+        elif what == 'formatter':
+            cls = find_formatter_class(name)
+            print("Help on the %s formatter:" % cls.name)
+            print(dedent(cls.__doc__))
+        elif what == 'filter':
+            cls = find_filter_class(name)
+            print("Help on the %s filter:" % name)
+            print(dedent(cls.__doc__))
+        return 0
+    except (AttributeError, ValueError):
+        print("%s not found!" % what, file=sys.stderr)
+        return 1
+
+
+def _print_list(what):
+    if what == 'lexer':
+        print()
+        print("Lexers:")
+        print("~~~~~~~")
+
+        info = []
+        for fullname, names, exts, _ in get_all_lexers():
+            tup = (', '.join(names)+':', fullname,
+                   exts and '(filenames ' + ', '.join(exts) + ')' or '')
+            info.append(tup)
+        info.sort()
+        for i in info:
+            print(('* %s\n    %s %s') % i)
+
+    elif what == 'formatter':
+        print()
+        print("Formatters:")
+        print("~~~~~~~~~~~")
+
+        info = []
+        for cls in get_all_formatters():
+            doc = docstring_headline(cls)
+            tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
+                   '(filenames ' + ', '.join(cls.filenames) + ')' or '')
+            info.append(tup)
+        info.sort()
+        for i in info:
+            print(('* %s\n    %s %s') % i)
+
+    elif what == 'filter':
+        print()
+        print("Filters:")
+        print("~~~~~~~~")
+
+        for name in get_all_filters():
+            cls = find_filter_class(name)
+            print("* " + name + ':')
+            print("    %s" % docstring_headline(cls))
+
+    elif what == 'style':
+        print()
+        print("Styles:")
+        print("~~~~~~~")
+
+        for name in get_all_styles():
+            cls = get_style_by_name(name)
+            print("* " + name + ':')
+            print("    %s" % docstring_headline(cls))
+
+
+def _print_list_as_json(requested_items):
+    import json
+    result = {}
+    if 'lexer' in requested_items:
+        info = {}
+        for fullname, names, filenames, mimetypes in get_all_lexers():
+            info[fullname] = {
+                'aliases': names,
+                'filenames': filenames,
+                'mimetypes': mimetypes
+            }
+        result['lexers'] = info
+
+    if 'formatter' in requested_items:
+        info = {}
+        for cls in get_all_formatters():
+            doc = docstring_headline(cls)
+            info[cls.name] = {
+                'aliases': cls.aliases,
+                'filenames': cls.filenames,
+                'doc': doc
+            }
+        result['formatters'] = info
+
+    if 'filter' in requested_items:
+        info = {}
+        for name in get_all_filters():
+            cls = find_filter_class(name)
+            info[name] = {
+                'doc': docstring_headline(cls)
+            }
+        result['filters'] = info
+
+    if 'style' in requested_items:
+        info = {}
+        for name in get_all_styles():
+            cls = get_style_by_name(name)
+            info[name] = {
+                'doc': docstring_headline(cls)
+            }
+        result['styles'] = info
+
+    json.dump(result, sys.stdout)
+
+def main_inner(parser, argns):
+    if argns.help:
+        parser.print_help()
+        return 0
+
+    if argns.V:
+        print('Pygments version %s, (c) 2006-2021 by Georg Brandl, Matthäus '
+              'Chajdas and contributors.' % __version__)
+        return 0
+
+    def is_only_option(opt):
+        return not any(v for (k, v) in vars(argns).items() if k != opt)
+
+    # handle ``pygmentize -L``
+    if argns.L is not None:
+        arg_set = set()
+        for k, v in vars(argns).items():
+            if v:
+                arg_set.add(k)
+
+        arg_set.discard('L')
+        arg_set.discard('json')
+
+        if arg_set:
+            parser.print_help(sys.stderr)
+            return 2
+
+        # print version
+        if not argns.json:
+            main(['', '-V'])
+        allowed_types = {'lexer', 'formatter', 'filter', 'style'}
+        largs = [arg.rstrip('s') for arg in argns.L]
+        if any(arg not in allowed_types for arg in largs):
+            parser.print_help(sys.stderr)
+            return 0
+        if not largs:
+            largs = allowed_types
+        if not argns.json:
+            for arg in largs:
+                _print_list(arg)
+        else:
+            _print_list_as_json(largs)
+        return 0
+
+    # handle ``pygmentize -H``
+    if argns.H:
+        if not is_only_option('H'):
+            parser.print_help(sys.stderr)
+            return 2
+        what, name = argns.H
+        if what not in ('lexer', 'formatter', 'filter'):
+            parser.print_help(sys.stderr)
+            return 2
+        return _print_help(what, name)
+
+    # parse -O options
+    parsed_opts = _parse_options(argns.O or [])
+
+    # parse -P options
+    for p_opt in argns.P or []:
+        try:
+            name, value = p_opt.split('=', 1)
+        except ValueError:
+            parsed_opts[p_opt] = True
+        else:
+            parsed_opts[name] = value
+
+    # encodings
+    inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
+    outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
+
+    # handle ``pygmentize -N``
+    if argns.N:
+        lexer = find_lexer_class_for_filename(argns.N)
+        if lexer is None:
+            lexer = TextLexer
+
+        print(lexer.aliases[0])
+        return 0
+
+    # handle ``pygmentize -C``
+    if argns.C:
+        inp = sys.stdin.buffer.read()
+        try:
+            lexer = guess_lexer(inp, inencoding=inencoding)
+        except ClassNotFound:
+            lexer = TextLexer
+
+        print(lexer.aliases[0])
+        return 0
+
+    # handle ``pygmentize -S``
+    S_opt = argns.S
+    a_opt = argns.a
+    if S_opt is not None:
+        f_opt = argns.f
+        if not f_opt:
+            parser.print_help(sys.stderr)
+            return 2
+        if argns.l or argns.INPUTFILE:
+            parser.print_help(sys.stderr)
+            return 2
+
+        try:
+            parsed_opts['style'] = S_opt
+            fmter = get_formatter_by_name(f_opt, **parsed_opts)
+        except ClassNotFound as err:
+            print(err, file=sys.stderr)
+            return 1
+
+        print(fmter.get_style_defs(a_opt or ''))
+        return 0
+
+    # if no -S is given, -a is not allowed
+    if argns.a is not None:
+        parser.print_help(sys.stderr)
+        return 2
+
+    # parse -F options
+    F_opts = _parse_filters(argns.F or [])
+
+    # -x: allow custom (eXternal) lexers and formatters
+    allow_custom_lexer_formatter = bool(argns.x)
+
+    # select lexer
+    lexer = None
+
+    # given by name?
+    lexername = argns.l
+    if lexername:
+        # custom lexer, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in lexername:
+            try:
+                filename = None
+                name = None
+                if ':' in lexername:
+                    filename, name = lexername.rsplit(':', 1)
+
+                    if '.py' in name:
+                        # This can happen on Windows: If the lexername is
+                        # C:\lexer.py -- return to normal load path in that case
+                        name = None
+
+                if filename and name:
+                    lexer = load_lexer_from_file(filename, name,
+                                                 **parsed_opts)
+                else:
+                    lexer = load_lexer_from_file(lexername, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                lexer = get_lexer_by_name(lexername, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+
+    # read input code
+    code = None
+
+    if argns.INPUTFILE:
+        if argns.s:
+            print('Error: -s option not usable when input file specified',
+                  file=sys.stderr)
+            return 2
+
+        infn = argns.INPUTFILE
+        try:
+            with open(infn, 'rb') as infp:
+                code = infp.read()
+        except Exception as err:
+            print('Error: cannot read infile:', err, file=sys.stderr)
+            return 1
+        if not inencoding:
+            code, inencoding = guess_decode(code)
+
+        # do we have to guess the lexer?
+        if not lexer:
+            try:
+                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
+            except ClassNotFound as err:
+                if argns.g:
+                    try:
+                        lexer = guess_lexer(code, **parsed_opts)
+                    except ClassNotFound:
+                        lexer = TextLexer(**parsed_opts)
+                else:
+                    print('Error:', err, file=sys.stderr)
+                    return 1
+            except OptionError as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+
+    elif not argns.s:  # treat stdin as full file (-s support is later)
+        # read code from terminal, always in binary mode since we want to
+        # decode ourselves and be tolerant with it
+        code = sys.stdin.buffer.read()  # use .buffer to get a binary stream
+        if not inencoding:
+            code, inencoding = guess_decode_from_terminal(code, sys.stdin)
+            # else the lexer will do the decoding
+        if not lexer:
+            try:
+                lexer = guess_lexer(code, **parsed_opts)
+            except ClassNotFound:
+                lexer = TextLexer(**parsed_opts)
+
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
+    # select formatter
+    outfn = argns.o
+    fmter = argns.f
+    if fmter:
+        # custom formatter, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in fmter:
+            try:
+                filename = None
+                name = None
+                if ':' in fmter:
+                    # Same logic as above for custom lexer
+                    filename, name = fmter.rsplit(':', 1)
+
+                    if '.py' in name:
+                        name = None
+
+                if filename and name:
+                    fmter = load_formatter_from_file(filename, name,
+                                                     **parsed_opts)
+                else:
+                    fmter = load_formatter_from_file(fmter, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                fmter = get_formatter_by_name(fmter, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+
+    if outfn:
+        if not fmter:
+            try:
+                fmter = get_formatter_for_filename(outfn, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        try:
+            outfile = open(outfn, 'wb')
+        except Exception as err:
+            print('Error: cannot open outfile:', err, file=sys.stderr)
+            return 1
+    else:
+        if not fmter:
+            if '256' in os.environ.get('TERM', ''):
+                fmter = Terminal256Formatter(**parsed_opts)
+            else:
+                fmter = TerminalFormatter(**parsed_opts)
+        outfile = sys.stdout.buffer
+
+    # determine output encoding if not explicitly selected
+    if not outencoding:
+        if outfn:
+            # output file? use lexer encoding for now (can still be None)
+            fmter.encoding = inencoding
+        else:
+            # else use terminal encoding
+            fmter.encoding = terminal_encoding(sys.stdout)
+
+    # provide coloring under Windows, if possible
+    if not outfn and sys.platform in ('win32', 'cygwin') and \
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
+        # unfortunately colorama doesn't support binary streams on Py3
+        outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
+        fmter.encoding = None
+        try:
+            import pip._vendor.colorama.initialise as colorama_initialise
+        except ImportError:
+            pass
+        else:
+            outfile = colorama_initialise.wrap_stream(
+                outfile, convert=None, strip=None, autoreset=False, wrap=True)
+
+    # When using the LaTeX formatter and the option `escapeinside` is
+    # specified, we need a special lexer which collects escaped text
+    # before running the chosen language lexer.
+    escapeinside = parsed_opts.get('escapeinside', '')
+    if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
+        left = escapeinside[0]
+        right = escapeinside[1]
+        lexer = LatexEmbeddedLexer(left, right, lexer)
+
+    # ... and do it!
+    if not argns.s:
+        # process whole input as per normal...
+        try:
+            highlight(code, lexer, fmter, outfile)
+        finally:
+            if outfn:
+                outfile.close()
+        return 0
+    else:
+        # line by line processing of stdin (eg: for 'tail -f')...
+        try:
+            while 1:
+                line = sys.stdin.buffer.readline()
+                if not line:
+                    break
+                if not inencoding:
+                    line = guess_decode_from_terminal(line, sys.stdin)[0]
+                highlight(line, lexer, fmter, outfile)
+                if hasattr(outfile, 'flush'):
+                    outfile.flush()
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
+            return 0
+        finally:
+            if outfn:
+                outfile.close()
+
+
+class HelpFormatter(argparse.HelpFormatter):
+    def __init__(self, prog, indent_increment=2, max_help_position=16, width=None):
+        if width is None:
+            try:
+                width = shutil.get_terminal_size().columns - 2
+            except Exception:
+                pass
+        argparse.HelpFormatter.__init__(self, prog, indent_increment,
+                                        max_help_position, width)
+
+
+def main(args=sys.argv):
+    """
+    Main command line entry point.
+    """
+    desc = "Highlight an input file and write the result to an output file."
+    parser = argparse.ArgumentParser(description=desc, add_help=False,
+                                     formatter_class=HelpFormatter)
+
+    operation = parser.add_argument_group('Main operation')
+    lexersel = operation.add_mutually_exclusive_group()
+    lexersel.add_argument(
+        '-l', metavar='LEXER',
+        help='Specify the lexer to use.  (Query names with -L.)  If not '
+        'given and -g is not present, the lexer is guessed from the filename.')
+    lexersel.add_argument(
+        '-g', action='store_true',
+        help='Guess the lexer from the file contents, or pass through '
+        'as plain text if nothing can be guessed.')
+    operation.add_argument(
+        '-F', metavar='FILTER[:options]', action='append',
+        help='Add a filter to the token stream.  (Query names with -L.) '
+        'Filter options are given after a colon if necessary.')
+    operation.add_argument(
+        '-f', metavar='FORMATTER',
+        help='Specify the formatter to use.  (Query names with -L.) '
+        'If not given, the formatter is guessed from the output filename, '
+        'and defaults to the terminal formatter if the output is to the '
+        'terminal or an unknown file extension.')
+    operation.add_argument(
+        '-O', metavar='OPTION=value[,OPTION=value,...]', action='append',
+        help='Give options to the lexer and formatter as a comma-separated '
+        'list of key-value pairs. '
+        'Example: `-O bg=light,python=cool`.')
+    operation.add_argument(
+        '-P', metavar='OPTION=value', action='append',
+        help='Give a single option to the lexer and formatter - with this '
+        'you can pass options whose value contains commas and equal signs. '
+        'Example: `-P "heading=Pygments, the Python highlighter"`.')
+    operation.add_argument(
+        '-o', metavar='OUTPUTFILE',
+        help='Where to write the output.  Defaults to standard output.')
+
+    operation.add_argument(
+        'INPUTFILE', nargs='?',
+        help='Where to read the input.  Defaults to standard input.')
+
+    flags = parser.add_argument_group('Operation flags')
+    flags.add_argument(
+        '-v', action='store_true',
+        help='Print a detailed traceback on unhandled exceptions, which '
+        'is useful for debugging and bug reports.')
+    flags.add_argument(
+        '-s', action='store_true',
+        help='Process lines one at a time until EOF, rather than waiting to '
+        'process the entire file.  This only works for stdin, only for lexers '
+        'with no line-spanning constructs, and is intended for streaming '
+        'input such as you get from `tail -f`. '
+        'Example usage: `tail -f sql.log | pygmentize -s -l sql`.')
+    flags.add_argument(
+        '-x', action='store_true',
+        help='Allow custom lexers and formatters to be loaded from a .py file '
+        'relative to the current working directory. For example, '
+        '`-l ./customlexer.py -x`. By default, this option expects a file '
+        'with a class named CustomLexer or CustomFormatter; you can also '
+        'specify your own class name with a colon (`-l ./lexer.py:MyLexer`). '
+        'Users should be very careful not to use this option with untrusted '
+        'files, because it will import and run them.')
+    flags.add_argument('--json', help='Output as JSON. This can '
+        'be only used in conjunction with -L.',
+        default=False,
+        action='store_true')
+
+    special_modes_group = parser.add_argument_group(
+        'Special modes - do not do any highlighting')
+    special_modes = special_modes_group.add_mutually_exclusive_group()
+    special_modes.add_argument(
+        '-S', metavar='STYLE -f formatter',
+        help='Print style definitions for STYLE for a formatter '
+        'given with -f. The argument given by -a is formatter '
+        'dependent.')
+    special_modes.add_argument(
+        '-L', nargs='*', metavar='WHAT',
+        help='List lexers, formatters, styles or filters -- '
+        'give additional arguments for the thing(s) you want to list '
+        '(e.g. "styles"), or omit them to list everything.')
+    special_modes.add_argument(
+        '-N', metavar='FILENAME',
+        help='Guess and print out a lexer name based solely on the given '
+        'filename. Does not take input or highlight anything. If no specific '
+        'lexer can be determined, "text" is printed.')
+    special_modes.add_argument(
+        '-C', action='store_true',
+        help='Like -N, but print out a lexer name based solely on '
+        'a given content from standard input.')
+    special_modes.add_argument(
+        '-H', action='store', nargs=2, metavar=('NAME', 'TYPE'),
+        help='Print detailed help for the object  of type , '
+        'where  is one of "lexer", "formatter" or "filter".')
+    special_modes.add_argument(
+        '-V', action='store_true',
+        help='Print the package version.')
+    special_modes.add_argument(
+        '-h', '--help', action='store_true',
+        help='Print this help.')
+    special_modes_group.add_argument(
+        '-a', metavar='ARG',
+        help='Formatter-specific additional argument for the -S (print '
+        'style sheet) mode.')
+
+    argns = parser.parse_args(args[1:])
+
+    try:
+        return main_inner(parser, argns)
+    except Exception:
+        if argns.v:
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
+        import traceback
+        info = traceback.format_exception(*sys.exc_info())
+        msg = info[-1].strip()
+        if len(info) >= 3:
+            # extract relevant file and position info
+            msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
+        print(file=sys.stderr)
+        print('*** Error while highlighting:', file=sys.stderr)
+        print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
+        return 1
diff --git a/src/pip/_vendor/pygments/console.py b/src/pip/_vendor/pygments/console.py
new file mode 100644
index 00000000000..8dd08abebce
--- /dev/null
+++ b/src/pip/_vendor/pygments/console.py
@@ -0,0 +1,70 @@
+"""
+    pygments.console
+    ~~~~~~~~~~~~~~~~
+
+    Format colored console output.
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+esc = "\x1b["
+
+codes = {}
+codes[""] = ""
+codes["reset"] = esc + "39;49;00m"
+
+codes["bold"] = esc + "01m"
+codes["faint"] = esc + "02m"
+codes["standout"] = esc + "03m"
+codes["underline"] = esc + "04m"
+codes["blink"] = esc + "05m"
+codes["overline"] = esc + "06m"
+
+dark_colors = ["black", "red", "green", "yellow", "blue",
+               "magenta", "cyan", "gray"]
+light_colors = ["brightblack", "brightred", "brightgreen", "brightyellow", "brightblue",
+                "brightmagenta", "brightcyan", "white"]
+
+x = 30
+for d, l in zip(dark_colors, light_colors):
+    codes[d] = esc + "%im" % x
+    codes[l] = esc + "%im" % (60 + x)
+    x += 1
+
+del d, l, x
+
+codes["white"] = codes["bold"]
+
+
+def reset_color():
+    return codes["reset"]
+
+
+def colorize(color_key, text):
+    return codes[color_key] + text + codes["reset"]
+
+
+def ansiformat(attr, text):
+    """
+    Format ``text`` with a color and/or some attributes::
+
+        color       normal color
+        *color*     bold color
+        _color_     underlined color
+        +color+     blinking color
+    """
+    result = []
+    if attr[:1] == attr[-1:] == '+':
+        result.append(codes['blink'])
+        attr = attr[1:-1]
+    if attr[:1] == attr[-1:] == '*':
+        result.append(codes['bold'])
+        attr = attr[1:-1]
+    if attr[:1] == attr[-1:] == '_':
+        result.append(codes['underline'])
+        attr = attr[1:-1]
+    result.append(codes[attr])
+    result.append(text)
+    result.append(codes['reset'])
+    return ''.join(result)
diff --git a/src/pip/_vendor/pygments/filter.py b/src/pip/_vendor/pygments/filter.py
new file mode 100644
index 00000000000..85b4829878f
--- /dev/null
+++ b/src/pip/_vendor/pygments/filter.py
@@ -0,0 +1,71 @@
+"""
+    pygments.filter
+    ~~~~~~~~~~~~~~~
+
+    Module that implements the default filter.
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+def apply_filters(stream, filters, lexer=None):
+    """
+    Use this method to apply an iterable of filters to
+    a stream. If lexer is given it's forwarded to the
+    filter, otherwise the filter receives `None`.
+    """
+    def _apply(filter_, stream):
+        yield from filter_.filter(lexer, stream)
+    for filter_ in filters:
+        stream = _apply(filter_, stream)
+    return stream
+
+
+def simplefilter(f):
+    """
+    Decorator that converts a function into a filter::
+
+        @simplefilter
+        def lowercase(self, lexer, stream, options):
+            for ttype, value in stream:
+                yield ttype, value.lower()
+    """
+    return type(f.__name__, (FunctionFilter,), {
+        '__module__': getattr(f, '__module__'),
+        '__doc__': f.__doc__,
+        'function': f,
+    })
+
+
+class Filter:
+    """
+    Default filter. Subclass this class or use the `simplefilter`
+    decorator to create own filters.
+    """
+
+    def __init__(self, **options):
+        self.options = options
+
+    def filter(self, lexer, stream):
+        raise NotImplementedError()
+
+
+class FunctionFilter(Filter):
+    """
+    Abstract class used by `simplefilter` to create simple
+    function filters on the fly. The `simplefilter` decorator
+    automatically creates subclasses of this class for
+    functions passed to it.
+    """
+    function = None
+
+    def __init__(self, **options):
+        if not hasattr(self, 'function'):
+            raise TypeError('%r used without bound function' %
+                            self.__class__.__name__)
+        Filter.__init__(self, **options)
+
+    def filter(self, lexer, stream):
+        # pylint: disable=not-callable
+        yield from self.function(lexer, stream, self.options)
diff --git a/src/pip/_vendor/pygments/filters/__init__.py b/src/pip/_vendor/pygments/filters/__init__.py
new file mode 100644
index 00000000000..1d5a808f6f5
--- /dev/null
+++ b/src/pip/_vendor/pygments/filters/__init__.py
@@ -0,0 +1,937 @@
+"""
+    pygments.filters
+    ~~~~~~~~~~~~~~~~
+
+    Module containing filter lookup functions and default
+    filters.
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pip._vendor.pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
+    string_to_tokentype
+from pip._vendor.pygments.filter import Filter
+from pip._vendor.pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
+    get_choice_opt, ClassNotFound, OptionError
+from pip._vendor.pygments.plugin import find_plugin_filters
+
+
+def find_filter_class(filtername):
+    """Lookup a filter by name. Return None if not found."""
+    if filtername in FILTERS:
+        return FILTERS[filtername]
+    for name, cls in find_plugin_filters():
+        if name == filtername:
+            return cls
+    return None
+
+
+def get_filter_by_name(filtername, **options):
+    """Return an instantiated filter.
+
+    Options are passed to the filter initializer if wanted.
+    Raise a ClassNotFound if not found.
+    """
+    cls = find_filter_class(filtername)
+    if cls:
+        return cls(**options)
+    else:
+        raise ClassNotFound('filter %r not found' % filtername)
+
+
+def get_all_filters():
+    """Return a generator of all filter names."""
+    yield from FILTERS
+    for name, _ in find_plugin_filters():
+        yield name
+
+
+def _replace_special(ttype, value, regex, specialttype,
+                     replacefunc=lambda x: x):
+    last = 0
+    for match in regex.finditer(value):
+        start, end = match.start(), match.end()
+        if start != last:
+            yield ttype, value[last:start]
+        yield specialttype, replacefunc(value[start:end])
+        last = end
+    if last != len(value):
+        yield ttype, value[last:]
+
+
+class CodeTagFilter(Filter):
+    """Highlight special code tags in comments and docstrings.
+
+    Options accepted:
+
+    `codetags` : list of strings
+       A list of strings that are flagged as code tags.  The default is to
+       highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``.
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        tags = get_list_opt(options, 'codetags',
+                            ['XXX', 'TODO', 'BUG', 'NOTE'])
+        self.tag_re = re.compile(r'\b(%s)\b' % '|'.join([
+            re.escape(tag) for tag in tags if tag
+        ]))
+
+    def filter(self, lexer, stream):
+        regex = self.tag_re
+        for ttype, value in stream:
+            if ttype in String.Doc or \
+               ttype in Comment and \
+               ttype not in Comment.Preproc:
+                yield from _replace_special(ttype, value, regex, Comment.Special)
+            else:
+                yield ttype, value
+
+
+class SymbolFilter(Filter):
+    """Convert mathematical symbols such as \\ in Isabelle
+    or \\longrightarrow in LaTeX into Unicode characters.
+
+    This is mostly useful for HTML or console output when you want to
+    approximate the source rendering you'd see in an IDE.
+
+    Options accepted:
+
+    `lang` : string
+       The symbol language. Must be one of ``'isabelle'`` or
+       ``'latex'``.  The default is ``'isabelle'``.
+    """
+
+    latex_symbols = {
+        '\\alpha'                : '\U000003b1',
+        '\\beta'                 : '\U000003b2',
+        '\\gamma'                : '\U000003b3',
+        '\\delta'                : '\U000003b4',
+        '\\varepsilon'           : '\U000003b5',
+        '\\zeta'                 : '\U000003b6',
+        '\\eta'                  : '\U000003b7',
+        '\\vartheta'             : '\U000003b8',
+        '\\iota'                 : '\U000003b9',
+        '\\kappa'                : '\U000003ba',
+        '\\lambda'               : '\U000003bb',
+        '\\mu'                   : '\U000003bc',
+        '\\nu'                   : '\U000003bd',
+        '\\xi'                   : '\U000003be',
+        '\\pi'                   : '\U000003c0',
+        '\\varrho'               : '\U000003c1',
+        '\\sigma'                : '\U000003c3',
+        '\\tau'                  : '\U000003c4',
+        '\\upsilon'              : '\U000003c5',
+        '\\varphi'               : '\U000003c6',
+        '\\chi'                  : '\U000003c7',
+        '\\psi'                  : '\U000003c8',
+        '\\omega'                : '\U000003c9',
+        '\\Gamma'                : '\U00000393',
+        '\\Delta'                : '\U00000394',
+        '\\Theta'                : '\U00000398',
+        '\\Lambda'               : '\U0000039b',
+        '\\Xi'                   : '\U0000039e',
+        '\\Pi'                   : '\U000003a0',
+        '\\Sigma'                : '\U000003a3',
+        '\\Upsilon'              : '\U000003a5',
+        '\\Phi'                  : '\U000003a6',
+        '\\Psi'                  : '\U000003a8',
+        '\\Omega'                : '\U000003a9',
+        '\\leftarrow'            : '\U00002190',
+        '\\longleftarrow'        : '\U000027f5',
+        '\\rightarrow'           : '\U00002192',
+        '\\longrightarrow'       : '\U000027f6',
+        '\\Leftarrow'            : '\U000021d0',
+        '\\Longleftarrow'        : '\U000027f8',
+        '\\Rightarrow'           : '\U000021d2',
+        '\\Longrightarrow'       : '\U000027f9',
+        '\\leftrightarrow'       : '\U00002194',
+        '\\longleftrightarrow'   : '\U000027f7',
+        '\\Leftrightarrow'       : '\U000021d4',
+        '\\Longleftrightarrow'   : '\U000027fa',
+        '\\mapsto'               : '\U000021a6',
+        '\\longmapsto'           : '\U000027fc',
+        '\\relbar'               : '\U00002500',
+        '\\Relbar'               : '\U00002550',
+        '\\hookleftarrow'        : '\U000021a9',
+        '\\hookrightarrow'       : '\U000021aa',
+        '\\leftharpoondown'      : '\U000021bd',
+        '\\rightharpoondown'     : '\U000021c1',
+        '\\leftharpoonup'        : '\U000021bc',
+        '\\rightharpoonup'       : '\U000021c0',
+        '\\rightleftharpoons'    : '\U000021cc',
+        '\\leadsto'              : '\U0000219d',
+        '\\downharpoonleft'      : '\U000021c3',
+        '\\downharpoonright'     : '\U000021c2',
+        '\\upharpoonleft'        : '\U000021bf',
+        '\\upharpoonright'       : '\U000021be',
+        '\\restriction'          : '\U000021be',
+        '\\uparrow'              : '\U00002191',
+        '\\Uparrow'              : '\U000021d1',
+        '\\downarrow'            : '\U00002193',
+        '\\Downarrow'            : '\U000021d3',
+        '\\updownarrow'          : '\U00002195',
+        '\\Updownarrow'          : '\U000021d5',
+        '\\langle'               : '\U000027e8',
+        '\\rangle'               : '\U000027e9',
+        '\\lceil'                : '\U00002308',
+        '\\rceil'                : '\U00002309',
+        '\\lfloor'               : '\U0000230a',
+        '\\rfloor'               : '\U0000230b',
+        '\\flqq'                 : '\U000000ab',
+        '\\frqq'                 : '\U000000bb',
+        '\\bot'                  : '\U000022a5',
+        '\\top'                  : '\U000022a4',
+        '\\wedge'                : '\U00002227',
+        '\\bigwedge'             : '\U000022c0',
+        '\\vee'                  : '\U00002228',
+        '\\bigvee'               : '\U000022c1',
+        '\\forall'               : '\U00002200',
+        '\\exists'               : '\U00002203',
+        '\\nexists'              : '\U00002204',
+        '\\neg'                  : '\U000000ac',
+        '\\Box'                  : '\U000025a1',
+        '\\Diamond'              : '\U000025c7',
+        '\\vdash'                : '\U000022a2',
+        '\\models'               : '\U000022a8',
+        '\\dashv'                : '\U000022a3',
+        '\\surd'                 : '\U0000221a',
+        '\\le'                   : '\U00002264',
+        '\\ge'                   : '\U00002265',
+        '\\ll'                   : '\U0000226a',
+        '\\gg'                   : '\U0000226b',
+        '\\lesssim'              : '\U00002272',
+        '\\gtrsim'               : '\U00002273',
+        '\\lessapprox'           : '\U00002a85',
+        '\\gtrapprox'            : '\U00002a86',
+        '\\in'                   : '\U00002208',
+        '\\notin'                : '\U00002209',
+        '\\subset'               : '\U00002282',
+        '\\supset'               : '\U00002283',
+        '\\subseteq'             : '\U00002286',
+        '\\supseteq'             : '\U00002287',
+        '\\sqsubset'             : '\U0000228f',
+        '\\sqsupset'             : '\U00002290',
+        '\\sqsubseteq'           : '\U00002291',
+        '\\sqsupseteq'           : '\U00002292',
+        '\\cap'                  : '\U00002229',
+        '\\bigcap'               : '\U000022c2',
+        '\\cup'                  : '\U0000222a',
+        '\\bigcup'               : '\U000022c3',
+        '\\sqcup'                : '\U00002294',
+        '\\bigsqcup'             : '\U00002a06',
+        '\\sqcap'                : '\U00002293',
+        '\\Bigsqcap'             : '\U00002a05',
+        '\\setminus'             : '\U00002216',
+        '\\propto'               : '\U0000221d',
+        '\\uplus'                : '\U0000228e',
+        '\\bigplus'              : '\U00002a04',
+        '\\sim'                  : '\U0000223c',
+        '\\doteq'                : '\U00002250',
+        '\\simeq'                : '\U00002243',
+        '\\approx'               : '\U00002248',
+        '\\asymp'                : '\U0000224d',
+        '\\cong'                 : '\U00002245',
+        '\\equiv'                : '\U00002261',
+        '\\Join'                 : '\U000022c8',
+        '\\bowtie'               : '\U00002a1d',
+        '\\prec'                 : '\U0000227a',
+        '\\succ'                 : '\U0000227b',
+        '\\preceq'               : '\U0000227c',
+        '\\succeq'               : '\U0000227d',
+        '\\parallel'             : '\U00002225',
+        '\\mid'                  : '\U000000a6',
+        '\\pm'                   : '\U000000b1',
+        '\\mp'                   : '\U00002213',
+        '\\times'                : '\U000000d7',
+        '\\div'                  : '\U000000f7',
+        '\\cdot'                 : '\U000022c5',
+        '\\star'                 : '\U000022c6',
+        '\\circ'                 : '\U00002218',
+        '\\dagger'               : '\U00002020',
+        '\\ddagger'              : '\U00002021',
+        '\\lhd'                  : '\U000022b2',
+        '\\rhd'                  : '\U000022b3',
+        '\\unlhd'                : '\U000022b4',
+        '\\unrhd'                : '\U000022b5',
+        '\\triangleleft'         : '\U000025c3',
+        '\\triangleright'        : '\U000025b9',
+        '\\triangle'             : '\U000025b3',
+        '\\triangleq'            : '\U0000225c',
+        '\\oplus'                : '\U00002295',
+        '\\bigoplus'             : '\U00002a01',
+        '\\otimes'               : '\U00002297',
+        '\\bigotimes'            : '\U00002a02',
+        '\\odot'                 : '\U00002299',
+        '\\bigodot'              : '\U00002a00',
+        '\\ominus'               : '\U00002296',
+        '\\oslash'               : '\U00002298',
+        '\\dots'                 : '\U00002026',
+        '\\cdots'                : '\U000022ef',
+        '\\sum'                  : '\U00002211',
+        '\\prod'                 : '\U0000220f',
+        '\\coprod'               : '\U00002210',
+        '\\infty'                : '\U0000221e',
+        '\\int'                  : '\U0000222b',
+        '\\oint'                 : '\U0000222e',
+        '\\clubsuit'             : '\U00002663',
+        '\\diamondsuit'          : '\U00002662',
+        '\\heartsuit'            : '\U00002661',
+        '\\spadesuit'            : '\U00002660',
+        '\\aleph'                : '\U00002135',
+        '\\emptyset'             : '\U00002205',
+        '\\nabla'                : '\U00002207',
+        '\\partial'              : '\U00002202',
+        '\\flat'                 : '\U0000266d',
+        '\\natural'              : '\U0000266e',
+        '\\sharp'                : '\U0000266f',
+        '\\angle'                : '\U00002220',
+        '\\copyright'            : '\U000000a9',
+        '\\textregistered'       : '\U000000ae',
+        '\\textonequarter'       : '\U000000bc',
+        '\\textonehalf'          : '\U000000bd',
+        '\\textthreequarters'    : '\U000000be',
+        '\\textordfeminine'      : '\U000000aa',
+        '\\textordmasculine'     : '\U000000ba',
+        '\\euro'                 : '\U000020ac',
+        '\\pounds'               : '\U000000a3',
+        '\\yen'                  : '\U000000a5',
+        '\\textcent'             : '\U000000a2',
+        '\\textcurrency'         : '\U000000a4',
+        '\\textdegree'           : '\U000000b0',
+    }
+
+    isabelle_symbols = {
+        '\\'                 : '\U0001d7ec',
+        '\\'                  : '\U0001d7ed',
+        '\\'                  : '\U0001d7ee',
+        '\\'                : '\U0001d7ef',
+        '\\'                 : '\U0001d7f0',
+        '\\'                 : '\U0001d7f1',
+        '\\'                  : '\U0001d7f2',
+        '\\'                : '\U0001d7f3',
+        '\\'                : '\U0001d7f4',
+        '\\'                 : '\U0001d7f5',
+        '\\'                    : '\U0001d49c',
+        '\\'                    : '\U0000212c',
+        '\\'                    : '\U0001d49e',
+        '\\'                    : '\U0001d49f',
+        '\\'                    : '\U00002130',
+        '\\'                    : '\U00002131',
+        '\\'                    : '\U0001d4a2',
+        '\\'                    : '\U0000210b',
+        '\\'                    : '\U00002110',
+        '\\'                    : '\U0001d4a5',
+        '\\'                    : '\U0001d4a6',
+        '\\'                    : '\U00002112',
+        '\\'                    : '\U00002133',
+        '\\'                    : '\U0001d4a9',
+        '\\'                    : '\U0001d4aa',
+        '\\

' : '\U0001d5c9', + '\\' : '\U0001d5ca', + '\\' : '\U0001d5cb', + '\\' : '\U0001d5cc', + '\\' : '\U0001d5cd', + '\\' : '\U0001d5ce', + '\\' : '\U0001d5cf', + '\\' : '\U0001d5d0', + '\\' : '\U0001d5d1', + '\\' : '\U0001d5d2', + '\\' : '\U0001d5d3', + '\\' : '\U0001d504', + '\\' : '\U0001d505', + '\\' : '\U0000212d', + '\\

' : '\U0001d507', + '\\' : '\U0001d508', + '\\' : '\U0001d509', + '\\' : '\U0001d50a', + '\\' : '\U0000210c', + '\\' : '\U00002111', + '\\' : '\U0001d50d', + '\\' : '\U0001d50e', + '\\' : '\U0001d50f', + '\\' : '\U0001d510', + '\\' : '\U0001d511', + '\\' : '\U0001d512', + '\\' : '\U0001d513', + '\\' : '\U0001d514', + '\\' : '\U0000211c', + '\\' : '\U0001d516', + '\\' : '\U0001d517', + '\\' : '\U0001d518', + '\\' : '\U0001d519', + '\\' : '\U0001d51a', + '\\' : '\U0001d51b', + '\\' : '\U0001d51c', + '\\' : '\U00002128', + '\\' : '\U0001d51e', + '\\' : '\U0001d51f', + '\\' : '\U0001d520', + '\\
' : '\U0001d521', + '\\' : '\U0001d522', + '\\' : '\U0001d523', + '\\' : '\U0001d524', + '\\' : '\U0001d525', + '\\' : '\U0001d526', + '\\' : '\U0001d527', + '\\' : '\U0001d528', + '\\' : '\U0001d529', + '\\' : '\U0001d52a', + '\\' : '\U0001d52b', + '\\' : '\U0001d52c', + '\\' : '\U0001d52d', + '\\' : '\U0001d52e', + '\\' : '\U0001d52f', + '\\' : '\U0001d530', + '\\' : '\U0001d531', + '\\' : '\U0001d532', + '\\' : '\U0001d533', + '\\' : '\U0001d534', + '\\' : '\U0001d535', + '\\' : '\U0001d536', + '\\' : '\U0001d537', + '\\' : '\U000003b1', + '\\' : '\U000003b2', + '\\' : '\U000003b3', + '\\' : '\U000003b4', + '\\' : '\U000003b5', + '\\' : '\U000003b6', + '\\' : '\U000003b7', + '\\' : '\U000003b8', + '\\' : '\U000003b9', + '\\' : '\U000003ba', + '\\' : '\U000003bb', + '\\' : '\U000003bc', + '\\' : '\U000003bd', + '\\' : '\U000003be', + '\\' : '\U000003c0', + '\\' : '\U000003c1', + '\\' : '\U000003c3', + '\\' : '\U000003c4', + '\\' : '\U000003c5', + '\\' : '\U000003c6', + '\\' : '\U000003c7', + '\\' : '\U000003c8', + '\\' : '\U000003c9', + '\\' : '\U00000393', + '\\' : '\U00000394', + '\\' : '\U00000398', + '\\' : '\U0000039b', + '\\' : '\U0000039e', + '\\' : '\U000003a0', + '\\' : '\U000003a3', + '\\' : '\U000003a5', + '\\' : '\U000003a6', + '\\' : '\U000003a8', + '\\' : '\U000003a9', + '\\' : '\U0001d539', + '\\' : '\U00002102', + '\\' : '\U00002115', + '\\' : '\U0000211a', + '\\' : '\U0000211d', + '\\' : '\U00002124', + '\\' : '\U00002190', + '\\' : '\U000027f5', + '\\' : '\U00002192', + '\\' : '\U000027f6', + '\\' : '\U000021d0', + '\\' : '\U000027f8', + '\\' : '\U000021d2', + '\\' : '\U000027f9', + '\\' : '\U00002194', + '\\' : '\U000027f7', + '\\' : '\U000021d4', + '\\' : '\U000027fa', + '\\' : '\U000021a6', + '\\' : '\U000027fc', + '\\' : '\U00002500', + '\\' : '\U00002550', + '\\' : '\U000021a9', + '\\' : '\U000021aa', + '\\' : '\U000021bd', + '\\' : '\U000021c1', + '\\' : '\U000021bc', + '\\' : '\U000021c0', + '\\' : '\U000021cc', + '\\' : '\U0000219d', + '\\' : '\U000021c3', + '\\' : '\U000021c2', + '\\' : '\U000021bf', + '\\' : '\U000021be', + '\\' : '\U000021be', + '\\' : '\U00002237', + '\\' : '\U00002191', + '\\' : '\U000021d1', + '\\' : '\U00002193', + '\\' : '\U000021d3', + '\\' : '\U00002195', + '\\' : '\U000021d5', + '\\' : '\U000027e8', + '\\' : '\U000027e9', + '\\' : '\U00002308', + '\\' : '\U00002309', + '\\' : '\U0000230a', + '\\' : '\U0000230b', + '\\' : '\U00002987', + '\\' : '\U00002988', + '\\' : '\U000027e6', + '\\' : '\U000027e7', + '\\' : '\U00002983', + '\\' : '\U00002984', + '\\' : '\U000000ab', + '\\' : '\U000000bb', + '\\' : '\U000022a5', + '\\' : '\U000022a4', + '\\' : '\U00002227', + '\\' : '\U000022c0', + '\\' : '\U00002228', + '\\' : '\U000022c1', + '\\' : '\U00002200', + '\\' : '\U00002203', + '\\' : '\U00002204', + '\\' : '\U000000ac', + '\\' : '\U000025a1', + '\\' : '\U000025c7', + '\\' : '\U000022a2', + '\\' : '\U000022a8', + '\\' : '\U000022a9', + '\\' : '\U000022ab', + '\\' : '\U000022a3', + '\\' : '\U0000221a', + '\\' : '\U00002264', + '\\' : '\U00002265', + '\\' : '\U0000226a', + '\\' : '\U0000226b', + '\\' : '\U00002272', + '\\' : '\U00002273', + '\\' : '\U00002a85', + '\\' : '\U00002a86', + '\\' : '\U00002208', + '\\' : '\U00002209', + '\\' : '\U00002282', + '\\' : '\U00002283', + '\\' : '\U00002286', + '\\' : '\U00002287', + '\\' : '\U0000228f', + '\\' : '\U00002290', + '\\' : '\U00002291', + '\\' : '\U00002292', + '\\' : '\U00002229', + '\\' : '\U000022c2', + '\\' : '\U0000222a', + '\\' : '\U000022c3', + '\\' : '\U00002294', + '\\' : '\U00002a06', + '\\' : '\U00002293', + '\\' : '\U00002a05', + '\\' : '\U00002216', + '\\' : '\U0000221d', + '\\' : '\U0000228e', + '\\' : '\U00002a04', + '\\' : '\U00002260', + '\\' : '\U0000223c', + '\\' : '\U00002250', + '\\' : '\U00002243', + '\\' : '\U00002248', + '\\' : '\U0000224d', + '\\' : '\U00002245', + '\\' : '\U00002323', + '\\' : '\U00002261', + '\\' : '\U00002322', + '\\' : '\U000022c8', + '\\' : '\U00002a1d', + '\\' : '\U0000227a', + '\\' : '\U0000227b', + '\\' : '\U0000227c', + '\\' : '\U0000227d', + '\\' : '\U00002225', + '\\' : '\U000000a6', + '\\' : '\U000000b1', + '\\' : '\U00002213', + '\\' : '\U000000d7', + '\\
' : '\U000000f7', + '\\' : '\U000022c5', + '\\' : '\U000022c6', + '\\' : '\U00002219', + '\\' : '\U00002218', + '\\' : '\U00002020', + '\\' : '\U00002021', + '\\' : '\U000022b2', + '\\' : '\U000022b3', + '\\' : '\U000022b4', + '\\' : '\U000022b5', + '\\' : '\U000025c3', + '\\' : '\U000025b9', + '\\' : '\U000025b3', + '\\' : '\U0000225c', + '\\' : '\U00002295', + '\\' : '\U00002a01', + '\\' : '\U00002297', + '\\' : '\U00002a02', + '\\' : '\U00002299', + '\\' : '\U00002a00', + '\\' : '\U00002296', + '\\' : '\U00002298', + '\\' : '\U00002026', + '\\' : '\U000022ef', + '\\' : '\U00002211', + '\\' : '\U0000220f', + '\\' : '\U00002210', + '\\' : '\U0000221e', + '\\' : '\U0000222b', + '\\' : '\U0000222e', + '\\' : '\U00002663', + '\\' : '\U00002662', + '\\' : '\U00002661', + '\\' : '\U00002660', + '\\' : '\U00002135', + '\\' : '\U00002205', + '\\' : '\U00002207', + '\\' : '\U00002202', + '\\' : '\U0000266d', + '\\' : '\U0000266e', + '\\' : '\U0000266f', + '\\' : '\U00002220', + '\\' : '\U000000a9', + '\\' : '\U000000ae', + '\\' : '\U000000ad', + '\\' : '\U000000af', + '\\' : '\U000000bc', + '\\' : '\U000000bd', + '\\' : '\U000000be', + '\\' : '\U000000aa', + '\\' : '\U000000ba', + '\\
' : '\U000000a7', + '\\' : '\U000000b6', + '\\' : '\U000000a1', + '\\' : '\U000000bf', + '\\' : '\U000020ac', + '\\' : '\U000000a3', + '\\' : '\U000000a5', + '\\' : '\U000000a2', + '\\' : '\U000000a4', + '\\' : '\U000000b0', + '\\' : '\U00002a3f', + '\\' : '\U00002127', + '\\' : '\U000025ca', + '\\' : '\U00002118', + '\\' : '\U00002240', + '\\' : '\U000022c4', + '\\' : '\U000000b4', + '\\' : '\U00000131', + '\\' : '\U000000a8', + '\\' : '\U000000b8', + '\\' : '\U000002dd', + '\\' : '\U000003f5', + '\\' : '\U000023ce', + '\\' : '\U00002039', + '\\' : '\U0000203a', + '\\' : '\U00002302', + '\\<^sub>' : '\U000021e9', + '\\<^sup>' : '\U000021e7', + '\\<^bold>' : '\U00002759', + '\\<^bsub>' : '\U000021d8', + '\\<^esub>' : '\U000021d9', + '\\<^bsup>' : '\U000021d7', + '\\<^esup>' : '\U000021d6', + } + + lang_map = {'isabelle' : isabelle_symbols, 'latex' : latex_symbols} + + def __init__(self, **options): + Filter.__init__(self, **options) + lang = get_choice_opt(options, 'lang', + ['isabelle', 'latex'], 'isabelle') + self.symbols = self.lang_map[lang] + + def filter(self, lexer, stream): + for ttype, value in stream: + if value in self.symbols: + yield ttype, self.symbols[value] + else: + yield ttype, value + + +class KeywordCaseFilter(Filter): + """Convert keywords to lowercase or uppercase or capitalize them, which + means first letter uppercase, rest lowercase. + + This can be useful e.g. if you highlight Pascal code and want to adapt the + code to your styleguide. + + Options accepted: + + `case` : string + The casing to convert keywords to. Must be one of ``'lower'``, + ``'upper'`` or ``'capitalize'``. The default is ``'lower'``. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + case = get_choice_opt(options, 'case', + ['lower', 'upper', 'capitalize'], 'lower') + self.convert = getattr(str, case) + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype in Keyword: + yield ttype, self.convert(value) + else: + yield ttype, value + + +class NameHighlightFilter(Filter): + """Highlight a normal Name (and Name.*) token with a different token type. + + Example:: + + filter = NameHighlightFilter( + names=['foo', 'bar', 'baz'], + tokentype=Name.Function, + ) + + This would highlight the names "foo", "bar" and "baz" + as functions. `Name.Function` is the default token type. + + Options accepted: + + `names` : list of strings + A list of names that should be given the different token type. + There is no default. + `tokentype` : TokenType or string + A token type or a string containing a token type name that is + used for highlighting the strings in `names`. The default is + `Name.Function`. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + self.names = set(get_list_opt(options, 'names', [])) + tokentype = options.get('tokentype') + if tokentype: + self.tokentype = string_to_tokentype(tokentype) + else: + self.tokentype = Name.Function + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype in Name and value in self.names: + yield self.tokentype, value + else: + yield ttype, value + + +class ErrorToken(Exception): + pass + + +class RaiseOnErrorTokenFilter(Filter): + """Raise an exception when the lexer generates an error token. + + Options accepted: + + `excclass` : Exception class + The exception class to raise. + The default is `pygments.filters.ErrorToken`. + + .. versionadded:: 0.8 + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + self.exception = options.get('excclass', ErrorToken) + try: + # issubclass() will raise TypeError if first argument is not a class + if not issubclass(self.exception, Exception): + raise TypeError + except TypeError: + raise OptionError('excclass option is not an exception class') + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype is Error: + raise self.exception(value) + yield ttype, value + + +class VisibleWhitespaceFilter(Filter): + """Convert tabs, newlines and/or spaces to visible characters. + + Options accepted: + + `spaces` : string or bool + If this is a one-character string, spaces will be replaces by this string. + If it is another true value, spaces will be replaced by ``·`` (unicode + MIDDLE DOT). If it is a false value, spaces will not be replaced. The + default is ``False``. + `tabs` : string or bool + The same as for `spaces`, but the default replacement character is ``»`` + (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value + is ``False``. Note: this will not work if the `tabsize` option for the + lexer is nonzero, as tabs will already have been expanded then. + `tabsize` : int + If tabs are to be replaced by this filter (see the `tabs` option), this + is the total number of characters that a tab should be expanded to. + The default is ``8``. + `newlines` : string or bool + The same as for `spaces`, but the default replacement character is ``¶`` + (unicode PILCROW SIGN). The default value is ``False``. + `wstokentype` : bool + If true, give whitespace the special `Whitespace` token type. This allows + styling the visible whitespace differently (e.g. greyed out), but it can + disrupt background colors. The default is ``True``. + + .. versionadded:: 0.8 + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + for name, default in [('spaces', '·'), + ('tabs', '»'), + ('newlines', '¶')]: + opt = options.get(name, False) + if isinstance(opt, str) and len(opt) == 1: + setattr(self, name, opt) + else: + setattr(self, name, (opt and default or '')) + tabsize = get_int_opt(options, 'tabsize', 8) + if self.tabs: + self.tabs += ' ' * (tabsize - 1) + if self.newlines: + self.newlines += '\n' + self.wstt = get_bool_opt(options, 'wstokentype', True) + + def filter(self, lexer, stream): + if self.wstt: + spaces = self.spaces or ' ' + tabs = self.tabs or '\t' + newlines = self.newlines or '\n' + regex = re.compile(r'\s') + + def replacefunc(wschar): + if wschar == ' ': + return spaces + elif wschar == '\t': + return tabs + elif wschar == '\n': + return newlines + return wschar + + for ttype, value in stream: + yield from _replace_special(ttype, value, regex, Whitespace, + replacefunc) + else: + spaces, tabs, newlines = self.spaces, self.tabs, self.newlines + # simpler processing + for ttype, value in stream: + if spaces: + value = value.replace(' ', spaces) + if tabs: + value = value.replace('\t', tabs) + if newlines: + value = value.replace('\n', newlines) + yield ttype, value + + +class GobbleFilter(Filter): + """Gobbles source code lines (eats initial characters). + + This filter drops the first ``n`` characters off every line of code. This + may be useful when the source code fed to the lexer is indented by a fixed + amount of space that isn't desired in the output. + + Options accepted: + + `n` : int + The number of characters to gobble. + + .. versionadded:: 1.2 + """ + def __init__(self, **options): + Filter.__init__(self, **options) + self.n = get_int_opt(options, 'n', 0) + + def gobble(self, value, left): + if left < len(value): + return value[left:], 0 + else: + return '', left - len(value) + + def filter(self, lexer, stream): + n = self.n + left = n # How many characters left to gobble. + for ttype, value in stream: + # Remove ``left`` tokens from first line, ``n`` from all others. + parts = value.split('\n') + (parts[0], left) = self.gobble(parts[0], left) + for i in range(1, len(parts)): + (parts[i], left) = self.gobble(parts[i], n) + value = '\n'.join(parts) + + if value != '': + yield ttype, value + + +class TokenMergeFilter(Filter): + """Merges consecutive tokens with the same token type in the output + stream of a lexer. + + .. versionadded:: 1.2 + """ + def __init__(self, **options): + Filter.__init__(self, **options) + + def filter(self, lexer, stream): + current_type = None + current_value = None + for ttype, value in stream: + if ttype is current_type: + current_value += value + else: + if current_type is not None: + yield current_type, current_value + current_type = ttype + current_value = value + if current_type is not None: + yield current_type, current_value + + +FILTERS = { + 'codetagify': CodeTagFilter, + 'keywordcase': KeywordCaseFilter, + 'highlight': NameHighlightFilter, + 'raiseonerror': RaiseOnErrorTokenFilter, + 'whitespace': VisibleWhitespaceFilter, + 'gobble': GobbleFilter, + 'tokenmerge': TokenMergeFilter, + 'symbols': SymbolFilter, +} diff --git a/src/pip/_vendor/pygments/formatter.py b/src/pip/_vendor/pygments/formatter.py new file mode 100644 index 00000000000..b585562b588 --- /dev/null +++ b/src/pip/_vendor/pygments/formatter.py @@ -0,0 +1,94 @@ +""" + pygments.formatter + ~~~~~~~~~~~~~~~~~~ + + Base formatter class. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import codecs + +from pip._vendor.pygments.util import get_bool_opt +from pip._vendor.pygments.styles import get_style_by_name + +__all__ = ['Formatter'] + + +def _lookup_style(style): + if isinstance(style, str): + return get_style_by_name(style) + return style + + +class Formatter: + """ + Converts a token stream to text. + + Options accepted: + + ``style`` + The style to use, can be a string or a Style subclass + (default: "default"). Not used by e.g. the + TerminalFormatter. + ``full`` + Tells the formatter to output a "full" document, i.e. + a complete self-contained document. This doesn't have + any effect for some formatters (default: false). + ``title`` + If ``full`` is true, the title that should be used to + caption the document (default: ''). + ``encoding`` + If given, must be an encoding name. This will be used to + convert the Unicode token strings to byte strings in the + output. If it is "" or None, Unicode strings will be written + to the output file, which most file-like objects do not + support (default: None). + ``outencoding`` + Overrides ``encoding`` if given. + """ + + #: Name of the formatter + name = None + + #: Shortcuts for the formatter + aliases = [] + + #: fn match rules + filenames = [] + + #: If True, this formatter outputs Unicode strings when no encoding + #: option is given. + unicodeoutput = True + + def __init__(self, **options): + self.style = _lookup_style(options.get('style', 'default')) + self.full = get_bool_opt(options, 'full', False) + self.title = options.get('title', '') + self.encoding = options.get('encoding', None) or None + if self.encoding in ('guess', 'chardet'): + # can happen for e.g. pygmentize -O encoding=guess + self.encoding = 'utf-8' + self.encoding = options.get('outencoding') or self.encoding + self.options = options + + def get_style_defs(self, arg=''): + """ + Return the style definitions for the current style as a string. + + ``arg`` is an additional argument whose meaning depends on the + formatter used. Note that ``arg`` can also be a list or tuple + for some formatters like the html formatter. + """ + return '' + + def format(self, tokensource, outfile): + """ + Format ``tokensource``, an iterable of ``(tokentype, tokenstring)`` + tuples and write it into ``outfile``. + """ + if self.encoding: + # wrap the outfile in a StreamWriter + outfile = codecs.lookup(self.encoding)[3](outfile) + return self.format_unencoded(tokensource, outfile) diff --git a/src/pip/_vendor/pygments/formatters/__init__.py b/src/pip/_vendor/pygments/formatters/__init__.py new file mode 100644 index 00000000000..34f2bec5981 --- /dev/null +++ b/src/pip/_vendor/pygments/formatters/__init__.py @@ -0,0 +1,153 @@ +""" + pygments.formatters + ~~~~~~~~~~~~~~~~~~~ + + Pygments formatters. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +import sys +import types +import fnmatch +from os.path import basename + +from pip._vendor.pygments.formatters._mapping import FORMATTERS +from pip._vendor.pygments.plugin import find_plugin_formatters +from pip._vendor.pygments.util import ClassNotFound + +__all__ = ['get_formatter_by_name', 'get_formatter_for_filename', + 'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS) + +_formatter_cache = {} # classes by name +_pattern_cache = {} + + +def _fn_matches(fn, glob): + """Return whether the supplied file name fn matches pattern filename.""" + if glob not in _pattern_cache: + pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob)) + return pattern.match(fn) + return _pattern_cache[glob].match(fn) + + +def _load_formatters(module_name): + """Load a formatter (and all others in the module too).""" + mod = __import__(module_name, None, None, ['__all__']) + for formatter_name in mod.__all__: + cls = getattr(mod, formatter_name) + _formatter_cache[cls.name] = cls + + +def get_all_formatters(): + """Return a generator for all formatter classes.""" + # NB: this returns formatter classes, not info like get_all_lexers(). + for info in FORMATTERS.values(): + if info[1] not in _formatter_cache: + _load_formatters(info[0]) + yield _formatter_cache[info[1]] + for _, formatter in find_plugin_formatters(): + yield formatter + + +def find_formatter_class(alias): + """Lookup a formatter by alias. + + Returns None if not found. + """ + for module_name, name, aliases, _, _ in FORMATTERS.values(): + if alias in aliases: + if name not in _formatter_cache: + _load_formatters(module_name) + return _formatter_cache[name] + for _, cls in find_plugin_formatters(): + if alias in cls.aliases: + return cls + + +def get_formatter_by_name(_alias, **options): + """Lookup and instantiate a formatter by alias. + + Raises ClassNotFound if not found. + """ + cls = find_formatter_class(_alias) + if cls is None: + raise ClassNotFound("no formatter found for name %r" % _alias) + return cls(**options) + + +def load_formatter_from_file(filename, formattername="CustomFormatter", + **options): + """Load a formatter from a file. + + This method expects a file located relative to the current working + directory, which contains a class named CustomFormatter. By default, + it expects the Formatter to be named CustomFormatter; you can specify + your own class name as the second argument to this function. + + Users should be very careful with the input, because this method + is equivalent to running eval on the input file. + + Raises ClassNotFound if there are any problems importing the Formatter. + + .. versionadded:: 2.2 + """ + try: + # This empty dict will contain the namespace for the exec'd file + custom_namespace = {} + with open(filename, 'rb') as f: + exec(f.read(), custom_namespace) + # Retrieve the class `formattername` from that namespace + if formattername not in custom_namespace: + raise ClassNotFound('no valid %s class found in %s' % + (formattername, filename)) + formatter_class = custom_namespace[formattername] + # And finally instantiate it with the options + return formatter_class(**options) + except OSError as err: + raise ClassNotFound('cannot read %s: %s' % (filename, err)) + except ClassNotFound: + raise + except Exception as err: + raise ClassNotFound('error when loading custom formatter: %s' % err) + + +def get_formatter_for_filename(fn, **options): + """Lookup and instantiate a formatter by filename pattern. + + Raises ClassNotFound if not found. + """ + fn = basename(fn) + for modname, name, _, filenames, _ in FORMATTERS.values(): + for filename in filenames: + if _fn_matches(fn, filename): + if name not in _formatter_cache: + _load_formatters(modname) + return _formatter_cache[name](**options) + for cls in find_plugin_formatters(): + for filename in cls.filenames: + if _fn_matches(fn, filename): + return cls(**options) + raise ClassNotFound("no formatter found for file name %r" % fn) + + +class _automodule(types.ModuleType): + """Automatically import formatters.""" + + def __getattr__(self, name): + info = FORMATTERS.get(name) + if info: + _load_formatters(info[0]) + cls = _formatter_cache[info[1]] + setattr(self, name, cls) + return cls + raise AttributeError(name) + + +oldmod = sys.modules[__name__] +newmod = _automodule(__name__) +newmod.__dict__.update(oldmod.__dict__) +sys.modules[__name__] = newmod +del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types diff --git a/src/pip/_vendor/pygments/formatters/_mapping.py b/src/pip/_vendor/pygments/formatters/_mapping.py new file mode 100644 index 00000000000..57445dc98ed --- /dev/null +++ b/src/pip/_vendor/pygments/formatters/_mapping.py @@ -0,0 +1,84 @@ +""" + pygments.formatters._mapping + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Formatter mapping definitions. This file is generated by itself. Everytime + you change something on a builtin formatter definition, run this script from + the formatters folder to update it. + + Do not alter the FORMATTERS dictionary by hand. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +FORMATTERS = { + 'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'), + 'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'GroffFormatter': ('pygments.formatters.groff', 'groff', ('groff', 'troff', 'roff'), (), 'Format tokens with groff escapes to change their color and font style.'), + 'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ```` tags within a ``
`` tag, wrapped in a ``
`` tag. The ``
``'s CSS class can be set by the `cssclass` option."), + 'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'), + 'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'), + 'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'), + 'PangoMarkupFormatter': ('pygments.formatters.pangomarkup', 'Pango Markup', ('pango', 'pangomarkup'), (), 'Format tokens as Pango Markup code. It can then be rendered to an SVG.'), + 'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'), + 'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'), + 'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental. Each line of code is a ```` element with explicit ``x`` and ``y`` coordinates containing ```` elements with the individual token styles.'), + 'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'), + 'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'), + 'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'), + 'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.') +} + +if __name__ == '__main__': # pragma: no cover + import sys + import os + + # lookup formatters + found_formatters = [] + imports = [] + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + from pip._vendor.pygments.util import docstring_headline + + for root, dirs, files in os.walk('.'): + for filename in files: + if filename.endswith('.py') and not filename.startswith('_'): + module_name = 'pygments.formatters%s.%s' % ( + root[1:].replace('/', '.'), filename[:-3]) + print(module_name) + module = __import__(module_name, None, None, ['']) + for formatter_name in module.__all__: + formatter = getattr(module, formatter_name) + found_formatters.append( + '%r: %r' % (formatter_name, + (module_name, + formatter.name, + tuple(formatter.aliases), + tuple(formatter.filenames), + docstring_headline(formatter)))) + # sort them to make the diff minimal + found_formatters.sort() + + # extract useful sourcecode from this file + with open(__file__) as fp: + content = fp.read() + # replace crnl to nl for Windows. + # + # Note that, originally, contributers should keep nl of master + # repository, for example by using some kind of automatic + # management EOL, like `EolExtension + # `. + content = content.replace("\r\n", "\n") + header = content[:content.find('FORMATTERS = {')] + footer = content[content.find("if __name__ == '__main__':"):] + + # write new file + with open(__file__, 'w') as fp: + fp.write(header) + fp.write('FORMATTERS = {\n %s\n}\n\n' % ',\n '.join(found_formatters)) + fp.write(footer) + + print ('=== %d formatters processed.' % len(found_formatters)) diff --git a/src/pip/_vendor/pygments/formatters/bbcode.py b/src/pip/_vendor/pygments/formatters/bbcode.py new file mode 100644 index 00000000000..35a37328ec7 --- /dev/null +++ b/src/pip/_vendor/pygments/formatters/bbcode.py @@ -0,0 +1,108 @@ +""" + pygments.formatters.bbcode + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BBcode formatter. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +from pip._vendor.pygments.formatter import Formatter +from pip._vendor.pygments.util import get_bool_opt + +__all__ = ['BBCodeFormatter'] + + +class BBCodeFormatter(Formatter): + """ + Format tokens with BBcodes. These formatting codes are used by many + bulletin boards, so you can highlight your sourcecode with pygments before + posting it there. + + This formatter has no support for background colors and borders, as there + are no common BBcode tags for that. + + Some board systems (e.g. phpBB) don't support colors in their [code] tag, + so you can't use the highlighting together with that tag. + Text in a [code] tag usually is shown with a monospace font (which this + formatter can do with the ``monofont`` option) and no spaces (which you + need for indentation) are removed. + + Additional options accepted: + + `style` + The style to use, can be a string or a Style subclass (default: + ``'default'``). + + `codetag` + If set to true, put the output into ``[code]`` tags (default: + ``false``) + + `monofont` + If set to true, add a tag to show the code with a monospace font + (default: ``false``). + """ + name = 'BBCode' + aliases = ['bbcode', 'bb'] + filenames = [] + + def __init__(self, **options): + Formatter.__init__(self, **options) + self._code = get_bool_opt(options, 'codetag', False) + self._mono = get_bool_opt(options, 'monofont', False) + + self.styles = {} + self._make_styles() + + def _make_styles(self): + for ttype, ndef in self.style: + start = end = '' + if ndef['color']: + start += '[color=#%s]' % ndef['color'] + end = '[/color]' + end + if ndef['bold']: + start += '[b]' + end = '[/b]' + end + if ndef['italic']: + start += '[i]' + end = '[/i]' + end + if ndef['underline']: + start += '[u]' + end = '[/u]' + end + # there are no common BBcodes for background-color and border + + self.styles[ttype] = start, end + + def format_unencoded(self, tokensource, outfile): + if self._code: + outfile.write('[code]') + if self._mono: + outfile.write('[font=monospace]') + + lastval = '' + lasttype = None + + for ttype, value in tokensource: + while ttype not in self.styles: + ttype = ttype.parent + if ttype == lasttype: + lastval += value + else: + if lastval: + start, end = self.styles[lasttype] + outfile.write(''.join((start, lastval, end))) + lastval = value + lasttype = ttype + + if lastval: + start, end = self.styles[lasttype] + outfile.write(''.join((start, lastval, end))) + + if self._mono: + outfile.write('[/font]') + if self._code: + outfile.write('[/code]') + if self._code or self._mono: + outfile.write('\n') diff --git a/src/pip/_vendor/pygments/formatters/groff.py b/src/pip/_vendor/pygments/formatters/groff.py new file mode 100644 index 00000000000..da9d7fca069 --- /dev/null +++ b/src/pip/_vendor/pygments/formatters/groff.py @@ -0,0 +1,168 @@ +""" + pygments.formatters.groff + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Formatter for groff output. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import math +from pip._vendor.pygments.formatter import Formatter +from pip._vendor.pygments.util import get_bool_opt, get_int_opt + +__all__ = ['GroffFormatter'] + + +class GroffFormatter(Formatter): + """ + Format tokens with groff escapes to change their color and font style. + + .. versionadded:: 2.11 + + Additional options accepted: + + `style` + The style to use, can be a string or a Style subclass (default: + ``'default'``). + + `monospaced` + If set to true, monospace font will be used (default: ``true``). + + `linenos` + If set to true, print the line numbers (default: ``false``). + + `wrap` + Wrap lines to the specified number of characters. Disabled if set to 0 + (default: ``0``). + """ + + name = 'groff' + aliases = ['groff','troff','roff'] + filenames = [] + + def __init__(self, **options): + Formatter.__init__(self, **options) + + self.monospaced = get_bool_opt(options, 'monospaced', True) + self.linenos = get_bool_opt(options, 'linenos', False) + self._lineno = 0 + self.wrap = get_int_opt(options, 'wrap', 0) + self._linelen = 0 + + self.styles = {} + self._make_styles() + + + def _make_styles(self): + regular = '\\f[CR]' if self.monospaced else '\\f[R]' + bold = '\\f[CB]' if self.monospaced else '\\f[B]' + italic = '\\f[CI]' if self.monospaced else '\\f[I]' + + for ttype, ndef in self.style: + start = end = '' + if ndef['color']: + start += '\\m[%s]' % ndef['color'] + end = '\\m[]' + end + if ndef['bold']: + start += bold + end = regular + end + if ndef['italic']: + start += italic + end = regular + end + if ndef['bgcolor']: + start += '\\M[%s]' % ndef['bgcolor'] + end = '\\M[]' + end + + self.styles[ttype] = start, end + + + def _define_colors(self, outfile): + colors = set() + for _, ndef in self.style: + if ndef['color'] is not None: + colors.add(ndef['color']) + + for color in colors: + outfile.write('.defcolor ' + color + ' rgb #' + color + '\n') + + + def _write_lineno(self, outfile): + self._lineno += 1 + outfile.write("%s% 4d " % (self._lineno != 1 and '\n' or '', self._lineno)) + + + def _wrap_line(self, line): + length = len(line.rstrip('\n')) + space = ' ' if self.linenos else '' + newline = '' + + if length > self.wrap: + for i in range(0, math.floor(length / self.wrap)): + chunk = line[i*self.wrap:i*self.wrap+self.wrap] + newline += (chunk + '\n' + space) + remainder = length % self.wrap + if remainder > 0: + newline += line[-remainder-1:] + self._linelen = remainder + elif self._linelen + length > self.wrap: + newline = ('\n' + space) + line + self._linelen = length + else: + newline = line + self._linelen += length + + return newline + + + def _escape_chars(self, text): + text = text.replace('\\', '\\[u005C]'). \ + replace('.', '\\[char46]'). \ + replace('\'', '\\[u0027]'). \ + replace('`', '\\[u0060]'). \ + replace('~', '\\[u007E]') + copy = text + + for char in copy: + if len(char) != len(char.encode()): + uni = char.encode('unicode_escape') \ + .decode()[1:] \ + .replace('x', 'u00') \ + .upper() + text = text.replace(char, '\\[u' + uni[1:] + ']') + + return text + + + def format_unencoded(self, tokensource, outfile): + self._define_colors(outfile) + + outfile.write('.nf\n\\f[CR]\n') + + if self.linenos: + self._write_lineno(outfile) + + for ttype, value in tokensource: + start, end = self.styles[ttype] + + for line in value.splitlines(True): + if self.wrap > 0: + line = self._wrap_line(line) + + if start and end: + text = self._escape_chars(line.rstrip('\n')) + if text != '': + outfile.write(''.join((start, text, end))) + else: + outfile.write(self._escape_chars(line.rstrip('\n'))) + + if line.endswith('\n'): + if self.linenos: + self._write_lineno(outfile) + self._linelen = 0 + else: + outfile.write('\n') + self._linelen = 0 + + outfile.write('\n.fi') diff --git a/src/pip/_vendor/pygments/formatters/html.py b/src/pip/_vendor/pygments/formatters/html.py new file mode 100644 index 00000000000..47f5d9c17fb --- /dev/null +++ b/src/pip/_vendor/pygments/formatters/html.py @@ -0,0 +1,983 @@ +""" + pygments.formatters.html + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Formatter for HTML output. + + :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import functools +import os +import sys +import os.path +from io import StringIO + +from pip._vendor.pygments.formatter import Formatter +from pip._vendor.pygments.token import Token, Text, STANDARD_TYPES +from pip._vendor.pygments.util import get_bool_opt, get_int_opt, get_list_opt + +try: + import ctags +except ImportError: + ctags = None + +__all__ = ['HtmlFormatter'] + + +_escape_html_table = { + ord('&'): '&', + ord('<'): '<', + ord('>'): '>', + ord('"'): '"', + ord("'"): ''', +} + + +def escape_html(text, table=_escape_html_table): + """Escape &, <, > as well as single and double quotes for HTML.""" + return text.translate(table) + + +def webify(color): + if color.startswith('calc') or color.startswith('var'): + return color + else: + return '#' + color + + +def _get_ttype_class(ttype): + fname = STANDARD_TYPES.get(ttype) + if fname: + return fname + aname = '' + while fname is None: + aname = '-' + ttype[-1] + aname + ttype = ttype.parent + fname = STANDARD_TYPES.get(ttype) + return fname + aname + + +CSSFILE_TEMPLATE = '''\ +/* +generated by Pygments +Copyright 2006-2021 by the Pygments team. +Licensed under the BSD license, see LICENSE for details. +*/ +%(styledefs)s +''' + +DOC_HEADER = '''\ + + + + + %(title)s + + + + +

%(title)s

+ +''' + +DOC_HEADER_EXTERNALCSS = '''\ + + + + + %(title)s + + + + +

%(title)s

+ +''' + +DOC_FOOTER = '''\ + + +''' + + +class HtmlFormatter(Formatter): + r""" + Format tokens as HTML 4 ```` tags within a ``
`` tag, wrapped
+    in a ``
`` tag. The ``
``'s CSS class can be set by the `cssclass` + option. + + If the `linenos` option is set to ``"table"``, the ``
`` is
+    additionally wrapped inside a ```` which has one row and two
+    cells: one containing the line numbers and one containing the code.
+    Example:
+
+    .. sourcecode:: html
+
+        
+
+ + +
+
1
+            2
+
+
def foo(bar):
+              pass
+            
+
+ + (whitespace added to improve clarity). + + Wrapping can be disabled using the `nowrap` option. + + A list of lines can be specified using the `hl_lines` option to make these + lines highlighted (as of Pygments 0.11). + + With the `full` option, a complete HTML 4 document is output, including + the style definitions inside a `` + {% else %} + {{ hear | safe }} + {% endif %} + + +{{ body | safe }} +{% for diagram in diagrams %} +
+

{{ diagram.title }}

+
{{ diagram.text }}
+
+ {{ diagram.svg }} +
+
+{% endfor %} + + diff --git a/src/pip/_vendor/pyparsing/exceptions.py b/src/pip/_vendor/pyparsing/exceptions.py new file mode 100644 index 00000000000..e06513eb00f --- /dev/null +++ b/src/pip/_vendor/pyparsing/exceptions.py @@ -0,0 +1,267 @@ +# exceptions.py + +import re +import sys +from typing import Optional + +from .util import col, line, lineno, _collapse_string_to_ranges +from .unicode import pyparsing_unicode as ppu + + +class ExceptionWordUnicode(ppu.Latin1, ppu.LatinA, ppu.LatinB, ppu.Greek, ppu.Cyrillic): + pass + + +_extract_alphanums = _collapse_string_to_ranges(ExceptionWordUnicode.alphanums) +_exception_word_extractor = re.compile("([" + _extract_alphanums + "]{1,16})|.") + + +class ParseBaseException(Exception): + """base exception class for all parsing runtime exceptions""" + + # Performance tuning: we construct a *lot* of these, so keep this + # constructor as small and fast as possible + def __init__( + self, + pstr: str, + loc: int = 0, + msg: Optional[str] = None, + elem=None, + ): + self.loc = loc + if msg is None: + self.msg = pstr + self.pstr = "" + else: + self.msg = msg + self.pstr = pstr + self.parser_element = self.parserElement = elem + self.args = (pstr, loc, msg) + + @staticmethod + def explain_exception(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + """ + import inspect + from .core import ParserElement + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(" " * (exc.column - 1) + "^") + ret.append("{}: {}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff[0] + + f_self = frm.f_locals.get("self", None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ("parseImpl", "_parseNoCache"): + continue + if id(f_self) in seen: + continue + seen.add(id(f_self)) + + self_type = type(f_self) + ret.append( + "{}.{} - {}".format( + self_type.__module__, self_type.__name__, f_self + ) + ) + + elif f_self is not None: + self_type = type(f_self) + ret.append("{}.{}".format(self_type.__module__, self_type.__name__)) + + else: + code = frm.f_code + if code.co_name in ("wrapper", ""): + continue + + ret.append("{}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return "\n".join(ret) + + @classmethod + def _from_exception(cls, pe): + """ + internal factory method to simplify creating one type of ParseException + from another - avoids having __init__ signature conflicts among subclasses + """ + return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) + + @property + def line(self) -> str: + """ + Return the line of text where the exception occurred. + """ + return line(self.loc, self.pstr) + + @property + def lineno(self) -> int: + """ + Return the 1-based line number of text where the exception occurred. + """ + return lineno(self.loc, self.pstr) + + @property + def col(self) -> int: + """ + Return the 1-based column on the line of text where the exception occurred. + """ + return col(self.loc, self.pstr) + + @property + def column(self) -> int: + """ + Return the 1-based column on the line of text where the exception occurred. + """ + return col(self.loc, self.pstr) + + def __str__(self) -> str: + if self.pstr: + if self.loc >= len(self.pstr): + foundstr = ", found end of text" + else: + # pull out next word at error location + found_match = _exception_word_extractor.match(self.pstr, self.loc) + if found_match is not None: + found = found_match.group(0) + else: + found = self.pstr[self.loc : self.loc + 1] + foundstr = (", found %r" % found).replace(r"\\", "\\") + else: + foundstr = "" + return "{}{} (at char {}), (line:{}, col:{})".format( + self.msg, foundstr, self.loc, self.lineno, self.column + ) + + def __repr__(self): + return str(self) + + def mark_input_line(self, marker_string: str = None, *, markerString=">!<") -> str: + """ + Extracts the exception line from the input string, and marks + the location of the exception with a special symbol. + """ + markerString = marker_string if marker_string is not None else markerString + line_str = self.line + line_column = self.column - 1 + if markerString: + line_str = "".join( + (line_str[:line_column], markerString, line_str[line_column:]) + ) + return line_str.strip() + + def explain(self, depth=16) -> str: + """ + Method to translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Example:: + + expr = pp.Word(pp.nums) * 3 + try: + expr.parse_string("123 456 A789") + except pp.ParseException as pe: + print(pe.explain(depth=0)) + + prints:: + + 123 456 A789 + ^ + ParseException: Expected W:(0-9), found 'A' (at char 8), (line:1, col:9) + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `set_name` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + Note: pyparsing's default truncation of exception tracebacks may also truncate the + stack of expressions that are displayed in the ``explain`` output. To get the full listing + of parser expressions, you may have to set ``ParserElement.verbose_stacktrace = True`` + """ + return self.explain_exception(self, depth) + + markInputline = mark_input_line + + +class ParseException(ParseBaseException): + """ + Exception thrown when a parse expression doesn't match the input string + + Example:: + + try: + Word(nums).set_name("integer").parse_string("ABC") + except ParseException as pe: + print(pe) + print("column: {}".format(pe.column)) + + prints:: + + Expected integer (at char 0), (line:1, col:1) + column: 1 + + """ + + +class ParseFatalException(ParseBaseException): + """ + User-throwable exception thrown when inconsistent parse content + is found; stops all parsing immediately + """ + + +class ParseSyntaxException(ParseFatalException): + """ + Just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ + + +class RecursiveGrammarException(Exception): + """ + Exception thrown by :class:`ParserElement.validate` if the + grammar could be left-recursive; parser may need to enable + left recursion using :class:`ParserElement.enable_left_recursion` + """ + + def __init__(self, parseElementList): + self.parseElementTrace = parseElementList + + def __str__(self) -> str: + return "RecursiveGrammarException: {}".format(self.parseElementTrace) diff --git a/src/pip/_vendor/pyparsing/helpers.py b/src/pip/_vendor/pyparsing/helpers.py new file mode 100644 index 00000000000..5e7b3ad05eb --- /dev/null +++ b/src/pip/_vendor/pyparsing/helpers.py @@ -0,0 +1,1069 @@ +# helpers.py +import html.entities +import re + +from . import __diag__ +from .core import * +from .util import _bslash, _flatten, _escape_regex_range_chars + + +# +# global helpers +# +def delimited_list( + expr: Union[str, ParserElement], + delim: Union[str, ParserElement] = ",", + combine: bool = False, + min: OptionalType[int] = None, + max: OptionalType[int] = None, + *, + allow_trailing_delim: bool = False, +) -> ParserElement: + """Helper to define a delimited list of expressions - the delimiter + defaults to ','. By default, the list elements and delimiters can + have intervening whitespace, and comments, but this can be + overridden by passing ``combine=True`` in the constructor. If + ``combine`` is set to ``True``, the matching tokens are + returned as a single token string, with the delimiters included; + otherwise, the matching tokens are returned as a list of tokens, + with the delimiters suppressed. + + If ``allow_trailing_delim`` is set to True, then the list may end with + a delimiter. + + Example:: + + delimited_list(Word(alphas)).parse_string("aa,bb,cc") # -> ['aa', 'bb', 'cc'] + delimited_list(Word(hexnums), delim=':', combine=True).parse_string("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] + """ + if isinstance(expr, str_type): + expr = ParserElement._literalStringClass(expr) + + dlName = "{expr} [{delim} {expr}]...{end}".format( + expr=str(expr.copy().streamline()), + delim=str(delim), + end=" [{}]".format(str(delim)) if allow_trailing_delim else "", + ) + + if not combine: + delim = Suppress(delim) + + if min is not None: + if min < 1: + raise ValueError("min must be greater than 0") + min -= 1 + if max is not None: + if min is not None and max <= min: + raise ValueError("max must be greater than, or equal to min") + max -= 1 + delimited_list_expr = expr + (delim + expr)[min, max] + + if allow_trailing_delim: + delimited_list_expr += Opt(delim) + + if combine: + return Combine(delimited_list_expr).set_name(dlName) + else: + return delimited_list_expr.set_name(dlName) + + +def counted_array( + expr: ParserElement, + int_expr: OptionalType[ParserElement] = None, + *, + intExpr: OptionalType[ParserElement] = None, +) -> ParserElement: + """Helper to define a counted list of expressions. + + This helper defines a pattern of the form:: + + integer expr expr expr... + + where the leading integer tells how many expr expressions follow. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``int_expr`` is specified, it should be a pyparsing expression + that produces an integer value. + + Example:: + + counted_array(Word(alphas)).parse_string('2 ab cd ef') # -> ['ab', 'cd'] + + # in this parser, the leading integer value is given in binary, + # '10' indicating that 2 values are in the array + binary_constant = Word('01').set_parse_action(lambda t: int(t[0], 2)) + counted_array(Word(alphas), int_expr=binary_constant).parse_string('10 ab cd ef') # -> ['ab', 'cd'] + + # if other fields must be parsed after the count but before the + # list items, give the fields results names and they will + # be preserved in the returned ParseResults: + count_with_metadata = integer + Word(alphas)("type") + typed_array = counted_array(Word(alphanums), int_expr=count_with_metadata)("items") + result = typed_array.parse_string("3 bool True True False") + print(result.dump()) + + # prints + # ['True', 'True', 'False'] + # - items: ['True', 'True', 'False'] + # - type: 'bool' + """ + intExpr = intExpr or int_expr + array_expr = Forward() + + def count_field_parse_action(s, l, t): + nonlocal array_expr + n = t[0] + array_expr <<= (expr * n) if n else Empty() + # clear list contents, but keep any named results + del t[:] + + if intExpr is None: + intExpr = Word(nums).set_parse_action(lambda t: int(t[0])) + else: + intExpr = intExpr.copy() + intExpr.set_name("arrayLen") + intExpr.add_parse_action(count_field_parse_action, call_during_try=True) + return (intExpr + array_expr).set_name("(len) " + str(expr) + "...") + + +def match_previous_literal(expr: ParserElement) -> ParserElement: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + + first = Word(nums) + second = match_previous_literal(first) + match_expr = first + ":" + second + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`match_previous_expr`. Do *not* use with packrat parsing + enabled. + """ + rep = Forward() + + def copy_token_to_repeater(s, l, t): + if t: + if len(t) == 1: + rep << t[0] + else: + # flatten t tokens + tflat = _flatten(t.as_list()) + rep << And(Literal(tt) for tt in tflat) + else: + rep << Empty() + + expr.add_parse_action(copy_token_to_repeater, callDuringTry=True) + rep.set_name("(prev) " + str(expr)) + return rep + + +def match_previous_expr(expr: ParserElement) -> ParserElement: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + + first = Word(nums) + second = match_previous_expr(first) + match_expr = first + ":" + second + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. + """ + rep = Forward() + e2 = expr.copy() + rep <<= e2 + + def copy_token_to_repeater(s, l, t): + matchTokens = _flatten(t.as_list()) + + def must_match_these_tokens(s, l, t): + theseTokens = _flatten(t.as_list()) + if theseTokens != matchTokens: + raise ParseException(s, l, "Expected {}, found{}".format(matchTokens, theseTokens)) + + rep.set_parse_action(must_match_these_tokens, callDuringTry=True) + + expr.add_parse_action(copy_token_to_repeater, callDuringTry=True) + rep.set_name("(prev) " + str(expr)) + return rep + + +def one_of( + strs: Union[IterableType[str], str], + caseless: bool = False, + use_regex: bool = True, + as_keyword: bool = False, + *, + useRegex: bool = True, + asKeyword: bool = False, +) -> ParserElement: + """Helper to quickly define a set of alternative :class:`Literal` s, + and makes sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. + + Parameters: + + - ``strs`` - a string of space-delimited literals, or a collection of + string literals + - ``caseless`` - treat all literals as caseless - (default= ``False``) + - ``use_regex`` - as an optimization, will + generate a :class:`Regex` object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True`` or ``asKeyword=True``, or if + creating a :class:`Regex` raises an exception) - (default= ``True``) + - ``as_keyword`` - enforce :class:`Keyword`-style matching on the + generated expressions - (default= ``False``) + - ``asKeyword`` and ``useRegex`` are retained for pre-PEP8 compatibility, + but will be removed in a future release + + Example:: + + comp_oper = one_of("< = > <= >= !=") + var = Word(alphas) + number = Word(nums) + term = var | number + comparison_expr = term + comp_oper + term + print(comparison_expr.search_string("B = 12 AA=23 B<=AA AA>12")) + + prints:: + + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] + """ + asKeyword = asKeyword or as_keyword + useRegex = useRegex and use_regex + + if ( + isinstance(caseless, str_type) + and __diag__.warn_on_multiple_string_args_to_oneof + ): + warnings.warn( + "More than one string argument passed to one_of, pass" + " choices as a list or space-delimited string", + stacklevel=2, + ) + + if caseless: + isequal = lambda a, b: a.upper() == b.upper() + masks = lambda a, b: b.upper().startswith(a.upper()) + parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral + else: + isequal = lambda a, b: a == b + masks = lambda a, b: b.startswith(a) + parseElementClass = Keyword if asKeyword else Literal + + symbols: List[str] = [] + if isinstance(strs, str_type): + symbols = strs.split() + elif isinstance(strs, Iterable): + symbols = list(strs) + else: + raise TypeError("Invalid argument to one_of, expected string or iterable") + if not symbols: + return NoMatch() + + # reorder given symbols to take care to avoid masking longer choices with shorter ones + # (but only if the given symbols are not just single characters) + if any(len(sym) > 1 for sym in symbols): + i = 0 + while i < len(symbols) - 1: + cur = symbols[i] + for j, other in enumerate(symbols[i + 1 :]): + if isequal(other, cur): + del symbols[i + j + 1] + break + elif masks(cur, other): + del symbols[i + j + 1] + symbols.insert(i, other) + break + else: + i += 1 + + if useRegex: + re_flags: int = re.IGNORECASE if caseless else 0 + + try: + if all(len(sym) == 1 for sym in symbols): + # symbols are just single characters, create range regex pattern + patt = "[{}]".format( + "".join(_escape_regex_range_chars(sym) for sym in symbols) + ) + else: + patt = "|".join(re.escape(sym) for sym in symbols) + + # wrap with \b word break markers if defining as keywords + if asKeyword: + patt = r"\b(?:{})\b".format(patt) + + ret = Regex(patt, flags=re_flags).set_name(" | ".join(symbols)) + + if caseless: + # add parse action to return symbols as specified, not in random + # casing as found in input string + symbol_map = {sym.lower(): sym for sym in symbols} + ret.add_parse_action(lambda s, l, t: symbol_map[t[0].lower()]) + + return ret + + except sre_constants.error: + warnings.warn( + "Exception creating Regex for one_of, building MatchFirst", stacklevel=2 + ) + + # last resort, just use MatchFirst + return MatchFirst(parseElementClass(sym) for sym in symbols).set_name( + " | ".join(symbols) + ) + + +def dict_of(key: ParserElement, value: ParserElement) -> ParserElement: + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. + + Example:: + + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" + attr_expr = (label + Suppress(':') + OneOrMore(data_word, stop_on=label).set_parse_action(' '.join)) + print(OneOrMore(attr_expr).parse_string(text).dump()) + + attr_label = label + attr_value = Suppress(':') + OneOrMore(data_word, stop_on=label).set_parse_action(' '.join) + + # similar to Dict, but simpler call format + result = dict_of(attr_label, attr_value).parse_string(text) + print(result.dump()) + print(result['shape']) + print(result.shape) # object attribute access works too + print(result.as_dict()) + + prints:: + + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] + - color: light blue + - posn: upper left + - shape: SQUARE + - texture: burlap + SQUARE + SQUARE + {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} + """ + return Dict(OneOrMore(Group(key + value))) + + +def original_text_for( + expr: ParserElement, as_string: bool = True, *, asString: bool = True +) -> ParserElement: + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``as_string`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`original_text_for` contains expressions with defined + results names, you must set ``as_string`` to ``False`` if you + want to preserve those results name values. + + The ``asString`` pre-PEP8 argument is retained for compatibility, + but will be removed in a future release. + + Example:: + + src = "this is test bold text normal text " + for tag in ("b", "i"): + opener, closer = make_html_tags(tag) + patt = original_text_for(opener + SkipTo(closer) + closer) + print(patt.search_string(src)[0]) + + prints:: + + [' bold text '] + ['text'] + """ + asString = asString and as_string + + locMarker = Empty().set_parse_action(lambda s, loc, t: loc) + endlocMarker = locMarker.copy() + endlocMarker.callPreparse = False + matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") + if asString: + extractText = lambda s, l, t: s[t._original_start : t._original_end] + else: + + def extractText(s, l, t): + t[:] = [s[t.pop("_original_start") : t.pop("_original_end")]] + + matchExpr.set_parse_action(extractText) + matchExpr.ignoreExprs = expr.ignoreExprs + matchExpr.suppress_warning(Diagnostics.warn_ungrouped_named_tokens_in_collection) + return matchExpr + + +def ungroup(expr: ParserElement) -> ParserElement: + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. + """ + return TokenConverter(expr).add_parse_action(lambda t: t[0]) + + +def locatedExpr(expr: ParserElement) -> ParserElement: + """ + (DEPRECATED - future code should use the Located class) + Helper to decorate a returned token with its starting and ending + locations in the input string. + + This helper adds the following results names: + + - ``locn_start`` - location where matched expression begins + - ``locn_end`` - location where matched expression ends + - ``value`` - the actual parsed results + + Be careful if the input text contains ```` characters, you + may want to call :class:`ParserElement.parseWithTabs` + + Example:: + + wd = Word(alphas) + for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): + print(match) + + prints:: + + [[0, 'ljsdf', 5]] + [[8, 'lksdjjf', 15]] + [[18, 'lkkjj', 23]] + """ + locator = Empty().set_parse_action(lambda ss, ll, tt: ll) + return Group( + locator("locn_start") + + expr("value") + + locator.copy().leaveWhitespace()("locn_end") + ) + + +def nested_expr( + opener: Union[str, ParserElement] = "(", + closer: Union[str, ParserElement] = ")", + content: OptionalType[ParserElement] = None, + ignore_expr: ParserElement = quoted_string(), + *, + ignoreExpr: ParserElement = quoted_string(), +) -> ParserElement: + """Helper method for defining nested lists enclosed in opening and + closing delimiters (``"("`` and ``")"`` are the default). + + Parameters: + - ``opener`` - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - ``closer`` - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - ``content`` - expression for items within the nested lists + (default= ``None``) + - ``ignore_expr`` - expression for ignoring opening and closing delimiters + (default= :class:`quoted_string`) + - ``ignoreExpr`` - this pre-PEP8 argument is retained for compatibility + but will be removed in a future release + + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. + + Use the ``ignore_expr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quoted_string or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quoted_string`, but if no expressions are to be ignored, then + pass ``None`` for this argument. + + Example:: + + data_type = one_of("void int short long char float double") + decl_data_type = Combine(data_type + Opt(Word('*'))) + ident = Word(alphas+'_', alphanums+'_') + number = pyparsing_common.number + arg = Group(decl_data_type + ident) + LPAR, RPAR = map(Suppress, "()") + + code_body = nested_expr('{', '}', ignore_expr=(quoted_string | c_style_comment)) + + c_function = (decl_data_type("type") + + ident("name") + + LPAR + Opt(delimited_list(arg), [])("args") + RPAR + + code_body("body")) + c_function.ignore(c_style_comment) + + source_code = ''' + int is_odd(int x) { + return (x%2); + } + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { + return (10+ord(hchar)-ord('A')); + } + } + ''' + for func in c_function.search_string(source_code): + print("%(name)s (%(type)s) args: %(args)s" % func) + + + prints:: + + is_odd (int) args: [['int', 'x']] + dec_to_hex (int) args: [['char', 'hchar']] + """ + if ignoreExpr != ignore_expr: + ignoreExpr = ignore_expr if ignoreExpr == quoted_string() else ignoreExpr + if opener == closer: + raise ValueError("opening and closing strings cannot be the same") + if content is None: + if isinstance(opener, str_type) and isinstance(closer, str_type): + if len(opener) == 1 and len(closer) == 1: + if ignoreExpr is not None: + content = Combine( + OneOrMore( + ~ignoreExpr + + CharsNotIn( + opener + closer + ParserElement.DEFAULT_WHITE_CHARS, + exact=1, + ) + ) + ).set_parse_action(lambda t: t[0].strip()) + else: + content = empty.copy() + CharsNotIn( + opener + closer + ParserElement.DEFAULT_WHITE_CHARS + ).set_parse_action(lambda t: t[0].strip()) + else: + if ignoreExpr is not None: + content = Combine( + OneOrMore( + ~ignoreExpr + + ~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1) + ) + ).set_parse_action(lambda t: t[0].strip()) + else: + content = Combine( + OneOrMore( + ~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1) + ) + ).set_parse_action(lambda t: t[0].strip()) + else: + raise ValueError( + "opening and closing arguments must be strings if no content expression is given" + ) + ret = Forward() + if ignoreExpr is not None: + ret <<= Group( + Suppress(opener) + ZeroOrMore(ignoreExpr | ret | content) + Suppress(closer) + ) + else: + ret <<= Group(Suppress(opener) + ZeroOrMore(ret | content) + Suppress(closer)) + ret.set_name("nested %s%s expression" % (opener, closer)) + return ret + + +def _makeTags(tagStr, xml, suppress_LT=Suppress("<"), suppress_GT=Suppress(">")): + """Internal helper to construct opening and closing tag expressions, given a tag name""" + if isinstance(tagStr, str_type): + resname = tagStr + tagStr = Keyword(tagStr, caseless=not xml) + else: + resname = tagStr.name + + tagAttrName = Word(alphas, alphanums + "_-:") + if xml: + tagAttrValue = dbl_quoted_string.copy().set_parse_action(remove_quotes) + openTag = ( + suppress_LT + + tagStr("tag") + + Dict(ZeroOrMore(Group(tagAttrName + Suppress("=") + tagAttrValue))) + + Opt("/", default=[False])("empty").set_parse_action( + lambda s, l, t: t[0] == "/" + ) + + suppress_GT + ) + else: + tagAttrValue = quoted_string.copy().set_parse_action(remove_quotes) | Word( + printables, exclude_chars=">" + ) + openTag = ( + suppress_LT + + tagStr("tag") + + Dict( + ZeroOrMore( + Group( + tagAttrName.set_parse_action(lambda t: t[0].lower()) + + Opt(Suppress("=") + tagAttrValue) + ) + ) + ) + + Opt("/", default=[False])("empty").set_parse_action( + lambda s, l, t: t[0] == "/" + ) + + suppress_GT + ) + closeTag = Combine(Literal("", adjacent=False) + + openTag.set_name("<%s>" % resname) + # add start results name in parse action now that ungrouped names are not reported at two levels + openTag.add_parse_action( + lambda t: t.__setitem__( + "start" + "".join(resname.replace(":", " ").title().split()), t.copy() + ) + ) + closeTag = closeTag( + "end" + "".join(resname.replace(":", " ").title().split()) + ).set_name("" % resname) + openTag.tag = resname + closeTag.tag = resname + openTag.tag_body = SkipTo(closeTag()) + return openTag, closeTag + + +def make_html_tags( + tag_str: Union[str, ParserElement] +) -> Tuple[ParserElement, ParserElement]: + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. + + Example:: + + text = 'More info at the
pyparsing wiki page' + # make_html_tags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple + a, a_end = make_html_tags("A") + link_expr = a + SkipTo(a_end)("link_text") + a_end + + for link in link_expr.search_string(text): + # attributes in the tag (like "href" shown here) are + # also accessible as named results + print(link.link_text, '->', link.href) + + prints:: + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki + """ + return _makeTags(tag_str, False) + + +def make_xml_tags( + tag_str: Union[str, ParserElement] +) -> Tuple[ParserElement, ParserElement]: + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. + + Example: similar to :class:`make_html_tags` + """ + return _makeTags(tag_str, True) + + +any_open_tag, any_close_tag = make_html_tags( + Word(alphas, alphanums + "_:").set_name("any tag") +) + +_htmlEntityMap = {k.rstrip(";"): v for k, v in html.entities.html5.items()} +common_html_entity = Regex("&(?P" + "|".join(_htmlEntityMap) + ");").set_name( + "common HTML entity" +) + + +def replace_html_entity(t): + """Helper parser action to replace common HTML entities with their special characters""" + return _htmlEntityMap.get(t.entity) + + +class OpAssoc(Enum): + LEFT = 1 + RIGHT = 2 + + +InfixNotationOperatorArgType = Union[ + ParserElement, str, Tuple[Union[ParserElement, str], Union[ParserElement, str]] +] +InfixNotationOperatorSpec = Union[ + Tuple[ + InfixNotationOperatorArgType, + int, + OpAssoc, + OptionalType[ParseAction], + ], + Tuple[ + InfixNotationOperatorArgType, + int, + OpAssoc, + ], +] + + +def infix_notation( + base_expr: ParserElement, + op_list: List[InfixNotationOperatorSpec], + lpar: Union[str, ParserElement] = Suppress("("), + rpar: Union[str, ParserElement] = Suppress(")"), +) -> ParserElement: + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infix_notation. See + :class:`ParserElement.enable_packrat` for a mechanism to potentially + improve your parser performance. + + Parameters: + - ``base_expr`` - expression representing the most basic operand to + be used in the expression + - ``op_list`` - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(op_expr, + num_operands, right_left_assoc, (optional)parse_action)``, where: + + - ``op_expr`` is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if ``num_operands`` + is 3, ``op_expr`` is a tuple of two expressions, for the two + operators separating the 3 terms + - ``num_operands`` is the number of terms for this operator (must be 1, + 2, or 3) + - ``right_left_assoc`` is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``OpAssoc.RIGHT`` and ``OpAssoc.LEFT``. + - ``parse_action`` is the parse action to be associated with + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``set_parse_action(*fn)`` + (:class:`ParserElement.set_parse_action`) + - ``lpar`` - expression for matching left-parentheses + (default= ``Suppress('(')``) + - ``rpar`` - expression for matching right-parentheses + (default= ``Suppress(')')``) + + Example:: + + # simple example of four-function arithmetic with ints and + # variable names + integer = pyparsing_common.signed_integer + varname = pyparsing_common.identifier + + arith_expr = infix_notation(integer | varname, + [ + ('-', 1, OpAssoc.RIGHT), + (one_of('* /'), 2, OpAssoc.LEFT), + (one_of('+ -'), 2, OpAssoc.LEFT), + ]) + + arith_expr.run_tests(''' + 5+3*6 + (5+3)*6 + -2--11 + ''', full_dump=False) + + prints:: + + 5+3*6 + [[5, '+', [3, '*', 6]]] + + (5+3)*6 + [[[5, '+', 3], '*', 6]] + + -2--11 + [[['-', 2], '-', ['-', 11]]] + """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.try_parse(instring, loc) + return loc, [] + + _FB.__name__ = "FollowedBy>" + + ret = Forward() + lpar = Suppress(lpar) + rpar = Suppress(rpar) + lastExpr = base_expr | (lpar + ret + rpar) + for i, operDef in enumerate(op_list): + opExpr, arity, rightLeftAssoc, pa = (operDef + (None,))[:4] + if isinstance(opExpr, str_type): + opExpr = ParserElement._literalStringClass(opExpr) + if arity == 3: + if not isinstance(opExpr, (tuple, list)) or len(opExpr) != 2: + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions" + ) + opExpr1, opExpr2 = opExpr + term_name = "{}{} term".format(opExpr1, opExpr2) + else: + term_name = "{} term".format(opExpr) + + if not 1 <= arity <= 3: + raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + + if rightLeftAssoc not in (OpAssoc.LEFT, OpAssoc.RIGHT): + raise ValueError("operator must indicate right or left associativity") + + thisExpr = Forward().set_name(term_name) + if rightLeftAssoc is OpAssoc.LEFT: + if arity == 1: + matchExpr = _FB(lastExpr + opExpr) + Group(lastExpr + opExpr[1, ...]) + elif arity == 2: + if opExpr is not None: + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group( + lastExpr + (opExpr + lastExpr)[1, ...] + ) + else: + matchExpr = _FB(lastExpr + lastExpr) + Group(lastExpr[2, ...]) + elif arity == 3: + matchExpr = _FB( + lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr + ) + Group(lastExpr + OneOrMore(opExpr1 + lastExpr + opExpr2 + lastExpr)) + elif rightLeftAssoc is OpAssoc.RIGHT: + if arity == 1: + # try to avoid LR with this extra test + if not isinstance(opExpr, Opt): + opExpr = Opt(opExpr) + matchExpr = _FB(opExpr.expr + thisExpr) + Group(opExpr + thisExpr) + elif arity == 2: + if opExpr is not None: + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group( + lastExpr + (opExpr + thisExpr)[1, ...] + ) + else: + matchExpr = _FB(lastExpr + thisExpr) + Group( + lastExpr + thisExpr[1, ...] + ) + elif arity == 3: + matchExpr = _FB( + lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr + ) + Group(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + if pa: + if isinstance(pa, (tuple, list)): + matchExpr.set_parse_action(*pa) + else: + matchExpr.set_parse_action(pa) + thisExpr <<= (matchExpr | lastExpr).setName(term_name) + lastExpr = thisExpr + ret <<= lastExpr + return ret + + +def indentedBlock(blockStatementExpr, indentStack, indent=True, backup_stacks=[]): + """ + (DEPRECATED - use IndentedBlock class instead) + Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. + + Parameters: + + - ``blockStatementExpr`` - expression defining syntax of statement that + is repeated within the indented block + - ``indentStack`` - list created by caller to manage indentation stack + (multiple ``statementWithIndentedBlock`` expressions within a single + grammar should share a common ``indentStack``) + - ``indent`` - boolean indicating whether block must be indented beyond + the current level; set to ``False`` for block of left-most statements + (default= ``True``) + + A valid block must contain at least one ``blockStatement``. + + (Note that indentedBlock uses internal parse actions which make it + incompatible with packrat parsing.) + + Example:: + + data = ''' + def A(z): + A1 + B = 100 + G = A2 + A2 + A3 + B + def BB(a,b,c): + BB1 + def BBA(): + bba1 + bba2 + bba3 + C + D + def spam(x,y): + def eggs(z): + pass + ''' + + + indentStack = [1] + stmt = Forward() + + identifier = Word(alphas, alphanums) + funcDecl = ("def" + identifier + Group("(" + Opt(delimitedList(identifier)) + ")") + ":") + func_body = indentedBlock(stmt, indentStack) + funcDef = Group(funcDecl + func_body) + + rvalue = Forward() + funcCall = Group(identifier + "(" + Opt(delimitedList(rvalue)) + ")") + rvalue << (funcCall | identifier | Word(nums)) + assignment = Group(identifier + "=" + rvalue) + stmt << (funcDef | assignment | identifier) + + module_body = OneOrMore(stmt) + + parseTree = module_body.parseString(data) + parseTree.pprint() + + prints:: + + [['def', + 'A', + ['(', 'z', ')'], + ':', + [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]], + 'B', + ['def', + 'BB', + ['(', 'a', 'b', 'c', ')'], + ':', + [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]], + 'C', + 'D', + ['def', + 'spam', + ['(', 'x', 'y', ')'], + ':', + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + """ + backup_stacks.append(indentStack[:]) + + def reset_stack(): + indentStack[:] = backup_stacks[-1] + + def checkPeerIndent(s, l, t): + if l >= len(s): + return + curCol = col(l, s) + if curCol != indentStack[-1]: + if curCol > indentStack[-1]: + raise ParseException(s, l, "illegal nesting") + raise ParseException(s, l, "not a peer entry") + + def checkSubIndent(s, l, t): + curCol = col(l, s) + if curCol > indentStack[-1]: + indentStack.append(curCol) + else: + raise ParseException(s, l, "not a subentry") + + def checkUnindent(s, l, t): + if l >= len(s): + return + curCol = col(l, s) + if not (indentStack and curCol in indentStack): + raise ParseException(s, l, "not an unindent") + if curCol < indentStack[-1]: + indentStack.pop() + + NL = OneOrMore(LineEnd().set_whitespace_chars("\t ").suppress()) + INDENT = (Empty() + Empty().set_parse_action(checkSubIndent)).set_name("INDENT") + PEER = Empty().set_parse_action(checkPeerIndent).set_name("") + UNDENT = Empty().set_parse_action(checkUnindent).set_name("UNINDENT") + if indent: + smExpr = Group( + Opt(NL) + + INDENT + + OneOrMore(PEER + Group(blockStatementExpr) + Opt(NL)) + + UNDENT + ) + else: + smExpr = Group( + Opt(NL) + + OneOrMore(PEER + Group(blockStatementExpr) + Opt(NL)) + + Opt(UNDENT) + ) + + # add a parse action to remove backup_stack from list of backups + smExpr.add_parse_action( + lambda: backup_stacks.pop(-1) and None if backup_stacks else None + ) + smExpr.set_fail_action(lambda a, b, c, d: reset_stack()) + blockStatementExpr.ignore(_bslash + LineEnd()) + return smExpr.set_name("indented block") + + +# it's easy to get these comment structures wrong - they're very common, so may as well make them available +c_style_comment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + "*/").set_name( + "C style comment" +) +"Comment of the form ``/* ... */``" + +html_comment = Regex(r"").set_name("HTML comment") +"Comment of the form ````" + +rest_of_line = Regex(r".*").leave_whitespace().set_name("rest of line") +dbl_slash_comment = Regex(r"//(?:\\\n|[^\n])*").set_name("// comment") +"Comment of the form ``// ... (to end of line)``" + +cpp_style_comment = Combine( + Regex(r"/\*(?:[^*]|\*(?!/))*") + "*/" | dbl_slash_comment +).set_name("C++ style comment") +"Comment of either form :class:`c_style_comment` or :class:`dbl_slash_comment`" + +java_style_comment = cpp_style_comment +"Same as :class:`cpp_style_comment`" + +python_style_comment = Regex(r"#.*").set_name("Python style comment") +"Comment of the form ``# ... (to end of line)``" + + +# build list of built-in expressions, for future reference if a global default value +# gets updated +_builtin_exprs = [v for v in vars().values() if isinstance(v, ParserElement)] + + +# pre-PEP8 compatible names +delimitedList = delimited_list +countedArray = counted_array +matchPreviousLiteral = match_previous_literal +matchPreviousExpr = match_previous_expr +oneOf = one_of +dictOf = dict_of +originalTextFor = original_text_for +nestedExpr = nested_expr +makeHTMLTags = make_html_tags +makeXMLTags = make_xml_tags +anyOpenTag, anyCloseTag = any_open_tag, any_close_tag +commonHTMLEntity = common_html_entity +replaceHTMLEntity = replace_html_entity +opAssoc = OpAssoc +infixNotation = infix_notation +cStyleComment = c_style_comment +htmlComment = html_comment +restOfLine = rest_of_line +dblSlashComment = dbl_slash_comment +cppStyleComment = cpp_style_comment +javaStyleComment = java_style_comment +pythonStyleComment = python_style_comment diff --git a/src/pip/_vendor/pyparsing/results.py b/src/pip/_vendor/pyparsing/results.py new file mode 100644 index 00000000000..9676f45b88b --- /dev/null +++ b/src/pip/_vendor/pyparsing/results.py @@ -0,0 +1,760 @@ +# results.py +from collections.abc import MutableMapping, Mapping, MutableSequence, Iterator +import pprint +from weakref import ref as wkref +from typing import Tuple, Any + +str_type: Tuple[type, ...] = (str, bytes) +_generator_type = type((_ for _ in ())) + + +class _ParseResultsWithOffset: + __slots__ = ["tup"] + + def __init__(self, p1, p2): + self.tup = (p1, p2) + + def __getitem__(self, i): + return self.tup[i] + + def __getstate__(self): + return self.tup + + def __setstate__(self, *args): + self.tup = args[0] + + +class ParseResults: + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.`` - see :class:`ParserElement.set_results_name`) + + Example:: + + integer = Word(nums) + date_str = (integer.set_results_name("year") + '/' + + integer.set_results_name("month") + '/' + + integer.set_results_name("day")) + # equivalent form: + # date_str = (integer("year") + '/' + # + integer("month") + '/' + # + integer("day")) + + # parse_string returns a ParseResults object + result = date_str.parse_string("1999/12/31") + + def test(s, fn=repr): + print("{} -> {}".format(s, fn(eval(s)))) + test("list(result)") + test("result[0]") + test("result['month']") + test("result.day") + test("'month' in result") + test("'minutes' in result") + test("result.dump()", str) + + prints:: + + list(result) -> ['1999', '/', '12', '/', '31'] + result[0] -> '1999' + result['month'] -> '12' + result.day -> '31' + 'month' in result -> True + 'minutes' in result -> False + result.dump() -> ['1999', '/', '12', '/', '31'] + - day: 31 + - month: 12 + - year: 1999 + """ + + _null_values: Tuple[Any, ...] = (None, [], "", ()) + + __slots__ = [ + "_name", + "_parent", + "_all_names", + "_modal", + "_toklist", + "_tokdict", + "__weakref__", + ] + + class List(list): + """ + Simple wrapper class to distinguish parsed list results that should be preserved + as actual Python lists, instead of being converted to :class:`ParseResults`: + + LBRACK, RBRACK = map(pp.Suppress, "[]") + element = pp.Forward() + item = ppc.integer + element_list = LBRACK + pp.delimited_list(element) + RBRACK + + # add parse actions to convert from ParseResults to actual Python collection types + def as_python_list(t): + return pp.ParseResults.List(t.as_list()) + element_list.add_parse_action(as_python_list) + + element <<= item | element_list + + element.run_tests(''' + 100 + [2,3,4] + [[2, 1],3,4] + [(2, 1),3,4] + (2,3,4) + ''', post_parse=lambda s, r: (r[0], type(r[0]))) + + prints: + + 100 + (100, ) + + [2,3,4] + ([2, 3, 4], ) + + [[2, 1],3,4] + ([[2, 1], 3, 4], ) + + (Used internally by :class:`Group` when `aslist=True`.) + """ + + def __new__(cls, contained=None): + if contained is None: + contained = [] + + if not isinstance(contained, list): + raise TypeError( + "{} may only be constructed with a list," + " not {}".format(cls.__name__, type(contained).__name__) + ) + + return list.__new__(cls) + + def __new__(cls, toklist=None, name=None, **kwargs): + if isinstance(toklist, ParseResults): + return toklist + self = object.__new__(cls) + self._name = None + self._parent = None + self._all_names = set() + + if toklist is None: + self._toklist = [] + elif isinstance(toklist, (list, _generator_type)): + self._toklist = ( + [toklist[:]] + if isinstance(toklist, ParseResults.List) + else list(toklist) + ) + else: + self._toklist = [toklist] + self._tokdict = dict() + return self + + # Performance tuning: we construct a *lot* of these, so keep this + # constructor as small and fast as possible + def __init__( + self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance + ): + self._modal = modal + if name is not None and name != "": + if isinstance(name, int): + name = str(name) + if not modal: + self._all_names = {name} + self._name = name + if toklist not in self._null_values: + if isinstance(toklist, (str_type, type)): + toklist = [toklist] + if asList: + if isinstance(toklist, ParseResults): + self[name] = _ParseResultsWithOffset( + ParseResults(toklist._toklist), 0 + ) + else: + self[name] = _ParseResultsWithOffset( + ParseResults(toklist[0]), 0 + ) + self[name]._name = name + else: + try: + self[name] = toklist[0] + except (KeyError, TypeError, IndexError): + if toklist is not self: + self[name] = toklist + else: + self._name = name + + def __getitem__(self, i): + if isinstance(i, (int, slice)): + return self._toklist[i] + else: + if i not in self._all_names: + return self._tokdict[i][-1][0] + else: + return ParseResults([v[0] for v in self._tokdict[i]]) + + def __setitem__(self, k, v, isinstance=isinstance): + if isinstance(v, _ParseResultsWithOffset): + self._tokdict[k] = self._tokdict.get(k, list()) + [v] + sub = v[0] + elif isinstance(k, (int, slice)): + self._toklist[k] = v + sub = v + else: + self._tokdict[k] = self._tokdict.get(k, list()) + [ + _ParseResultsWithOffset(v, 0) + ] + sub = v + if isinstance(sub, ParseResults): + sub._parent = wkref(self) + + def __delitem__(self, i): + if isinstance(i, (int, slice)): + mylen = len(self._toklist) + del self._toklist[i] + + # convert int to slice + if isinstance(i, int): + if i < 0: + i += mylen + i = slice(i, i + 1) + # get removed indices + removed = list(range(*i.indices(mylen))) + removed.reverse() + # fixup indices in token dictionary + for name, occurrences in self._tokdict.items(): + for j in removed: + for k, (value, position) in enumerate(occurrences): + occurrences[k] = _ParseResultsWithOffset( + value, position - (position > j) + ) + else: + del self._tokdict[i] + + def __contains__(self, k) -> bool: + return k in self._tokdict + + def __len__(self) -> int: + return len(self._toklist) + + def __bool__(self) -> bool: + return not not (self._toklist or self._tokdict) + + def __iter__(self) -> Iterator: + return iter(self._toklist) + + def __reversed__(self) -> Iterator: + return iter(self._toklist[::-1]) + + def keys(self): + return iter(self._tokdict) + + def values(self): + return (self[k] for k in self.keys()) + + def items(self): + return ((k, self[k]) for k in self.keys()) + + def haskeys(self) -> bool: + """ + Since ``keys()`` returns an iterator, this method is helpful in bypassing + code that looks for the existence of any defined results names.""" + return bool(self._tokdict) + + def pop(self, *args, **kwargs): + """ + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. + + Example:: + + numlist = Word(nums)[...] + print(numlist.parse_string("0 123 321")) # -> ['0', '123', '321'] + + def remove_first(tokens): + tokens.pop(0) + numlist.add_parse_action(remove_first) + print(numlist.parse_string("0 123 321")) # -> ['123', '321'] + + label = Word(alphas) + patt = label("LABEL") + OneOrMore(Word(nums)) + print(patt.parse_string("AAB 123 321").dump()) + + # Use pop() in a parse action to remove named result (note that corresponding value is not + # removed from list form of results) + def remove_LABEL(tokens): + tokens.pop("LABEL") + return tokens + patt.add_parse_action(remove_LABEL) + print(patt.parse_string("AAB 123 321").dump()) + + prints:: + + ['AAB', '123', '321'] + - LABEL: AAB + + ['AAB', '123', '321'] + """ + if not args: + args = [-1] + for k, v in kwargs.items(): + if k == "default": + args = (args[0], v) + else: + raise TypeError( + "pop() got an unexpected keyword argument {!r}".format(k) + ) + if isinstance(args[0], int) or len(args) == 1 or args[0] in self: + index = args[0] + ret = self[index] + del self[index] + return ret + else: + defaultvalue = args[1] + return defaultvalue + + def get(self, key, default_value=None): + """ + Returns named result matching the given key, or if there is no + such name, then returns the given ``default_value`` or ``None`` if no + ``default_value`` is specified. + + Similar to ``dict.get()``. + + Example:: + + integer = Word(nums) + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + + result = date_str.parse_string("1999/12/31") + print(result.get("year")) # -> '1999' + print(result.get("hour", "not specified")) # -> 'not specified' + print(result.get("hour")) # -> None + """ + if key in self: + return self[key] + else: + return default_value + + def insert(self, index, ins_string): + """ + Inserts new element at location index in the list of parsed tokens. + + Similar to ``list.insert()``. + + Example:: + + numlist = Word(nums)[...] + print(numlist.parse_string("0 123 321")) # -> ['0', '123', '321'] + + # use a parse action to insert the parse location in the front of the parsed results + def insert_locn(locn, tokens): + tokens.insert(0, locn) + numlist.add_parse_action(insert_locn) + print(numlist.parse_string("0 123 321")) # -> [0, '0', '123', '321'] + """ + self._toklist.insert(index, ins_string) + # fixup indices in token dictionary + for name, occurrences in self._tokdict.items(): + for k, (value, position) in enumerate(occurrences): + occurrences[k] = _ParseResultsWithOffset( + value, position + (position > index) + ) + + def append(self, item): + """ + Add single element to end of ``ParseResults`` list of elements. + + Example:: + + numlist = Word(nums)[...] + print(numlist.parse_string("0 123 321")) # -> ['0', '123', '321'] + + # use a parse action to compute the sum of the parsed integers, and add it to the end + def append_sum(tokens): + tokens.append(sum(map(int, tokens))) + numlist.add_parse_action(append_sum) + print(numlist.parse_string("0 123 321")) # -> ['0', '123', '321', 444] + """ + self._toklist.append(item) + + def extend(self, itemseq): + """ + Add sequence of elements to end of ``ParseResults`` list of elements. + + Example:: + + patt = OneOrMore(Word(alphas)) + + # use a parse action to append the reverse of the matched strings, to make a palindrome + def make_palindrome(tokens): + tokens.extend(reversed([t[::-1] for t in tokens])) + return ''.join(tokens) + patt.add_parse_action(make_palindrome) + print(patt.parse_string("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl' + """ + if isinstance(itemseq, ParseResults): + self.__iadd__(itemseq) + else: + self._toklist.extend(itemseq) + + def clear(self): + """ + Clear all elements and results names. + """ + del self._toklist[:] + self._tokdict.clear() + + def __getattr__(self, name): + try: + return self[name] + except KeyError: + if name.startswith("__"): + raise AttributeError(name) + return "" + + def __add__(self, other) -> "ParseResults": + ret = self.copy() + ret += other + return ret + + def __iadd__(self, other) -> "ParseResults": + if other._tokdict: + offset = len(self._toklist) + addoffset = lambda a: offset if a < 0 else a + offset + otheritems = other._tokdict.items() + otherdictitems = [ + (k, _ParseResultsWithOffset(v[0], addoffset(v[1]))) + for k, vlist in otheritems + for v in vlist + ] + for k, v in otherdictitems: + self[k] = v + if isinstance(v[0], ParseResults): + v[0]._parent = wkref(self) + + self._toklist += other._toklist + self._all_names |= other._all_names + return self + + def __radd__(self, other) -> "ParseResults": + if isinstance(other, int) and other == 0: + # useful for merging many ParseResults using sum() builtin + return self.copy() + else: + # this may raise a TypeError - so be it + return other + self + + def __repr__(self) -> str: + return "{}({!r}, {})".format(type(self).__name__, self._toklist, self.as_dict()) + + def __str__(self) -> str: + return ( + "[" + + ", ".join( + [ + str(i) if isinstance(i, ParseResults) else repr(i) + for i in self._toklist + ] + ) + + "]" + ) + + def _asStringList(self, sep=""): + out = [] + for item in self._toklist: + if out and sep: + out.append(sep) + if isinstance(item, ParseResults): + out += item._asStringList() + else: + out.append(str(item)) + return out + + def as_list(self) -> list: + """ + Returns the parse results as a nested list of matching tokens, all converted to strings. + + Example:: + + patt = OneOrMore(Word(alphas)) + result = patt.parse_string("sldkj lsdkj sldkj") + # even though the result prints in string-like form, it is actually a pyparsing ParseResults + print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] + + # Use as_list() to create an actual list + result_list = result.as_list() + print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] + """ + return [ + res.as_list() if isinstance(res, ParseResults) else res + for res in self._toklist + ] + + def as_dict(self) -> dict: + """ + Returns the named parse results as a nested dictionary. + + Example:: + + integer = Word(nums) + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + + result = date_str.parse_string('12/31/1999') + print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) + + result_dict = result.as_dict() + print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} + + # even though a ParseResults supports dict-like access, sometime you just need to have a dict + import json + print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable + print(json.dumps(result.as_dict())) # -> {"month": "31", "day": "1999", "year": "12"} + """ + + def to_item(obj): + if isinstance(obj, ParseResults): + return obj.as_dict() if obj.haskeys() else [to_item(v) for v in obj] + else: + return obj + + return dict((k, to_item(v)) for k, v in self.items()) + + def copy(self) -> "ParseResults": + """ + Returns a new copy of a :class:`ParseResults` object. + """ + ret = ParseResults(self._toklist) + ret._tokdict = self._tokdict.copy() + ret._parent = self._parent + ret._all_names |= self._all_names + ret._name = self._name + return ret + + def get_name(self): + r""" + Returns the results name for this token expression. Useful when several + different expressions might match at a particular location. + + Example:: + + integer = Word(nums) + ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") + house_number_expr = Suppress('#') + Word(nums, alphanums) + user_data = (Group(house_number_expr)("house_number") + | Group(ssn_expr)("ssn") + | Group(integer)("age")) + user_info = OneOrMore(user_data) + + result = user_info.parse_string("22 111-22-3333 #221B") + for item in result: + print(item.get_name(), ':', item[0]) + + prints:: + + age : 22 + ssn : 111-22-3333 + house_number : 221B + """ + if self._name: + return self._name + elif self._parent: + par = self._parent() + + def find_in_parent(sub): + return next( + ( + k + for k, vlist in par._tokdict.items() + for v, loc in vlist + if sub is v + ), + None, + ) + + return find_in_parent(self) if par else None + elif ( + len(self) == 1 + and len(self._tokdict) == 1 + and next(iter(self._tokdict.values()))[0][1] in (0, -1) + ): + return next(iter(self._tokdict.keys())) + else: + return None + + def dump(self, indent="", full=True, include_list=True, _depth=0) -> str: + """ + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. + + Example:: + + integer = Word(nums) + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + + result = date_str.parse_string('12/31/1999') + print(result.dump()) + + prints:: + + ['12', '/', '31', '/', '1999'] + - day: 1999 + - month: 31 + - year: 12 + """ + out = [] + NL = "\n" + out.append(indent + str(self.as_list()) if include_list else "") + + if full: + if self.haskeys(): + items = sorted((str(k), v) for k, v in self.items()) + for k, v in items: + if out: + out.append(NL) + out.append("{}{}- {}: ".format(indent, (" " * _depth), k)) + if isinstance(v, ParseResults): + if v: + out.append( + v.dump( + indent=indent, + full=full, + include_list=include_list, + _depth=_depth + 1, + ) + ) + else: + out.append(str(v)) + else: + out.append(repr(v)) + if any(isinstance(vv, ParseResults) for vv in self): + v = self + for i, vv in enumerate(v): + if isinstance(vv, ParseResults): + out.append( + "\n{}{}[{}]:\n{}{}{}".format( + indent, + (" " * (_depth)), + i, + indent, + (" " * (_depth + 1)), + vv.dump( + indent=indent, + full=full, + include_list=include_list, + _depth=_depth + 1, + ), + ) + ) + else: + out.append( + "\n%s%s[%d]:\n%s%s%s" + % ( + indent, + (" " * (_depth)), + i, + indent, + (" " * (_depth + 1)), + str(vv), + ) + ) + + return "".join(out) + + def pprint(self, *args, **kwargs): + """ + Pretty-printer for parsed results as a list, using the + `pprint `_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint `_ . + + Example:: + + ident = Word(alphas, alphanums) + num = Word(nums) + func = Forward() + term = ident | num | Group('(' + func + ')') + func <<= ident + Group(Optional(delimited_list(term))) + result = func.parse_string("fna a,b,(fnb c,d,200),100") + result.pprint(width=40) + + prints:: + + ['fna', + ['a', + 'b', + ['(', 'fnb', ['c', 'd', '200'], ')'], + '100']] + """ + pprint.pprint(self.as_list(), *args, **kwargs) + + # add support for pickle protocol + def __getstate__(self): + return ( + self._toklist, + ( + self._tokdict.copy(), + self._parent is not None and self._parent() or None, + self._all_names, + self._name, + ), + ) + + def __setstate__(self, state): + self._toklist, (self._tokdict, par, inAccumNames, self._name) = state + self._all_names = set(inAccumNames) + if par is not None: + self._parent = wkref(par) + else: + self._parent = None + + def __getnewargs__(self): + return self._toklist, self._name + + def __dir__(self): + return dir(type(self)) + list(self.keys()) + + @classmethod + def from_dict(cls, other, name=None) -> "ParseResults": + """ + Helper classmethod to construct a ``ParseResults`` from a ``dict``, preserving the + name-value relations as results names. If an optional ``name`` argument is + given, a nested ``ParseResults`` will be returned. + """ + + def is_iterable(obj): + try: + iter(obj) + except Exception: + return False + else: + return not isinstance(obj, str_type) + + ret = cls([]) + for k, v in other.items(): + if isinstance(v, Mapping): + ret += cls.from_dict(v, name=k) + else: + ret += cls([v], name=k, asList=is_iterable(v)) + if name is not None: + ret = cls([ret], name=name) + return ret + + asList = as_list + asDict = as_dict + getName = get_name + + +MutableMapping.register(ParseResults) +MutableSequence.register(ParseResults) diff --git a/src/pip/_vendor/pyparsing/testing.py b/src/pip/_vendor/pyparsing/testing.py new file mode 100644 index 00000000000..991972f3fb2 --- /dev/null +++ b/src/pip/_vendor/pyparsing/testing.py @@ -0,0 +1,331 @@ +# testing.py + +from contextlib import contextmanager +from typing import Optional + +from .core import ( + ParserElement, + ParseException, + Keyword, + __diag__, + __compat__, +) + + +class pyparsing_test: + """ + namespace class for classes useful in writing unit tests + """ + + class reset_pyparsing_context: + """ + Context manager to be used when writing unit tests that modify pyparsing config values: + - packrat parsing + - bounded recursion parsing + - default whitespace characters. + - default keyword characters + - literal string auto-conversion class + - __diag__ settings + + Example:: + + with reset_pyparsing_context(): + # test that literals used to construct a grammar are automatically suppressed + ParserElement.inlineLiteralsUsing(Suppress) + + term = Word(alphas) | Word(nums) + group = Group('(' + term[...] + ')') + + # assert that the '()' characters are not included in the parsed tokens + self.assertParseAndCheckList(group, "(abc 123 def)", ['abc', '123', 'def']) + + # after exiting context manager, literals are converted to Literal expressions again + """ + + def __init__(self): + self._save_context = {} + + def save(self): + self._save_context["default_whitespace"] = ParserElement.DEFAULT_WHITE_CHARS + self._save_context["default_keyword_chars"] = Keyword.DEFAULT_KEYWORD_CHARS + + self._save_context[ + "literal_string_class" + ] = ParserElement._literalStringClass + + self._save_context["verbose_stacktrace"] = ParserElement.verbose_stacktrace + + self._save_context["packrat_enabled"] = ParserElement._packratEnabled + if ParserElement._packratEnabled: + self._save_context[ + "packrat_cache_size" + ] = ParserElement.packrat_cache.size + else: + self._save_context["packrat_cache_size"] = None + self._save_context["packrat_parse"] = ParserElement._parse + self._save_context[ + "recursion_enabled" + ] = ParserElement._left_recursion_enabled + + self._save_context["__diag__"] = { + name: getattr(__diag__, name) for name in __diag__._all_names + } + + self._save_context["__compat__"] = { + "collect_all_And_tokens": __compat__.collect_all_And_tokens + } + + return self + + def restore(self): + # reset pyparsing global state + if ( + ParserElement.DEFAULT_WHITE_CHARS + != self._save_context["default_whitespace"] + ): + ParserElement.set_default_whitespace_chars( + self._save_context["default_whitespace"] + ) + + ParserElement.verbose_stacktrace = self._save_context["verbose_stacktrace"] + + Keyword.DEFAULT_KEYWORD_CHARS = self._save_context["default_keyword_chars"] + ParserElement.inlineLiteralsUsing( + self._save_context["literal_string_class"] + ) + + for name, value in self._save_context["__diag__"].items(): + (__diag__.enable if value else __diag__.disable)(name) + + ParserElement._packratEnabled = False + if self._save_context["packrat_enabled"]: + ParserElement.enable_packrat(self._save_context["packrat_cache_size"]) + else: + ParserElement._parse = self._save_context["packrat_parse"] + ParserElement._left_recursion_enabled = self._save_context[ + "recursion_enabled" + ] + + __compat__.collect_all_And_tokens = self._save_context["__compat__"] + + return self + + def copy(self): + ret = type(self)() + ret._save_context.update(self._save_context) + return ret + + def __enter__(self): + return self.save() + + def __exit__(self, *args): + self.restore() + + class TestParseResultsAsserts: + """ + A mixin class to add parse results assertion methods to normal unittest.TestCase classes. + """ + + def assertParseResultsEquals( + self, result, expected_list=None, expected_dict=None, msg=None + ): + """ + Unit test assertion to compare a :class:`ParseResults` object with an optional ``expected_list``, + and compare any defined results names with an optional ``expected_dict``. + """ + if expected_list is not None: + self.assertEqual(expected_list, result.as_list(), msg=msg) + if expected_dict is not None: + self.assertEqual(expected_dict, result.as_dict(), msg=msg) + + def assertParseAndCheckList( + self, expr, test_string, expected_list, msg=None, verbose=True + ): + """ + Convenience wrapper assert to test a parser element and input string, and assert that + the resulting ``ParseResults.asList()`` is equal to the ``expected_list``. + """ + result = expr.parse_string(test_string, parse_all=True) + if verbose: + print(result.dump()) + else: + print(result.as_list()) + self.assertParseResultsEquals(result, expected_list=expected_list, msg=msg) + + def assertParseAndCheckDict( + self, expr, test_string, expected_dict, msg=None, verbose=True + ): + """ + Convenience wrapper assert to test a parser element and input string, and assert that + the resulting ``ParseResults.asDict()`` is equal to the ``expected_dict``. + """ + result = expr.parse_string(test_string, parseAll=True) + if verbose: + print(result.dump()) + else: + print(result.as_list()) + self.assertParseResultsEquals(result, expected_dict=expected_dict, msg=msg) + + def assertRunTestResults( + self, run_tests_report, expected_parse_results=None, msg=None + ): + """ + Unit test assertion to evaluate output of ``ParserElement.runTests()``. If a list of + list-dict tuples is given as the ``expected_parse_results`` argument, then these are zipped + with the report tuples returned by ``runTests`` and evaluated using ``assertParseResultsEquals``. + Finally, asserts that the overall ``runTests()`` success value is ``True``. + + :param run_tests_report: tuple(bool, [tuple(str, ParseResults or Exception)]) returned from runTests + :param expected_parse_results (optional): [tuple(str, list, dict, Exception)] + """ + run_test_success, run_test_results = run_tests_report + + if expected_parse_results is not None: + merged = [ + (*rpt, expected) + for rpt, expected in zip(run_test_results, expected_parse_results) + ] + for test_string, result, expected in merged: + # expected should be a tuple containing a list and/or a dict or an exception, + # and optional failure message string + # an empty tuple will skip any result validation + fail_msg = next( + (exp for exp in expected if isinstance(exp, str)), None + ) + expected_exception = next( + ( + exp + for exp in expected + if isinstance(exp, type) and issubclass(exp, Exception) + ), + None, + ) + if expected_exception is not None: + with self.assertRaises( + expected_exception=expected_exception, msg=fail_msg or msg + ): + if isinstance(result, Exception): + raise result + else: + expected_list = next( + (exp for exp in expected if isinstance(exp, list)), None + ) + expected_dict = next( + (exp for exp in expected if isinstance(exp, dict)), None + ) + if (expected_list, expected_dict) != (None, None): + self.assertParseResultsEquals( + result, + expected_list=expected_list, + expected_dict=expected_dict, + msg=fail_msg or msg, + ) + else: + # warning here maybe? + print("no validation for {!r}".format(test_string)) + + # do this last, in case some specific test results can be reported instead + self.assertTrue( + run_test_success, msg=msg if msg is not None else "failed runTests" + ) + + @contextmanager + def assertRaisesParseException(self, exc_type=ParseException, msg=None): + with self.assertRaises(exc_type, msg=msg): + yield + + @staticmethod + def with_line_numbers( + s: str, + start_line: Optional[int] = None, + end_line: Optional[int] = None, + expand_tabs: bool = True, + eol_mark: str = "|", + mark_spaces: Optional[str] = None, + mark_control: Optional[str] = None, + ) -> str: + """ + Helpful method for debugging a parser - prints a string with line and column numbers. + (Line and column numbers are 1-based.) + + :param s: tuple(bool, str - string to be printed with line and column numbers + :param start_line: int - (optional) starting line number in s to print (default=1) + :param end_line: int - (optional) ending line number in s to print (default=len(s)) + :param expand_tabs: bool - (optional) expand tabs to spaces, to match the pyparsing default + :param eol_mark: str - (optional) string to mark the end of lines, helps visualize trailing spaces (default="|") + :param mark_spaces: str - (optional) special character to display in place of spaces + :param mark_control: str - (optional) convert non-printing control characters to a placeholding + character; valid values: + - "unicode" - replaces control chars with Unicode symbols, such as "␍" and "␊" + - any single character string - replace control characters with given string + - None (default) - string is displayed as-is + + :return: str - input string with leading line numbers and column number headers + """ + if expand_tabs: + s = s.expandtabs() + if mark_control is not None: + if mark_control == "unicode": + tbl = str.maketrans( + {c: u for c, u in zip(range(0, 33), range(0x2400, 0x2433))} + | {127: 0x2421} + ) + eol_mark = "" + else: + tbl = str.maketrans( + {c: mark_control for c in list(range(0, 32)) + [127]} + ) + s = s.translate(tbl) + if mark_spaces is not None and mark_spaces != " ": + if mark_spaces == "unicode": + tbl = str.maketrans({9: 0x2409, 32: 0x2423}) + s = s.translate(tbl) + else: + s = s.replace(" ", mark_spaces) + if start_line is None: + start_line = 1 + if end_line is None: + end_line = len(s) + end_line = min(end_line, len(s)) + start_line = min(max(1, start_line), end_line) + + if mark_control != "unicode": + s_lines = s.splitlines()[start_line - 1 : end_line] + else: + s_lines = [line + "␊" for line in s.split("␊")[start_line - 1 : end_line]] + if not s_lines: + return "" + + lineno_width = len(str(end_line)) + max_line_len = max(len(line) for line in s_lines) + lead = " " * (lineno_width + 1) + if max_line_len >= 99: + header0 = ( + lead + + "".join( + "{}{}".format(" " * 99, (i + 1) % 100) + for i in range(max(max_line_len // 100, 1)) + ) + + "\n" + ) + else: + header0 = "" + header1 = ( + header0 + + lead + + "".join( + " {}".format((i + 1) % 10) + for i in range(-(-max_line_len // 10)) + ) + + "\n" + ) + header2 = lead + "1234567890" * (-(-max_line_len // 10)) + "\n" + return ( + header1 + + header2 + + "\n".join( + "{:{}d}:{}{}".format(i, lineno_width, line, eol_mark) + for i, line in enumerate(s_lines, start=start_line) + ) + + "\n" + ) diff --git a/src/pip/_vendor/pyparsing/unicode.py b/src/pip/_vendor/pyparsing/unicode.py new file mode 100644 index 00000000000..92261487c7a --- /dev/null +++ b/src/pip/_vendor/pyparsing/unicode.py @@ -0,0 +1,332 @@ +# unicode.py + +import sys +from itertools import filterfalse +from typing import List, Tuple, Union + + +class _lazyclassproperty: + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, "_intern") or any( + cls._intern is getattr(superclass, "_intern", []) + for superclass in cls.__mro__[1:] + ): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +UnicodeRangeList = List[Union[Tuple[int, int], Tuple[int]]] + + +class unicode_set: + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``. Ranges can be specified using + 2-tuples or a 1-tuple, such as:: + + _ranges = [ + (0x0020, 0x007e), + (0x00a0, 0x00ff), + (0x0100,), + ] + + Ranges are left- and right-inclusive. A 1-tuple of (x,) is treated as (x, x). + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + + _ranges: UnicodeRangeList = [] + + @_lazyclassproperty + def _chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in getattr(cc, "_ranges", ()): + ret.extend(range(rr[0], rr[-1] + 1)) + return [chr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return "".join(filterfalse(str.isspace, cls._chars_for_ranges)) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return "".join(filter(str.isalpha, cls._chars_for_ranges)) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return "".join(filter(str.isdigit, cls._chars_for_ranges)) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + @_lazyclassproperty + def identchars(cls): + "all characters in this range that are valid identifier characters, plus underscore '_'" + return "".join( + sorted( + set( + "".join(filter(str.isidentifier, cls._chars_for_ranges)) + + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzªµº" + + "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ" + + "_" + ) + ) + ) + + @_lazyclassproperty + def identbodychars(cls): + """ + all characters in this range that are valid identifier body characters, + plus the digits 0-9 + """ + return "".join( + sorted( + set( + cls.identchars + + "0123456789" + + "".join( + [c for c in cls._chars_for_ranges if ("_" + c).isidentifier()] + ) + ) + ) + ) + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + + _ranges: UnicodeRangeList = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x0020, 0x007E), + (0x00A0, 0x00FF), + ] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x0100, 0x017F), + ] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x0180, 0x024F), + ] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges: UnicodeRangeList = [ + (0x0342, 0x0345), + (0x0370, 0x0377), + (0x037A, 0x037F), + (0x0384, 0x038A), + (0x038C,), + (0x038E, 0x03A1), + (0x03A3, 0x03E1), + (0x03F0, 0x03FF), + (0x1D26, 0x1D2A), + (0x1D5E,), + (0x1D60,), + (0x1D66, 0x1D6A), + (0x1F00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59,), + (0x1F5B,), + (0x1F5D,), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FC4), + (0x1FC6, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FDD, 0x1FEF), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFE), + (0x2129,), + (0x2719, 0x271A), + (0xAB65,), + (0x10140, 0x1018D), + (0x101A0,), + (0x1D200, 0x1D245), + (0x1F7A1, 0x1F7A7), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x0400, 0x052F), + (0x1C80, 0x1C88), + (0x1D2B,), + (0x1D78,), + (0x2DE0, 0x2DFF), + (0xA640, 0xA672), + (0xA674, 0xA69F), + (0xFE2E, 0xFE2F), + ] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x2E80, 0x2E99), + (0x2E9B, 0x2EF3), + (0x31C0, 0x31E3), + (0x3400, 0x4DB5), + (0x4E00, 0x9FEF), + (0xA700, 0xA707), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0x16FE2, 0x16FE3), + (0x1F210, 0x1F212), + (0x1F214, 0x1F23B), + (0x1F240, 0x1F248), + (0x20000, 0x2A6D6), + (0x2A700, 0x2B734), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2F800, 0x2FA1D), + ] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges: UnicodeRangeList = [] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x4E00, 0x9FBF), + (0x3000, 0x303F), + ] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x3041, 0x3096), + (0x3099, 0x30A0), + (0x30FC,), + (0xFF70,), + (0x1B001,), + (0x1B150, 0x1B152), + (0x1F200,), + ] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x3099, 0x309C), + (0x30A0, 0x30FF), + (0x31F0, 0x31FF), + (0x32D0, 0x32FE), + (0xFF65, 0xFF9F), + (0x1B000,), + (0x1B164, 0x1B167), + (0x1F201, 0x1F202), + (0x1F213,), + ] + + class Hangul(unicode_set): + "Unicode set for Hangul (Korean) Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x1100, 0x11FF), + (0x302E, 0x302F), + (0x3131, 0x318E), + (0x3200, 0x321C), + (0x3260, 0x327B), + (0x327E,), + (0xA960, 0xA97C), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xFFA0, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + ] + + Korean = Hangul + + class CJK(Chinese, Japanese, Hangul): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges: UnicodeRangeList = [(0x0E01, 0x0E3A), (0x0E3F, 0x0E5B)] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x0600, 0x061B), + (0x061E, 0x06FF), + (0x0700, 0x077F), + ] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges: UnicodeRangeList = [ + (0x0591, 0x05C7), + (0x05D0, 0x05EA), + (0x05EF, 0x05F4), + (0xFB1D, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E,), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFB4F), + ] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges: UnicodeRangeList = [(0x0900, 0x097F), (0xA8E0, 0xA8FF)] + + +pyparsing_unicode.Japanese._ranges = ( + pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges +) + +# define ranges in language character sets +pyparsing_unicode.العربية = pyparsing_unicode.Arabic +pyparsing_unicode.中文 = pyparsing_unicode.Chinese +pyparsing_unicode.кириллица = pyparsing_unicode.Cyrillic +pyparsing_unicode.Ελληνικά = pyparsing_unicode.Greek +pyparsing_unicode.עִברִית = pyparsing_unicode.Hebrew +pyparsing_unicode.日本語 = pyparsing_unicode.Japanese +pyparsing_unicode.Japanese.漢字 = pyparsing_unicode.Japanese.Kanji +pyparsing_unicode.Japanese.カタカナ = pyparsing_unicode.Japanese.Katakana +pyparsing_unicode.Japanese.ひらがな = pyparsing_unicode.Japanese.Hiragana +pyparsing_unicode.한국어 = pyparsing_unicode.Korean +pyparsing_unicode.ไทย = pyparsing_unicode.Thai +pyparsing_unicode.देवनागरी = pyparsing_unicode.Devanagari diff --git a/src/pip/_vendor/pyparsing/util.py b/src/pip/_vendor/pyparsing/util.py new file mode 100644 index 00000000000..34ce092c6d0 --- /dev/null +++ b/src/pip/_vendor/pyparsing/util.py @@ -0,0 +1,235 @@ +# util.py +import warnings +import types +import collections +import itertools +from functools import lru_cache +from typing import List, Union, Iterable + +_bslash = chr(92) + + +class __config_flags: + """Internal class for defining compatibility and debugging flags""" + + _all_names: List[str] = [] + _fixed_names: List[str] = [] + _type_desc = "configuration" + + @classmethod + def _set(cls, dname, value): + if dname in cls._fixed_names: + warnings.warn( + "{}.{} {} is {} and cannot be overridden".format( + cls.__name__, + dname, + cls._type_desc, + str(getattr(cls, dname)).upper(), + ) + ) + return + if dname in cls._all_names: + setattr(cls, dname, value) + else: + raise ValueError("no such {} {!r}".format(cls._type_desc, dname)) + + enable = classmethod(lambda cls, name: cls._set(name, True)) + disable = classmethod(lambda cls, name: cls._set(name, False)) + + +@lru_cache(maxsize=128) +def col(loc: int, strg: str) -> int: + """ + Returns current column within a string, counting newlines as line separators. + The first column is number 1. + + Note: the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + """ + s = strg + return 1 if 0 < loc < len(s) and s[loc - 1] == "\n" else loc - s.rfind("\n", 0, loc) + + +@lru_cache(maxsize=128) +def lineno(loc: int, strg: str) -> int: + """Returns current line number within a string, counting newlines as line separators. + The first line is number 1. + + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ```` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ + return strg.count("\n", 0, loc) + 1 + + +@lru_cache(maxsize=128) +def line(loc: int, strg: str) -> str: + """ + Returns the line of text containing loc within a string, counting newlines as line separators. + """ + last_cr = strg.rfind("\n", 0, loc) + next_cr = strg.find("\n", loc) + return strg[last_cr + 1 : next_cr] if next_cr >= 0 else strg[last_cr + 1 :] + + +class _UnboundedCache: + def __init__(self): + cache = {} + cache_get = cache.get + self.not_in_cache = not_in_cache = object() + + def get(_, key): + return cache_get(key, not_in_cache) + + def set_(_, key, value): + cache[key] = value + + def clear(_): + cache.clear() + + self.size = None + self.get = types.MethodType(get, self) + self.set = types.MethodType(set_, self) + self.clear = types.MethodType(clear, self) + + +class _FifoCache: + def __init__(self, size): + self.not_in_cache = not_in_cache = object() + cache = collections.OrderedDict() + cache_get = cache.get + + def get(_, key): + return cache_get(key, not_in_cache) + + def set_(_, key, value): + cache[key] = value + while len(cache) > size: + cache.popitem(last=False) + + def clear(_): + cache.clear() + + self.size = size + self.get = types.MethodType(get, self) + self.set = types.MethodType(set_, self) + self.clear = types.MethodType(clear, self) + + +class LRUMemo: + """ + A memoizing mapping that retains `capacity` deleted items + + The memo tracks retained items by their access order; once `capacity` items + are retained, the least recently used item is discarded. + """ + + def __init__(self, capacity): + self._capacity = capacity + self._active = {} + self._memory = collections.OrderedDict() + + def __getitem__(self, key): + try: + return self._active[key] + except KeyError: + self._memory.move_to_end(key) + return self._memory[key] + + def __setitem__(self, key, value): + self._memory.pop(key, None) + self._active[key] = value + + def __delitem__(self, key): + try: + value = self._active.pop(key) + except KeyError: + pass + else: + while len(self._memory) >= self._capacity: + self._memory.popitem(last=False) + self._memory[key] = value + + def clear(self): + self._active.clear() + self._memory.clear() + + +class UnboundedMemo(dict): + """ + A memoizing mapping that retains all deleted items + """ + + def __delitem__(self, key): + pass + + +def _escape_regex_range_chars(s: str) -> str: + # escape these chars: ^-[] + for c in r"\^-[]": + s = s.replace(c, _bslash + c) + s = s.replace("\n", r"\n") + s = s.replace("\t", r"\t") + return str(s) + + +def _collapse_string_to_ranges( + s: Union[str, Iterable[str]], re_escape: bool = True +) -> str: + def is_consecutive(c): + c_int = ord(c) + is_consecutive.prev, prev = c_int, is_consecutive.prev + if c_int - prev > 1: + is_consecutive.value = next(is_consecutive.counter) + return is_consecutive.value + + is_consecutive.prev = 0 + is_consecutive.counter = itertools.count() + is_consecutive.value = -1 + + def escape_re_range_char(c): + return "\\" + c if c in r"\^-][" else c + + def no_escape_re_range_char(c): + return c + + if not re_escape: + escape_re_range_char = no_escape_re_range_char + + ret = [] + s = "".join(sorted(set(s))) + if len(s) > 3: + for _, chars in itertools.groupby(s, key=is_consecutive): + first = last = next(chars) + last = collections.deque( + itertools.chain(iter([last]), chars), maxlen=1 + ).pop() + if first == last: + ret.append(escape_re_range_char(first)) + else: + sep = "" if ord(last) == ord(first) + 1 else "-" + ret.append( + "{}{}{}".format( + escape_re_range_char(first), sep, escape_re_range_char(last) + ) + ) + else: + ret = [escape_re_range_char(c) for c in s] + + return "".join(ret) + + +def _flatten(ll: list) -> list: + ret = [] + for i in ll: + if isinstance(i, list): + ret.extend(_flatten(i)) + else: + ret.append(i) + return ret diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py index 4bea577a36f..75a633bf9dc 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py @@ -41,12 +41,17 @@ """ from pip._vendor import urllib3 -from pip._vendor import chardet import warnings from .exceptions import RequestsDependencyWarning +charset_normalizer_version = None -def check_compatibility(urllib3_version, chardet_version): +try: + from pip._vendor.chardet import __version__ as chardet_version +except ImportError: + chardet_version = None + +def check_compatibility(urllib3_version, chardet_version, charset_normalizer_version): urllib3_version = urllib3_version.split('.') assert urllib3_version != ['dev'] # Verify urllib3 isn't installed from git. @@ -62,14 +67,19 @@ def check_compatibility(urllib3_version, chardet_version): assert minor >= 21 assert minor <= 26 - # Check chardet for compatibility. - major, minor, patch = chardet_version.split('.')[:3] - major, minor, patch = int(major), int(minor), int(patch) - # chardet >= 3.0.2, < 3.1.0 - assert major == 3 - assert minor < 1 - assert patch >= 2 - + # Check charset_normalizer for compatibility. + if chardet_version: + major, minor, patch = chardet_version.split('.')[:3] + major, minor, patch = int(major), int(minor), int(patch) + # chardet_version >= 3.0.2, < 5.0.0 + assert (3, 0, 2) <= (major, minor, patch) < (5, 0, 0) + elif charset_normalizer_version: + major, minor, patch = charset_normalizer_version.split('.')[:3] + major, minor, patch = int(major), int(minor), int(patch) + # charset_normalizer >= 2.0.0 < 3.0.0 + assert (2, 0, 0) <= (major, minor, patch) < (3, 0, 0) + else: + raise Exception("You need either charset_normalizer or chardet installed") def _check_cryptography(cryptography_version): # cryptography < 1.3.4 @@ -84,10 +94,10 @@ def _check_cryptography(cryptography_version): # Check imported dependencies for compatibility. try: - check_compatibility(urllib3.__version__, chardet.__version__) + check_compatibility(urllib3.__version__, chardet_version, charset_normalizer_version) except (AssertionError, ValueError): - warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " - "version!".format(urllib3.__version__, chardet.__version__), + warnings.warn("urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported " + "version!".format(urllib3.__version__, chardet_version, charset_normalizer_version), RequestsDependencyWarning) # Attempt to enable urllib3's fallback for SNI support @@ -131,7 +141,7 @@ def _check_cryptography(cryptography_version): from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, ConnectTimeout, ReadTimeout + FileModeWarning, ConnectTimeout, ReadTimeout, JSONDecodeError ) # Set default logging handler to avoid "No handler found" warnings. diff --git a/src/pip/_vendor/requests/__version__.py b/src/pip/_vendor/requests/__version__.py index 71085207750..e973b03b5ff 100644 --- a/src/pip/_vendor/requests/__version__.py +++ b/src/pip/_vendor/requests/__version__.py @@ -5,10 +5,10 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'https://requests.readthedocs.io' -__version__ = '2.25.0' -__build__ = 0x022500 +__version__ = '2.27.1' +__build__ = 0x022701 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2020 Kenneth Reitz' +__copyright__ = 'Copyright 2022 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/src/pip/_vendor/requests/adapters.py b/src/pip/_vendor/requests/adapters.py index c30e7c92dc7..b3dfa570637 100644 --- a/src/pip/_vendor/requests/adapters.py +++ b/src/pip/_vendor/requests/adapters.py @@ -19,6 +19,7 @@ from pip._vendor.urllib3.exceptions import ClosedPoolError from pip._vendor.urllib3.exceptions import ConnectTimeoutError from pip._vendor.urllib3.exceptions import HTTPError as _HTTPError +from pip._vendor.urllib3.exceptions import InvalidHeader as _InvalidHeader from pip._vendor.urllib3.exceptions import MaxRetryError from pip._vendor.urllib3.exceptions import NewConnectionError from pip._vendor.urllib3.exceptions import ProxyError as _ProxyError @@ -37,7 +38,7 @@ from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, ProxyError, RetryError, InvalidSchema, InvalidProxyURL, - InvalidURL) + InvalidURL, InvalidHeader) from .auth import _basic_auth_str try: @@ -457,9 +458,11 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) try: + skip_host = 'Host' in request.headers low_conn.putrequest(request.method, url, - skip_accept_encoding=True) + skip_accept_encoding=True, + skip_host=skip_host) for header, value in request.headers.items(): low_conn.putheader(header, value) @@ -527,6 +530,8 @@ def send(self, request, stream=False, timeout=None, verify=True, cert=None, prox raise SSLError(e, request=request) elif isinstance(e, ReadTimeoutError): raise ReadTimeout(e, request=request) + elif isinstance(e, _InvalidHeader): + raise InvalidHeader(e, request=request) else: raise diff --git a/src/pip/_vendor/requests/api.py b/src/pip/_vendor/requests/api.py index e978e203118..4cba90eefe8 100644 --- a/src/pip/_vendor/requests/api.py +++ b/src/pip/_vendor/requests/api.py @@ -72,7 +72,6 @@ def get(url, params=None, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return request('get', url, params=params, **kwargs) @@ -85,7 +84,6 @@ def options(url, **kwargs): :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return request('options', url, **kwargs) diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py index 9e29371678b..f98cc910f9b 100644 --- a/src/pip/_vendor/requests/compat.py +++ b/src/pip/_vendor/requests/compat.py @@ -50,13 +50,13 @@ # Keep OrderedDict for backwards compatibility. from collections import Callable, Mapping, MutableMapping, OrderedDict - builtin_str = str bytes = str str = unicode basestring = basestring numeric_types = (int, long, float) integer_types = (int, long) + JSONDecodeError = ValueError elif is_py3: from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag @@ -67,6 +67,7 @@ # Keep OrderedDict for backwards compatibility. from collections import OrderedDict from collections.abc import Callable, Mapping, MutableMapping + from json import JSONDecodeError builtin_str = str str = str diff --git a/src/pip/_vendor/requests/exceptions.py b/src/pip/_vendor/requests/exceptions.py index 9ef9e6e97b8..83b9232e4cb 100644 --- a/src/pip/_vendor/requests/exceptions.py +++ b/src/pip/_vendor/requests/exceptions.py @@ -8,6 +8,8 @@ """ from pip._vendor.urllib3.exceptions import HTTPError as BaseHTTPError +from .compat import JSONDecodeError as CompatJSONDecodeError + class RequestException(IOError): """There was an ambiguous exception that occurred while handling your @@ -25,6 +27,14 @@ def __init__(self, *args, **kwargs): super(RequestException, self).__init__(*args, **kwargs) +class InvalidJSONError(RequestException): + """A JSON error occurred.""" + + +class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError): + """Couldn't decode the text into json""" + + class HTTPError(RequestException): """An HTTP error occurred.""" @@ -70,11 +80,11 @@ class TooManyRedirects(RequestException): class MissingSchema(RequestException, ValueError): - """The URL schema (e.g. http or https) is missing.""" + """The URL scheme (e.g. http or https) is missing.""" class InvalidSchema(RequestException, ValueError): - """See defaults.py for valid schemas.""" + """The URL scheme provided is either invalid or unsupported.""" class InvalidURL(RequestException, ValueError): diff --git a/src/pip/_vendor/requests/help.py b/src/pip/_vendor/requests/help.py index 3c3072ba141..745f0d7b346 100644 --- a/src/pip/_vendor/requests/help.py +++ b/src/pip/_vendor/requests/help.py @@ -8,10 +8,16 @@ from pip._vendor import idna from pip._vendor import urllib3 -from pip._vendor import chardet from . import __version__ as requests_version +charset_normalizer = None + +try: + from pip._vendor import chardet +except ImportError: + chardet = None + try: from pip._vendor.urllib3.contrib import pyopenssl except ImportError: @@ -71,7 +77,12 @@ def info(): implementation_info = _implementation() urllib3_info = {'version': urllib3.__version__} - chardet_info = {'version': chardet.__version__} + charset_normalizer_info = {'version': None} + chardet_info = {'version': None} + if charset_normalizer: + charset_normalizer_info = {'version': charset_normalizer.__version__} + if chardet: + chardet_info = {'version': chardet.__version__} pyopenssl_info = { 'version': None, @@ -99,9 +110,11 @@ def info(): 'implementation': implementation_info, 'system_ssl': system_ssl_info, 'using_pyopenssl': pyopenssl is not None, + 'using_charset_normalizer': chardet is None, 'pyOpenSSL': pyopenssl_info, 'urllib3': urllib3_info, 'chardet': chardet_info, + 'charset_normalizer': charset_normalizer_info, 'cryptography': cryptography_info, 'idna': idna_info, 'requests': { diff --git a/src/pip/_vendor/requests/models.py b/src/pip/_vendor/requests/models.py index b0ce2950f25..f538c1054d5 100644 --- a/src/pip/_vendor/requests/models.py +++ b/src/pip/_vendor/requests/models.py @@ -29,7 +29,9 @@ from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .exceptions import ( HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError) + ContentDecodingError, ConnectionError, StreamConsumedError, + InvalidJSONError) +from .exceptions import JSONDecodeError as RequestsJSONDecodeError from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( guess_filename, get_auth_from_url, requote_uri, @@ -38,7 +40,7 @@ from .compat import ( Callable, Mapping, cookielib, urlunparse, urlsplit, urlencode, str, bytes, - is_py2, chardet, builtin_str, basestring) + is_py2, chardet, builtin_str, basestring, JSONDecodeError) from .compat import json as complexjson from .status_codes import codes @@ -384,7 +386,7 @@ def prepare_url(self, url, params): raise InvalidURL(*e.args) if not scheme: - error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") + error = ("Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?") error = error.format(to_native_string(url, 'utf8')) raise MissingSchema(error) @@ -401,7 +403,7 @@ def prepare_url(self, url, params): host = self._get_idna_encoded_host(host) except UnicodeError: raise InvalidURL('URL has an invalid label.') - elif host.startswith(u'*'): + elif host.startswith((u'*', u'.')): raise InvalidURL('URL has an invalid label.') # Carefully reconstruct the network location @@ -466,7 +468,12 @@ def prepare_body(self, data, files, json=None): # urllib3 requires a bytes-like body. Python 2's json.dumps # provides this natively, but Python 3 gives a Unicode string. content_type = 'application/json' - body = complexjson.dumps(json) + + try: + body = complexjson.dumps(json, allow_nan=False) + except ValueError as ve: + raise InvalidJSONError(ve, request=self) + if not isinstance(body, bytes): body = body.encode('utf-8') @@ -726,7 +733,7 @@ def next(self): @property def apparent_encoding(self): - """The apparent encoding, provided by the chardet library.""" + """The apparent encoding, provided by the charset_normalizer or chardet libraries.""" return chardet.detect(self.content)['encoding'] def iter_content(self, chunk_size=1, decode_unicode=False): @@ -840,7 +847,7 @@ def text(self): """Content of the response, in unicode. If Response.encoding is None, encoding will be guessed using - ``chardet``. + ``charset_normalizer`` or ``chardet``. The encoding of the response content is determined based solely on HTTP headers, following RFC 2616 to the letter. If you can take advantage of @@ -877,13 +884,14 @@ def json(self, **kwargs): r"""Returns the json-encoded content of a response, if any. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. - :raises ValueError: If the response body does not contain valid json. + :raises requests.exceptions.JSONDecodeError: If the response body does not + contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or - # decoding fails, fall back to `self.text` (using chardet to make + # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: @@ -897,7 +905,16 @@ def json(self, **kwargs): # and the server didn't bother to tell us what codec *was* # used. pass - return complexjson.loads(self.text, **kwargs) + + try: + return complexjson.loads(self.text, **kwargs) + except JSONDecodeError as e: + # Catch JSON-related errors and raise as requests.JSONDecodeError + # This aliases json.JSONDecodeError and simplejson.JSONDecodeError + if is_py2: # e is a ValueError + raise RequestsJSONDecodeError(e.message) + else: + raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) @property def links(self): diff --git a/src/pip/_vendor/requests/sessions.py b/src/pip/_vendor/requests/sessions.py index fdf7e9fe35d..3f59cab9225 100644 --- a/src/pip/_vendor/requests/sessions.py +++ b/src/pip/_vendor/requests/sessions.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -requests.session -~~~~~~~~~~~~~~~~ +requests.sessions +~~~~~~~~~~~~~~~~~ This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). @@ -29,7 +29,7 @@ from .utils import ( requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, - get_auth_from_url, rewind_body + get_auth_from_url, rewind_body, resolve_proxies ) from .status_codes import codes @@ -269,7 +269,6 @@ def rebuild_auth(self, prepared_request, response): if new_auth is not None: prepared_request.prepare_auth(new_auth) - def rebuild_proxies(self, prepared_request, proxies): """This method re-evaluates the proxy configuration by considering the environment variables. If we are redirected to a URL covered by @@ -282,21 +281,9 @@ def rebuild_proxies(self, prepared_request, proxies): :rtype: dict """ - proxies = proxies if proxies is not None else {} headers = prepared_request.headers - url = prepared_request.url - scheme = urlparse(url).scheme - new_proxies = proxies.copy() - no_proxy = proxies.get('no_proxy') - - bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) - if self.trust_env and not bypass_proxy: - environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) - - proxy = environ_proxies.get(scheme, environ_proxies.get('all')) - - if proxy: - new_proxies.setdefault(scheme, proxy) + scheme = urlparse(prepared_request.url).scheme + new_proxies = resolve_proxies(prepared_request, proxies, self.trust_env) if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] @@ -633,7 +620,10 @@ def send(self, request, **kwargs): kwargs.setdefault('stream', self.stream) kwargs.setdefault('verify', self.verify) kwargs.setdefault('cert', self.cert) - kwargs.setdefault('proxies', self.proxies) + if 'proxies' not in kwargs: + kwargs['proxies'] = resolve_proxies( + request, self.proxies, self.trust_env + ) # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. diff --git a/src/pip/_vendor/requests/utils.py b/src/pip/_vendor/requests/utils.py index 16d5776201d..1e5857ad8af 100644 --- a/src/pip/_vendor/requests/utils.py +++ b/src/pip/_vendor/requests/utils.py @@ -20,6 +20,8 @@ import warnings import zipfile from collections import OrderedDict +from pip._vendor.urllib3.util import make_headers +from pip._vendor.urllib3.util import parse_url from .__version__ import __version__ from . import certs @@ -41,6 +43,11 @@ DEFAULT_PORTS = {'http': 80, 'https': 443} +# Ensure that ', ' is used to preserve previous delimiter behavior. +DEFAULT_ACCEPT_ENCODING = ", ".join( + re.split(r",\s*", make_headers(accept_encoding=True)["accept-encoding"]) +) + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -118,7 +125,10 @@ def super_len(o): elif hasattr(o, 'fileno'): try: fileno = o.fileno() - except io.UnsupportedOperation: + except (io.UnsupportedOperation, AttributeError): + # AttributeError is a surprising exception, seeing as how we've just checked + # that `hasattr(o, 'fileno')`. It happens for objects obtained via + # `Tarfile.extractfile()`, per issue 5229. pass else: total_length = os.fstat(fileno).st_size @@ -148,7 +158,7 @@ def super_len(o): current_position = total_length else: if hasattr(o, 'seek') and total_length is None: - # StringIO and BytesIO have seek but no useable fileno + # StringIO and BytesIO have seek but no usable fileno try: # seek to end of file o.seek(0, 2) @@ -245,6 +255,10 @@ def extract_zipped_paths(path): archive, member = os.path.split(path) while archive and not os.path.exists(archive): archive, prefix = os.path.split(archive) + if not prefix: + # If we don't check for an empty prefix after the split (in other words, archive remains unchanged after the split), + # we _can_ end up in an infinite loop on a rare corner case affecting a small number of users + break member = '/'.join([prefix, member]) if not zipfile.is_zipfile(archive): @@ -256,13 +270,28 @@ def extract_zipped_paths(path): # we have a valid zip archive and a valid member of that archive tmp = tempfile.gettempdir() - extracted_path = os.path.join(tmp, *member.split('/')) + extracted_path = os.path.join(tmp, member.split('/')[-1]) if not os.path.exists(extracted_path): - extracted_path = zip_file.extract(member, path=tmp) - + # use read + write to avoid the creating nested folders, we only want the file, avoids mkdir racing condition + with atomic_open(extracted_path) as file_handler: + file_handler.write(zip_file.read(member)) return extracted_path +@contextlib.contextmanager +def atomic_open(filename): + """Write a file to the disk in an atomic fashion""" + replacer = os.rename if sys.version_info[0] == 2 else os.replace + tmp_descriptor, tmp_name = tempfile.mkstemp(dir=os.path.dirname(filename)) + try: + with os.fdopen(tmp_descriptor, 'wb') as tmp_handler: + yield tmp_handler + replacer(tmp_name, filename) + except BaseException: + os.remove(tmp_name) + raise + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an @@ -503,6 +532,10 @@ def get_encoding_from_headers(headers): if 'text' in content_type: return 'ISO-8859-1' + if 'application/json' in content_type: + # Assume UTF-8 based on RFC 4627: https://www.ietf.org/rfc/rfc4627.txt since the charset was unset + return 'utf-8' + def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" @@ -801,6 +834,33 @@ def select_proxy(url, proxies): return proxy +def resolve_proxies(request, proxies, trust_env=True): + """This method takes proxy information from a request and configuration + input to resolve a mapping of target proxies. This will consider settings + such a NO_PROXY to strip proxy configurations. + + :param request: Request or PreparedRequest + :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs + :param trust_env: Boolean declaring whether to trust environment configs + + :rtype: dict + """ + proxies = proxies if proxies is not None else {} + url = request.url + scheme = urlparse(url).scheme + no_proxy = proxies.get('no_proxy') + new_proxies = proxies.copy() + + if trust_env and not should_bypass_proxies(url, no_proxy=no_proxy): + environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) + + proxy = environ_proxies.get(scheme, environ_proxies.get('all')) + + if proxy: + new_proxies.setdefault(scheme, proxy) + return new_proxies + + def default_user_agent(name="python-requests"): """ Return a string representing the default user agent. @@ -816,7 +876,7 @@ def default_headers(): """ return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate')), + 'Accept-Encoding': DEFAULT_ACCEPT_ENCODING, 'Accept': '*/*', 'Connection': 'keep-alive', }) @@ -903,15 +963,27 @@ def prepend_scheme_if_needed(url, new_scheme): :rtype: str """ - scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) - - # urlparse is a finicky beast, and sometimes decides that there isn't a - # netloc present. Assume that it's being over-cautious, and switch netloc - # and path if urlparse decided there was no netloc. + parsed = parse_url(url) + scheme, auth, host, port, path, query, fragment = parsed + + # A defect in urlparse determines that there isn't a netloc present in some + # urls. We previously assumed parsing was overly cautious, and swapped the + # netloc and path. Due to a lack of tests on the original defect, this is + # maintained with parse_url for backwards compatibility. + netloc = parsed.netloc if not netloc: netloc, path = path, netloc - return urlunparse((scheme, netloc, path, params, query, fragment)) + if auth: + # parse_url doesn't provide the netloc with auth + # so we'll add it ourselves. + netloc = '@'.join([auth, netloc]) + if scheme is None: + scheme = new_scheme + if path is None: + path = '' + + return urlunparse((scheme, netloc, path, '', query, fragment)) def get_auth_from_url(url): diff --git a/src/pip/_vendor/resolvelib.pyi b/src/pip/_vendor/resolvelib.pyi deleted file mode 100644 index b4ef4e108c4..00000000000 --- a/src/pip/_vendor/resolvelib.pyi +++ /dev/null @@ -1 +0,0 @@ -from resolvelib import * \ No newline at end of file diff --git a/src/pip/_vendor/resolvelib/__init__.py b/src/pip/_vendor/resolvelib/__init__.py index f023ad63154..ce05fd30274 100644 --- a/src/pip/_vendor/resolvelib/__init__.py +++ b/src/pip/_vendor/resolvelib/__init__.py @@ -11,7 +11,7 @@ "ResolutionTooDeep", ] -__version__ = "0.5.4" +__version__ = "0.8.1" from .providers import AbstractProvider, AbstractResolver @@ -19,8 +19,8 @@ from .resolvers import ( InconsistentCandidate, RequirementsConflicted, - Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, + Resolver, ) diff --git a/src/pip/_vendor/resolvelib/__init__.pyi b/src/pip/_vendor/resolvelib/__init__.pyi new file mode 100644 index 00000000000..d64c52ced00 --- /dev/null +++ b/src/pip/_vendor/resolvelib/__init__.pyi @@ -0,0 +1,11 @@ +__version__: str + +from .providers import AbstractProvider as AbstractProvider +from .providers import AbstractResolver as AbstractResolver +from .reporters import BaseReporter as BaseReporter +from .resolvers import InconsistentCandidate as InconsistentCandidate +from .resolvers import RequirementsConflicted as RequirementsConflicted +from .resolvers import ResolutionError as ResolutionError +from .resolvers import ResolutionImpossible as ResolutionImpossible +from .resolvers import ResolutionTooDeep as ResolutionTooDeep +from .resolvers import Resolver as Resolver diff --git a/src/pip/_vendor/resolvelib/compat/collections_abc.py b/src/pip/_vendor/resolvelib/compat/collections_abc.py index 366cc5e2e12..1becc5093c5 100644 --- a/src/pip/_vendor/resolvelib/compat/collections_abc.py +++ b/src/pip/_vendor/resolvelib/compat/collections_abc.py @@ -1,6 +1,6 @@ -__all__ = ["Sequence"] +__all__ = ["Mapping", "Sequence"] try: - from collections.abc import Sequence + from collections.abc import Mapping, Sequence except ImportError: - from collections import Sequence + from collections import Mapping, Sequence diff --git a/src/pip/_vendor/resolvelib/providers.py b/src/pip/_vendor/resolvelib/providers.py index 965cf9c138f..7d0a9c22a46 100644 --- a/src/pip/_vendor/resolvelib/providers.py +++ b/src/pip/_vendor/resolvelib/providers.py @@ -2,37 +2,45 @@ class AbstractProvider(object): """Delegate class to provide requirement interface for the resolver.""" def identify(self, requirement_or_candidate): - """Given a requirement or candidate, return an identifier for it. + """Given a requirement, return an identifier for it. - This is used in many places to identify a requirement or candidate, - e.g. whether two requirements should have their specifier parts merged, - whether two candidates would conflict with each other (because they - have same name but different versions). + This is used to identify a requirement, e.g. whether two requirements + should have their specifier parts merged. """ raise NotImplementedError - def get_preference(self, resolution, candidates, information): + def get_preference( + self, + identifier, + resolutions, + candidates, + information, + backtrack_causes, + ): """Produce a sort key for given requirement based on preference. The preference is defined as "I think this requirement should be resolved first". The lower the return value is, the more preferred this group of arguments is. - :param resolution: Currently pinned candidate, or `None`. - :param candidates: An iterable of possible candidates. - :param information: A list of requirement information. - - The `candidates` iterable's exact type depends on the return type of - `find_matches()`. A sequence is passed-in as-is if possible. If it - returns a callble, the iterator returned by that callable is passed - in here. - - Each element in `information` is a named tuple with two entries: - - * `requirement` specifies a requirement contributing to the current - candidate list. - * `parent` specifies the candidate that provides (dependend on) the - requirement, or `None` to indicate a root requirement. + :param identifier: An identifier as returned by ``identify()``. This + identifies the dependency matches of which should be returned. + :param resolutions: Mapping of candidates currently pinned by the + resolver. Each key is an identifier, and the value a candidate. + The candidate may conflict with requirements from ``information``. + :param candidates: Mapping of each dependency's possible candidates. + Each value is an iterator of candidates. + :param information: Mapping of requirement information of each package. + Each value is an iterator of *requirement information*. + :param backtrack_causes: Sequence of requirement information that were + the requirements that caused the resolver to most recently backtrack. + + A *requirement information* instance is a named tuple with two members: + + * ``requirement`` specifies a requirement contributing to the current + list of candidates. + * ``parent`` specifies the candidate that provides (dependend on) the + requirement, or ``None`` to indicate a root requirement. The preference could depend on a various of issues, including (not necessarily in this order): @@ -45,15 +53,25 @@ def get_preference(self, resolution, candidates, information): * Are there any known conflicts for this requirement? We should probably work on those with the most known conflicts. - A sortable value should be returned (this will be used as the `key` + A sortable value should be returned (this will be used as the ``key`` parameter of the built-in sorting function). The smaller the value is, the more preferred this requirement is (i.e. the sorting function - is called with `reverse=False`). + is called with ``reverse=False``). """ raise NotImplementedError - def find_matches(self, requirements): - """Find all possible candidates that satisfy the given requirements. + def find_matches(self, identifier, requirements, incompatibilities): + """Find all possible candidates that satisfy given constraints. + + :param identifier: An identifier as returned by ``identify()``. This + identifies the dependency matches of which should be returned. + :param requirements: A mapping of requirements that all returned + candidates must satisfy. Each key is an identifier, and the value + an iterator of requirements for that dependency. + :param incompatibilities: A mapping of known incompatibilities of + each dependency. Each key is an identifier, and the value an + iterator of incompatibilities known to the resolver. All + incompatibilities *must* be excluded from the return value. This should try to get candidates based on the requirements' types. For VCS, local, and archive requirements, the one-and-only match is @@ -68,10 +86,6 @@ def find_matches(self, requirements): * An collection of candidates. * An iterable of candidates. This will be consumed immediately into a list of candidates. - - :param requirements: A collection of requirements which all of the - returned candidates must match. All requirements are guaranteed to - have the same identifier. The collection is never empty. """ raise NotImplementedError @@ -81,7 +95,7 @@ def is_satisfied_by(self, requirement, candidate): The candidate is guarenteed to have been generated from the requirement. - A boolean should be returned to indicate whether `candidate` is a + A boolean should be returned to indicate whether ``candidate`` is a viable solution to the requirement. """ raise NotImplementedError diff --git a/src/pip/_vendor/resolvelib/providers.pyi b/src/pip/_vendor/resolvelib/providers.pyi new file mode 100644 index 00000000000..47d6f8abad7 --- /dev/null +++ b/src/pip/_vendor/resolvelib/providers.pyi @@ -0,0 +1,44 @@ +from typing import ( + Any, + Collection, + Generic, + Iterable, + Iterator, + Mapping, + Optional, + Protocol, + Union, +) + +from .reporters import BaseReporter +from .resolvers import RequirementInformation +from .structs import CT, KT, RT, Matches + +class Preference(Protocol): + def __lt__(self, __other: Any) -> bool: ... + +class AbstractProvider(Generic[RT, CT, KT]): + def identify(self, requirement_or_candidate: Union[RT, CT]) -> KT: ... + def get_preference( + self, + identifier: KT, + resolutions: Mapping[KT, CT], + candidates: Mapping[KT, Iterator[CT]], + information: Mapping[KT, Iterator[RequirementInformation[RT, CT]]], + ) -> Preference: ... + def find_matches( + self, + identifier: KT, + requirements: Mapping[KT, Iterator[RT]], + incompatibilities: Mapping[KT, Iterator[CT]], + ) -> Matches: ... + def is_satisfied_by(self, requirement: RT, candidate: CT) -> bool: ... + def get_dependencies(self, candidate: CT) -> Iterable[RT]: ... + +class AbstractResolver(Generic[RT, CT, KT]): + base_exception = Exception + provider: AbstractProvider[RT, CT, KT] + reporter: BaseReporter + def __init__( + self, provider: AbstractProvider[RT, CT, KT], reporter: BaseReporter + ): ... diff --git a/news/24193261-eaf9-4117-a1a9-d5bb7f93b447.trivial.rst b/src/pip/_vendor/resolvelib/py.typed similarity index 100% rename from news/24193261-eaf9-4117-a1a9-d5bb7f93b447.trivial.rst rename to src/pip/_vendor/resolvelib/py.typed diff --git a/src/pip/_vendor/resolvelib/reporters.py b/src/pip/_vendor/resolvelib/reporters.py index 563489e133b..6695480fff4 100644 --- a/src/pip/_vendor/resolvelib/reporters.py +++ b/src/pip/_vendor/resolvelib/reporters.py @@ -30,6 +30,12 @@ def adding_requirement(self, requirement, parent): requirements passed in from ``Resolver.resolve()``. """ + def resolving_conflicts(self, causes): + """Called when starting to attempt requirement conflict resolution. + + :param causes: The information on the collision that caused the backtracking. + """ + def backtracking(self, candidate): """Called when rejecting a candidate during backtracking.""" diff --git a/src/pip/_vendor/resolvelib/reporters.pyi b/src/pip/_vendor/resolvelib/reporters.pyi new file mode 100644 index 00000000000..03d4f09a390 --- /dev/null +++ b/src/pip/_vendor/resolvelib/reporters.pyi @@ -0,0 +1,11 @@ +from typing import Any + +class BaseReporter: + def starting(self) -> Any: ... + def starting_round(self, index: int) -> Any: ... + def ending_round(self, index: int, state: Any) -> Any: ... + def ending(self, state: Any) -> Any: ... + def adding_requirement(self, requirement: Any, parent: Any) -> Any: ... + def backtracking(self, candidate: Any) -> Any: ... + def resolving_conflicts(self, causes: Any) -> Any: ... + def pinning(self, candidate: Any) -> Any: ... diff --git a/src/pip/_vendor/resolvelib/resolvers.py b/src/pip/_vendor/resolvelib/resolvers.py index bb88d8c2c75..787681b03e9 100644 --- a/src/pip/_vendor/resolvelib/resolvers.py +++ b/src/pip/_vendor/resolvelib/resolvers.py @@ -1,8 +1,8 @@ import collections +import operator from .providers import AbstractResolver -from .structs import DirectedGraph, build_iter_view - +from .structs import DirectedGraph, IteratorMapping, build_iter_view RequirementInformation = collections.namedtuple( "RequirementInformation", ["requirement", "parent"] @@ -73,43 +73,12 @@ def __repr__(self): ) return "Criterion({})".format(requirements) - @classmethod - def from_requirement(cls, provider, requirement, parent): - """Build an instance from a requirement.""" - cands = build_iter_view(provider.find_matches([requirement])) - infos = [RequirementInformation(requirement, parent)] - criterion = cls(cands, infos, incompatibilities=[]) - if not cands: - raise RequirementsConflicted(criterion) - return criterion - def iter_requirement(self): return (i.requirement for i in self.information) def iter_parent(self): return (i.parent for i in self.information) - def merged_with(self, provider, requirement, parent): - """Build a new instance from this and a new requirement.""" - infos = list(self.information) - infos.append(RequirementInformation(requirement, parent)) - cands = build_iter_view(provider.find_matches([r for r, _ in infos])) - criterion = type(self)(cands, infos, list(self.incompatibilities)) - if not cands: - raise RequirementsConflicted(criterion) - return criterion - - def excluded_of(self, candidates): - """Build a new instance from this, but excluding specified candidates. - - Returns the new instance, or None if we still have no valid candidates. - """ - cands = self.candidates.excluding(candidates) - if not cands: - return None - incompats = self.incompatibilities + candidates - return type(self)(cands, list(self.information), incompats) - class ResolutionError(ResolverException): pass @@ -129,7 +98,7 @@ def __init__(self, round_count): # Resolution state in a round. -State = collections.namedtuple("State", "mapping criteria") +State = collections.namedtuple("State", "mapping criteria backtrack_causes") class Resolution(object): @@ -161,26 +130,62 @@ def _push_new_state(self): state = State( mapping=base.mapping.copy(), criteria=base.criteria.copy(), + backtrack_causes=base.backtrack_causes[:], ) self._states.append(state) - def _merge_into_criterion(self, requirement, parent): - self._r.adding_requirement(requirement, parent) - name = self._p.identify(requirement) - try: - crit = self.state.criteria[name] - except KeyError: - crit = Criterion.from_requirement(self._p, requirement, parent) + def _add_to_criteria(self, criteria, requirement, parent): + self._r.adding_requirement(requirement=requirement, parent=parent) + + identifier = self._p.identify(requirement_or_candidate=requirement) + criterion = criteria.get(identifier) + if criterion: + incompatibilities = list(criterion.incompatibilities) else: - crit = crit.merged_with(self._p, requirement, parent) - return name, crit + incompatibilities = [] + + matches = self._p.find_matches( + identifier=identifier, + requirements=IteratorMapping( + criteria, + operator.methodcaller("iter_requirement"), + {identifier: [requirement]}, + ), + incompatibilities=IteratorMapping( + criteria, + operator.attrgetter("incompatibilities"), + {identifier: incompatibilities}, + ), + ) - def _get_criterion_item_preference(self, item): - name, criterion = item + if criterion: + information = list(criterion.information) + information.append(RequirementInformation(requirement, parent)) + else: + information = [RequirementInformation(requirement, parent)] + + criterion = Criterion( + candidates=build_iter_view(matches), + information=information, + incompatibilities=incompatibilities, + ) + if not criterion.candidates: + raise RequirementsConflicted(criterion) + criteria[identifier] = criterion + + def _get_preference(self, name): return self._p.get_preference( - self.state.mapping.get(name), - criterion.candidates.for_preference(), - criterion.information, + identifier=name, + resolutions=self.state.mapping, + candidates=IteratorMapping( + self.state.criteria, + operator.attrgetter("candidates"), + ), + information=IteratorMapping( + self.state.criteria, + operator.attrgetter("information"), + ), + backtrack_causes=self.state.backtrack_causes, ) def _is_current_pin_satisfying(self, name, criterion): @@ -189,22 +194,23 @@ def _is_current_pin_satisfying(self, name, criterion): except KeyError: return False return all( - self._p.is_satisfied_by(r, current_pin) + self._p.is_satisfied_by(requirement=r, candidate=current_pin) for r in criterion.iter_requirement() ) - def _get_criteria_to_update(self, candidate): - criteria = {} - for r in self._p.get_dependencies(candidate): - name, crit = self._merge_into_criterion(r, parent=candidate) - criteria[name] = crit + def _get_updated_criteria(self, candidate): + criteria = self.state.criteria.copy() + for requirement in self._p.get_dependencies(candidate=candidate): + self._add_to_criteria(criteria, requirement, parent=candidate) return criteria - def _attempt_to_pin_criterion(self, name, criterion): + def _attempt_to_pin_criterion(self, name): + criterion = self.state.criteria[name] + causes = [] for candidate in criterion.candidates: try: - criteria = self._get_criteria_to_update(candidate) + criteria = self._get_updated_criteria(candidate) except RequirementsConflicted as e: causes.append(e.criterion) continue @@ -214,18 +220,19 @@ def _attempt_to_pin_criterion(self, name, criterion): # faulty provider, we will raise an error to notify the implementer # to fix find_matches() and/or is_satisfied_by(). satisfied = all( - self._p.is_satisfied_by(r, candidate) + self._p.is_satisfied_by(requirement=r, candidate=candidate) for r in criterion.iter_requirement() ) if not satisfied: raise InconsistentCandidate(candidate, criterion) + self._r.pinning(candidate=candidate) + self.state.criteria.update(criteria) + # Put newly-pinned candidate at the end. This is essential because # backtracking looks at this mapping to get the last pin. - self._r.pinning(candidate) self.state.mapping.pop(name, None) self.state.mapping[name] = candidate - self.state.criteria.update(criteria) return [] @@ -267,14 +274,14 @@ def _backtrack(self): broken_state = self._states.pop() name, candidate = broken_state.mapping.popitem() incompatibilities_from_broken = [ - (k, v.incompatibilities) + (k, list(v.incompatibilities)) for k, v in broken_state.criteria.items() ] # Also mark the newly known incompatibility. incompatibilities_from_broken.append((name, [candidate])) - self._r.backtracking(candidate) + self._r.backtracking(candidate=candidate) # Create a new state from the last known-to-work one, and apply # the previously gathered incompatibility information. @@ -286,10 +293,27 @@ def _patch_criteria(): criterion = self.state.criteria[k] except KeyError: continue - criterion = criterion.excluded_of(incompatibilities) - if criterion is None: + matches = self._p.find_matches( + identifier=k, + requirements=IteratorMapping( + self.state.criteria, + operator.methodcaller("iter_requirement"), + ), + incompatibilities=IteratorMapping( + self.state.criteria, + operator.attrgetter("incompatibilities"), + {k: incompatibilities}, + ), + ) + candidates = build_iter_view(matches) + if not candidates: return False - self.state.criteria[k] = criterion + incompatibilities.extend(criterion.incompatibilities) + self.state.criteria[k] = Criterion( + candidates=candidates, + information=list(criterion.information), + incompatibilities=incompatibilities, + ) return True self._push_new_state() @@ -312,13 +336,18 @@ def resolve(self, requirements, max_rounds): self._r.starting() # Initialize the root state. - self._states = [State(mapping=collections.OrderedDict(), criteria={})] + self._states = [ + State( + mapping=collections.OrderedDict(), + criteria={}, + backtrack_causes=[], + ) + ] for r in requirements: try: - name, crit = self._merge_into_criterion(r, parent=None) + self._add_to_criteria(self.state.criteria, r, parent=None) except RequirementsConflicted as e: raise ResolutionImpossible(e.criterion.information) - self.state.criteria[name] = crit # The root state is saved as a sentinel so the first ever pin can have # something to backtrack to if it fails. The root state is basically @@ -326,40 +355,39 @@ def resolve(self, requirements, max_rounds): self._push_new_state() for round_index in range(max_rounds): - self._r.starting_round(round_index) + self._r.starting_round(index=round_index) - unsatisfied_criterion_items = [ - item - for item in self.state.criteria.items() - if not self._is_current_pin_satisfying(*item) + unsatisfied_names = [ + key + for key, criterion in self.state.criteria.items() + if not self._is_current_pin_satisfying(key, criterion) ] # All criteria are accounted for. Nothing more to pin, we are done! - if not unsatisfied_criterion_items: - self._r.ending(self.state) + if not unsatisfied_names: + self._r.ending(state=self.state) return self.state # Choose the most preferred unpinned criterion to try. - name, criterion = min( - unsatisfied_criterion_items, - key=self._get_criterion_item_preference, - ) - failure_causes = self._attempt_to_pin_criterion(name, criterion) + name = min(unsatisfied_names, key=self._get_preference) + failure_causes = self._attempt_to_pin_criterion(name) if failure_causes: + causes = [i for c in failure_causes for i in c.information] # Backtrack if pinning fails. The backtrack process puts us in # an unpinned state, so we can work on it in the next round. + self._r.resolving_conflicts(causes=causes) success = self._backtrack() + self.state.backtrack_causes[:] = causes # Dead ends everywhere. Give up. if not success: - causes = [i for c in failure_causes for i in c.information] - raise ResolutionImpossible(causes) + raise ResolutionImpossible(self.state.backtrack_causes) else: # Pinning was successful. Push a new state to do another pin. self._push_new_state() - self._r.ending_round(round_index, self.state) + self._r.ending_round(index=round_index, state=self.state) raise ResolutionTooDeep(max_rounds) diff --git a/src/pip/_vendor/resolvelib/resolvers.pyi b/src/pip/_vendor/resolvelib/resolvers.pyi new file mode 100644 index 00000000000..0eb5b2162c1 --- /dev/null +++ b/src/pip/_vendor/resolvelib/resolvers.pyi @@ -0,0 +1,67 @@ +from typing import ( + Collection, + Generic, + Iterable, + Iterator, + List, + Mapping, + Optional, +) + +from .providers import AbstractProvider, AbstractResolver +from .structs import CT, KT, RT, DirectedGraph, IterableView + +# This should be a NamedTuple, but Python 3.6 has a bug that prevents it. +# https://stackoverflow.com/a/50531189/1376863 +class RequirementInformation(tuple, Generic[RT, CT]): + requirement: RT + parent: Optional[CT] + +class Criterion(Generic[RT, CT, KT]): + candidates: IterableView[CT] + information: Collection[RequirementInformation[RT, CT]] + incompatibilities: List[CT] + @classmethod + def from_requirement( + cls, + provider: AbstractProvider[RT, CT, KT], + requirement: RT, + parent: Optional[CT], + ) -> Criterion[RT, CT, KT]: ... + def iter_requirement(self) -> Iterator[RT]: ... + def iter_parent(self) -> Iterator[Optional[CT]]: ... + def merged_with( + self, + provider: AbstractProvider[RT, CT, KT], + requirement: RT, + parent: Optional[CT], + ) -> Criterion[RT, CT, KT]: ... + def excluded_of(self, candidates: List[CT]) -> Criterion[RT, CT, KT]: ... + +class ResolverException(Exception): ... + +class RequirementsConflicted(ResolverException, Generic[RT, CT, KT]): + criterion: Criterion[RT, CT, KT] + +class ResolutionError(ResolverException): ... + +class InconsistentCandidate(ResolverException, Generic[RT, CT, KT]): + candidate: CT + criterion: Criterion[RT, CT, KT] + +class ResolutionImpossible(ResolutionError, Generic[RT, CT]): + causes: List[RequirementInformation[RT, CT]] + +class ResolutionTooDeep(ResolutionError): + round_count: int + +class Result(Generic[RT, CT, KT]): + mapping: Mapping[KT, CT] + graph: DirectedGraph[Optional[KT]] + criteria: Mapping[KT, Criterion[RT, CT, KT]] + +class Resolver(AbstractResolver, Generic[RT, CT, KT]): + base_exception = ResolverException + def resolve( + self, requirements: Iterable[RT], max_rounds: int = 100 + ) -> Result[RT, CT, KT]: ... diff --git a/src/pip/_vendor/resolvelib/structs.py b/src/pip/_vendor/resolvelib/structs.py index c4542f08a06..93d1568bd4d 100644 --- a/src/pip/_vendor/resolvelib/structs.py +++ b/src/pip/_vendor/resolvelib/structs.py @@ -1,3 +1,5 @@ +import itertools + from .compat import collections_abc @@ -67,6 +69,43 @@ def iter_parents(self, key): return iter(self._backwards[key]) +class IteratorMapping(collections_abc.Mapping): + def __init__(self, mapping, accessor, appends=None): + self._mapping = mapping + self._accessor = accessor + self._appends = appends or {} + + def __repr__(self): + return "IteratorMapping({!r}, {!r}, {!r})".format( + self._mapping, + self._accessor, + self._appends, + ) + + def __bool__(self): + return bool(self._mapping or self._appends) + + __nonzero__ = __bool__ # XXX: Python 2. + + def __contains__(self, key): + return key in self._mapping or key in self._appends + + def __getitem__(self, k): + try: + v = self._mapping[k] + except KeyError: + return iter(self._appends[k]) + return itertools.chain(self._accessor(v), self._appends.get(k, ())) + + def __iter__(self): + more = (k for k in self._appends if k not in self._mapping) + return itertools.chain(self._mapping, more) + + def __len__(self): + more = sum(1 for k in self._appends if k not in self._mapping) + return len(self._mapping) + more + + class _FactoryIterableView(object): """Wrap an iterator factory returned by `find_matches()`. @@ -94,18 +133,6 @@ def __bool__(self): def __iter__(self): return self._factory() - def for_preference(self): - """Provide an candidate iterable for `get_preference()`""" - return self._factory() - - def excluding(self, candidates): - """Create a new instance excluding specified candidates.""" - - def factory(): - return (c for c in self._factory() if c not in candidates) - - return type(self)(factory) - class _SequenceIterableView(object): """Wrap an iterable returned by find_matches(). @@ -128,17 +155,6 @@ def __bool__(self): def __iter__(self): return iter(self._sequence) - def __len__(self): - return len(self._sequence) - - def for_preference(self): - """Provide an candidate iterable for `get_preference()`""" - return self._sequence - - def excluding(self, candidates): - """Create a new instance excluding specified candidates.""" - return type(self)([c for c in self._sequence if c not in candidates]) - def build_iter_view(matches): """Build an iterable view from the value returned by `find_matches()`.""" diff --git a/src/pip/_vendor/resolvelib/structs.pyi b/src/pip/_vendor/resolvelib/structs.pyi new file mode 100644 index 00000000000..fae2a2fcefc --- /dev/null +++ b/src/pip/_vendor/resolvelib/structs.pyi @@ -0,0 +1,40 @@ +from abc import ABCMeta +from typing import ( + Callable, + Container, + Generic, + Iterable, + Iterator, + Mapping, + Tuple, + TypeVar, + Union, +) + +KT = TypeVar("KT") # Identifier. +RT = TypeVar("RT") # Requirement. +CT = TypeVar("CT") # Candidate. +_T = TypeVar("_T") + +Matches = Union[Iterable[CT], Callable[[], Iterator[CT]]] + +class IteratorMapping(Mapping[KT, _T], metaclass=ABCMeta): + pass + +class IterableView(Container[CT], Iterable[CT], metaclass=ABCMeta): + pass + +class DirectedGraph(Generic[KT]): + def __iter__(self) -> Iterator[KT]: ... + def __len__(self) -> int: ... + def __contains__(self, key: KT) -> bool: ... + def copy(self) -> "DirectedGraph[KT]": ... + def add(self, key: KT) -> None: ... + def remove(self, key: KT) -> None: ... + def connected(self, f: KT, t: KT) -> bool: ... + def connect(self, f: KT, t: KT) -> None: ... + def iter_edges(self) -> Iterable[Tuple[KT, KT]]: ... + def iter_children(self, key: KT) -> Iterable[KT]: ... + def iter_parents(self, key: KT) -> Iterable[KT]: ... + +def build_iter_view(matches: Matches) -> IterableView[CT]: ... diff --git a/src/pip/_vendor/retrying.py b/src/pip/_vendor/retrying.py deleted file mode 100644 index 6d1e627aae8..00000000000 --- a/src/pip/_vendor/retrying.py +++ /dev/null @@ -1,267 +0,0 @@ -## Copyright 2013-2014 Ray Holder -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - -import random -from pip._vendor import six -import sys -import time -import traceback - - -# sys.maxint / 2, since Python 3.2 doesn't have a sys.maxint... -MAX_WAIT = 1073741823 - - -def retry(*dargs, **dkw): - """ - Decorator function that instantiates the Retrying object - @param *dargs: positional arguments passed to Retrying object - @param **dkw: keyword arguments passed to the Retrying object - """ - # support both @retry and @retry() as valid syntax - if len(dargs) == 1 and callable(dargs[0]): - def wrap_simple(f): - - @six.wraps(f) - def wrapped_f(*args, **kw): - return Retrying().call(f, *args, **kw) - - return wrapped_f - - return wrap_simple(dargs[0]) - - else: - def wrap(f): - - @six.wraps(f) - def wrapped_f(*args, **kw): - return Retrying(*dargs, **dkw).call(f, *args, **kw) - - return wrapped_f - - return wrap - - -class Retrying(object): - - def __init__(self, - stop=None, wait=None, - stop_max_attempt_number=None, - stop_max_delay=None, - wait_fixed=None, - wait_random_min=None, wait_random_max=None, - wait_incrementing_start=None, wait_incrementing_increment=None, - wait_exponential_multiplier=None, wait_exponential_max=None, - retry_on_exception=None, - retry_on_result=None, - wrap_exception=False, - stop_func=None, - wait_func=None, - wait_jitter_max=None): - - self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number - self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay - self._wait_fixed = 1000 if wait_fixed is None else wait_fixed - self._wait_random_min = 0 if wait_random_min is None else wait_random_min - self._wait_random_max = 1000 if wait_random_max is None else wait_random_max - self._wait_incrementing_start = 0 if wait_incrementing_start is None else wait_incrementing_start - self._wait_incrementing_increment = 100 if wait_incrementing_increment is None else wait_incrementing_increment - self._wait_exponential_multiplier = 1 if wait_exponential_multiplier is None else wait_exponential_multiplier - self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max - self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max - - # TODO add chaining of stop behaviors - # stop behavior - stop_funcs = [] - if stop_max_attempt_number is not None: - stop_funcs.append(self.stop_after_attempt) - - if stop_max_delay is not None: - stop_funcs.append(self.stop_after_delay) - - if stop_func is not None: - self.stop = stop_func - - elif stop is None: - self.stop = lambda attempts, delay: any(f(attempts, delay) for f in stop_funcs) - - else: - self.stop = getattr(self, stop) - - # TODO add chaining of wait behaviors - # wait behavior - wait_funcs = [lambda *args, **kwargs: 0] - if wait_fixed is not None: - wait_funcs.append(self.fixed_sleep) - - if wait_random_min is not None or wait_random_max is not None: - wait_funcs.append(self.random_sleep) - - if wait_incrementing_start is not None or wait_incrementing_increment is not None: - wait_funcs.append(self.incrementing_sleep) - - if wait_exponential_multiplier is not None or wait_exponential_max is not None: - wait_funcs.append(self.exponential_sleep) - - if wait_func is not None: - self.wait = wait_func - - elif wait is None: - self.wait = lambda attempts, delay: max(f(attempts, delay) for f in wait_funcs) - - else: - self.wait = getattr(self, wait) - - # retry on exception filter - if retry_on_exception is None: - self._retry_on_exception = self.always_reject - else: - self._retry_on_exception = retry_on_exception - - # TODO simplify retrying by Exception types - # retry on result filter - if retry_on_result is None: - self._retry_on_result = self.never_reject - else: - self._retry_on_result = retry_on_result - - self._wrap_exception = wrap_exception - - def stop_after_attempt(self, previous_attempt_number, delay_since_first_attempt_ms): - """Stop after the previous attempt >= stop_max_attempt_number.""" - return previous_attempt_number >= self._stop_max_attempt_number - - def stop_after_delay(self, previous_attempt_number, delay_since_first_attempt_ms): - """Stop after the time from the first attempt >= stop_max_delay.""" - return delay_since_first_attempt_ms >= self._stop_max_delay - - def no_sleep(self, previous_attempt_number, delay_since_first_attempt_ms): - """Don't sleep at all before retrying.""" - return 0 - - def fixed_sleep(self, previous_attempt_number, delay_since_first_attempt_ms): - """Sleep a fixed amount of time between each retry.""" - return self._wait_fixed - - def random_sleep(self, previous_attempt_number, delay_since_first_attempt_ms): - """Sleep a random amount of time between wait_random_min and wait_random_max""" - return random.randint(self._wait_random_min, self._wait_random_max) - - def incrementing_sleep(self, previous_attempt_number, delay_since_first_attempt_ms): - """ - Sleep an incremental amount of time after each attempt, starting at - wait_incrementing_start and incrementing by wait_incrementing_increment - """ - result = self._wait_incrementing_start + (self._wait_incrementing_increment * (previous_attempt_number - 1)) - if result < 0: - result = 0 - return result - - def exponential_sleep(self, previous_attempt_number, delay_since_first_attempt_ms): - exp = 2 ** previous_attempt_number - result = self._wait_exponential_multiplier * exp - if result > self._wait_exponential_max: - result = self._wait_exponential_max - if result < 0: - result = 0 - return result - - def never_reject(self, result): - return False - - def always_reject(self, result): - return True - - def should_reject(self, attempt): - reject = False - if attempt.has_exception: - reject |= self._retry_on_exception(attempt.value[1]) - else: - reject |= self._retry_on_result(attempt.value) - - return reject - - def call(self, fn, *args, **kwargs): - start_time = int(round(time.time() * 1000)) - attempt_number = 1 - while True: - try: - attempt = Attempt(fn(*args, **kwargs), attempt_number, False) - except: - tb = sys.exc_info() - attempt = Attempt(tb, attempt_number, True) - - if not self.should_reject(attempt): - return attempt.get(self._wrap_exception) - - delay_since_first_attempt_ms = int(round(time.time() * 1000)) - start_time - if self.stop(attempt_number, delay_since_first_attempt_ms): - if not self._wrap_exception and attempt.has_exception: - # get() on an attempt with an exception should cause it to be raised, but raise just in case - raise attempt.get() - else: - raise RetryError(attempt) - else: - sleep = self.wait(attempt_number, delay_since_first_attempt_ms) - if self._wait_jitter_max: - jitter = random.random() * self._wait_jitter_max - sleep = sleep + max(0, jitter) - time.sleep(sleep / 1000.0) - - attempt_number += 1 - - -class Attempt(object): - """ - An Attempt encapsulates a call to a target function that may end as a - normal return value from the function or an Exception depending on what - occurred during the execution. - """ - - def __init__(self, value, attempt_number, has_exception): - self.value = value - self.attempt_number = attempt_number - self.has_exception = has_exception - - def get(self, wrap_exception=False): - """ - Return the return value of this Attempt instance or raise an Exception. - If wrap_exception is true, this Attempt is wrapped inside of a - RetryError before being raised. - """ - if self.has_exception: - if wrap_exception: - raise RetryError(self) - else: - six.reraise(self.value[0], self.value[1], self.value[2]) - else: - return self.value - - def __repr__(self): - if self.has_exception: - return "Attempts: {0}, Error:\n{1}".format(self.attempt_number, "".join(traceback.format_tb(self.value[2]))) - else: - return "Attempts: {0}, Value: {1}".format(self.attempt_number, self.value) - - -class RetryError(Exception): - """ - A RetryError encapsulates the last Attempt instance right before giving up. - """ - - def __init__(self, last_attempt): - self.last_attempt = last_attempt - - def __str__(self): - return "RetryError[{0}]".format(self.last_attempt) diff --git a/src/pip/_vendor/retrying.pyi b/src/pip/_vendor/retrying.pyi deleted file mode 100644 index 90f20c6dbc1..00000000000 --- a/src/pip/_vendor/retrying.pyi +++ /dev/null @@ -1 +0,0 @@ -from retrying import * \ No newline at end of file diff --git a/src/pip/_vendor/rich/LICENSE b/src/pip/_vendor/rich/LICENSE new file mode 100644 index 00000000000..4415505566f --- /dev/null +++ b/src/pip/_vendor/rich/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2020 Will McGugan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/pip/_vendor/rich/__init__.py b/src/pip/_vendor/rich/__init__.py new file mode 100644 index 00000000000..50f38157611 --- /dev/null +++ b/src/pip/_vendor/rich/__init__.py @@ -0,0 +1,172 @@ +"""Rich text and beautiful formatting in the terminal.""" + +import os +from typing import Callable, IO, TYPE_CHECKING, Any, Optional + +from ._extension import load_ipython_extension + +__all__ = ["get_console", "reconfigure", "print", "inspect"] + +if TYPE_CHECKING: + from .console import Console + +# Global console used by alternative print +_console: Optional["Console"] = None + +_IMPORT_CWD = os.path.abspath(os.getcwd()) + + +def get_console() -> "Console": + """Get a global :class:`~rich.console.Console` instance. This function is used when Rich requires a Console, + and hasn't been explicitly given one. + + Returns: + Console: A console instance. + """ + global _console + if _console is None: + from .console import Console + + _console = Console() + + return _console + + +def reconfigure(*args: Any, **kwargs: Any) -> None: + """Reconfigures the global console by replacing it with another. + + Args: + console (Console): Replacement console instance. + """ + from pip._vendor.rich.console import Console + + new_console = Console(*args, **kwargs) + _console = get_console() + _console.__dict__ = new_console.__dict__ + + +def print( + *objects: Any, + sep: str = " ", + end: str = "\n", + file: Optional[IO[str]] = None, + flush: bool = False, +) -> None: + r"""Print object(s) supplied via positional arguments. + This function has an identical signature to the built-in print. + For more advanced features, see the :class:`~rich.console.Console` class. + + Args: + sep (str, optional): Separator between printed objects. Defaults to " ". + end (str, optional): Character to write at end of output. Defaults to "\\n". + file (IO[str], optional): File to write to, or None for stdout. Defaults to None. + flush (bool, optional): Has no effect as Rich always flushes output. Defaults to False. + + """ + from .console import Console + + write_console = get_console() if file is None else Console(file=file) + return write_console.print(*objects, sep=sep, end=end) + + +def print_json( + json: Optional[str] = None, + *, + data: Any = None, + indent: int = 2, + highlight: bool = True, + skip_keys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + default: Optional[Callable[[Any], Any]] = None, + sort_keys: bool = False, +) -> None: + """Pretty prints JSON. Output will be valid JSON. + + Args: + json (str): A string containing JSON. + data (Any): If json is not supplied, then encode this data. + indent (int, optional): Number of spaces to indent. Defaults to 2. + highlight (bool, optional): Enable highlighting of output: Defaults to True. + skip_keys (bool, optional): Skip keys not of a basic type. Defaults to False. + ensure_ascii (bool, optional): Escape all non-ascii characters. Defaults to False. + check_circular (bool, optional): Check for circular references. Defaults to True. + allow_nan (bool, optional): Allow NaN and Infinity values. Defaults to True. + default (Callable, optional): A callable that converts values that can not be encoded + in to something that can be JSON encoded. Defaults to None. + sort_keys (bool, optional): Sort dictionary keys. Defaults to False. + """ + + get_console().print_json( + json, + data=data, + indent=indent, + highlight=highlight, + skip_keys=skip_keys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + default=default, + sort_keys=sort_keys, + ) + + +def inspect( + obj: Any, + *, + console: Optional["Console"] = None, + title: Optional[str] = None, + help: bool = False, + methods: bool = False, + docs: bool = True, + private: bool = False, + dunder: bool = False, + sort: bool = True, + all: bool = False, + value: bool = True, +) -> None: + """Inspect any Python object. + + * inspect() to see summarized info. + * inspect(, methods=True) to see methods. + * inspect(, help=True) to see full (non-abbreviated) help. + * inspect(, private=True) to see private attributes (single underscore). + * inspect(, dunder=True) to see attributes beginning with double underscore. + * inspect(, all=True) to see all attributes. + + Args: + obj (Any): An object to inspect. + title (str, optional): Title to display over inspect result, or None use type. Defaults to None. + help (bool, optional): Show full help text rather than just first paragraph. Defaults to False. + methods (bool, optional): Enable inspection of callables. Defaults to False. + docs (bool, optional): Also render doc strings. Defaults to True. + private (bool, optional): Show private attributes (beginning with underscore). Defaults to False. + dunder (bool, optional): Show attributes starting with double underscore. Defaults to False. + sort (bool, optional): Sort attributes alphabetically. Defaults to True. + all (bool, optional): Show all attributes. Defaults to False. + value (bool, optional): Pretty print value. Defaults to True. + """ + _console = console or get_console() + from pip._vendor.rich._inspect import Inspect + + # Special case for inspect(inspect) + is_inspect = obj is inspect + + _inspect = Inspect( + obj, + title=title, + help=is_inspect or help, + methods=is_inspect or methods, + docs=is_inspect or docs, + private=private, + dunder=dunder, + sort=sort, + all=all, + value=value, + ) + _console.print(_inspect) + + +if __name__ == "__main__": # pragma: no cover + print("Hello, **World**") diff --git a/src/pip/_vendor/rich/__main__.py b/src/pip/_vendor/rich/__main__.py new file mode 100644 index 00000000000..8692d37e00b --- /dev/null +++ b/src/pip/_vendor/rich/__main__.py @@ -0,0 +1,280 @@ +import colorsys +import io +from time import process_time + +from pip._vendor.rich import box +from pip._vendor.rich.color import Color +from pip._vendor.rich.console import Console, ConsoleOptions, Group, RenderableType, RenderResult +from pip._vendor.rich.markdown import Markdown +from pip._vendor.rich.measure import Measurement +from pip._vendor.rich.pretty import Pretty +from pip._vendor.rich.segment import Segment +from pip._vendor.rich.style import Style +from pip._vendor.rich.syntax import Syntax +from pip._vendor.rich.table import Table +from pip._vendor.rich.text import Text + + +class ColorBox: + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + for y in range(0, 5): + for x in range(options.max_width): + h = x / options.max_width + l = 0.1 + ((y / 5) * 0.7) + r1, g1, b1 = colorsys.hls_to_rgb(h, l, 1.0) + r2, g2, b2 = colorsys.hls_to_rgb(h, l + 0.7 / 10, 1.0) + bgcolor = Color.from_rgb(r1 * 255, g1 * 255, b1 * 255) + color = Color.from_rgb(r2 * 255, g2 * 255, b2 * 255) + yield Segment("▄", Style(color=color, bgcolor=bgcolor)) + yield Segment.line() + + def __rich_measure__( + self, console: "Console", options: ConsoleOptions + ) -> Measurement: + return Measurement(1, options.max_width) + + +def make_test_card() -> Table: + """Get a renderable that demonstrates a number of features.""" + table = Table.grid(padding=1, pad_edge=True) + table.title = "Rich features" + table.add_column("Feature", no_wrap=True, justify="center", style="bold red") + table.add_column("Demonstration") + + color_table = Table( + box=None, + expand=False, + show_header=False, + show_edge=False, + pad_edge=False, + ) + color_table.add_row( + # "[bold yellow]256[/] colors or [bold green]16.7 million[/] colors [blue](if supported by your terminal)[/].", + ( + "✓ [bold green]4-bit color[/]\n" + "✓ [bold blue]8-bit color[/]\n" + "✓ [bold magenta]Truecolor (16.7 million)[/]\n" + "✓ [bold yellow]Dumb terminals[/]\n" + "✓ [bold cyan]Automatic color conversion" + ), + ColorBox(), + ) + + table.add_row("Colors", color_table) + + table.add_row( + "Styles", + "All ansi styles: [bold]bold[/], [dim]dim[/], [italic]italic[/italic], [underline]underline[/], [strike]strikethrough[/], [reverse]reverse[/], and even [blink]blink[/].", + ) + + lorem = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque in metus sed sapien ultricies pretium a at justo. Maecenas luctus velit et auctor maximus." + lorem_table = Table.grid(padding=1, collapse_padding=True) + lorem_table.pad_edge = False + lorem_table.add_row( + Text(lorem, justify="left", style="green"), + Text(lorem, justify="center", style="yellow"), + Text(lorem, justify="right", style="blue"), + Text(lorem, justify="full", style="red"), + ) + table.add_row( + "Text", + Group( + Text.from_markup( + """Word wrap text. Justify [green]left[/], [yellow]center[/], [blue]right[/] or [red]full[/].\n""" + ), + lorem_table, + ), + ) + + def comparison(renderable1: RenderableType, renderable2: RenderableType) -> Table: + table = Table(show_header=False, pad_edge=False, box=None, expand=True) + table.add_column("1", ratio=1) + table.add_column("2", ratio=1) + table.add_row(renderable1, renderable2) + return table + + table.add_row( + "Asian\nlanguage\nsupport", + ":flag_for_china: 该库支持中文,日文和韩文文本!\n:flag_for_japan: ライブラリは中国語、日本語、韓国語のテキストをサポートしています\n:flag_for_south_korea: 이 라이브러리는 중국어, 일본어 및 한국어 텍스트를 지원합니다", + ) + + markup_example = ( + "[bold magenta]Rich[/] supports a simple [i]bbcode[/i]-like [b]markup[/b] for [yellow]color[/], [underline]style[/], and emoji! " + ":+1: :apple: :ant: :bear: :baguette_bread: :bus: " + ) + table.add_row("Markup", markup_example) + + example_table = Table( + show_edge=False, + show_header=True, + expand=False, + row_styles=["none", "dim"], + box=box.SIMPLE, + ) + example_table.add_column("[green]Date", style="green", no_wrap=True) + example_table.add_column("[blue]Title", style="blue") + example_table.add_column( + "[cyan]Production Budget", + style="cyan", + justify="right", + no_wrap=True, + ) + example_table.add_column( + "[magenta]Box Office", + style="magenta", + justify="right", + no_wrap=True, + ) + example_table.add_row( + "Dec 20, 2019", + "Star Wars: The Rise of Skywalker", + "$275,000,000", + "$375,126,118", + ) + example_table.add_row( + "May 25, 2018", + "[b]Solo[/]: A Star Wars Story", + "$275,000,000", + "$393,151,347", + ) + example_table.add_row( + "Dec 15, 2017", + "Star Wars Ep. VIII: The Last Jedi", + "$262,000,000", + "[bold]$1,332,539,889[/bold]", + ) + example_table.add_row( + "May 19, 1999", + "Star Wars Ep. [b]I[/b]: [i]The phantom Menace", + "$115,000,000", + "$1,027,044,677", + ) + + table.add_row("Tables", example_table) + + code = '''\ +def iter_last(values: Iterable[T]) -> Iterable[Tuple[bool, T]]: + """Iterate and generate a tuple with a flag for last value.""" + iter_values = iter(values) + try: + previous_value = next(iter_values) + except StopIteration: + return + for value in iter_values: + yield False, previous_value + previous_value = value + yield True, previous_value''' + + pretty_data = { + "foo": [ + 3.1427, + ( + "Paul Atreides", + "Vladimir Harkonnen", + "Thufir Hawat", + ), + ], + "atomic": (False, True, None), + } + table.add_row( + "Syntax\nhighlighting\n&\npretty\nprinting", + comparison( + Syntax(code, "python3", line_numbers=True, indent_guides=True), + Pretty(pretty_data, indent_guides=True), + ), + ) + + markdown_example = """\ +# Markdown + +Supports much of the *markdown* __syntax__! + +- Headers +- Basic formatting: **bold**, *italic*, `code` +- Block quotes +- Lists, and more... + """ + table.add_row( + "Markdown", comparison("[cyan]" + markdown_example, Markdown(markdown_example)) + ) + + table.add_row( + "+more!", + """Progress bars, columns, styled logging handler, tracebacks, etc...""", + ) + return table + + +if __name__ == "__main__": # pragma: no cover + + console = Console( + file=io.StringIO(), + force_terminal=True, + ) + test_card = make_test_card() + + # Print once to warm cache + start = process_time() + console.print(test_card) + pre_cache_taken = round((process_time() - start) * 1000.0, 1) + + console.file = io.StringIO() + + start = process_time() + console.print(test_card) + taken = round((process_time() - start) * 1000.0, 1) + + text = console.file.getvalue() + # https://bugs.python.org/issue37871 + for line in text.splitlines(True): + print(line, end="") + + print(f"rendered in {pre_cache_taken}ms (cold cache)") + print(f"rendered in {taken}ms (warm cache)") + + from pip._vendor.rich.panel import Panel + + console = Console() + + sponsor_message = Table.grid(padding=1) + sponsor_message.add_column(style="green", justify="right") + sponsor_message.add_column(no_wrap=True) + + sponsor_message.add_row( + "Buy devs a :coffee:", + "[u blue link=https://ko-fi.com/textualize]https://ko-fi.com/textualize", + ) + sponsor_message.add_row( + "Twitter", + "[u blue link=https://twitter.com/willmcgugan]https://twitter.com/willmcgugan", + ) + sponsor_message.add_row( + "Blog", "[u blue link=https://www.willmcgugan.com]https://www.willmcgugan.com" + ) + + intro_message = Text.from_markup( + """\ +We hope you enjoy using Rich! + +Rich is maintained with :heart: by [link=https://www.textualize.io]Textualize.io[/] + +- Will McGugan""" + ) + + message = Table.grid(padding=2) + message.add_column() + message.add_column(no_wrap=True) + message.add_row(intro_message, sponsor_message) + + console.print( + Panel.fit( + message, + box=box.ROUNDED, + padding=(1, 2), + title="[b red]Thanks for trying out Rich!", + border_style="bright_blue", + ), + justify="center", + ) diff --git a/src/pip/_vendor/rich/_cell_widths.py b/src/pip/_vendor/rich/_cell_widths.py new file mode 100644 index 00000000000..36286df379e --- /dev/null +++ b/src/pip/_vendor/rich/_cell_widths.py @@ -0,0 +1,451 @@ +# Auto generated by make_terminal_widths.py + +CELL_WIDTHS = [ + (0, 0, 0), + (1, 31, -1), + (127, 159, -1), + (768, 879, 0), + (1155, 1161, 0), + (1425, 1469, 0), + (1471, 1471, 0), + (1473, 1474, 0), + (1476, 1477, 0), + (1479, 1479, 0), + (1552, 1562, 0), + (1611, 1631, 0), + (1648, 1648, 0), + (1750, 1756, 0), + (1759, 1764, 0), + (1767, 1768, 0), + (1770, 1773, 0), + (1809, 1809, 0), + (1840, 1866, 0), + (1958, 1968, 0), + (2027, 2035, 0), + (2045, 2045, 0), + (2070, 2073, 0), + (2075, 2083, 0), + (2085, 2087, 0), + (2089, 2093, 0), + (2137, 2139, 0), + (2259, 2273, 0), + (2275, 2306, 0), + (2362, 2362, 0), + (2364, 2364, 0), + (2369, 2376, 0), + (2381, 2381, 0), + (2385, 2391, 0), + (2402, 2403, 0), + (2433, 2433, 0), + (2492, 2492, 0), + (2497, 2500, 0), + (2509, 2509, 0), + (2530, 2531, 0), + (2558, 2558, 0), + (2561, 2562, 0), + (2620, 2620, 0), + (2625, 2626, 0), + (2631, 2632, 0), + (2635, 2637, 0), + (2641, 2641, 0), + (2672, 2673, 0), + (2677, 2677, 0), + (2689, 2690, 0), + (2748, 2748, 0), + (2753, 2757, 0), + (2759, 2760, 0), + (2765, 2765, 0), + (2786, 2787, 0), + (2810, 2815, 0), + (2817, 2817, 0), + (2876, 2876, 0), + (2879, 2879, 0), + (2881, 2884, 0), + (2893, 2893, 0), + (2901, 2902, 0), + (2914, 2915, 0), + (2946, 2946, 0), + (3008, 3008, 0), + (3021, 3021, 0), + (3072, 3072, 0), + (3076, 3076, 0), + (3134, 3136, 0), + (3142, 3144, 0), + (3146, 3149, 0), + (3157, 3158, 0), + (3170, 3171, 0), + (3201, 3201, 0), + (3260, 3260, 0), + (3263, 3263, 0), + (3270, 3270, 0), + (3276, 3277, 0), + (3298, 3299, 0), + (3328, 3329, 0), + (3387, 3388, 0), + (3393, 3396, 0), + (3405, 3405, 0), + (3426, 3427, 0), + (3457, 3457, 0), + (3530, 3530, 0), + (3538, 3540, 0), + (3542, 3542, 0), + (3633, 3633, 0), + (3636, 3642, 0), + (3655, 3662, 0), + (3761, 3761, 0), + (3764, 3772, 0), + (3784, 3789, 0), + (3864, 3865, 0), + (3893, 3893, 0), + (3895, 3895, 0), + (3897, 3897, 0), + (3953, 3966, 0), + (3968, 3972, 0), + (3974, 3975, 0), + (3981, 3991, 0), + (3993, 4028, 0), + (4038, 4038, 0), + (4141, 4144, 0), + (4146, 4151, 0), + (4153, 4154, 0), + (4157, 4158, 0), + (4184, 4185, 0), + (4190, 4192, 0), + (4209, 4212, 0), + (4226, 4226, 0), + (4229, 4230, 0), + (4237, 4237, 0), + (4253, 4253, 0), + (4352, 4447, 2), + (4957, 4959, 0), + (5906, 5908, 0), + (5938, 5940, 0), + (5970, 5971, 0), + (6002, 6003, 0), + (6068, 6069, 0), + (6071, 6077, 0), + (6086, 6086, 0), + (6089, 6099, 0), + (6109, 6109, 0), + (6155, 6157, 0), + (6277, 6278, 0), + (6313, 6313, 0), + (6432, 6434, 0), + (6439, 6440, 0), + (6450, 6450, 0), + (6457, 6459, 0), + (6679, 6680, 0), + (6683, 6683, 0), + (6742, 6742, 0), + (6744, 6750, 0), + (6752, 6752, 0), + (6754, 6754, 0), + (6757, 6764, 0), + (6771, 6780, 0), + (6783, 6783, 0), + (6832, 6848, 0), + (6912, 6915, 0), + (6964, 6964, 0), + (6966, 6970, 0), + (6972, 6972, 0), + (6978, 6978, 0), + (7019, 7027, 0), + (7040, 7041, 0), + (7074, 7077, 0), + (7080, 7081, 0), + (7083, 7085, 0), + (7142, 7142, 0), + (7144, 7145, 0), + (7149, 7149, 0), + (7151, 7153, 0), + (7212, 7219, 0), + (7222, 7223, 0), + (7376, 7378, 0), + (7380, 7392, 0), + (7394, 7400, 0), + (7405, 7405, 0), + (7412, 7412, 0), + (7416, 7417, 0), + (7616, 7673, 0), + (7675, 7679, 0), + (8203, 8207, 0), + (8232, 8238, 0), + (8288, 8291, 0), + (8400, 8432, 0), + (8986, 8987, 2), + (9001, 9002, 2), + (9193, 9196, 2), + (9200, 9200, 2), + (9203, 9203, 2), + (9725, 9726, 2), + (9748, 9749, 2), + (9800, 9811, 2), + (9855, 9855, 2), + (9875, 9875, 2), + (9889, 9889, 2), + (9898, 9899, 2), + (9917, 9918, 2), + (9924, 9925, 2), + (9934, 9934, 2), + (9940, 9940, 2), + (9962, 9962, 2), + (9970, 9971, 2), + (9973, 9973, 2), + (9978, 9978, 2), + (9981, 9981, 2), + (9989, 9989, 2), + (9994, 9995, 2), + (10024, 10024, 2), + (10060, 10060, 2), + (10062, 10062, 2), + (10067, 10069, 2), + (10071, 10071, 2), + (10133, 10135, 2), + (10160, 10160, 2), + (10175, 10175, 2), + (11035, 11036, 2), + (11088, 11088, 2), + (11093, 11093, 2), + (11503, 11505, 0), + (11647, 11647, 0), + (11744, 11775, 0), + (11904, 11929, 2), + (11931, 12019, 2), + (12032, 12245, 2), + (12272, 12283, 2), + (12288, 12329, 2), + (12330, 12333, 0), + (12334, 12350, 2), + (12353, 12438, 2), + (12441, 12442, 0), + (12443, 12543, 2), + (12549, 12591, 2), + (12593, 12686, 2), + (12688, 12771, 2), + (12784, 12830, 2), + (12832, 12871, 2), + (12880, 19903, 2), + (19968, 42124, 2), + (42128, 42182, 2), + (42607, 42610, 0), + (42612, 42621, 0), + (42654, 42655, 0), + (42736, 42737, 0), + (43010, 43010, 0), + (43014, 43014, 0), + (43019, 43019, 0), + (43045, 43046, 0), + (43052, 43052, 0), + (43204, 43205, 0), + (43232, 43249, 0), + (43263, 43263, 0), + (43302, 43309, 0), + (43335, 43345, 0), + (43360, 43388, 2), + (43392, 43394, 0), + (43443, 43443, 0), + (43446, 43449, 0), + (43452, 43453, 0), + (43493, 43493, 0), + (43561, 43566, 0), + (43569, 43570, 0), + (43573, 43574, 0), + (43587, 43587, 0), + (43596, 43596, 0), + (43644, 43644, 0), + (43696, 43696, 0), + (43698, 43700, 0), + (43703, 43704, 0), + (43710, 43711, 0), + (43713, 43713, 0), + (43756, 43757, 0), + (43766, 43766, 0), + (44005, 44005, 0), + (44008, 44008, 0), + (44013, 44013, 0), + (44032, 55203, 2), + (63744, 64255, 2), + (64286, 64286, 0), + (65024, 65039, 0), + (65040, 65049, 2), + (65056, 65071, 0), + (65072, 65106, 2), + (65108, 65126, 2), + (65128, 65131, 2), + (65281, 65376, 2), + (65504, 65510, 2), + (66045, 66045, 0), + (66272, 66272, 0), + (66422, 66426, 0), + (68097, 68099, 0), + (68101, 68102, 0), + (68108, 68111, 0), + (68152, 68154, 0), + (68159, 68159, 0), + (68325, 68326, 0), + (68900, 68903, 0), + (69291, 69292, 0), + (69446, 69456, 0), + (69633, 69633, 0), + (69688, 69702, 0), + (69759, 69761, 0), + (69811, 69814, 0), + (69817, 69818, 0), + (69888, 69890, 0), + (69927, 69931, 0), + (69933, 69940, 0), + (70003, 70003, 0), + (70016, 70017, 0), + (70070, 70078, 0), + (70089, 70092, 0), + (70095, 70095, 0), + (70191, 70193, 0), + (70196, 70196, 0), + (70198, 70199, 0), + (70206, 70206, 0), + (70367, 70367, 0), + (70371, 70378, 0), + (70400, 70401, 0), + (70459, 70460, 0), + (70464, 70464, 0), + (70502, 70508, 0), + (70512, 70516, 0), + (70712, 70719, 0), + (70722, 70724, 0), + (70726, 70726, 0), + (70750, 70750, 0), + (70835, 70840, 0), + (70842, 70842, 0), + (70847, 70848, 0), + (70850, 70851, 0), + (71090, 71093, 0), + (71100, 71101, 0), + (71103, 71104, 0), + (71132, 71133, 0), + (71219, 71226, 0), + (71229, 71229, 0), + (71231, 71232, 0), + (71339, 71339, 0), + (71341, 71341, 0), + (71344, 71349, 0), + (71351, 71351, 0), + (71453, 71455, 0), + (71458, 71461, 0), + (71463, 71467, 0), + (71727, 71735, 0), + (71737, 71738, 0), + (71995, 71996, 0), + (71998, 71998, 0), + (72003, 72003, 0), + (72148, 72151, 0), + (72154, 72155, 0), + (72160, 72160, 0), + (72193, 72202, 0), + (72243, 72248, 0), + (72251, 72254, 0), + (72263, 72263, 0), + (72273, 72278, 0), + (72281, 72283, 0), + (72330, 72342, 0), + (72344, 72345, 0), + (72752, 72758, 0), + (72760, 72765, 0), + (72767, 72767, 0), + (72850, 72871, 0), + (72874, 72880, 0), + (72882, 72883, 0), + (72885, 72886, 0), + (73009, 73014, 0), + (73018, 73018, 0), + (73020, 73021, 0), + (73023, 73029, 0), + (73031, 73031, 0), + (73104, 73105, 0), + (73109, 73109, 0), + (73111, 73111, 0), + (73459, 73460, 0), + (92912, 92916, 0), + (92976, 92982, 0), + (94031, 94031, 0), + (94095, 94098, 0), + (94176, 94179, 2), + (94180, 94180, 0), + (94192, 94193, 2), + (94208, 100343, 2), + (100352, 101589, 2), + (101632, 101640, 2), + (110592, 110878, 2), + (110928, 110930, 2), + (110948, 110951, 2), + (110960, 111355, 2), + (113821, 113822, 0), + (119143, 119145, 0), + (119163, 119170, 0), + (119173, 119179, 0), + (119210, 119213, 0), + (119362, 119364, 0), + (121344, 121398, 0), + (121403, 121452, 0), + (121461, 121461, 0), + (121476, 121476, 0), + (121499, 121503, 0), + (121505, 121519, 0), + (122880, 122886, 0), + (122888, 122904, 0), + (122907, 122913, 0), + (122915, 122916, 0), + (122918, 122922, 0), + (123184, 123190, 0), + (123628, 123631, 0), + (125136, 125142, 0), + (125252, 125258, 0), + (126980, 126980, 2), + (127183, 127183, 2), + (127374, 127374, 2), + (127377, 127386, 2), + (127488, 127490, 2), + (127504, 127547, 2), + (127552, 127560, 2), + (127568, 127569, 2), + (127584, 127589, 2), + (127744, 127776, 2), + (127789, 127797, 2), + (127799, 127868, 2), + (127870, 127891, 2), + (127904, 127946, 2), + (127951, 127955, 2), + (127968, 127984, 2), + (127988, 127988, 2), + (127992, 128062, 2), + (128064, 128064, 2), + (128066, 128252, 2), + (128255, 128317, 2), + (128331, 128334, 2), + (128336, 128359, 2), + (128378, 128378, 2), + (128405, 128406, 2), + (128420, 128420, 2), + (128507, 128591, 2), + (128640, 128709, 2), + (128716, 128716, 2), + (128720, 128722, 2), + (128725, 128727, 2), + (128747, 128748, 2), + (128756, 128764, 2), + (128992, 129003, 2), + (129292, 129338, 2), + (129340, 129349, 2), + (129351, 129400, 2), + (129402, 129483, 2), + (129485, 129535, 2), + (129648, 129652, 2), + (129656, 129658, 2), + (129664, 129670, 2), + (129680, 129704, 2), + (129712, 129718, 2), + (129728, 129730, 2), + (129744, 129750, 2), + (131072, 196605, 2), + (196608, 262141, 2), + (917760, 917999, 0), +] diff --git a/src/pip/_vendor/rich/_emoji_codes.py b/src/pip/_vendor/rich/_emoji_codes.py new file mode 100644 index 00000000000..1f2877bb2bd --- /dev/null +++ b/src/pip/_vendor/rich/_emoji_codes.py @@ -0,0 +1,3610 @@ +EMOJI = { + "1st_place_medal": "🥇", + "2nd_place_medal": "🥈", + "3rd_place_medal": "🥉", + "ab_button_(blood_type)": "🆎", + "atm_sign": "🏧", + "a_button_(blood_type)": "🅰", + "afghanistan": "🇦🇫", + "albania": "🇦🇱", + "algeria": "🇩🇿", + "american_samoa": "🇦🇸", + "andorra": "🇦🇩", + "angola": "🇦🇴", + "anguilla": "🇦🇮", + "antarctica": "🇦🇶", + "antigua_&_barbuda": "🇦🇬", + "aquarius": "♒", + "argentina": "🇦🇷", + "aries": "♈", + "armenia": "🇦🇲", + "aruba": "🇦🇼", + "ascension_island": "🇦🇨", + "australia": "🇦🇺", + "austria": "🇦🇹", + "azerbaijan": "🇦🇿", + "back_arrow": "🔙", + "b_button_(blood_type)": "🅱", + "bahamas": "🇧🇸", + "bahrain": "🇧🇭", + "bangladesh": "🇧🇩", + "barbados": "🇧🇧", + "belarus": "🇧🇾", + "belgium": "🇧🇪", + "belize": "🇧🇿", + "benin": "🇧🇯", + "bermuda": "🇧🇲", + "bhutan": "🇧🇹", + "bolivia": "🇧🇴", + "bosnia_&_herzegovina": "🇧🇦", + "botswana": "🇧🇼", + "bouvet_island": "🇧🇻", + "brazil": "🇧🇷", + "british_indian_ocean_territory": "🇮🇴", + "british_virgin_islands": "🇻🇬", + "brunei": "🇧🇳", + "bulgaria": "🇧🇬", + "burkina_faso": "🇧🇫", + "burundi": "🇧🇮", + "cl_button": "🆑", + "cool_button": "🆒", + "cambodia": "🇰🇭", + "cameroon": "🇨🇲", + "canada": "🇨🇦", + "canary_islands": "🇮🇨", + "cancer": "♋", + "cape_verde": "🇨🇻", + "capricorn": "♑", + "caribbean_netherlands": "🇧🇶", + "cayman_islands": "🇰🇾", + "central_african_republic": "🇨🇫", + "ceuta_&_melilla": "🇪🇦", + "chad": "🇹🇩", + "chile": "🇨🇱", + "china": "🇨🇳", + "christmas_island": "🇨🇽", + "christmas_tree": "🎄", + "clipperton_island": "🇨🇵", + "cocos_(keeling)_islands": "🇨🇨", + "colombia": "🇨🇴", + "comoros": "🇰🇲", + "congo_-_brazzaville": "🇨🇬", + "congo_-_kinshasa": "🇨🇩", + "cook_islands": "🇨🇰", + "costa_rica": "🇨🇷", + "croatia": "🇭🇷", + "cuba": "🇨🇺", + "curaçao": "🇨🇼", + "cyprus": "🇨🇾", + "czechia": "🇨🇿", + "côte_d’ivoire": "🇨🇮", + "denmark": "🇩🇰", + "diego_garcia": "🇩🇬", + "djibouti": "🇩🇯", + "dominica": "🇩🇲", + "dominican_republic": "🇩🇴", + "end_arrow": "🔚", + "ecuador": "🇪🇨", + "egypt": "🇪🇬", + "el_salvador": "🇸🇻", + "england": "🏴\U000e0067\U000e0062\U000e0065\U000e006e\U000e0067\U000e007f", + "equatorial_guinea": "🇬🇶", + "eritrea": "🇪🇷", + "estonia": "🇪🇪", + "ethiopia": "🇪🇹", + "european_union": "🇪🇺", + "free_button": "🆓", + "falkland_islands": "🇫🇰", + "faroe_islands": "🇫🇴", + "fiji": "🇫🇯", + "finland": "🇫🇮", + "france": "🇫🇷", + "french_guiana": "🇬🇫", + "french_polynesia": "🇵🇫", + "french_southern_territories": "🇹🇫", + "gabon": "🇬🇦", + "gambia": "🇬🇲", + "gemini": "♊", + "georgia": "🇬🇪", + "germany": "🇩🇪", + "ghana": "🇬🇭", + "gibraltar": "🇬🇮", + "greece": "🇬🇷", + "greenland": "🇬🇱", + "grenada": "🇬🇩", + "guadeloupe": "🇬🇵", + "guam": "🇬🇺", + "guatemala": "🇬🇹", + "guernsey": "🇬🇬", + "guinea": "🇬🇳", + "guinea-bissau": "🇬🇼", + "guyana": "🇬🇾", + "haiti": "🇭🇹", + "heard_&_mcdonald_islands": "🇭🇲", + "honduras": "🇭🇳", + "hong_kong_sar_china": "🇭🇰", + "hungary": "🇭🇺", + "id_button": "🆔", + "iceland": "🇮🇸", + "india": "🇮🇳", + "indonesia": "🇮🇩", + "iran": "🇮🇷", + "iraq": "🇮🇶", + "ireland": "🇮🇪", + "isle_of_man": "🇮🇲", + "israel": "🇮🇱", + "italy": "🇮🇹", + "jamaica": "🇯🇲", + "japan": "🗾", + "japanese_acceptable_button": "🉑", + "japanese_application_button": "🈸", + "japanese_bargain_button": "🉐", + "japanese_castle": "🏯", + "japanese_congratulations_button": "㊗", + "japanese_discount_button": "🈹", + "japanese_dolls": "🎎", + "japanese_free_of_charge_button": "🈚", + "japanese_here_button": "🈁", + "japanese_monthly_amount_button": "🈷", + "japanese_no_vacancy_button": "🈵", + "japanese_not_free_of_charge_button": "🈶", + "japanese_open_for_business_button": "🈺", + "japanese_passing_grade_button": "🈴", + "japanese_post_office": "🏣", + "japanese_prohibited_button": "🈲", + "japanese_reserved_button": "🈯", + "japanese_secret_button": "㊙", + "japanese_service_charge_button": "🈂", + "japanese_symbol_for_beginner": "🔰", + "japanese_vacancy_button": "🈳", + "jersey": "🇯🇪", + "jordan": "🇯🇴", + "kazakhstan": "🇰🇿", + "kenya": "🇰🇪", + "kiribati": "🇰🇮", + "kosovo": "🇽🇰", + "kuwait": "🇰🇼", + "kyrgyzstan": "🇰🇬", + "laos": "🇱🇦", + "latvia": "🇱🇻", + "lebanon": "🇱🇧", + "leo": "♌", + "lesotho": "🇱🇸", + "liberia": "🇱🇷", + "libra": "♎", + "libya": "🇱🇾", + "liechtenstein": "🇱🇮", + "lithuania": "🇱🇹", + "luxembourg": "🇱🇺", + "macau_sar_china": "🇲🇴", + "macedonia": "🇲🇰", + "madagascar": "🇲🇬", + "malawi": "🇲🇼", + "malaysia": "🇲🇾", + "maldives": "🇲🇻", + "mali": "🇲🇱", + "malta": "🇲🇹", + "marshall_islands": "🇲🇭", + "martinique": "🇲🇶", + "mauritania": "🇲🇷", + "mauritius": "🇲🇺", + "mayotte": "🇾🇹", + "mexico": "🇲🇽", + "micronesia": "🇫🇲", + "moldova": "🇲🇩", + "monaco": "🇲🇨", + "mongolia": "🇲🇳", + "montenegro": "🇲🇪", + "montserrat": "🇲🇸", + "morocco": "🇲🇦", + "mozambique": "🇲🇿", + "mrs._claus": "🤶", + "mrs._claus_dark_skin_tone": "🤶🏿", + "mrs._claus_light_skin_tone": "🤶🏻", + "mrs._claus_medium-dark_skin_tone": "🤶🏾", + "mrs._claus_medium-light_skin_tone": "🤶🏼", + "mrs._claus_medium_skin_tone": "🤶🏽", + "myanmar_(burma)": "🇲🇲", + "new_button": "🆕", + "ng_button": "🆖", + "namibia": "🇳🇦", + "nauru": "🇳🇷", + "nepal": "🇳🇵", + "netherlands": "🇳🇱", + "new_caledonia": "🇳🇨", + "new_zealand": "🇳🇿", + "nicaragua": "🇳🇮", + "niger": "🇳🇪", + "nigeria": "🇳🇬", + "niue": "🇳🇺", + "norfolk_island": "🇳🇫", + "north_korea": "🇰🇵", + "northern_mariana_islands": "🇲🇵", + "norway": "🇳🇴", + "ok_button": "🆗", + "ok_hand": "👌", + "ok_hand_dark_skin_tone": "👌🏿", + "ok_hand_light_skin_tone": "👌🏻", + "ok_hand_medium-dark_skin_tone": "👌🏾", + "ok_hand_medium-light_skin_tone": "👌🏼", + "ok_hand_medium_skin_tone": "👌🏽", + "on!_arrow": "🔛", + "o_button_(blood_type)": "🅾", + "oman": "🇴🇲", + "ophiuchus": "⛎", + "p_button": "🅿", + "pakistan": "🇵🇰", + "palau": "🇵🇼", + "palestinian_territories": "🇵🇸", + "panama": "🇵🇦", + "papua_new_guinea": "🇵🇬", + "paraguay": "🇵🇾", + "peru": "🇵🇪", + "philippines": "🇵🇭", + "pisces": "♓", + "pitcairn_islands": "🇵🇳", + "poland": "🇵🇱", + "portugal": "🇵🇹", + "puerto_rico": "🇵🇷", + "qatar": "🇶🇦", + "romania": "🇷🇴", + "russia": "🇷🇺", + "rwanda": "🇷🇼", + "réunion": "🇷🇪", + "soon_arrow": "🔜", + "sos_button": "🆘", + "sagittarius": "♐", + "samoa": "🇼🇸", + "san_marino": "🇸🇲", + "santa_claus": "🎅", + "santa_claus_dark_skin_tone": "🎅🏿", + "santa_claus_light_skin_tone": "🎅🏻", + "santa_claus_medium-dark_skin_tone": "🎅🏾", + "santa_claus_medium-light_skin_tone": "🎅🏼", + "santa_claus_medium_skin_tone": "🎅🏽", + "saudi_arabia": "🇸🇦", + "scorpio": "♏", + "scotland": "🏴\U000e0067\U000e0062\U000e0073\U000e0063\U000e0074\U000e007f", + "senegal": "🇸🇳", + "serbia": "🇷🇸", + "seychelles": "🇸🇨", + "sierra_leone": "🇸🇱", + "singapore": "🇸🇬", + "sint_maarten": "🇸🇽", + "slovakia": "🇸🇰", + "slovenia": "🇸🇮", + "solomon_islands": "🇸🇧", + "somalia": "🇸🇴", + "south_africa": "🇿🇦", + "south_georgia_&_south_sandwich_islands": "🇬🇸", + "south_korea": "🇰🇷", + "south_sudan": "🇸🇸", + "spain": "🇪🇸", + "sri_lanka": "🇱🇰", + "st._barthélemy": "🇧🇱", + "st._helena": "🇸🇭", + "st._kitts_&_nevis": "🇰🇳", + "st._lucia": "🇱🇨", + "st._martin": "🇲🇫", + "st._pierre_&_miquelon": "🇵🇲", + "st._vincent_&_grenadines": "🇻🇨", + "statue_of_liberty": "🗽", + "sudan": "🇸🇩", + "suriname": "🇸🇷", + "svalbard_&_jan_mayen": "🇸🇯", + "swaziland": "🇸🇿", + "sweden": "🇸🇪", + "switzerland": "🇨🇭", + "syria": "🇸🇾", + "são_tomé_&_príncipe": "🇸🇹", + "t-rex": "🦖", + "top_arrow": "🔝", + "taiwan": "🇹🇼", + "tajikistan": "🇹🇯", + "tanzania": "🇹🇿", + "taurus": "♉", + "thailand": "🇹🇭", + "timor-leste": "🇹🇱", + "togo": "🇹🇬", + "tokelau": "🇹🇰", + "tokyo_tower": "🗼", + "tonga": "🇹🇴", + "trinidad_&_tobago": "🇹🇹", + "tristan_da_cunha": "🇹🇦", + "tunisia": "🇹🇳", + "turkey": "🦃", + "turkmenistan": "🇹🇲", + "turks_&_caicos_islands": "🇹🇨", + "tuvalu": "🇹🇻", + "u.s._outlying_islands": "🇺🇲", + "u.s._virgin_islands": "🇻🇮", + "up!_button": "🆙", + "uganda": "🇺🇬", + "ukraine": "🇺🇦", + "united_arab_emirates": "🇦🇪", + "united_kingdom": "🇬🇧", + "united_nations": "🇺🇳", + "united_states": "🇺🇸", + "uruguay": "🇺🇾", + "uzbekistan": "🇺🇿", + "vs_button": "🆚", + "vanuatu": "🇻🇺", + "vatican_city": "🇻🇦", + "venezuela": "🇻🇪", + "vietnam": "🇻🇳", + "virgo": "♍", + "wales": "🏴\U000e0067\U000e0062\U000e0077\U000e006c\U000e0073\U000e007f", + "wallis_&_futuna": "🇼🇫", + "western_sahara": "🇪🇭", + "yemen": "🇾🇪", + "zambia": "🇿🇲", + "zimbabwe": "🇿🇼", + "abacus": "🧮", + "adhesive_bandage": "🩹", + "admission_tickets": "🎟", + "adult": "🧑", + "adult_dark_skin_tone": "🧑🏿", + "adult_light_skin_tone": "🧑🏻", + "adult_medium-dark_skin_tone": "🧑🏾", + "adult_medium-light_skin_tone": "🧑🏼", + "adult_medium_skin_tone": "🧑🏽", + "aerial_tramway": "🚡", + "airplane": "✈", + "airplane_arrival": "🛬", + "airplane_departure": "🛫", + "alarm_clock": "⏰", + "alembic": "⚗", + "alien": "👽", + "alien_monster": "👾", + "ambulance": "🚑", + "american_football": "🏈", + "amphora": "🏺", + "anchor": "⚓", + "anger_symbol": "💢", + "angry_face": "😠", + "angry_face_with_horns": "👿", + "anguished_face": "😧", + "ant": "🐜", + "antenna_bars": "📶", + "anxious_face_with_sweat": "😰", + "articulated_lorry": "🚛", + "artist_palette": "🎨", + "astonished_face": "😲", + "atom_symbol": "⚛", + "auto_rickshaw": "🛺", + "automobile": "🚗", + "avocado": "🥑", + "axe": "🪓", + "baby": "👶", + "baby_angel": "👼", + "baby_angel_dark_skin_tone": "👼🏿", + "baby_angel_light_skin_tone": "👼🏻", + "baby_angel_medium-dark_skin_tone": "👼🏾", + "baby_angel_medium-light_skin_tone": "👼🏼", + "baby_angel_medium_skin_tone": "👼🏽", + "baby_bottle": "🍼", + "baby_chick": "🐤", + "baby_dark_skin_tone": "👶🏿", + "baby_light_skin_tone": "👶🏻", + "baby_medium-dark_skin_tone": "👶🏾", + "baby_medium-light_skin_tone": "👶🏼", + "baby_medium_skin_tone": "👶🏽", + "baby_symbol": "🚼", + "backhand_index_pointing_down": "👇", + "backhand_index_pointing_down_dark_skin_tone": "👇🏿", + "backhand_index_pointing_down_light_skin_tone": "👇🏻", + "backhand_index_pointing_down_medium-dark_skin_tone": "👇🏾", + "backhand_index_pointing_down_medium-light_skin_tone": "👇🏼", + "backhand_index_pointing_down_medium_skin_tone": "👇🏽", + "backhand_index_pointing_left": "👈", + "backhand_index_pointing_left_dark_skin_tone": "👈🏿", + "backhand_index_pointing_left_light_skin_tone": "👈🏻", + "backhand_index_pointing_left_medium-dark_skin_tone": "👈🏾", + "backhand_index_pointing_left_medium-light_skin_tone": "👈🏼", + "backhand_index_pointing_left_medium_skin_tone": "👈🏽", + "backhand_index_pointing_right": "👉", + "backhand_index_pointing_right_dark_skin_tone": "👉🏿", + "backhand_index_pointing_right_light_skin_tone": "👉🏻", + "backhand_index_pointing_right_medium-dark_skin_tone": "👉🏾", + "backhand_index_pointing_right_medium-light_skin_tone": "👉🏼", + "backhand_index_pointing_right_medium_skin_tone": "👉🏽", + "backhand_index_pointing_up": "👆", + "backhand_index_pointing_up_dark_skin_tone": "👆🏿", + "backhand_index_pointing_up_light_skin_tone": "👆🏻", + "backhand_index_pointing_up_medium-dark_skin_tone": "👆🏾", + "backhand_index_pointing_up_medium-light_skin_tone": "👆🏼", + "backhand_index_pointing_up_medium_skin_tone": "👆🏽", + "bacon": "🥓", + "badger": "🦡", + "badminton": "🏸", + "bagel": "🥯", + "baggage_claim": "🛄", + "baguette_bread": "🥖", + "balance_scale": "⚖", + "bald": "🦲", + "bald_man": "👨\u200d🦲", + "bald_woman": "👩\u200d🦲", + "ballet_shoes": "🩰", + "balloon": "🎈", + "ballot_box_with_ballot": "🗳", + "ballot_box_with_check": "☑", + "banana": "🍌", + "banjo": "🪕", + "bank": "🏦", + "bar_chart": "📊", + "barber_pole": "💈", + "baseball": "⚾", + "basket": "🧺", + "basketball": "🏀", + "bat": "🦇", + "bathtub": "🛁", + "battery": "🔋", + "beach_with_umbrella": "🏖", + "beaming_face_with_smiling_eyes": "😁", + "bear_face": "🐻", + "bearded_person": "🧔", + "bearded_person_dark_skin_tone": "🧔🏿", + "bearded_person_light_skin_tone": "🧔🏻", + "bearded_person_medium-dark_skin_tone": "🧔🏾", + "bearded_person_medium-light_skin_tone": "🧔🏼", + "bearded_person_medium_skin_tone": "🧔🏽", + "beating_heart": "💓", + "bed": "🛏", + "beer_mug": "🍺", + "bell": "🔔", + "bell_with_slash": "🔕", + "bellhop_bell": "🛎", + "bento_box": "🍱", + "beverage_box": "🧃", + "bicycle": "🚲", + "bikini": "👙", + "billed_cap": "🧢", + "biohazard": "☣", + "bird": "🐦", + "birthday_cake": "🎂", + "black_circle": "⚫", + "black_flag": "🏴", + "black_heart": "🖤", + "black_large_square": "⬛", + "black_medium-small_square": "◾", + "black_medium_square": "◼", + "black_nib": "✒", + "black_small_square": "▪", + "black_square_button": "🔲", + "blond-haired_man": "👱\u200d♂️", + "blond-haired_man_dark_skin_tone": "👱🏿\u200d♂️", + "blond-haired_man_light_skin_tone": "👱🏻\u200d♂️", + "blond-haired_man_medium-dark_skin_tone": "👱🏾\u200d♂️", + "blond-haired_man_medium-light_skin_tone": "👱🏼\u200d♂️", + "blond-haired_man_medium_skin_tone": "👱🏽\u200d♂️", + "blond-haired_person": "👱", + "blond-haired_person_dark_skin_tone": "👱🏿", + "blond-haired_person_light_skin_tone": "👱🏻", + "blond-haired_person_medium-dark_skin_tone": "👱🏾", + "blond-haired_person_medium-light_skin_tone": "👱🏼", + "blond-haired_person_medium_skin_tone": "👱🏽", + "blond-haired_woman": "👱\u200d♀️", + "blond-haired_woman_dark_skin_tone": "👱🏿\u200d♀️", + "blond-haired_woman_light_skin_tone": "👱🏻\u200d♀️", + "blond-haired_woman_medium-dark_skin_tone": "👱🏾\u200d♀️", + "blond-haired_woman_medium-light_skin_tone": "👱🏼\u200d♀️", + "blond-haired_woman_medium_skin_tone": "👱🏽\u200d♀️", + "blossom": "🌼", + "blowfish": "🐡", + "blue_book": "📘", + "blue_circle": "🔵", + "blue_heart": "💙", + "blue_square": "🟦", + "boar": "🐗", + "bomb": "💣", + "bone": "🦴", + "bookmark": "🔖", + "bookmark_tabs": "📑", + "books": "📚", + "bottle_with_popping_cork": "🍾", + "bouquet": "💐", + "bow_and_arrow": "🏹", + "bowl_with_spoon": "🥣", + "bowling": "🎳", + "boxing_glove": "🥊", + "boy": "👦", + "boy_dark_skin_tone": "👦🏿", + "boy_light_skin_tone": "👦🏻", + "boy_medium-dark_skin_tone": "👦🏾", + "boy_medium-light_skin_tone": "👦🏼", + "boy_medium_skin_tone": "👦🏽", + "brain": "🧠", + "bread": "🍞", + "breast-feeding": "🤱", + "breast-feeding_dark_skin_tone": "🤱🏿", + "breast-feeding_light_skin_tone": "🤱🏻", + "breast-feeding_medium-dark_skin_tone": "🤱🏾", + "breast-feeding_medium-light_skin_tone": "🤱🏼", + "breast-feeding_medium_skin_tone": "🤱🏽", + "brick": "🧱", + "bride_with_veil": "👰", + "bride_with_veil_dark_skin_tone": "👰🏿", + "bride_with_veil_light_skin_tone": "👰🏻", + "bride_with_veil_medium-dark_skin_tone": "👰🏾", + "bride_with_veil_medium-light_skin_tone": "👰🏼", + "bride_with_veil_medium_skin_tone": "👰🏽", + "bridge_at_night": "🌉", + "briefcase": "💼", + "briefs": "🩲", + "bright_button": "🔆", + "broccoli": "🥦", + "broken_heart": "💔", + "broom": "🧹", + "brown_circle": "🟤", + "brown_heart": "🤎", + "brown_square": "🟫", + "bug": "🐛", + "building_construction": "🏗", + "bullet_train": "🚅", + "burrito": "🌯", + "bus": "🚌", + "bus_stop": "🚏", + "bust_in_silhouette": "👤", + "busts_in_silhouette": "👥", + "butter": "🧈", + "butterfly": "🦋", + "cactus": "🌵", + "calendar": "📆", + "call_me_hand": "🤙", + "call_me_hand_dark_skin_tone": "🤙🏿", + "call_me_hand_light_skin_tone": "🤙🏻", + "call_me_hand_medium-dark_skin_tone": "🤙🏾", + "call_me_hand_medium-light_skin_tone": "🤙🏼", + "call_me_hand_medium_skin_tone": "🤙🏽", + "camel": "🐫", + "camera": "📷", + "camera_with_flash": "📸", + "camping": "🏕", + "candle": "🕯", + "candy": "🍬", + "canned_food": "🥫", + "canoe": "🛶", + "card_file_box": "🗃", + "card_index": "📇", + "card_index_dividers": "🗂", + "carousel_horse": "🎠", + "carp_streamer": "🎏", + "carrot": "🥕", + "castle": "🏰", + "cat": "🐱", + "cat_face": "🐱", + "cat_face_with_tears_of_joy": "😹", + "cat_face_with_wry_smile": "😼", + "chains": "⛓", + "chair": "🪑", + "chart_decreasing": "📉", + "chart_increasing": "📈", + "chart_increasing_with_yen": "💹", + "cheese_wedge": "🧀", + "chequered_flag": "🏁", + "cherries": "🍒", + "cherry_blossom": "🌸", + "chess_pawn": "♟", + "chestnut": "🌰", + "chicken": "🐔", + "child": "🧒", + "child_dark_skin_tone": "🧒🏿", + "child_light_skin_tone": "🧒🏻", + "child_medium-dark_skin_tone": "🧒🏾", + "child_medium-light_skin_tone": "🧒🏼", + "child_medium_skin_tone": "🧒🏽", + "children_crossing": "🚸", + "chipmunk": "🐿", + "chocolate_bar": "🍫", + "chopsticks": "🥢", + "church": "⛪", + "cigarette": "🚬", + "cinema": "🎦", + "circled_m": "Ⓜ", + "circus_tent": "🎪", + "cityscape": "🏙", + "cityscape_at_dusk": "🌆", + "clamp": "🗜", + "clapper_board": "🎬", + "clapping_hands": "👏", + "clapping_hands_dark_skin_tone": "👏🏿", + "clapping_hands_light_skin_tone": "👏🏻", + "clapping_hands_medium-dark_skin_tone": "👏🏾", + "clapping_hands_medium-light_skin_tone": "👏🏼", + "clapping_hands_medium_skin_tone": "👏🏽", + "classical_building": "🏛", + "clinking_beer_mugs": "🍻", + "clinking_glasses": "🥂", + "clipboard": "📋", + "clockwise_vertical_arrows": "🔃", + "closed_book": "📕", + "closed_mailbox_with_lowered_flag": "📪", + "closed_mailbox_with_raised_flag": "📫", + "closed_umbrella": "🌂", + "cloud": "☁", + "cloud_with_lightning": "🌩", + "cloud_with_lightning_and_rain": "⛈", + "cloud_with_rain": "🌧", + "cloud_with_snow": "🌨", + "clown_face": "🤡", + "club_suit": "♣", + "clutch_bag": "👝", + "coat": "🧥", + "cocktail_glass": "🍸", + "coconut": "🥥", + "coffin": "⚰", + "cold_face": "🥶", + "collision": "💥", + "comet": "☄", + "compass": "🧭", + "computer_disk": "💽", + "computer_mouse": "🖱", + "confetti_ball": "🎊", + "confounded_face": "😖", + "confused_face": "😕", + "construction": "🚧", + "construction_worker": "👷", + "construction_worker_dark_skin_tone": "👷🏿", + "construction_worker_light_skin_tone": "👷🏻", + "construction_worker_medium-dark_skin_tone": "👷🏾", + "construction_worker_medium-light_skin_tone": "👷🏼", + "construction_worker_medium_skin_tone": "👷🏽", + "control_knobs": "🎛", + "convenience_store": "🏪", + "cooked_rice": "🍚", + "cookie": "🍪", + "cooking": "🍳", + "copyright": "©", + "couch_and_lamp": "🛋", + "counterclockwise_arrows_button": "🔄", + "couple_with_heart": "💑", + "couple_with_heart_man_man": "👨\u200d❤️\u200d👨", + "couple_with_heart_woman_man": "👩\u200d❤️\u200d👨", + "couple_with_heart_woman_woman": "👩\u200d❤️\u200d👩", + "cow": "🐮", + "cow_face": "🐮", + "cowboy_hat_face": "🤠", + "crab": "🦀", + "crayon": "🖍", + "credit_card": "💳", + "crescent_moon": "🌙", + "cricket": "🦗", + "cricket_game": "🏏", + "crocodile": "🐊", + "croissant": "🥐", + "cross_mark": "❌", + "cross_mark_button": "❎", + "crossed_fingers": "🤞", + "crossed_fingers_dark_skin_tone": "🤞🏿", + "crossed_fingers_light_skin_tone": "🤞🏻", + "crossed_fingers_medium-dark_skin_tone": "🤞🏾", + "crossed_fingers_medium-light_skin_tone": "🤞🏼", + "crossed_fingers_medium_skin_tone": "🤞🏽", + "crossed_flags": "🎌", + "crossed_swords": "⚔", + "crown": "👑", + "crying_cat_face": "😿", + "crying_face": "😢", + "crystal_ball": "🔮", + "cucumber": "🥒", + "cupcake": "🧁", + "cup_with_straw": "🥤", + "curling_stone": "🥌", + "curly_hair": "🦱", + "curly-haired_man": "👨\u200d🦱", + "curly-haired_woman": "👩\u200d🦱", + "curly_loop": "➰", + "currency_exchange": "💱", + "curry_rice": "🍛", + "custard": "🍮", + "customs": "🛃", + "cut_of_meat": "🥩", + "cyclone": "🌀", + "dagger": "🗡", + "dango": "🍡", + "dashing_away": "💨", + "deaf_person": "🧏", + "deciduous_tree": "🌳", + "deer": "🦌", + "delivery_truck": "🚚", + "department_store": "🏬", + "derelict_house": "🏚", + "desert": "🏜", + "desert_island": "🏝", + "desktop_computer": "🖥", + "detective": "🕵", + "detective_dark_skin_tone": "🕵🏿", + "detective_light_skin_tone": "🕵🏻", + "detective_medium-dark_skin_tone": "🕵🏾", + "detective_medium-light_skin_tone": "🕵🏼", + "detective_medium_skin_tone": "🕵🏽", + "diamond_suit": "♦", + "diamond_with_a_dot": "💠", + "dim_button": "🔅", + "direct_hit": "🎯", + "disappointed_face": "😞", + "diving_mask": "🤿", + "diya_lamp": "🪔", + "dizzy": "💫", + "dizzy_face": "😵", + "dna": "🧬", + "dog": "🐶", + "dog_face": "🐶", + "dollar_banknote": "💵", + "dolphin": "🐬", + "door": "🚪", + "dotted_six-pointed_star": "🔯", + "double_curly_loop": "➿", + "double_exclamation_mark": "‼", + "doughnut": "🍩", + "dove": "🕊", + "down-left_arrow": "↙", + "down-right_arrow": "↘", + "down_arrow": "⬇", + "downcast_face_with_sweat": "😓", + "downwards_button": "🔽", + "dragon": "🐉", + "dragon_face": "🐲", + "dress": "👗", + "drooling_face": "🤤", + "drop_of_blood": "🩸", + "droplet": "💧", + "drum": "🥁", + "duck": "🦆", + "dumpling": "🥟", + "dvd": "📀", + "e-mail": "📧", + "eagle": "🦅", + "ear": "👂", + "ear_dark_skin_tone": "👂🏿", + "ear_light_skin_tone": "👂🏻", + "ear_medium-dark_skin_tone": "👂🏾", + "ear_medium-light_skin_tone": "👂🏼", + "ear_medium_skin_tone": "👂🏽", + "ear_of_corn": "🌽", + "ear_with_hearing_aid": "🦻", + "egg": "🍳", + "eggplant": "🍆", + "eight-pointed_star": "✴", + "eight-spoked_asterisk": "✳", + "eight-thirty": "🕣", + "eight_o’clock": "🕗", + "eject_button": "⏏", + "electric_plug": "🔌", + "elephant": "🐘", + "eleven-thirty": "🕦", + "eleven_o’clock": "🕚", + "elf": "🧝", + "elf_dark_skin_tone": "🧝🏿", + "elf_light_skin_tone": "🧝🏻", + "elf_medium-dark_skin_tone": "🧝🏾", + "elf_medium-light_skin_tone": "🧝🏼", + "elf_medium_skin_tone": "🧝🏽", + "envelope": "✉", + "envelope_with_arrow": "📩", + "euro_banknote": "💶", + "evergreen_tree": "🌲", + "ewe": "🐑", + "exclamation_mark": "❗", + "exclamation_question_mark": "⁉", + "exploding_head": "🤯", + "expressionless_face": "😑", + "eye": "👁", + "eye_in_speech_bubble": "👁️\u200d🗨️", + "eyes": "👀", + "face_blowing_a_kiss": "😘", + "face_savoring_food": "😋", + "face_screaming_in_fear": "😱", + "face_vomiting": "🤮", + "face_with_hand_over_mouth": "🤭", + "face_with_head-bandage": "🤕", + "face_with_medical_mask": "😷", + "face_with_monocle": "🧐", + "face_with_open_mouth": "😮", + "face_with_raised_eyebrow": "🤨", + "face_with_rolling_eyes": "🙄", + "face_with_steam_from_nose": "😤", + "face_with_symbols_on_mouth": "🤬", + "face_with_tears_of_joy": "😂", + "face_with_thermometer": "🤒", + "face_with_tongue": "😛", + "face_without_mouth": "😶", + "factory": "🏭", + "fairy": "🧚", + "fairy_dark_skin_tone": "🧚🏿", + "fairy_light_skin_tone": "🧚🏻", + "fairy_medium-dark_skin_tone": "🧚🏾", + "fairy_medium-light_skin_tone": "🧚🏼", + "fairy_medium_skin_tone": "🧚🏽", + "falafel": "🧆", + "fallen_leaf": "🍂", + "family": "👪", + "family_man_boy": "👨\u200d👦", + "family_man_boy_boy": "👨\u200d👦\u200d👦", + "family_man_girl": "👨\u200d👧", + "family_man_girl_boy": "👨\u200d👧\u200d👦", + "family_man_girl_girl": "👨\u200d👧\u200d👧", + "family_man_man_boy": "👨\u200d👨\u200d👦", + "family_man_man_boy_boy": "👨\u200d👨\u200d👦\u200d👦", + "family_man_man_girl": "👨\u200d👨\u200d👧", + "family_man_man_girl_boy": "👨\u200d👨\u200d👧\u200d👦", + "family_man_man_girl_girl": "👨\u200d👨\u200d👧\u200d👧", + "family_man_woman_boy": "👨\u200d👩\u200d👦", + "family_man_woman_boy_boy": "👨\u200d👩\u200d👦\u200d👦", + "family_man_woman_girl": "👨\u200d👩\u200d👧", + "family_man_woman_girl_boy": "👨\u200d👩\u200d👧\u200d👦", + "family_man_woman_girl_girl": "👨\u200d👩\u200d👧\u200d👧", + "family_woman_boy": "👩\u200d👦", + "family_woman_boy_boy": "👩\u200d👦\u200d👦", + "family_woman_girl": "👩\u200d👧", + "family_woman_girl_boy": "👩\u200d👧\u200d👦", + "family_woman_girl_girl": "👩\u200d👧\u200d👧", + "family_woman_woman_boy": "👩\u200d👩\u200d👦", + "family_woman_woman_boy_boy": "👩\u200d👩\u200d👦\u200d👦", + "family_woman_woman_girl": "👩\u200d👩\u200d👧", + "family_woman_woman_girl_boy": "👩\u200d👩\u200d👧\u200d👦", + "family_woman_woman_girl_girl": "👩\u200d👩\u200d👧\u200d👧", + "fast-forward_button": "⏩", + "fast_down_button": "⏬", + "fast_reverse_button": "⏪", + "fast_up_button": "⏫", + "fax_machine": "📠", + "fearful_face": "😨", + "female_sign": "♀", + "ferris_wheel": "🎡", + "ferry": "⛴", + "field_hockey": "🏑", + "file_cabinet": "🗄", + "file_folder": "📁", + "film_frames": "🎞", + "film_projector": "📽", + "fire": "🔥", + "fire_extinguisher": "🧯", + "firecracker": "🧨", + "fire_engine": "🚒", + "fireworks": "🎆", + "first_quarter_moon": "🌓", + "first_quarter_moon_face": "🌛", + "fish": "🐟", + "fish_cake_with_swirl": "🍥", + "fishing_pole": "🎣", + "five-thirty": "🕠", + "five_o’clock": "🕔", + "flag_in_hole": "⛳", + "flamingo": "🦩", + "flashlight": "🔦", + "flat_shoe": "🥿", + "fleur-de-lis": "⚜", + "flexed_biceps": "💪", + "flexed_biceps_dark_skin_tone": "💪🏿", + "flexed_biceps_light_skin_tone": "💪🏻", + "flexed_biceps_medium-dark_skin_tone": "💪🏾", + "flexed_biceps_medium-light_skin_tone": "💪🏼", + "flexed_biceps_medium_skin_tone": "💪🏽", + "floppy_disk": "💾", + "flower_playing_cards": "🎴", + "flushed_face": "😳", + "flying_disc": "🥏", + "flying_saucer": "🛸", + "fog": "🌫", + "foggy": "🌁", + "folded_hands": "🙏", + "folded_hands_dark_skin_tone": "🙏🏿", + "folded_hands_light_skin_tone": "🙏🏻", + "folded_hands_medium-dark_skin_tone": "🙏🏾", + "folded_hands_medium-light_skin_tone": "🙏🏼", + "folded_hands_medium_skin_tone": "🙏🏽", + "foot": "🦶", + "footprints": "👣", + "fork_and_knife": "🍴", + "fork_and_knife_with_plate": "🍽", + "fortune_cookie": "🥠", + "fountain": "⛲", + "fountain_pen": "🖋", + "four-thirty": "🕟", + "four_leaf_clover": "🍀", + "four_o’clock": "🕓", + "fox_face": "🦊", + "framed_picture": "🖼", + "french_fries": "🍟", + "fried_shrimp": "🍤", + "frog_face": "🐸", + "front-facing_baby_chick": "🐥", + "frowning_face": "☹", + "frowning_face_with_open_mouth": "😦", + "fuel_pump": "⛽", + "full_moon": "🌕", + "full_moon_face": "🌝", + "funeral_urn": "⚱", + "game_die": "🎲", + "garlic": "🧄", + "gear": "⚙", + "gem_stone": "💎", + "genie": "🧞", + "ghost": "👻", + "giraffe": "🦒", + "girl": "👧", + "girl_dark_skin_tone": "👧🏿", + "girl_light_skin_tone": "👧🏻", + "girl_medium-dark_skin_tone": "👧🏾", + "girl_medium-light_skin_tone": "👧🏼", + "girl_medium_skin_tone": "👧🏽", + "glass_of_milk": "🥛", + "glasses": "👓", + "globe_showing_americas": "🌎", + "globe_showing_asia-australia": "🌏", + "globe_showing_europe-africa": "🌍", + "globe_with_meridians": "🌐", + "gloves": "🧤", + "glowing_star": "🌟", + "goal_net": "🥅", + "goat": "🐐", + "goblin": "👺", + "goggles": "🥽", + "gorilla": "🦍", + "graduation_cap": "🎓", + "grapes": "🍇", + "green_apple": "🍏", + "green_book": "📗", + "green_circle": "🟢", + "green_heart": "💚", + "green_salad": "🥗", + "green_square": "🟩", + "grimacing_face": "😬", + "grinning_cat_face": "😺", + "grinning_cat_face_with_smiling_eyes": "😸", + "grinning_face": "😀", + "grinning_face_with_big_eyes": "😃", + "grinning_face_with_smiling_eyes": "😄", + "grinning_face_with_sweat": "😅", + "grinning_squinting_face": "😆", + "growing_heart": "💗", + "guard": "💂", + "guard_dark_skin_tone": "💂🏿", + "guard_light_skin_tone": "💂🏻", + "guard_medium-dark_skin_tone": "💂🏾", + "guard_medium-light_skin_tone": "💂🏼", + "guard_medium_skin_tone": "💂🏽", + "guide_dog": "🦮", + "guitar": "🎸", + "hamburger": "🍔", + "hammer": "🔨", + "hammer_and_pick": "⚒", + "hammer_and_wrench": "🛠", + "hamster_face": "🐹", + "hand_with_fingers_splayed": "🖐", + "hand_with_fingers_splayed_dark_skin_tone": "🖐🏿", + "hand_with_fingers_splayed_light_skin_tone": "🖐🏻", + "hand_with_fingers_splayed_medium-dark_skin_tone": "🖐🏾", + "hand_with_fingers_splayed_medium-light_skin_tone": "🖐🏼", + "hand_with_fingers_splayed_medium_skin_tone": "🖐🏽", + "handbag": "👜", + "handshake": "🤝", + "hatching_chick": "🐣", + "headphone": "🎧", + "hear-no-evil_monkey": "🙉", + "heart_decoration": "💟", + "heart_suit": "♥", + "heart_with_arrow": "💘", + "heart_with_ribbon": "💝", + "heavy_check_mark": "✔", + "heavy_division_sign": "➗", + "heavy_dollar_sign": "💲", + "heavy_heart_exclamation": "❣", + "heavy_large_circle": "⭕", + "heavy_minus_sign": "➖", + "heavy_multiplication_x": "✖", + "heavy_plus_sign": "➕", + "hedgehog": "🦔", + "helicopter": "🚁", + "herb": "🌿", + "hibiscus": "🌺", + "high-heeled_shoe": "👠", + "high-speed_train": "🚄", + "high_voltage": "⚡", + "hiking_boot": "🥾", + "hindu_temple": "🛕", + "hippopotamus": "🦛", + "hole": "🕳", + "honey_pot": "🍯", + "honeybee": "🐝", + "horizontal_traffic_light": "🚥", + "horse": "🐴", + "horse_face": "🐴", + "horse_racing": "🏇", + "horse_racing_dark_skin_tone": "🏇🏿", + "horse_racing_light_skin_tone": "🏇🏻", + "horse_racing_medium-dark_skin_tone": "🏇🏾", + "horse_racing_medium-light_skin_tone": "🏇🏼", + "horse_racing_medium_skin_tone": "🏇🏽", + "hospital": "🏥", + "hot_beverage": "☕", + "hot_dog": "🌭", + "hot_face": "🥵", + "hot_pepper": "🌶", + "hot_springs": "♨", + "hotel": "🏨", + "hourglass_done": "⌛", + "hourglass_not_done": "⏳", + "house": "🏠", + "house_with_garden": "🏡", + "houses": "🏘", + "hugging_face": "🤗", + "hundred_points": "💯", + "hushed_face": "😯", + "ice": "🧊", + "ice_cream": "🍨", + "ice_hockey": "🏒", + "ice_skate": "⛸", + "inbox_tray": "📥", + "incoming_envelope": "📨", + "index_pointing_up": "☝", + "index_pointing_up_dark_skin_tone": "☝🏿", + "index_pointing_up_light_skin_tone": "☝🏻", + "index_pointing_up_medium-dark_skin_tone": "☝🏾", + "index_pointing_up_medium-light_skin_tone": "☝🏼", + "index_pointing_up_medium_skin_tone": "☝🏽", + "infinity": "♾", + "information": "ℹ", + "input_latin_letters": "🔤", + "input_latin_lowercase": "🔡", + "input_latin_uppercase": "🔠", + "input_numbers": "🔢", + "input_symbols": "🔣", + "jack-o-lantern": "🎃", + "jeans": "👖", + "jigsaw": "🧩", + "joker": "🃏", + "joystick": "🕹", + "kaaba": "🕋", + "kangaroo": "🦘", + "key": "🔑", + "keyboard": "⌨", + "keycap_#": "#️⃣", + "keycap_*": "*️⃣", + "keycap_0": "0️⃣", + "keycap_1": "1️⃣", + "keycap_10": "🔟", + "keycap_2": "2️⃣", + "keycap_3": "3️⃣", + "keycap_4": "4️⃣", + "keycap_5": "5️⃣", + "keycap_6": "6️⃣", + "keycap_7": "7️⃣", + "keycap_8": "8️⃣", + "keycap_9": "9️⃣", + "kick_scooter": "🛴", + "kimono": "👘", + "kiss": "💋", + "kiss_man_man": "👨\u200d❤️\u200d💋\u200d👨", + "kiss_mark": "💋", + "kiss_woman_man": "👩\u200d❤️\u200d💋\u200d👨", + "kiss_woman_woman": "👩\u200d❤️\u200d💋\u200d👩", + "kissing_cat_face": "😽", + "kissing_face": "😗", + "kissing_face_with_closed_eyes": "😚", + "kissing_face_with_smiling_eyes": "😙", + "kitchen_knife": "🔪", + "kite": "🪁", + "kiwi_fruit": "🥝", + "koala": "🐨", + "lab_coat": "🥼", + "label": "🏷", + "lacrosse": "🥍", + "lady_beetle": "🐞", + "laptop_computer": "💻", + "large_blue_diamond": "🔷", + "large_orange_diamond": "🔶", + "last_quarter_moon": "🌗", + "last_quarter_moon_face": "🌜", + "last_track_button": "⏮", + "latin_cross": "✝", + "leaf_fluttering_in_wind": "🍃", + "leafy_green": "🥬", + "ledger": "📒", + "left-facing_fist": "🤛", + "left-facing_fist_dark_skin_tone": "🤛🏿", + "left-facing_fist_light_skin_tone": "🤛🏻", + "left-facing_fist_medium-dark_skin_tone": "🤛🏾", + "left-facing_fist_medium-light_skin_tone": "🤛🏼", + "left-facing_fist_medium_skin_tone": "🤛🏽", + "left-right_arrow": "↔", + "left_arrow": "⬅", + "left_arrow_curving_right": "↪", + "left_luggage": "🛅", + "left_speech_bubble": "🗨", + "leg": "🦵", + "lemon": "🍋", + "leopard": "🐆", + "level_slider": "🎚", + "light_bulb": "💡", + "light_rail": "🚈", + "link": "🔗", + "linked_paperclips": "🖇", + "lion_face": "🦁", + "lipstick": "💄", + "litter_in_bin_sign": "🚮", + "lizard": "🦎", + "llama": "🦙", + "lobster": "🦞", + "locked": "🔒", + "locked_with_key": "🔐", + "locked_with_pen": "🔏", + "locomotive": "🚂", + "lollipop": "🍭", + "lotion_bottle": "🧴", + "loudly_crying_face": "😭", + "loudspeaker": "📢", + "love-you_gesture": "🤟", + "love-you_gesture_dark_skin_tone": "🤟🏿", + "love-you_gesture_light_skin_tone": "🤟🏻", + "love-you_gesture_medium-dark_skin_tone": "🤟🏾", + "love-you_gesture_medium-light_skin_tone": "🤟🏼", + "love-you_gesture_medium_skin_tone": "🤟🏽", + "love_hotel": "🏩", + "love_letter": "💌", + "luggage": "🧳", + "lying_face": "🤥", + "mage": "🧙", + "mage_dark_skin_tone": "🧙🏿", + "mage_light_skin_tone": "🧙🏻", + "mage_medium-dark_skin_tone": "🧙🏾", + "mage_medium-light_skin_tone": "🧙🏼", + "mage_medium_skin_tone": "🧙🏽", + "magnet": "🧲", + "magnifying_glass_tilted_left": "🔍", + "magnifying_glass_tilted_right": "🔎", + "mahjong_red_dragon": "🀄", + "male_sign": "♂", + "man": "👨", + "man_and_woman_holding_hands": "👫", + "man_artist": "👨\u200d🎨", + "man_artist_dark_skin_tone": "👨🏿\u200d🎨", + "man_artist_light_skin_tone": "👨🏻\u200d🎨", + "man_artist_medium-dark_skin_tone": "👨🏾\u200d🎨", + "man_artist_medium-light_skin_tone": "👨🏼\u200d🎨", + "man_artist_medium_skin_tone": "👨🏽\u200d🎨", + "man_astronaut": "👨\u200d🚀", + "man_astronaut_dark_skin_tone": "👨🏿\u200d🚀", + "man_astronaut_light_skin_tone": "👨🏻\u200d🚀", + "man_astronaut_medium-dark_skin_tone": "👨🏾\u200d🚀", + "man_astronaut_medium-light_skin_tone": "👨🏼\u200d🚀", + "man_astronaut_medium_skin_tone": "👨🏽\u200d🚀", + "man_biking": "🚴\u200d♂️", + "man_biking_dark_skin_tone": "🚴🏿\u200d♂️", + "man_biking_light_skin_tone": "🚴🏻\u200d♂️", + "man_biking_medium-dark_skin_tone": "🚴🏾\u200d♂️", + "man_biking_medium-light_skin_tone": "🚴🏼\u200d♂️", + "man_biking_medium_skin_tone": "🚴🏽\u200d♂️", + "man_bouncing_ball": "⛹️\u200d♂️", + "man_bouncing_ball_dark_skin_tone": "⛹🏿\u200d♂️", + "man_bouncing_ball_light_skin_tone": "⛹🏻\u200d♂️", + "man_bouncing_ball_medium-dark_skin_tone": "⛹🏾\u200d♂️", + "man_bouncing_ball_medium-light_skin_tone": "⛹🏼\u200d♂️", + "man_bouncing_ball_medium_skin_tone": "⛹🏽\u200d♂️", + "man_bowing": "🙇\u200d♂️", + "man_bowing_dark_skin_tone": "🙇🏿\u200d♂️", + "man_bowing_light_skin_tone": "🙇🏻\u200d♂️", + "man_bowing_medium-dark_skin_tone": "🙇🏾\u200d♂️", + "man_bowing_medium-light_skin_tone": "🙇🏼\u200d♂️", + "man_bowing_medium_skin_tone": "🙇🏽\u200d♂️", + "man_cartwheeling": "🤸\u200d♂️", + "man_cartwheeling_dark_skin_tone": "🤸🏿\u200d♂️", + "man_cartwheeling_light_skin_tone": "🤸🏻\u200d♂️", + "man_cartwheeling_medium-dark_skin_tone": "🤸🏾\u200d♂️", + "man_cartwheeling_medium-light_skin_tone": "🤸🏼\u200d♂️", + "man_cartwheeling_medium_skin_tone": "🤸🏽\u200d♂️", + "man_climbing": "🧗\u200d♂️", + "man_climbing_dark_skin_tone": "🧗🏿\u200d♂️", + "man_climbing_light_skin_tone": "🧗🏻\u200d♂️", + "man_climbing_medium-dark_skin_tone": "🧗🏾\u200d♂️", + "man_climbing_medium-light_skin_tone": "🧗🏼\u200d♂️", + "man_climbing_medium_skin_tone": "🧗🏽\u200d♂️", + "man_construction_worker": "👷\u200d♂️", + "man_construction_worker_dark_skin_tone": "👷🏿\u200d♂️", + "man_construction_worker_light_skin_tone": "👷🏻\u200d♂️", + "man_construction_worker_medium-dark_skin_tone": "👷🏾\u200d♂️", + "man_construction_worker_medium-light_skin_tone": "👷🏼\u200d♂️", + "man_construction_worker_medium_skin_tone": "👷🏽\u200d♂️", + "man_cook": "👨\u200d🍳", + "man_cook_dark_skin_tone": "👨🏿\u200d🍳", + "man_cook_light_skin_tone": "👨🏻\u200d🍳", + "man_cook_medium-dark_skin_tone": "👨🏾\u200d🍳", + "man_cook_medium-light_skin_tone": "👨🏼\u200d🍳", + "man_cook_medium_skin_tone": "👨🏽\u200d🍳", + "man_dancing": "🕺", + "man_dancing_dark_skin_tone": "🕺🏿", + "man_dancing_light_skin_tone": "🕺🏻", + "man_dancing_medium-dark_skin_tone": "🕺🏾", + "man_dancing_medium-light_skin_tone": "🕺🏼", + "man_dancing_medium_skin_tone": "🕺🏽", + "man_dark_skin_tone": "👨🏿", + "man_detective": "🕵️\u200d♂️", + "man_detective_dark_skin_tone": "🕵🏿\u200d♂️", + "man_detective_light_skin_tone": "🕵🏻\u200d♂️", + "man_detective_medium-dark_skin_tone": "🕵🏾\u200d♂️", + "man_detective_medium-light_skin_tone": "🕵🏼\u200d♂️", + "man_detective_medium_skin_tone": "🕵🏽\u200d♂️", + "man_elf": "🧝\u200d♂️", + "man_elf_dark_skin_tone": "🧝🏿\u200d♂️", + "man_elf_light_skin_tone": "🧝🏻\u200d♂️", + "man_elf_medium-dark_skin_tone": "🧝🏾\u200d♂️", + "man_elf_medium-light_skin_tone": "🧝🏼\u200d♂️", + "man_elf_medium_skin_tone": "🧝🏽\u200d♂️", + "man_facepalming": "🤦\u200d♂️", + "man_facepalming_dark_skin_tone": "🤦🏿\u200d♂️", + "man_facepalming_light_skin_tone": "🤦🏻\u200d♂️", + "man_facepalming_medium-dark_skin_tone": "🤦🏾\u200d♂️", + "man_facepalming_medium-light_skin_tone": "🤦🏼\u200d♂️", + "man_facepalming_medium_skin_tone": "🤦🏽\u200d♂️", + "man_factory_worker": "👨\u200d🏭", + "man_factory_worker_dark_skin_tone": "👨🏿\u200d🏭", + "man_factory_worker_light_skin_tone": "👨🏻\u200d🏭", + "man_factory_worker_medium-dark_skin_tone": "👨🏾\u200d🏭", + "man_factory_worker_medium-light_skin_tone": "👨🏼\u200d🏭", + "man_factory_worker_medium_skin_tone": "👨🏽\u200d🏭", + "man_fairy": "🧚\u200d♂️", + "man_fairy_dark_skin_tone": "🧚🏿\u200d♂️", + "man_fairy_light_skin_tone": "🧚🏻\u200d♂️", + "man_fairy_medium-dark_skin_tone": "🧚🏾\u200d♂️", + "man_fairy_medium-light_skin_tone": "🧚🏼\u200d♂️", + "man_fairy_medium_skin_tone": "🧚🏽\u200d♂️", + "man_farmer": "👨\u200d🌾", + "man_farmer_dark_skin_tone": "👨🏿\u200d🌾", + "man_farmer_light_skin_tone": "👨🏻\u200d🌾", + "man_farmer_medium-dark_skin_tone": "👨🏾\u200d🌾", + "man_farmer_medium-light_skin_tone": "👨🏼\u200d🌾", + "man_farmer_medium_skin_tone": "👨🏽\u200d🌾", + "man_firefighter": "👨\u200d🚒", + "man_firefighter_dark_skin_tone": "👨🏿\u200d🚒", + "man_firefighter_light_skin_tone": "👨🏻\u200d🚒", + "man_firefighter_medium-dark_skin_tone": "👨🏾\u200d🚒", + "man_firefighter_medium-light_skin_tone": "👨🏼\u200d🚒", + "man_firefighter_medium_skin_tone": "👨🏽\u200d🚒", + "man_frowning": "🙍\u200d♂️", + "man_frowning_dark_skin_tone": "🙍🏿\u200d♂️", + "man_frowning_light_skin_tone": "🙍🏻\u200d♂️", + "man_frowning_medium-dark_skin_tone": "🙍🏾\u200d♂️", + "man_frowning_medium-light_skin_tone": "🙍🏼\u200d♂️", + "man_frowning_medium_skin_tone": "🙍🏽\u200d♂️", + "man_genie": "🧞\u200d♂️", + "man_gesturing_no": "🙅\u200d♂️", + "man_gesturing_no_dark_skin_tone": "🙅🏿\u200d♂️", + "man_gesturing_no_light_skin_tone": "🙅🏻\u200d♂️", + "man_gesturing_no_medium-dark_skin_tone": "🙅🏾\u200d♂️", + "man_gesturing_no_medium-light_skin_tone": "🙅🏼\u200d♂️", + "man_gesturing_no_medium_skin_tone": "🙅🏽\u200d♂️", + "man_gesturing_ok": "🙆\u200d♂️", + "man_gesturing_ok_dark_skin_tone": "🙆🏿\u200d♂️", + "man_gesturing_ok_light_skin_tone": "🙆🏻\u200d♂️", + "man_gesturing_ok_medium-dark_skin_tone": "🙆🏾\u200d♂️", + "man_gesturing_ok_medium-light_skin_tone": "🙆🏼\u200d♂️", + "man_gesturing_ok_medium_skin_tone": "🙆🏽\u200d♂️", + "man_getting_haircut": "💇\u200d♂️", + "man_getting_haircut_dark_skin_tone": "💇🏿\u200d♂️", + "man_getting_haircut_light_skin_tone": "💇🏻\u200d♂️", + "man_getting_haircut_medium-dark_skin_tone": "💇🏾\u200d♂️", + "man_getting_haircut_medium-light_skin_tone": "💇🏼\u200d♂️", + "man_getting_haircut_medium_skin_tone": "💇🏽\u200d♂️", + "man_getting_massage": "💆\u200d♂️", + "man_getting_massage_dark_skin_tone": "💆🏿\u200d♂️", + "man_getting_massage_light_skin_tone": "💆🏻\u200d♂️", + "man_getting_massage_medium-dark_skin_tone": "💆🏾\u200d♂️", + "man_getting_massage_medium-light_skin_tone": "💆🏼\u200d♂️", + "man_getting_massage_medium_skin_tone": "💆🏽\u200d♂️", + "man_golfing": "🏌️\u200d♂️", + "man_golfing_dark_skin_tone": "🏌🏿\u200d♂️", + "man_golfing_light_skin_tone": "🏌🏻\u200d♂️", + "man_golfing_medium-dark_skin_tone": "🏌🏾\u200d♂️", + "man_golfing_medium-light_skin_tone": "🏌🏼\u200d♂️", + "man_golfing_medium_skin_tone": "🏌🏽\u200d♂️", + "man_guard": "💂\u200d♂️", + "man_guard_dark_skin_tone": "💂🏿\u200d♂️", + "man_guard_light_skin_tone": "💂🏻\u200d♂️", + "man_guard_medium-dark_skin_tone": "💂🏾\u200d♂️", + "man_guard_medium-light_skin_tone": "💂🏼\u200d♂️", + "man_guard_medium_skin_tone": "💂🏽\u200d♂️", + "man_health_worker": "👨\u200d⚕️", + "man_health_worker_dark_skin_tone": "👨🏿\u200d⚕️", + "man_health_worker_light_skin_tone": "👨🏻\u200d⚕️", + "man_health_worker_medium-dark_skin_tone": "👨🏾\u200d⚕️", + "man_health_worker_medium-light_skin_tone": "👨🏼\u200d⚕️", + "man_health_worker_medium_skin_tone": "👨🏽\u200d⚕️", + "man_in_lotus_position": "🧘\u200d♂️", + "man_in_lotus_position_dark_skin_tone": "🧘🏿\u200d♂️", + "man_in_lotus_position_light_skin_tone": "🧘🏻\u200d♂️", + "man_in_lotus_position_medium-dark_skin_tone": "🧘🏾\u200d♂️", + "man_in_lotus_position_medium-light_skin_tone": "🧘🏼\u200d♂️", + "man_in_lotus_position_medium_skin_tone": "🧘🏽\u200d♂️", + "man_in_manual_wheelchair": "👨\u200d🦽", + "man_in_motorized_wheelchair": "👨\u200d🦼", + "man_in_steamy_room": "🧖\u200d♂️", + "man_in_steamy_room_dark_skin_tone": "🧖🏿\u200d♂️", + "man_in_steamy_room_light_skin_tone": "🧖🏻\u200d♂️", + "man_in_steamy_room_medium-dark_skin_tone": "🧖🏾\u200d♂️", + "man_in_steamy_room_medium-light_skin_tone": "🧖🏼\u200d♂️", + "man_in_steamy_room_medium_skin_tone": "🧖🏽\u200d♂️", + "man_in_suit_levitating": "🕴", + "man_in_suit_levitating_dark_skin_tone": "🕴🏿", + "man_in_suit_levitating_light_skin_tone": "🕴🏻", + "man_in_suit_levitating_medium-dark_skin_tone": "🕴🏾", + "man_in_suit_levitating_medium-light_skin_tone": "🕴🏼", + "man_in_suit_levitating_medium_skin_tone": "🕴🏽", + "man_in_tuxedo": "🤵", + "man_in_tuxedo_dark_skin_tone": "🤵🏿", + "man_in_tuxedo_light_skin_tone": "🤵🏻", + "man_in_tuxedo_medium-dark_skin_tone": "🤵🏾", + "man_in_tuxedo_medium-light_skin_tone": "🤵🏼", + "man_in_tuxedo_medium_skin_tone": "🤵🏽", + "man_judge": "👨\u200d⚖️", + "man_judge_dark_skin_tone": "👨🏿\u200d⚖️", + "man_judge_light_skin_tone": "👨🏻\u200d⚖️", + "man_judge_medium-dark_skin_tone": "👨🏾\u200d⚖️", + "man_judge_medium-light_skin_tone": "👨🏼\u200d⚖️", + "man_judge_medium_skin_tone": "👨🏽\u200d⚖️", + "man_juggling": "🤹\u200d♂️", + "man_juggling_dark_skin_tone": "🤹🏿\u200d♂️", + "man_juggling_light_skin_tone": "🤹🏻\u200d♂️", + "man_juggling_medium-dark_skin_tone": "🤹🏾\u200d♂️", + "man_juggling_medium-light_skin_tone": "🤹🏼\u200d♂️", + "man_juggling_medium_skin_tone": "🤹🏽\u200d♂️", + "man_lifting_weights": "🏋️\u200d♂️", + "man_lifting_weights_dark_skin_tone": "🏋🏿\u200d♂️", + "man_lifting_weights_light_skin_tone": "🏋🏻\u200d♂️", + "man_lifting_weights_medium-dark_skin_tone": "🏋🏾\u200d♂️", + "man_lifting_weights_medium-light_skin_tone": "🏋🏼\u200d♂️", + "man_lifting_weights_medium_skin_tone": "🏋🏽\u200d♂️", + "man_light_skin_tone": "👨🏻", + "man_mage": "🧙\u200d♂️", + "man_mage_dark_skin_tone": "🧙🏿\u200d♂️", + "man_mage_light_skin_tone": "🧙🏻\u200d♂️", + "man_mage_medium-dark_skin_tone": "🧙🏾\u200d♂️", + "man_mage_medium-light_skin_tone": "🧙🏼\u200d♂️", + "man_mage_medium_skin_tone": "🧙🏽\u200d♂️", + "man_mechanic": "👨\u200d🔧", + "man_mechanic_dark_skin_tone": "👨🏿\u200d🔧", + "man_mechanic_light_skin_tone": "👨🏻\u200d🔧", + "man_mechanic_medium-dark_skin_tone": "👨🏾\u200d🔧", + "man_mechanic_medium-light_skin_tone": "👨🏼\u200d🔧", + "man_mechanic_medium_skin_tone": "👨🏽\u200d🔧", + "man_medium-dark_skin_tone": "👨🏾", + "man_medium-light_skin_tone": "👨🏼", + "man_medium_skin_tone": "👨🏽", + "man_mountain_biking": "🚵\u200d♂️", + "man_mountain_biking_dark_skin_tone": "🚵🏿\u200d♂️", + "man_mountain_biking_light_skin_tone": "🚵🏻\u200d♂️", + "man_mountain_biking_medium-dark_skin_tone": "🚵🏾\u200d♂️", + "man_mountain_biking_medium-light_skin_tone": "🚵🏼\u200d♂️", + "man_mountain_biking_medium_skin_tone": "🚵🏽\u200d♂️", + "man_office_worker": "👨\u200d💼", + "man_office_worker_dark_skin_tone": "👨🏿\u200d💼", + "man_office_worker_light_skin_tone": "👨🏻\u200d💼", + "man_office_worker_medium-dark_skin_tone": "👨🏾\u200d💼", + "man_office_worker_medium-light_skin_tone": "👨🏼\u200d💼", + "man_office_worker_medium_skin_tone": "👨🏽\u200d💼", + "man_pilot": "👨\u200d✈️", + "man_pilot_dark_skin_tone": "👨🏿\u200d✈️", + "man_pilot_light_skin_tone": "👨🏻\u200d✈️", + "man_pilot_medium-dark_skin_tone": "👨🏾\u200d✈️", + "man_pilot_medium-light_skin_tone": "👨🏼\u200d✈️", + "man_pilot_medium_skin_tone": "👨🏽\u200d✈️", + "man_playing_handball": "🤾\u200d♂️", + "man_playing_handball_dark_skin_tone": "🤾🏿\u200d♂️", + "man_playing_handball_light_skin_tone": "🤾🏻\u200d♂️", + "man_playing_handball_medium-dark_skin_tone": "🤾🏾\u200d♂️", + "man_playing_handball_medium-light_skin_tone": "🤾🏼\u200d♂️", + "man_playing_handball_medium_skin_tone": "🤾🏽\u200d♂️", + "man_playing_water_polo": "🤽\u200d♂️", + "man_playing_water_polo_dark_skin_tone": "🤽🏿\u200d♂️", + "man_playing_water_polo_light_skin_tone": "🤽🏻\u200d♂️", + "man_playing_water_polo_medium-dark_skin_tone": "🤽🏾\u200d♂️", + "man_playing_water_polo_medium-light_skin_tone": "🤽🏼\u200d♂️", + "man_playing_water_polo_medium_skin_tone": "🤽🏽\u200d♂️", + "man_police_officer": "👮\u200d♂️", + "man_police_officer_dark_skin_tone": "👮🏿\u200d♂️", + "man_police_officer_light_skin_tone": "👮🏻\u200d♂️", + "man_police_officer_medium-dark_skin_tone": "👮🏾\u200d♂️", + "man_police_officer_medium-light_skin_tone": "👮🏼\u200d♂️", + "man_police_officer_medium_skin_tone": "👮🏽\u200d♂️", + "man_pouting": "🙎\u200d♂️", + "man_pouting_dark_skin_tone": "🙎🏿\u200d♂️", + "man_pouting_light_skin_tone": "🙎🏻\u200d♂️", + "man_pouting_medium-dark_skin_tone": "🙎🏾\u200d♂️", + "man_pouting_medium-light_skin_tone": "🙎🏼\u200d♂️", + "man_pouting_medium_skin_tone": "🙎🏽\u200d♂️", + "man_raising_hand": "🙋\u200d♂️", + "man_raising_hand_dark_skin_tone": "🙋🏿\u200d♂️", + "man_raising_hand_light_skin_tone": "🙋🏻\u200d♂️", + "man_raising_hand_medium-dark_skin_tone": "🙋🏾\u200d♂️", + "man_raising_hand_medium-light_skin_tone": "🙋🏼\u200d♂️", + "man_raising_hand_medium_skin_tone": "🙋🏽\u200d♂️", + "man_rowing_boat": "🚣\u200d♂️", + "man_rowing_boat_dark_skin_tone": "🚣🏿\u200d♂️", + "man_rowing_boat_light_skin_tone": "🚣🏻\u200d♂️", + "man_rowing_boat_medium-dark_skin_tone": "🚣🏾\u200d♂️", + "man_rowing_boat_medium-light_skin_tone": "🚣🏼\u200d♂️", + "man_rowing_boat_medium_skin_tone": "🚣🏽\u200d♂️", + "man_running": "🏃\u200d♂️", + "man_running_dark_skin_tone": "🏃🏿\u200d♂️", + "man_running_light_skin_tone": "🏃🏻\u200d♂️", + "man_running_medium-dark_skin_tone": "🏃🏾\u200d♂️", + "man_running_medium-light_skin_tone": "🏃🏼\u200d♂️", + "man_running_medium_skin_tone": "🏃🏽\u200d♂️", + "man_scientist": "👨\u200d🔬", + "man_scientist_dark_skin_tone": "👨🏿\u200d🔬", + "man_scientist_light_skin_tone": "👨🏻\u200d🔬", + "man_scientist_medium-dark_skin_tone": "👨🏾\u200d🔬", + "man_scientist_medium-light_skin_tone": "👨🏼\u200d🔬", + "man_scientist_medium_skin_tone": "👨🏽\u200d🔬", + "man_shrugging": "🤷\u200d♂️", + "man_shrugging_dark_skin_tone": "🤷🏿\u200d♂️", + "man_shrugging_light_skin_tone": "🤷🏻\u200d♂️", + "man_shrugging_medium-dark_skin_tone": "🤷🏾\u200d♂️", + "man_shrugging_medium-light_skin_tone": "🤷🏼\u200d♂️", + "man_shrugging_medium_skin_tone": "🤷🏽\u200d♂️", + "man_singer": "👨\u200d🎤", + "man_singer_dark_skin_tone": "👨🏿\u200d🎤", + "man_singer_light_skin_tone": "👨🏻\u200d🎤", + "man_singer_medium-dark_skin_tone": "👨🏾\u200d🎤", + "man_singer_medium-light_skin_tone": "👨🏼\u200d🎤", + "man_singer_medium_skin_tone": "👨🏽\u200d🎤", + "man_student": "👨\u200d🎓", + "man_student_dark_skin_tone": "👨🏿\u200d🎓", + "man_student_light_skin_tone": "👨🏻\u200d🎓", + "man_student_medium-dark_skin_tone": "👨🏾\u200d🎓", + "man_student_medium-light_skin_tone": "👨🏼\u200d🎓", + "man_student_medium_skin_tone": "👨🏽\u200d🎓", + "man_surfing": "🏄\u200d♂️", + "man_surfing_dark_skin_tone": "🏄🏿\u200d♂️", + "man_surfing_light_skin_tone": "🏄🏻\u200d♂️", + "man_surfing_medium-dark_skin_tone": "🏄🏾\u200d♂️", + "man_surfing_medium-light_skin_tone": "🏄🏼\u200d♂️", + "man_surfing_medium_skin_tone": "🏄🏽\u200d♂️", + "man_swimming": "🏊\u200d♂️", + "man_swimming_dark_skin_tone": "🏊🏿\u200d♂️", + "man_swimming_light_skin_tone": "🏊🏻\u200d♂️", + "man_swimming_medium-dark_skin_tone": "🏊🏾\u200d♂️", + "man_swimming_medium-light_skin_tone": "🏊🏼\u200d♂️", + "man_swimming_medium_skin_tone": "🏊🏽\u200d♂️", + "man_teacher": "👨\u200d🏫", + "man_teacher_dark_skin_tone": "👨🏿\u200d🏫", + "man_teacher_light_skin_tone": "👨🏻\u200d🏫", + "man_teacher_medium-dark_skin_tone": "👨🏾\u200d🏫", + "man_teacher_medium-light_skin_tone": "👨🏼\u200d🏫", + "man_teacher_medium_skin_tone": "👨🏽\u200d🏫", + "man_technologist": "👨\u200d💻", + "man_technologist_dark_skin_tone": "👨🏿\u200d💻", + "man_technologist_light_skin_tone": "👨🏻\u200d💻", + "man_technologist_medium-dark_skin_tone": "👨🏾\u200d💻", + "man_technologist_medium-light_skin_tone": "👨🏼\u200d💻", + "man_technologist_medium_skin_tone": "👨🏽\u200d💻", + "man_tipping_hand": "💁\u200d♂️", + "man_tipping_hand_dark_skin_tone": "💁🏿\u200d♂️", + "man_tipping_hand_light_skin_tone": "💁🏻\u200d♂️", + "man_tipping_hand_medium-dark_skin_tone": "💁🏾\u200d♂️", + "man_tipping_hand_medium-light_skin_tone": "💁🏼\u200d♂️", + "man_tipping_hand_medium_skin_tone": "💁🏽\u200d♂️", + "man_vampire": "🧛\u200d♂️", + "man_vampire_dark_skin_tone": "🧛🏿\u200d♂️", + "man_vampire_light_skin_tone": "🧛🏻\u200d♂️", + "man_vampire_medium-dark_skin_tone": "🧛🏾\u200d♂️", + "man_vampire_medium-light_skin_tone": "🧛🏼\u200d♂️", + "man_vampire_medium_skin_tone": "🧛🏽\u200d♂️", + "man_walking": "🚶\u200d♂️", + "man_walking_dark_skin_tone": "🚶🏿\u200d♂️", + "man_walking_light_skin_tone": "🚶🏻\u200d♂️", + "man_walking_medium-dark_skin_tone": "🚶🏾\u200d♂️", + "man_walking_medium-light_skin_tone": "🚶🏼\u200d♂️", + "man_walking_medium_skin_tone": "🚶🏽\u200d♂️", + "man_wearing_turban": "👳\u200d♂️", + "man_wearing_turban_dark_skin_tone": "👳🏿\u200d♂️", + "man_wearing_turban_light_skin_tone": "👳🏻\u200d♂️", + "man_wearing_turban_medium-dark_skin_tone": "👳🏾\u200d♂️", + "man_wearing_turban_medium-light_skin_tone": "👳🏼\u200d♂️", + "man_wearing_turban_medium_skin_tone": "👳🏽\u200d♂️", + "man_with_probing_cane": "👨\u200d🦯", + "man_with_chinese_cap": "👲", + "man_with_chinese_cap_dark_skin_tone": "👲🏿", + "man_with_chinese_cap_light_skin_tone": "👲🏻", + "man_with_chinese_cap_medium-dark_skin_tone": "👲🏾", + "man_with_chinese_cap_medium-light_skin_tone": "👲🏼", + "man_with_chinese_cap_medium_skin_tone": "👲🏽", + "man_zombie": "🧟\u200d♂️", + "mango": "🥭", + "mantelpiece_clock": "🕰", + "manual_wheelchair": "🦽", + "man’s_shoe": "👞", + "map_of_japan": "🗾", + "maple_leaf": "🍁", + "martial_arts_uniform": "🥋", + "mate": "🧉", + "meat_on_bone": "🍖", + "mechanical_arm": "🦾", + "mechanical_leg": "🦿", + "medical_symbol": "⚕", + "megaphone": "📣", + "melon": "🍈", + "memo": "📝", + "men_with_bunny_ears": "👯\u200d♂️", + "men_wrestling": "🤼\u200d♂️", + "menorah": "🕎", + "men’s_room": "🚹", + "mermaid": "🧜\u200d♀️", + "mermaid_dark_skin_tone": "🧜🏿\u200d♀️", + "mermaid_light_skin_tone": "🧜🏻\u200d♀️", + "mermaid_medium-dark_skin_tone": "🧜🏾\u200d♀️", + "mermaid_medium-light_skin_tone": "🧜🏼\u200d♀️", + "mermaid_medium_skin_tone": "🧜🏽\u200d♀️", + "merman": "🧜\u200d♂️", + "merman_dark_skin_tone": "🧜🏿\u200d♂️", + "merman_light_skin_tone": "🧜🏻\u200d♂️", + "merman_medium-dark_skin_tone": "🧜🏾\u200d♂️", + "merman_medium-light_skin_tone": "🧜🏼\u200d♂️", + "merman_medium_skin_tone": "🧜🏽\u200d♂️", + "merperson": "🧜", + "merperson_dark_skin_tone": "🧜🏿", + "merperson_light_skin_tone": "🧜🏻", + "merperson_medium-dark_skin_tone": "🧜🏾", + "merperson_medium-light_skin_tone": "🧜🏼", + "merperson_medium_skin_tone": "🧜🏽", + "metro": "🚇", + "microbe": "🦠", + "microphone": "🎤", + "microscope": "🔬", + "middle_finger": "🖕", + "middle_finger_dark_skin_tone": "🖕🏿", + "middle_finger_light_skin_tone": "🖕🏻", + "middle_finger_medium-dark_skin_tone": "🖕🏾", + "middle_finger_medium-light_skin_tone": "🖕🏼", + "middle_finger_medium_skin_tone": "🖕🏽", + "military_medal": "🎖", + "milky_way": "🌌", + "minibus": "🚐", + "moai": "🗿", + "mobile_phone": "📱", + "mobile_phone_off": "📴", + "mobile_phone_with_arrow": "📲", + "money-mouth_face": "🤑", + "money_bag": "💰", + "money_with_wings": "💸", + "monkey": "🐒", + "monkey_face": "🐵", + "monorail": "🚝", + "moon_cake": "🥮", + "moon_viewing_ceremony": "🎑", + "mosque": "🕌", + "mosquito": "🦟", + "motor_boat": "🛥", + "motor_scooter": "🛵", + "motorcycle": "🏍", + "motorized_wheelchair": "🦼", + "motorway": "🛣", + "mount_fuji": "🗻", + "mountain": "⛰", + "mountain_cableway": "🚠", + "mountain_railway": "🚞", + "mouse": "🐭", + "mouse_face": "🐭", + "mouth": "👄", + "movie_camera": "🎥", + "mushroom": "🍄", + "musical_keyboard": "🎹", + "musical_note": "🎵", + "musical_notes": "🎶", + "musical_score": "🎼", + "muted_speaker": "🔇", + "nail_polish": "💅", + "nail_polish_dark_skin_tone": "💅🏿", + "nail_polish_light_skin_tone": "💅🏻", + "nail_polish_medium-dark_skin_tone": "💅🏾", + "nail_polish_medium-light_skin_tone": "💅🏼", + "nail_polish_medium_skin_tone": "💅🏽", + "name_badge": "📛", + "national_park": "🏞", + "nauseated_face": "🤢", + "nazar_amulet": "🧿", + "necktie": "👔", + "nerd_face": "🤓", + "neutral_face": "😐", + "new_moon": "🌑", + "new_moon_face": "🌚", + "newspaper": "📰", + "next_track_button": "⏭", + "night_with_stars": "🌃", + "nine-thirty": "🕤", + "nine_o’clock": "🕘", + "no_bicycles": "🚳", + "no_entry": "⛔", + "no_littering": "🚯", + "no_mobile_phones": "📵", + "no_one_under_eighteen": "🔞", + "no_pedestrians": "🚷", + "no_smoking": "🚭", + "non-potable_water": "🚱", + "nose": "👃", + "nose_dark_skin_tone": "👃🏿", + "nose_light_skin_tone": "👃🏻", + "nose_medium-dark_skin_tone": "👃🏾", + "nose_medium-light_skin_tone": "👃🏼", + "nose_medium_skin_tone": "👃🏽", + "notebook": "📓", + "notebook_with_decorative_cover": "📔", + "nut_and_bolt": "🔩", + "octopus": "🐙", + "oden": "🍢", + "office_building": "🏢", + "ogre": "👹", + "oil_drum": "🛢", + "old_key": "🗝", + "old_man": "👴", + "old_man_dark_skin_tone": "👴🏿", + "old_man_light_skin_tone": "👴🏻", + "old_man_medium-dark_skin_tone": "👴🏾", + "old_man_medium-light_skin_tone": "👴🏼", + "old_man_medium_skin_tone": "👴🏽", + "old_woman": "👵", + "old_woman_dark_skin_tone": "👵🏿", + "old_woman_light_skin_tone": "👵🏻", + "old_woman_medium-dark_skin_tone": "👵🏾", + "old_woman_medium-light_skin_tone": "👵🏼", + "old_woman_medium_skin_tone": "👵🏽", + "older_adult": "🧓", + "older_adult_dark_skin_tone": "🧓🏿", + "older_adult_light_skin_tone": "🧓🏻", + "older_adult_medium-dark_skin_tone": "🧓🏾", + "older_adult_medium-light_skin_tone": "🧓🏼", + "older_adult_medium_skin_tone": "🧓🏽", + "om": "🕉", + "oncoming_automobile": "🚘", + "oncoming_bus": "🚍", + "oncoming_fist": "👊", + "oncoming_fist_dark_skin_tone": "👊🏿", + "oncoming_fist_light_skin_tone": "👊🏻", + "oncoming_fist_medium-dark_skin_tone": "👊🏾", + "oncoming_fist_medium-light_skin_tone": "👊🏼", + "oncoming_fist_medium_skin_tone": "👊🏽", + "oncoming_police_car": "🚔", + "oncoming_taxi": "🚖", + "one-piece_swimsuit": "🩱", + "one-thirty": "🕜", + "one_o’clock": "🕐", + "onion": "🧅", + "open_book": "📖", + "open_file_folder": "📂", + "open_hands": "👐", + "open_hands_dark_skin_tone": "👐🏿", + "open_hands_light_skin_tone": "👐🏻", + "open_hands_medium-dark_skin_tone": "👐🏾", + "open_hands_medium-light_skin_tone": "👐🏼", + "open_hands_medium_skin_tone": "👐🏽", + "open_mailbox_with_lowered_flag": "📭", + "open_mailbox_with_raised_flag": "📬", + "optical_disk": "💿", + "orange_book": "📙", + "orange_circle": "🟠", + "orange_heart": "🧡", + "orange_square": "🟧", + "orangutan": "🦧", + "orthodox_cross": "☦", + "otter": "🦦", + "outbox_tray": "📤", + "owl": "🦉", + "ox": "🐂", + "oyster": "🦪", + "package": "📦", + "page_facing_up": "📄", + "page_with_curl": "📃", + "pager": "📟", + "paintbrush": "🖌", + "palm_tree": "🌴", + "palms_up_together": "🤲", + "palms_up_together_dark_skin_tone": "🤲🏿", + "palms_up_together_light_skin_tone": "🤲🏻", + "palms_up_together_medium-dark_skin_tone": "🤲🏾", + "palms_up_together_medium-light_skin_tone": "🤲🏼", + "palms_up_together_medium_skin_tone": "🤲🏽", + "pancakes": "🥞", + "panda_face": "🐼", + "paperclip": "📎", + "parrot": "🦜", + "part_alternation_mark": "〽", + "party_popper": "🎉", + "partying_face": "🥳", + "passenger_ship": "🛳", + "passport_control": "🛂", + "pause_button": "⏸", + "paw_prints": "🐾", + "peace_symbol": "☮", + "peach": "🍑", + "peacock": "🦚", + "peanuts": "🥜", + "pear": "🍐", + "pen": "🖊", + "pencil": "📝", + "penguin": "🐧", + "pensive_face": "😔", + "people_holding_hands": "🧑\u200d🤝\u200d🧑", + "people_with_bunny_ears": "👯", + "people_wrestling": "🤼", + "performing_arts": "🎭", + "persevering_face": "😣", + "person_biking": "🚴", + "person_biking_dark_skin_tone": "🚴🏿", + "person_biking_light_skin_tone": "🚴🏻", + "person_biking_medium-dark_skin_tone": "🚴🏾", + "person_biking_medium-light_skin_tone": "🚴🏼", + "person_biking_medium_skin_tone": "🚴🏽", + "person_bouncing_ball": "⛹", + "person_bouncing_ball_dark_skin_tone": "⛹🏿", + "person_bouncing_ball_light_skin_tone": "⛹🏻", + "person_bouncing_ball_medium-dark_skin_tone": "⛹🏾", + "person_bouncing_ball_medium-light_skin_tone": "⛹🏼", + "person_bouncing_ball_medium_skin_tone": "⛹🏽", + "person_bowing": "🙇", + "person_bowing_dark_skin_tone": "🙇🏿", + "person_bowing_light_skin_tone": "🙇🏻", + "person_bowing_medium-dark_skin_tone": "🙇🏾", + "person_bowing_medium-light_skin_tone": "🙇🏼", + "person_bowing_medium_skin_tone": "🙇🏽", + "person_cartwheeling": "🤸", + "person_cartwheeling_dark_skin_tone": "🤸🏿", + "person_cartwheeling_light_skin_tone": "🤸🏻", + "person_cartwheeling_medium-dark_skin_tone": "🤸🏾", + "person_cartwheeling_medium-light_skin_tone": "🤸🏼", + "person_cartwheeling_medium_skin_tone": "🤸🏽", + "person_climbing": "🧗", + "person_climbing_dark_skin_tone": "🧗🏿", + "person_climbing_light_skin_tone": "🧗🏻", + "person_climbing_medium-dark_skin_tone": "🧗🏾", + "person_climbing_medium-light_skin_tone": "🧗🏼", + "person_climbing_medium_skin_tone": "🧗🏽", + "person_facepalming": "🤦", + "person_facepalming_dark_skin_tone": "🤦🏿", + "person_facepalming_light_skin_tone": "🤦🏻", + "person_facepalming_medium-dark_skin_tone": "🤦🏾", + "person_facepalming_medium-light_skin_tone": "🤦🏼", + "person_facepalming_medium_skin_tone": "🤦🏽", + "person_fencing": "🤺", + "person_frowning": "🙍", + "person_frowning_dark_skin_tone": "🙍🏿", + "person_frowning_light_skin_tone": "🙍🏻", + "person_frowning_medium-dark_skin_tone": "🙍🏾", + "person_frowning_medium-light_skin_tone": "🙍🏼", + "person_frowning_medium_skin_tone": "🙍🏽", + "person_gesturing_no": "🙅", + "person_gesturing_no_dark_skin_tone": "🙅🏿", + "person_gesturing_no_light_skin_tone": "🙅🏻", + "person_gesturing_no_medium-dark_skin_tone": "🙅🏾", + "person_gesturing_no_medium-light_skin_tone": "🙅🏼", + "person_gesturing_no_medium_skin_tone": "🙅🏽", + "person_gesturing_ok": "🙆", + "person_gesturing_ok_dark_skin_tone": "🙆🏿", + "person_gesturing_ok_light_skin_tone": "🙆🏻", + "person_gesturing_ok_medium-dark_skin_tone": "🙆🏾", + "person_gesturing_ok_medium-light_skin_tone": "🙆🏼", + "person_gesturing_ok_medium_skin_tone": "🙆🏽", + "person_getting_haircut": "💇", + "person_getting_haircut_dark_skin_tone": "💇🏿", + "person_getting_haircut_light_skin_tone": "💇🏻", + "person_getting_haircut_medium-dark_skin_tone": "💇🏾", + "person_getting_haircut_medium-light_skin_tone": "💇🏼", + "person_getting_haircut_medium_skin_tone": "💇🏽", + "person_getting_massage": "💆", + "person_getting_massage_dark_skin_tone": "💆🏿", + "person_getting_massage_light_skin_tone": "💆🏻", + "person_getting_massage_medium-dark_skin_tone": "💆🏾", + "person_getting_massage_medium-light_skin_tone": "💆🏼", + "person_getting_massage_medium_skin_tone": "💆🏽", + "person_golfing": "🏌", + "person_golfing_dark_skin_tone": "🏌🏿", + "person_golfing_light_skin_tone": "🏌🏻", + "person_golfing_medium-dark_skin_tone": "🏌🏾", + "person_golfing_medium-light_skin_tone": "🏌🏼", + "person_golfing_medium_skin_tone": "🏌🏽", + "person_in_bed": "🛌", + "person_in_bed_dark_skin_tone": "🛌🏿", + "person_in_bed_light_skin_tone": "🛌🏻", + "person_in_bed_medium-dark_skin_tone": "🛌🏾", + "person_in_bed_medium-light_skin_tone": "🛌🏼", + "person_in_bed_medium_skin_tone": "🛌🏽", + "person_in_lotus_position": "🧘", + "person_in_lotus_position_dark_skin_tone": "🧘🏿", + "person_in_lotus_position_light_skin_tone": "🧘🏻", + "person_in_lotus_position_medium-dark_skin_tone": "🧘🏾", + "person_in_lotus_position_medium-light_skin_tone": "🧘🏼", + "person_in_lotus_position_medium_skin_tone": "🧘🏽", + "person_in_steamy_room": "🧖", + "person_in_steamy_room_dark_skin_tone": "🧖🏿", + "person_in_steamy_room_light_skin_tone": "🧖🏻", + "person_in_steamy_room_medium-dark_skin_tone": "🧖🏾", + "person_in_steamy_room_medium-light_skin_tone": "🧖🏼", + "person_in_steamy_room_medium_skin_tone": "🧖🏽", + "person_juggling": "🤹", + "person_juggling_dark_skin_tone": "🤹🏿", + "person_juggling_light_skin_tone": "🤹🏻", + "person_juggling_medium-dark_skin_tone": "🤹🏾", + "person_juggling_medium-light_skin_tone": "🤹🏼", + "person_juggling_medium_skin_tone": "🤹🏽", + "person_kneeling": "🧎", + "person_lifting_weights": "🏋", + "person_lifting_weights_dark_skin_tone": "🏋🏿", + "person_lifting_weights_light_skin_tone": "🏋🏻", + "person_lifting_weights_medium-dark_skin_tone": "🏋🏾", + "person_lifting_weights_medium-light_skin_tone": "🏋🏼", + "person_lifting_weights_medium_skin_tone": "🏋🏽", + "person_mountain_biking": "🚵", + "person_mountain_biking_dark_skin_tone": "🚵🏿", + "person_mountain_biking_light_skin_tone": "🚵🏻", + "person_mountain_biking_medium-dark_skin_tone": "🚵🏾", + "person_mountain_biking_medium-light_skin_tone": "🚵🏼", + "person_mountain_biking_medium_skin_tone": "🚵🏽", + "person_playing_handball": "🤾", + "person_playing_handball_dark_skin_tone": "🤾🏿", + "person_playing_handball_light_skin_tone": "🤾🏻", + "person_playing_handball_medium-dark_skin_tone": "🤾🏾", + "person_playing_handball_medium-light_skin_tone": "🤾🏼", + "person_playing_handball_medium_skin_tone": "🤾🏽", + "person_playing_water_polo": "🤽", + "person_playing_water_polo_dark_skin_tone": "🤽🏿", + "person_playing_water_polo_light_skin_tone": "🤽🏻", + "person_playing_water_polo_medium-dark_skin_tone": "🤽🏾", + "person_playing_water_polo_medium-light_skin_tone": "🤽🏼", + "person_playing_water_polo_medium_skin_tone": "🤽🏽", + "person_pouting": "🙎", + "person_pouting_dark_skin_tone": "🙎🏿", + "person_pouting_light_skin_tone": "🙎🏻", + "person_pouting_medium-dark_skin_tone": "🙎🏾", + "person_pouting_medium-light_skin_tone": "🙎🏼", + "person_pouting_medium_skin_tone": "🙎🏽", + "person_raising_hand": "🙋", + "person_raising_hand_dark_skin_tone": "🙋🏿", + "person_raising_hand_light_skin_tone": "🙋🏻", + "person_raising_hand_medium-dark_skin_tone": "🙋🏾", + "person_raising_hand_medium-light_skin_tone": "🙋🏼", + "person_raising_hand_medium_skin_tone": "🙋🏽", + "person_rowing_boat": "🚣", + "person_rowing_boat_dark_skin_tone": "🚣🏿", + "person_rowing_boat_light_skin_tone": "🚣🏻", + "person_rowing_boat_medium-dark_skin_tone": "🚣🏾", + "person_rowing_boat_medium-light_skin_tone": "🚣🏼", + "person_rowing_boat_medium_skin_tone": "🚣🏽", + "person_running": "🏃", + "person_running_dark_skin_tone": "🏃🏿", + "person_running_light_skin_tone": "🏃🏻", + "person_running_medium-dark_skin_tone": "🏃🏾", + "person_running_medium-light_skin_tone": "🏃🏼", + "person_running_medium_skin_tone": "🏃🏽", + "person_shrugging": "🤷", + "person_shrugging_dark_skin_tone": "🤷🏿", + "person_shrugging_light_skin_tone": "🤷🏻", + "person_shrugging_medium-dark_skin_tone": "🤷🏾", + "person_shrugging_medium-light_skin_tone": "🤷🏼", + "person_shrugging_medium_skin_tone": "🤷🏽", + "person_standing": "🧍", + "person_surfing": "🏄", + "person_surfing_dark_skin_tone": "🏄🏿", + "person_surfing_light_skin_tone": "🏄🏻", + "person_surfing_medium-dark_skin_tone": "🏄🏾", + "person_surfing_medium-light_skin_tone": "🏄🏼", + "person_surfing_medium_skin_tone": "🏄🏽", + "person_swimming": "🏊", + "person_swimming_dark_skin_tone": "🏊🏿", + "person_swimming_light_skin_tone": "🏊🏻", + "person_swimming_medium-dark_skin_tone": "🏊🏾", + "person_swimming_medium-light_skin_tone": "🏊🏼", + "person_swimming_medium_skin_tone": "🏊🏽", + "person_taking_bath": "🛀", + "person_taking_bath_dark_skin_tone": "🛀🏿", + "person_taking_bath_light_skin_tone": "🛀🏻", + "person_taking_bath_medium-dark_skin_tone": "🛀🏾", + "person_taking_bath_medium-light_skin_tone": "🛀🏼", + "person_taking_bath_medium_skin_tone": "🛀🏽", + "person_tipping_hand": "💁", + "person_tipping_hand_dark_skin_tone": "💁🏿", + "person_tipping_hand_light_skin_tone": "💁🏻", + "person_tipping_hand_medium-dark_skin_tone": "💁🏾", + "person_tipping_hand_medium-light_skin_tone": "💁🏼", + "person_tipping_hand_medium_skin_tone": "💁🏽", + "person_walking": "🚶", + "person_walking_dark_skin_tone": "🚶🏿", + "person_walking_light_skin_tone": "🚶🏻", + "person_walking_medium-dark_skin_tone": "🚶🏾", + "person_walking_medium-light_skin_tone": "🚶🏼", + "person_walking_medium_skin_tone": "🚶🏽", + "person_wearing_turban": "👳", + "person_wearing_turban_dark_skin_tone": "👳🏿", + "person_wearing_turban_light_skin_tone": "👳🏻", + "person_wearing_turban_medium-dark_skin_tone": "👳🏾", + "person_wearing_turban_medium-light_skin_tone": "👳🏼", + "person_wearing_turban_medium_skin_tone": "👳🏽", + "petri_dish": "🧫", + "pick": "⛏", + "pie": "🥧", + "pig": "🐷", + "pig_face": "🐷", + "pig_nose": "🐽", + "pile_of_poo": "💩", + "pill": "💊", + "pinching_hand": "🤏", + "pine_decoration": "🎍", + "pineapple": "🍍", + "ping_pong": "🏓", + "pirate_flag": "🏴\u200d☠️", + "pistol": "🔫", + "pizza": "🍕", + "place_of_worship": "🛐", + "play_button": "▶", + "play_or_pause_button": "⏯", + "pleading_face": "🥺", + "police_car": "🚓", + "police_car_light": "🚨", + "police_officer": "👮", + "police_officer_dark_skin_tone": "👮🏿", + "police_officer_light_skin_tone": "👮🏻", + "police_officer_medium-dark_skin_tone": "👮🏾", + "police_officer_medium-light_skin_tone": "👮🏼", + "police_officer_medium_skin_tone": "👮🏽", + "poodle": "🐩", + "pool_8_ball": "🎱", + "popcorn": "🍿", + "post_office": "🏣", + "postal_horn": "📯", + "postbox": "📮", + "pot_of_food": "🍲", + "potable_water": "🚰", + "potato": "🥔", + "poultry_leg": "🍗", + "pound_banknote": "💷", + "pouting_cat_face": "😾", + "pouting_face": "😡", + "prayer_beads": "📿", + "pregnant_woman": "🤰", + "pregnant_woman_dark_skin_tone": "🤰🏿", + "pregnant_woman_light_skin_tone": "🤰🏻", + "pregnant_woman_medium-dark_skin_tone": "🤰🏾", + "pregnant_woman_medium-light_skin_tone": "🤰🏼", + "pregnant_woman_medium_skin_tone": "🤰🏽", + "pretzel": "🥨", + "probing_cane": "🦯", + "prince": "🤴", + "prince_dark_skin_tone": "🤴🏿", + "prince_light_skin_tone": "🤴🏻", + "prince_medium-dark_skin_tone": "🤴🏾", + "prince_medium-light_skin_tone": "🤴🏼", + "prince_medium_skin_tone": "🤴🏽", + "princess": "👸", + "princess_dark_skin_tone": "👸🏿", + "princess_light_skin_tone": "👸🏻", + "princess_medium-dark_skin_tone": "👸🏾", + "princess_medium-light_skin_tone": "👸🏼", + "princess_medium_skin_tone": "👸🏽", + "printer": "🖨", + "prohibited": "🚫", + "purple_circle": "🟣", + "purple_heart": "💜", + "purple_square": "🟪", + "purse": "👛", + "pushpin": "📌", + "question_mark": "❓", + "rabbit": "🐰", + "rabbit_face": "🐰", + "raccoon": "🦝", + "racing_car": "🏎", + "radio": "📻", + "radio_button": "🔘", + "radioactive": "☢", + "railway_car": "🚃", + "railway_track": "🛤", + "rainbow": "🌈", + "rainbow_flag": "🏳️\u200d🌈", + "raised_back_of_hand": "🤚", + "raised_back_of_hand_dark_skin_tone": "🤚🏿", + "raised_back_of_hand_light_skin_tone": "🤚🏻", + "raised_back_of_hand_medium-dark_skin_tone": "🤚🏾", + "raised_back_of_hand_medium-light_skin_tone": "🤚🏼", + "raised_back_of_hand_medium_skin_tone": "🤚🏽", + "raised_fist": "✊", + "raised_fist_dark_skin_tone": "✊🏿", + "raised_fist_light_skin_tone": "✊🏻", + "raised_fist_medium-dark_skin_tone": "✊🏾", + "raised_fist_medium-light_skin_tone": "✊🏼", + "raised_fist_medium_skin_tone": "✊🏽", + "raised_hand": "✋", + "raised_hand_dark_skin_tone": "✋🏿", + "raised_hand_light_skin_tone": "✋🏻", + "raised_hand_medium-dark_skin_tone": "✋🏾", + "raised_hand_medium-light_skin_tone": "✋🏼", + "raised_hand_medium_skin_tone": "✋🏽", + "raising_hands": "🙌", + "raising_hands_dark_skin_tone": "🙌🏿", + "raising_hands_light_skin_tone": "🙌🏻", + "raising_hands_medium-dark_skin_tone": "🙌🏾", + "raising_hands_medium-light_skin_tone": "🙌🏼", + "raising_hands_medium_skin_tone": "🙌🏽", + "ram": "🐏", + "rat": "🐀", + "razor": "🪒", + "ringed_planet": "🪐", + "receipt": "🧾", + "record_button": "⏺", + "recycling_symbol": "♻", + "red_apple": "🍎", + "red_circle": "🔴", + "red_envelope": "🧧", + "red_hair": "🦰", + "red-haired_man": "👨\u200d🦰", + "red-haired_woman": "👩\u200d🦰", + "red_heart": "❤", + "red_paper_lantern": "🏮", + "red_square": "🟥", + "red_triangle_pointed_down": "🔻", + "red_triangle_pointed_up": "🔺", + "registered": "®", + "relieved_face": "😌", + "reminder_ribbon": "🎗", + "repeat_button": "🔁", + "repeat_single_button": "🔂", + "rescue_worker’s_helmet": "⛑", + "restroom": "🚻", + "reverse_button": "◀", + "revolving_hearts": "💞", + "rhinoceros": "🦏", + "ribbon": "🎀", + "rice_ball": "🍙", + "rice_cracker": "🍘", + "right-facing_fist": "🤜", + "right-facing_fist_dark_skin_tone": "🤜🏿", + "right-facing_fist_light_skin_tone": "🤜🏻", + "right-facing_fist_medium-dark_skin_tone": "🤜🏾", + "right-facing_fist_medium-light_skin_tone": "🤜🏼", + "right-facing_fist_medium_skin_tone": "🤜🏽", + "right_anger_bubble": "🗯", + "right_arrow": "➡", + "right_arrow_curving_down": "⤵", + "right_arrow_curving_left": "↩", + "right_arrow_curving_up": "⤴", + "ring": "💍", + "roasted_sweet_potato": "🍠", + "robot_face": "🤖", + "rocket": "🚀", + "roll_of_paper": "🧻", + "rolled-up_newspaper": "🗞", + "roller_coaster": "🎢", + "rolling_on_the_floor_laughing": "🤣", + "rooster": "🐓", + "rose": "🌹", + "rosette": "🏵", + "round_pushpin": "📍", + "rugby_football": "🏉", + "running_shirt": "🎽", + "running_shoe": "👟", + "sad_but_relieved_face": "😥", + "safety_pin": "🧷", + "safety_vest": "🦺", + "salt": "🧂", + "sailboat": "⛵", + "sake": "🍶", + "sandwich": "🥪", + "sari": "🥻", + "satellite": "📡", + "satellite_antenna": "📡", + "sauropod": "🦕", + "saxophone": "🎷", + "scarf": "🧣", + "school": "🏫", + "school_backpack": "🎒", + "scissors": "✂", + "scorpion": "🦂", + "scroll": "📜", + "seat": "💺", + "see-no-evil_monkey": "🙈", + "seedling": "🌱", + "selfie": "🤳", + "selfie_dark_skin_tone": "🤳🏿", + "selfie_light_skin_tone": "🤳🏻", + "selfie_medium-dark_skin_tone": "🤳🏾", + "selfie_medium-light_skin_tone": "🤳🏼", + "selfie_medium_skin_tone": "🤳🏽", + "service_dog": "🐕\u200d🦺", + "seven-thirty": "🕢", + "seven_o’clock": "🕖", + "shallow_pan_of_food": "🥘", + "shamrock": "☘", + "shark": "🦈", + "shaved_ice": "🍧", + "sheaf_of_rice": "🌾", + "shield": "🛡", + "shinto_shrine": "⛩", + "ship": "🚢", + "shooting_star": "🌠", + "shopping_bags": "🛍", + "shopping_cart": "🛒", + "shortcake": "🍰", + "shorts": "🩳", + "shower": "🚿", + "shrimp": "🦐", + "shuffle_tracks_button": "🔀", + "shushing_face": "🤫", + "sign_of_the_horns": "🤘", + "sign_of_the_horns_dark_skin_tone": "🤘🏿", + "sign_of_the_horns_light_skin_tone": "🤘🏻", + "sign_of_the_horns_medium-dark_skin_tone": "🤘🏾", + "sign_of_the_horns_medium-light_skin_tone": "🤘🏼", + "sign_of_the_horns_medium_skin_tone": "🤘🏽", + "six-thirty": "🕡", + "six_o’clock": "🕕", + "skateboard": "🛹", + "skier": "⛷", + "skis": "🎿", + "skull": "💀", + "skull_and_crossbones": "☠", + "skunk": "🦨", + "sled": "🛷", + "sleeping_face": "😴", + "sleepy_face": "😪", + "slightly_frowning_face": "🙁", + "slightly_smiling_face": "🙂", + "slot_machine": "🎰", + "sloth": "🦥", + "small_airplane": "🛩", + "small_blue_diamond": "🔹", + "small_orange_diamond": "🔸", + "smiling_cat_face_with_heart-eyes": "😻", + "smiling_face": "☺", + "smiling_face_with_halo": "😇", + "smiling_face_with_3_hearts": "🥰", + "smiling_face_with_heart-eyes": "😍", + "smiling_face_with_horns": "😈", + "smiling_face_with_smiling_eyes": "😊", + "smiling_face_with_sunglasses": "😎", + "smirking_face": "😏", + "snail": "🐌", + "snake": "🐍", + "sneezing_face": "🤧", + "snow-capped_mountain": "🏔", + "snowboarder": "🏂", + "snowboarder_dark_skin_tone": "🏂🏿", + "snowboarder_light_skin_tone": "🏂🏻", + "snowboarder_medium-dark_skin_tone": "🏂🏾", + "snowboarder_medium-light_skin_tone": "🏂🏼", + "snowboarder_medium_skin_tone": "🏂🏽", + "snowflake": "❄", + "snowman": "☃", + "snowman_without_snow": "⛄", + "soap": "🧼", + "soccer_ball": "⚽", + "socks": "🧦", + "softball": "🥎", + "soft_ice_cream": "🍦", + "spade_suit": "♠", + "spaghetti": "🍝", + "sparkle": "❇", + "sparkler": "🎇", + "sparkles": "✨", + "sparkling_heart": "💖", + "speak-no-evil_monkey": "🙊", + "speaker_high_volume": "🔊", + "speaker_low_volume": "🔈", + "speaker_medium_volume": "🔉", + "speaking_head": "🗣", + "speech_balloon": "💬", + "speedboat": "🚤", + "spider": "🕷", + "spider_web": "🕸", + "spiral_calendar": "🗓", + "spiral_notepad": "🗒", + "spiral_shell": "🐚", + "spoon": "🥄", + "sponge": "🧽", + "sport_utility_vehicle": "🚙", + "sports_medal": "🏅", + "spouting_whale": "🐳", + "squid": "🦑", + "squinting_face_with_tongue": "😝", + "stadium": "🏟", + "star-struck": "🤩", + "star_and_crescent": "☪", + "star_of_david": "✡", + "station": "🚉", + "steaming_bowl": "🍜", + "stethoscope": "🩺", + "stop_button": "⏹", + "stop_sign": "🛑", + "stopwatch": "⏱", + "straight_ruler": "📏", + "strawberry": "🍓", + "studio_microphone": "🎙", + "stuffed_flatbread": "🥙", + "sun": "☀", + "sun_behind_cloud": "⛅", + "sun_behind_large_cloud": "🌥", + "sun_behind_rain_cloud": "🌦", + "sun_behind_small_cloud": "🌤", + "sun_with_face": "🌞", + "sunflower": "🌻", + "sunglasses": "😎", + "sunrise": "🌅", + "sunrise_over_mountains": "🌄", + "sunset": "🌇", + "superhero": "🦸", + "supervillain": "🦹", + "sushi": "🍣", + "suspension_railway": "🚟", + "swan": "🦢", + "sweat_droplets": "💦", + "synagogue": "🕍", + "syringe": "💉", + "t-shirt": "👕", + "taco": "🌮", + "takeout_box": "🥡", + "tanabata_tree": "🎋", + "tangerine": "🍊", + "taxi": "🚕", + "teacup_without_handle": "🍵", + "tear-off_calendar": "📆", + "teddy_bear": "🧸", + "telephone": "☎", + "telephone_receiver": "📞", + "telescope": "🔭", + "television": "📺", + "ten-thirty": "🕥", + "ten_o’clock": "🕙", + "tennis": "🎾", + "tent": "⛺", + "test_tube": "🧪", + "thermometer": "🌡", + "thinking_face": "🤔", + "thought_balloon": "💭", + "thread": "🧵", + "three-thirty": "🕞", + "three_o’clock": "🕒", + "thumbs_down": "👎", + "thumbs_down_dark_skin_tone": "👎🏿", + "thumbs_down_light_skin_tone": "👎🏻", + "thumbs_down_medium-dark_skin_tone": "👎🏾", + "thumbs_down_medium-light_skin_tone": "👎🏼", + "thumbs_down_medium_skin_tone": "👎🏽", + "thumbs_up": "👍", + "thumbs_up_dark_skin_tone": "👍🏿", + "thumbs_up_light_skin_tone": "👍🏻", + "thumbs_up_medium-dark_skin_tone": "👍🏾", + "thumbs_up_medium-light_skin_tone": "👍🏼", + "thumbs_up_medium_skin_tone": "👍🏽", + "ticket": "🎫", + "tiger": "🐯", + "tiger_face": "🐯", + "timer_clock": "⏲", + "tired_face": "😫", + "toolbox": "🧰", + "toilet": "🚽", + "tomato": "🍅", + "tongue": "👅", + "tooth": "🦷", + "top_hat": "🎩", + "tornado": "🌪", + "trackball": "🖲", + "tractor": "🚜", + "trade_mark": "™", + "train": "🚋", + "tram": "🚊", + "tram_car": "🚋", + "triangular_flag": "🚩", + "triangular_ruler": "📐", + "trident_emblem": "🔱", + "trolleybus": "🚎", + "trophy": "🏆", + "tropical_drink": "🍹", + "tropical_fish": "🐠", + "trumpet": "🎺", + "tulip": "🌷", + "tumbler_glass": "🥃", + "turtle": "🐢", + "twelve-thirty": "🕧", + "twelve_o’clock": "🕛", + "two-hump_camel": "🐫", + "two-thirty": "🕝", + "two_hearts": "💕", + "two_men_holding_hands": "👬", + "two_o’clock": "🕑", + "two_women_holding_hands": "👭", + "umbrella": "☂", + "umbrella_on_ground": "⛱", + "umbrella_with_rain_drops": "☔", + "unamused_face": "😒", + "unicorn_face": "🦄", + "unlocked": "🔓", + "up-down_arrow": "↕", + "up-left_arrow": "↖", + "up-right_arrow": "↗", + "up_arrow": "⬆", + "upside-down_face": "🙃", + "upwards_button": "🔼", + "vampire": "🧛", + "vampire_dark_skin_tone": "🧛🏿", + "vampire_light_skin_tone": "🧛🏻", + "vampire_medium-dark_skin_tone": "🧛🏾", + "vampire_medium-light_skin_tone": "🧛🏼", + "vampire_medium_skin_tone": "🧛🏽", + "vertical_traffic_light": "🚦", + "vibration_mode": "📳", + "victory_hand": "✌", + "victory_hand_dark_skin_tone": "✌🏿", + "victory_hand_light_skin_tone": "✌🏻", + "victory_hand_medium-dark_skin_tone": "✌🏾", + "victory_hand_medium-light_skin_tone": "✌🏼", + "victory_hand_medium_skin_tone": "✌🏽", + "video_camera": "📹", + "video_game": "🎮", + "videocassette": "📼", + "violin": "🎻", + "volcano": "🌋", + "volleyball": "🏐", + "vulcan_salute": "🖖", + "vulcan_salute_dark_skin_tone": "🖖🏿", + "vulcan_salute_light_skin_tone": "🖖🏻", + "vulcan_salute_medium-dark_skin_tone": "🖖🏾", + "vulcan_salute_medium-light_skin_tone": "🖖🏼", + "vulcan_salute_medium_skin_tone": "🖖🏽", + "waffle": "🧇", + "waning_crescent_moon": "🌘", + "waning_gibbous_moon": "🌖", + "warning": "⚠", + "wastebasket": "🗑", + "watch": "⌚", + "water_buffalo": "🐃", + "water_closet": "🚾", + "water_wave": "🌊", + "watermelon": "🍉", + "waving_hand": "👋", + "waving_hand_dark_skin_tone": "👋🏿", + "waving_hand_light_skin_tone": "👋🏻", + "waving_hand_medium-dark_skin_tone": "👋🏾", + "waving_hand_medium-light_skin_tone": "👋🏼", + "waving_hand_medium_skin_tone": "👋🏽", + "wavy_dash": "〰", + "waxing_crescent_moon": "🌒", + "waxing_gibbous_moon": "🌔", + "weary_cat_face": "🙀", + "weary_face": "😩", + "wedding": "💒", + "whale": "🐳", + "wheel_of_dharma": "☸", + "wheelchair_symbol": "♿", + "white_circle": "⚪", + "white_exclamation_mark": "❕", + "white_flag": "🏳", + "white_flower": "💮", + "white_hair": "🦳", + "white-haired_man": "👨\u200d🦳", + "white-haired_woman": "👩\u200d🦳", + "white_heart": "🤍", + "white_heavy_check_mark": "✅", + "white_large_square": "⬜", + "white_medium-small_square": "◽", + "white_medium_square": "◻", + "white_medium_star": "⭐", + "white_question_mark": "❔", + "white_small_square": "▫", + "white_square_button": "🔳", + "wilted_flower": "🥀", + "wind_chime": "🎐", + "wind_face": "🌬", + "wine_glass": "🍷", + "winking_face": "😉", + "winking_face_with_tongue": "😜", + "wolf_face": "🐺", + "woman": "👩", + "woman_artist": "👩\u200d🎨", + "woman_artist_dark_skin_tone": "👩🏿\u200d🎨", + "woman_artist_light_skin_tone": "👩🏻\u200d🎨", + "woman_artist_medium-dark_skin_tone": "👩🏾\u200d🎨", + "woman_artist_medium-light_skin_tone": "👩🏼\u200d🎨", + "woman_artist_medium_skin_tone": "👩🏽\u200d🎨", + "woman_astronaut": "👩\u200d🚀", + "woman_astronaut_dark_skin_tone": "👩🏿\u200d🚀", + "woman_astronaut_light_skin_tone": "👩🏻\u200d🚀", + "woman_astronaut_medium-dark_skin_tone": "👩🏾\u200d🚀", + "woman_astronaut_medium-light_skin_tone": "👩🏼\u200d🚀", + "woman_astronaut_medium_skin_tone": "👩🏽\u200d🚀", + "woman_biking": "🚴\u200d♀️", + "woman_biking_dark_skin_tone": "🚴🏿\u200d♀️", + "woman_biking_light_skin_tone": "🚴🏻\u200d♀️", + "woman_biking_medium-dark_skin_tone": "🚴🏾\u200d♀️", + "woman_biking_medium-light_skin_tone": "🚴🏼\u200d♀️", + "woman_biking_medium_skin_tone": "🚴🏽\u200d♀️", + "woman_bouncing_ball": "⛹️\u200d♀️", + "woman_bouncing_ball_dark_skin_tone": "⛹🏿\u200d♀️", + "woman_bouncing_ball_light_skin_tone": "⛹🏻\u200d♀️", + "woman_bouncing_ball_medium-dark_skin_tone": "⛹🏾\u200d♀️", + "woman_bouncing_ball_medium-light_skin_tone": "⛹🏼\u200d♀️", + "woman_bouncing_ball_medium_skin_tone": "⛹🏽\u200d♀️", + "woman_bowing": "🙇\u200d♀️", + "woman_bowing_dark_skin_tone": "🙇🏿\u200d♀️", + "woman_bowing_light_skin_tone": "🙇🏻\u200d♀️", + "woman_bowing_medium-dark_skin_tone": "🙇🏾\u200d♀️", + "woman_bowing_medium-light_skin_tone": "🙇🏼\u200d♀️", + "woman_bowing_medium_skin_tone": "🙇🏽\u200d♀️", + "woman_cartwheeling": "🤸\u200d♀️", + "woman_cartwheeling_dark_skin_tone": "🤸🏿\u200d♀️", + "woman_cartwheeling_light_skin_tone": "🤸🏻\u200d♀️", + "woman_cartwheeling_medium-dark_skin_tone": "🤸🏾\u200d♀️", + "woman_cartwheeling_medium-light_skin_tone": "🤸🏼\u200d♀️", + "woman_cartwheeling_medium_skin_tone": "🤸🏽\u200d♀️", + "woman_climbing": "🧗\u200d♀️", + "woman_climbing_dark_skin_tone": "🧗🏿\u200d♀️", + "woman_climbing_light_skin_tone": "🧗🏻\u200d♀️", + "woman_climbing_medium-dark_skin_tone": "🧗🏾\u200d♀️", + "woman_climbing_medium-light_skin_tone": "🧗🏼\u200d♀️", + "woman_climbing_medium_skin_tone": "🧗🏽\u200d♀️", + "woman_construction_worker": "👷\u200d♀️", + "woman_construction_worker_dark_skin_tone": "👷🏿\u200d♀️", + "woman_construction_worker_light_skin_tone": "👷🏻\u200d♀️", + "woman_construction_worker_medium-dark_skin_tone": "👷🏾\u200d♀️", + "woman_construction_worker_medium-light_skin_tone": "👷🏼\u200d♀️", + "woman_construction_worker_medium_skin_tone": "👷🏽\u200d♀️", + "woman_cook": "👩\u200d🍳", + "woman_cook_dark_skin_tone": "👩🏿\u200d🍳", + "woman_cook_light_skin_tone": "👩🏻\u200d🍳", + "woman_cook_medium-dark_skin_tone": "👩🏾\u200d🍳", + "woman_cook_medium-light_skin_tone": "👩🏼\u200d🍳", + "woman_cook_medium_skin_tone": "👩🏽\u200d🍳", + "woman_dancing": "💃", + "woman_dancing_dark_skin_tone": "💃🏿", + "woman_dancing_light_skin_tone": "💃🏻", + "woman_dancing_medium-dark_skin_tone": "💃🏾", + "woman_dancing_medium-light_skin_tone": "💃🏼", + "woman_dancing_medium_skin_tone": "💃🏽", + "woman_dark_skin_tone": "👩🏿", + "woman_detective": "🕵️\u200d♀️", + "woman_detective_dark_skin_tone": "🕵🏿\u200d♀️", + "woman_detective_light_skin_tone": "🕵🏻\u200d♀️", + "woman_detective_medium-dark_skin_tone": "🕵🏾\u200d♀️", + "woman_detective_medium-light_skin_tone": "🕵🏼\u200d♀️", + "woman_detective_medium_skin_tone": "🕵🏽\u200d♀️", + "woman_elf": "🧝\u200d♀️", + "woman_elf_dark_skin_tone": "🧝🏿\u200d♀️", + "woman_elf_light_skin_tone": "🧝🏻\u200d♀️", + "woman_elf_medium-dark_skin_tone": "🧝🏾\u200d♀️", + "woman_elf_medium-light_skin_tone": "🧝🏼\u200d♀️", + "woman_elf_medium_skin_tone": "🧝🏽\u200d♀️", + "woman_facepalming": "🤦\u200d♀️", + "woman_facepalming_dark_skin_tone": "🤦🏿\u200d♀️", + "woman_facepalming_light_skin_tone": "🤦🏻\u200d♀️", + "woman_facepalming_medium-dark_skin_tone": "🤦🏾\u200d♀️", + "woman_facepalming_medium-light_skin_tone": "🤦🏼\u200d♀️", + "woman_facepalming_medium_skin_tone": "🤦🏽\u200d♀️", + "woman_factory_worker": "👩\u200d🏭", + "woman_factory_worker_dark_skin_tone": "👩🏿\u200d🏭", + "woman_factory_worker_light_skin_tone": "👩🏻\u200d🏭", + "woman_factory_worker_medium-dark_skin_tone": "👩🏾\u200d🏭", + "woman_factory_worker_medium-light_skin_tone": "👩🏼\u200d🏭", + "woman_factory_worker_medium_skin_tone": "👩🏽\u200d🏭", + "woman_fairy": "🧚\u200d♀️", + "woman_fairy_dark_skin_tone": "🧚🏿\u200d♀️", + "woman_fairy_light_skin_tone": "🧚🏻\u200d♀️", + "woman_fairy_medium-dark_skin_tone": "🧚🏾\u200d♀️", + "woman_fairy_medium-light_skin_tone": "🧚🏼\u200d♀️", + "woman_fairy_medium_skin_tone": "🧚🏽\u200d♀️", + "woman_farmer": "👩\u200d🌾", + "woman_farmer_dark_skin_tone": "👩🏿\u200d🌾", + "woman_farmer_light_skin_tone": "👩🏻\u200d🌾", + "woman_farmer_medium-dark_skin_tone": "👩🏾\u200d🌾", + "woman_farmer_medium-light_skin_tone": "👩🏼\u200d🌾", + "woman_farmer_medium_skin_tone": "👩🏽\u200d🌾", + "woman_firefighter": "👩\u200d🚒", + "woman_firefighter_dark_skin_tone": "👩🏿\u200d🚒", + "woman_firefighter_light_skin_tone": "👩🏻\u200d🚒", + "woman_firefighter_medium-dark_skin_tone": "👩🏾\u200d🚒", + "woman_firefighter_medium-light_skin_tone": "👩🏼\u200d🚒", + "woman_firefighter_medium_skin_tone": "👩🏽\u200d🚒", + "woman_frowning": "🙍\u200d♀️", + "woman_frowning_dark_skin_tone": "🙍🏿\u200d♀️", + "woman_frowning_light_skin_tone": "🙍🏻\u200d♀️", + "woman_frowning_medium-dark_skin_tone": "🙍🏾\u200d♀️", + "woman_frowning_medium-light_skin_tone": "🙍🏼\u200d♀️", + "woman_frowning_medium_skin_tone": "🙍🏽\u200d♀️", + "woman_genie": "🧞\u200d♀️", + "woman_gesturing_no": "🙅\u200d♀️", + "woman_gesturing_no_dark_skin_tone": "🙅🏿\u200d♀️", + "woman_gesturing_no_light_skin_tone": "🙅🏻\u200d♀️", + "woman_gesturing_no_medium-dark_skin_tone": "🙅🏾\u200d♀️", + "woman_gesturing_no_medium-light_skin_tone": "🙅🏼\u200d♀️", + "woman_gesturing_no_medium_skin_tone": "🙅🏽\u200d♀️", + "woman_gesturing_ok": "🙆\u200d♀️", + "woman_gesturing_ok_dark_skin_tone": "🙆🏿\u200d♀️", + "woman_gesturing_ok_light_skin_tone": "🙆🏻\u200d♀️", + "woman_gesturing_ok_medium-dark_skin_tone": "🙆🏾\u200d♀️", + "woman_gesturing_ok_medium-light_skin_tone": "🙆🏼\u200d♀️", + "woman_gesturing_ok_medium_skin_tone": "🙆🏽\u200d♀️", + "woman_getting_haircut": "💇\u200d♀️", + "woman_getting_haircut_dark_skin_tone": "💇🏿\u200d♀️", + "woman_getting_haircut_light_skin_tone": "💇🏻\u200d♀️", + "woman_getting_haircut_medium-dark_skin_tone": "💇🏾\u200d♀️", + "woman_getting_haircut_medium-light_skin_tone": "💇🏼\u200d♀️", + "woman_getting_haircut_medium_skin_tone": "💇🏽\u200d♀️", + "woman_getting_massage": "💆\u200d♀️", + "woman_getting_massage_dark_skin_tone": "💆🏿\u200d♀️", + "woman_getting_massage_light_skin_tone": "💆🏻\u200d♀️", + "woman_getting_massage_medium-dark_skin_tone": "💆🏾\u200d♀️", + "woman_getting_massage_medium-light_skin_tone": "💆🏼\u200d♀️", + "woman_getting_massage_medium_skin_tone": "💆🏽\u200d♀️", + "woman_golfing": "🏌️\u200d♀️", + "woman_golfing_dark_skin_tone": "🏌🏿\u200d♀️", + "woman_golfing_light_skin_tone": "🏌🏻\u200d♀️", + "woman_golfing_medium-dark_skin_tone": "🏌🏾\u200d♀️", + "woman_golfing_medium-light_skin_tone": "🏌🏼\u200d♀️", + "woman_golfing_medium_skin_tone": "🏌🏽\u200d♀️", + "woman_guard": "💂\u200d♀️", + "woman_guard_dark_skin_tone": "💂🏿\u200d♀️", + "woman_guard_light_skin_tone": "💂🏻\u200d♀️", + "woman_guard_medium-dark_skin_tone": "💂🏾\u200d♀️", + "woman_guard_medium-light_skin_tone": "💂🏼\u200d♀️", + "woman_guard_medium_skin_tone": "💂🏽\u200d♀️", + "woman_health_worker": "👩\u200d⚕️", + "woman_health_worker_dark_skin_tone": "👩🏿\u200d⚕️", + "woman_health_worker_light_skin_tone": "👩🏻\u200d⚕️", + "woman_health_worker_medium-dark_skin_tone": "👩🏾\u200d⚕️", + "woman_health_worker_medium-light_skin_tone": "👩🏼\u200d⚕️", + "woman_health_worker_medium_skin_tone": "👩🏽\u200d⚕️", + "woman_in_lotus_position": "🧘\u200d♀️", + "woman_in_lotus_position_dark_skin_tone": "🧘🏿\u200d♀️", + "woman_in_lotus_position_light_skin_tone": "🧘🏻\u200d♀️", + "woman_in_lotus_position_medium-dark_skin_tone": "🧘🏾\u200d♀️", + "woman_in_lotus_position_medium-light_skin_tone": "🧘🏼\u200d♀️", + "woman_in_lotus_position_medium_skin_tone": "🧘🏽\u200d♀️", + "woman_in_manual_wheelchair": "👩\u200d🦽", + "woman_in_motorized_wheelchair": "👩\u200d🦼", + "woman_in_steamy_room": "🧖\u200d♀️", + "woman_in_steamy_room_dark_skin_tone": "🧖🏿\u200d♀️", + "woman_in_steamy_room_light_skin_tone": "🧖🏻\u200d♀️", + "woman_in_steamy_room_medium-dark_skin_tone": "🧖🏾\u200d♀️", + "woman_in_steamy_room_medium-light_skin_tone": "🧖🏼\u200d♀️", + "woman_in_steamy_room_medium_skin_tone": "🧖🏽\u200d♀️", + "woman_judge": "👩\u200d⚖️", + "woman_judge_dark_skin_tone": "👩🏿\u200d⚖️", + "woman_judge_light_skin_tone": "👩🏻\u200d⚖️", + "woman_judge_medium-dark_skin_tone": "👩🏾\u200d⚖️", + "woman_judge_medium-light_skin_tone": "👩🏼\u200d⚖️", + "woman_judge_medium_skin_tone": "👩🏽\u200d⚖️", + "woman_juggling": "🤹\u200d♀️", + "woman_juggling_dark_skin_tone": "🤹🏿\u200d♀️", + "woman_juggling_light_skin_tone": "🤹🏻\u200d♀️", + "woman_juggling_medium-dark_skin_tone": "🤹🏾\u200d♀️", + "woman_juggling_medium-light_skin_tone": "🤹🏼\u200d♀️", + "woman_juggling_medium_skin_tone": "🤹🏽\u200d♀️", + "woman_lifting_weights": "🏋️\u200d♀️", + "woman_lifting_weights_dark_skin_tone": "🏋🏿\u200d♀️", + "woman_lifting_weights_light_skin_tone": "🏋🏻\u200d♀️", + "woman_lifting_weights_medium-dark_skin_tone": "🏋🏾\u200d♀️", + "woman_lifting_weights_medium-light_skin_tone": "🏋🏼\u200d♀️", + "woman_lifting_weights_medium_skin_tone": "🏋🏽\u200d♀️", + "woman_light_skin_tone": "👩🏻", + "woman_mage": "🧙\u200d♀️", + "woman_mage_dark_skin_tone": "🧙🏿\u200d♀️", + "woman_mage_light_skin_tone": "🧙🏻\u200d♀️", + "woman_mage_medium-dark_skin_tone": "🧙🏾\u200d♀️", + "woman_mage_medium-light_skin_tone": "🧙🏼\u200d♀️", + "woman_mage_medium_skin_tone": "🧙🏽\u200d♀️", + "woman_mechanic": "👩\u200d🔧", + "woman_mechanic_dark_skin_tone": "👩🏿\u200d🔧", + "woman_mechanic_light_skin_tone": "👩🏻\u200d🔧", + "woman_mechanic_medium-dark_skin_tone": "👩🏾\u200d🔧", + "woman_mechanic_medium-light_skin_tone": "👩🏼\u200d🔧", + "woman_mechanic_medium_skin_tone": "👩🏽\u200d🔧", + "woman_medium-dark_skin_tone": "👩🏾", + "woman_medium-light_skin_tone": "👩🏼", + "woman_medium_skin_tone": "👩🏽", + "woman_mountain_biking": "🚵\u200d♀️", + "woman_mountain_biking_dark_skin_tone": "🚵🏿\u200d♀️", + "woman_mountain_biking_light_skin_tone": "🚵🏻\u200d♀️", + "woman_mountain_biking_medium-dark_skin_tone": "🚵🏾\u200d♀️", + "woman_mountain_biking_medium-light_skin_tone": "🚵🏼\u200d♀️", + "woman_mountain_biking_medium_skin_tone": "🚵🏽\u200d♀️", + "woman_office_worker": "👩\u200d💼", + "woman_office_worker_dark_skin_tone": "👩🏿\u200d💼", + "woman_office_worker_light_skin_tone": "👩🏻\u200d💼", + "woman_office_worker_medium-dark_skin_tone": "👩🏾\u200d💼", + "woman_office_worker_medium-light_skin_tone": "👩🏼\u200d💼", + "woman_office_worker_medium_skin_tone": "👩🏽\u200d💼", + "woman_pilot": "👩\u200d✈️", + "woman_pilot_dark_skin_tone": "👩🏿\u200d✈️", + "woman_pilot_light_skin_tone": "👩🏻\u200d✈️", + "woman_pilot_medium-dark_skin_tone": "👩🏾\u200d✈️", + "woman_pilot_medium-light_skin_tone": "👩🏼\u200d✈️", + "woman_pilot_medium_skin_tone": "👩🏽\u200d✈️", + "woman_playing_handball": "🤾\u200d♀️", + "woman_playing_handball_dark_skin_tone": "🤾🏿\u200d♀️", + "woman_playing_handball_light_skin_tone": "🤾🏻\u200d♀️", + "woman_playing_handball_medium-dark_skin_tone": "🤾🏾\u200d♀️", + "woman_playing_handball_medium-light_skin_tone": "🤾🏼\u200d♀️", + "woman_playing_handball_medium_skin_tone": "🤾🏽\u200d♀️", + "woman_playing_water_polo": "🤽\u200d♀️", + "woman_playing_water_polo_dark_skin_tone": "🤽🏿\u200d♀️", + "woman_playing_water_polo_light_skin_tone": "🤽🏻\u200d♀️", + "woman_playing_water_polo_medium-dark_skin_tone": "🤽🏾\u200d♀️", + "woman_playing_water_polo_medium-light_skin_tone": "🤽🏼\u200d♀️", + "woman_playing_water_polo_medium_skin_tone": "🤽🏽\u200d♀️", + "woman_police_officer": "👮\u200d♀️", + "woman_police_officer_dark_skin_tone": "👮🏿\u200d♀️", + "woman_police_officer_light_skin_tone": "👮🏻\u200d♀️", + "woman_police_officer_medium-dark_skin_tone": "👮🏾\u200d♀️", + "woman_police_officer_medium-light_skin_tone": "👮🏼\u200d♀️", + "woman_police_officer_medium_skin_tone": "👮🏽\u200d♀️", + "woman_pouting": "🙎\u200d♀️", + "woman_pouting_dark_skin_tone": "🙎🏿\u200d♀️", + "woman_pouting_light_skin_tone": "🙎🏻\u200d♀️", + "woman_pouting_medium-dark_skin_tone": "🙎🏾\u200d♀️", + "woman_pouting_medium-light_skin_tone": "🙎🏼\u200d♀️", + "woman_pouting_medium_skin_tone": "🙎🏽\u200d♀️", + "woman_raising_hand": "🙋\u200d♀️", + "woman_raising_hand_dark_skin_tone": "🙋🏿\u200d♀️", + "woman_raising_hand_light_skin_tone": "🙋🏻\u200d♀️", + "woman_raising_hand_medium-dark_skin_tone": "🙋🏾\u200d♀️", + "woman_raising_hand_medium-light_skin_tone": "🙋🏼\u200d♀️", + "woman_raising_hand_medium_skin_tone": "🙋🏽\u200d♀️", + "woman_rowing_boat": "🚣\u200d♀️", + "woman_rowing_boat_dark_skin_tone": "🚣🏿\u200d♀️", + "woman_rowing_boat_light_skin_tone": "🚣🏻\u200d♀️", + "woman_rowing_boat_medium-dark_skin_tone": "🚣🏾\u200d♀️", + "woman_rowing_boat_medium-light_skin_tone": "🚣🏼\u200d♀️", + "woman_rowing_boat_medium_skin_tone": "🚣🏽\u200d♀️", + "woman_running": "🏃\u200d♀️", + "woman_running_dark_skin_tone": "🏃🏿\u200d♀️", + "woman_running_light_skin_tone": "🏃🏻\u200d♀️", + "woman_running_medium-dark_skin_tone": "🏃🏾\u200d♀️", + "woman_running_medium-light_skin_tone": "🏃🏼\u200d♀️", + "woman_running_medium_skin_tone": "🏃🏽\u200d♀️", + "woman_scientist": "👩\u200d🔬", + "woman_scientist_dark_skin_tone": "👩🏿\u200d🔬", + "woman_scientist_light_skin_tone": "👩🏻\u200d🔬", + "woman_scientist_medium-dark_skin_tone": "👩🏾\u200d🔬", + "woman_scientist_medium-light_skin_tone": "👩🏼\u200d🔬", + "woman_scientist_medium_skin_tone": "👩🏽\u200d🔬", + "woman_shrugging": "🤷\u200d♀️", + "woman_shrugging_dark_skin_tone": "🤷🏿\u200d♀️", + "woman_shrugging_light_skin_tone": "🤷🏻\u200d♀️", + "woman_shrugging_medium-dark_skin_tone": "🤷🏾\u200d♀️", + "woman_shrugging_medium-light_skin_tone": "🤷🏼\u200d♀️", + "woman_shrugging_medium_skin_tone": "🤷🏽\u200d♀️", + "woman_singer": "👩\u200d🎤", + "woman_singer_dark_skin_tone": "👩🏿\u200d🎤", + "woman_singer_light_skin_tone": "👩🏻\u200d🎤", + "woman_singer_medium-dark_skin_tone": "👩🏾\u200d🎤", + "woman_singer_medium-light_skin_tone": "👩🏼\u200d🎤", + "woman_singer_medium_skin_tone": "👩🏽\u200d🎤", + "woman_student": "👩\u200d🎓", + "woman_student_dark_skin_tone": "👩🏿\u200d🎓", + "woman_student_light_skin_tone": "👩🏻\u200d🎓", + "woman_student_medium-dark_skin_tone": "👩🏾\u200d🎓", + "woman_student_medium-light_skin_tone": "👩🏼\u200d🎓", + "woman_student_medium_skin_tone": "👩🏽\u200d🎓", + "woman_surfing": "🏄\u200d♀️", + "woman_surfing_dark_skin_tone": "🏄🏿\u200d♀️", + "woman_surfing_light_skin_tone": "🏄🏻\u200d♀️", + "woman_surfing_medium-dark_skin_tone": "🏄🏾\u200d♀️", + "woman_surfing_medium-light_skin_tone": "🏄🏼\u200d♀️", + "woman_surfing_medium_skin_tone": "🏄🏽\u200d♀️", + "woman_swimming": "🏊\u200d♀️", + "woman_swimming_dark_skin_tone": "🏊🏿\u200d♀️", + "woman_swimming_light_skin_tone": "🏊🏻\u200d♀️", + "woman_swimming_medium-dark_skin_tone": "🏊🏾\u200d♀️", + "woman_swimming_medium-light_skin_tone": "🏊🏼\u200d♀️", + "woman_swimming_medium_skin_tone": "🏊🏽\u200d♀️", + "woman_teacher": "👩\u200d🏫", + "woman_teacher_dark_skin_tone": "👩🏿\u200d🏫", + "woman_teacher_light_skin_tone": "👩🏻\u200d🏫", + "woman_teacher_medium-dark_skin_tone": "👩🏾\u200d🏫", + "woman_teacher_medium-light_skin_tone": "👩🏼\u200d🏫", + "woman_teacher_medium_skin_tone": "👩🏽\u200d🏫", + "woman_technologist": "👩\u200d💻", + "woman_technologist_dark_skin_tone": "👩🏿\u200d💻", + "woman_technologist_light_skin_tone": "👩🏻\u200d💻", + "woman_technologist_medium-dark_skin_tone": "👩🏾\u200d💻", + "woman_technologist_medium-light_skin_tone": "👩🏼\u200d💻", + "woman_technologist_medium_skin_tone": "👩🏽\u200d💻", + "woman_tipping_hand": "💁\u200d♀️", + "woman_tipping_hand_dark_skin_tone": "💁🏿\u200d♀️", + "woman_tipping_hand_light_skin_tone": "💁🏻\u200d♀️", + "woman_tipping_hand_medium-dark_skin_tone": "💁🏾\u200d♀️", + "woman_tipping_hand_medium-light_skin_tone": "💁🏼\u200d♀️", + "woman_tipping_hand_medium_skin_tone": "💁🏽\u200d♀️", + "woman_vampire": "🧛\u200d♀️", + "woman_vampire_dark_skin_tone": "🧛🏿\u200d♀️", + "woman_vampire_light_skin_tone": "🧛🏻\u200d♀️", + "woman_vampire_medium-dark_skin_tone": "🧛🏾\u200d♀️", + "woman_vampire_medium-light_skin_tone": "🧛🏼\u200d♀️", + "woman_vampire_medium_skin_tone": "🧛🏽\u200d♀️", + "woman_walking": "🚶\u200d♀️", + "woman_walking_dark_skin_tone": "🚶🏿\u200d♀️", + "woman_walking_light_skin_tone": "🚶🏻\u200d♀️", + "woman_walking_medium-dark_skin_tone": "🚶🏾\u200d♀️", + "woman_walking_medium-light_skin_tone": "🚶🏼\u200d♀️", + "woman_walking_medium_skin_tone": "🚶🏽\u200d♀️", + "woman_wearing_turban": "👳\u200d♀️", + "woman_wearing_turban_dark_skin_tone": "👳🏿\u200d♀️", + "woman_wearing_turban_light_skin_tone": "👳🏻\u200d♀️", + "woman_wearing_turban_medium-dark_skin_tone": "👳🏾\u200d♀️", + "woman_wearing_turban_medium-light_skin_tone": "👳🏼\u200d♀️", + "woman_wearing_turban_medium_skin_tone": "👳🏽\u200d♀️", + "woman_with_headscarf": "🧕", + "woman_with_headscarf_dark_skin_tone": "🧕🏿", + "woman_with_headscarf_light_skin_tone": "🧕🏻", + "woman_with_headscarf_medium-dark_skin_tone": "🧕🏾", + "woman_with_headscarf_medium-light_skin_tone": "🧕🏼", + "woman_with_headscarf_medium_skin_tone": "🧕🏽", + "woman_with_probing_cane": "👩\u200d🦯", + "woman_zombie": "🧟\u200d♀️", + "woman’s_boot": "👢", + "woman’s_clothes": "👚", + "woman’s_hat": "👒", + "woman’s_sandal": "👡", + "women_with_bunny_ears": "👯\u200d♀️", + "women_wrestling": "🤼\u200d♀️", + "women’s_room": "🚺", + "woozy_face": "🥴", + "world_map": "🗺", + "worried_face": "😟", + "wrapped_gift": "🎁", + "wrench": "🔧", + "writing_hand": "✍", + "writing_hand_dark_skin_tone": "✍🏿", + "writing_hand_light_skin_tone": "✍🏻", + "writing_hand_medium-dark_skin_tone": "✍🏾", + "writing_hand_medium-light_skin_tone": "✍🏼", + "writing_hand_medium_skin_tone": "✍🏽", + "yarn": "🧶", + "yawning_face": "🥱", + "yellow_circle": "🟡", + "yellow_heart": "💛", + "yellow_square": "🟨", + "yen_banknote": "💴", + "yo-yo": "🪀", + "yin_yang": "☯", + "zany_face": "🤪", + "zebra": "🦓", + "zipper-mouth_face": "🤐", + "zombie": "🧟", + "zzz": "💤", + "åland_islands": "🇦🇽", + "keycap_asterisk": "*⃣", + "keycap_digit_eight": "8⃣", + "keycap_digit_five": "5⃣", + "keycap_digit_four": "4⃣", + "keycap_digit_nine": "9⃣", + "keycap_digit_one": "1⃣", + "keycap_digit_seven": "7⃣", + "keycap_digit_six": "6⃣", + "keycap_digit_three": "3⃣", + "keycap_digit_two": "2⃣", + "keycap_digit_zero": "0⃣", + "keycap_number_sign": "#⃣", + "light_skin_tone": "🏻", + "medium_light_skin_tone": "🏼", + "medium_skin_tone": "🏽", + "medium_dark_skin_tone": "🏾", + "dark_skin_tone": "🏿", + "regional_indicator_symbol_letter_a": "🇦", + "regional_indicator_symbol_letter_b": "🇧", + "regional_indicator_symbol_letter_c": "🇨", + "regional_indicator_symbol_letter_d": "🇩", + "regional_indicator_symbol_letter_e": "🇪", + "regional_indicator_symbol_letter_f": "🇫", + "regional_indicator_symbol_letter_g": "🇬", + "regional_indicator_symbol_letter_h": "🇭", + "regional_indicator_symbol_letter_i": "🇮", + "regional_indicator_symbol_letter_j": "🇯", + "regional_indicator_symbol_letter_k": "🇰", + "regional_indicator_symbol_letter_l": "🇱", + "regional_indicator_symbol_letter_m": "🇲", + "regional_indicator_symbol_letter_n": "🇳", + "regional_indicator_symbol_letter_o": "🇴", + "regional_indicator_symbol_letter_p": "🇵", + "regional_indicator_symbol_letter_q": "🇶", + "regional_indicator_symbol_letter_r": "🇷", + "regional_indicator_symbol_letter_s": "🇸", + "regional_indicator_symbol_letter_t": "🇹", + "regional_indicator_symbol_letter_u": "🇺", + "regional_indicator_symbol_letter_v": "🇻", + "regional_indicator_symbol_letter_w": "🇼", + "regional_indicator_symbol_letter_x": "🇽", + "regional_indicator_symbol_letter_y": "🇾", + "regional_indicator_symbol_letter_z": "🇿", + "airplane_arriving": "🛬", + "space_invader": "👾", + "football": "🏈", + "anger": "💢", + "angry": "😠", + "anguished": "😧", + "signal_strength": "📶", + "arrows_counterclockwise": "🔄", + "arrow_heading_down": "⤵", + "arrow_heading_up": "⤴", + "art": "🎨", + "astonished": "😲", + "athletic_shoe": "👟", + "atm": "🏧", + "car": "🚗", + "red_car": "🚗", + "angel": "👼", + "back": "🔙", + "badminton_racquet_and_shuttlecock": "🏸", + "dollar": "💵", + "euro": "💶", + "pound": "💷", + "yen": "💴", + "barber": "💈", + "bath": "🛀", + "bear": "🐻", + "heartbeat": "💓", + "beer": "🍺", + "no_bell": "🔕", + "bento": "🍱", + "bike": "🚲", + "bicyclist": "🚴", + "8ball": "🎱", + "biohazard_sign": "☣", + "birthday": "🎂", + "black_circle_for_record": "⏺", + "clubs": "♣", + "diamonds": "♦", + "arrow_double_down": "⏬", + "hearts": "♥", + "rewind": "⏪", + "black_left__pointing_double_triangle_with_vertical_bar": "⏮", + "arrow_backward": "◀", + "black_medium_small_square": "◾", + "question": "❓", + "fast_forward": "⏩", + "black_right__pointing_double_triangle_with_vertical_bar": "⏭", + "arrow_forward": "▶", + "black_right__pointing_triangle_with_double_vertical_bar": "⏯", + "arrow_right": "➡", + "spades": "♠", + "black_square_for_stop": "⏹", + "sunny": "☀", + "phone": "☎", + "recycle": "♻", + "arrow_double_up": "⏫", + "busstop": "🚏", + "date": "📅", + "flags": "🎏", + "cat2": "🐈", + "joy_cat": "😹", + "smirk_cat": "😼", + "chart_with_downwards_trend": "📉", + "chart_with_upwards_trend": "📈", + "chart": "💹", + "mega": "📣", + "checkered_flag": "🏁", + "accept": "🉑", + "ideograph_advantage": "🉐", + "congratulations": "㊗", + "secret": "㊙", + "m": "Ⓜ", + "city_sunset": "🌆", + "clapper": "🎬", + "clap": "👏", + "beers": "🍻", + "clock830": "🕣", + "clock8": "🕗", + "clock1130": "🕦", + "clock11": "🕚", + "clock530": "🕠", + "clock5": "🕔", + "clock430": "🕟", + "clock4": "🕓", + "clock930": "🕤", + "clock9": "🕘", + "clock130": "🕜", + "clock1": "🕐", + "clock730": "🕢", + "clock7": "🕖", + "clock630": "🕡", + "clock6": "🕕", + "clock1030": "🕥", + "clock10": "🕙", + "clock330": "🕞", + "clock3": "🕒", + "clock1230": "🕧", + "clock12": "🕛", + "clock230": "🕝", + "clock2": "🕑", + "arrows_clockwise": "🔃", + "repeat": "🔁", + "repeat_one": "🔂", + "closed_lock_with_key": "🔐", + "mailbox_closed": "📪", + "mailbox": "📫", + "cloud_with_tornado": "🌪", + "cocktail": "🍸", + "boom": "💥", + "compression": "🗜", + "confounded": "😖", + "confused": "😕", + "rice": "🍚", + "cow2": "🐄", + "cricket_bat_and_ball": "🏏", + "x": "❌", + "cry": "😢", + "curry": "🍛", + "dagger_knife": "🗡", + "dancer": "💃", + "dark_sunglasses": "🕶", + "dash": "💨", + "truck": "🚚", + "derelict_house_building": "🏚", + "diamond_shape_with_a_dot_inside": "💠", + "dart": "🎯", + "disappointed_relieved": "😥", + "disappointed": "😞", + "do_not_litter": "🚯", + "dog2": "🐕", + "flipper": "🐬", + "loop": "➿", + "bangbang": "‼", + "double_vertical_bar": "⏸", + "dove_of_peace": "🕊", + "small_red_triangle_down": "🔻", + "arrow_down_small": "🔽", + "arrow_down": "⬇", + "dromedary_camel": "🐪", + "e__mail": "📧", + "corn": "🌽", + "ear_of_rice": "🌾", + "earth_americas": "🌎", + "earth_asia": "🌏", + "earth_africa": "🌍", + "eight_pointed_black_star": "✴", + "eight_spoked_asterisk": "✳", + "eject_symbol": "⏏", + "bulb": "💡", + "emoji_modifier_fitzpatrick_type__1__2": "🏻", + "emoji_modifier_fitzpatrick_type__3": "🏼", + "emoji_modifier_fitzpatrick_type__4": "🏽", + "emoji_modifier_fitzpatrick_type__5": "🏾", + "emoji_modifier_fitzpatrick_type__6": "🏿", + "end": "🔚", + "email": "✉", + "european_castle": "🏰", + "european_post_office": "🏤", + "interrobang": "⁉", + "expressionless": "😑", + "eyeglasses": "👓", + "massage": "💆", + "yum": "😋", + "scream": "😱", + "kissing_heart": "😘", + "sweat": "😓", + "face_with_head__bandage": "🤕", + "triumph": "😤", + "mask": "😷", + "no_good": "🙅", + "ok_woman": "🙆", + "open_mouth": "😮", + "cold_sweat": "😰", + "stuck_out_tongue": "😛", + "stuck_out_tongue_closed_eyes": "😝", + "stuck_out_tongue_winking_eye": "😜", + "joy": "😂", + "no_mouth": "😶", + "santa": "🎅", + "fax": "📠", + "fearful": "😨", + "field_hockey_stick_and_ball": "🏑", + "first_quarter_moon_with_face": "🌛", + "fish_cake": "🍥", + "fishing_pole_and_fish": "🎣", + "facepunch": "👊", + "punch": "👊", + "flag_for_afghanistan": "🇦🇫", + "flag_for_albania": "🇦🇱", + "flag_for_algeria": "🇩🇿", + "flag_for_american_samoa": "🇦🇸", + "flag_for_andorra": "🇦🇩", + "flag_for_angola": "🇦🇴", + "flag_for_anguilla": "🇦🇮", + "flag_for_antarctica": "🇦🇶", + "flag_for_antigua_&_barbuda": "🇦🇬", + "flag_for_argentina": "🇦🇷", + "flag_for_armenia": "🇦🇲", + "flag_for_aruba": "🇦🇼", + "flag_for_ascension_island": "🇦🇨", + "flag_for_australia": "🇦🇺", + "flag_for_austria": "🇦🇹", + "flag_for_azerbaijan": "🇦🇿", + "flag_for_bahamas": "🇧🇸", + "flag_for_bahrain": "🇧🇭", + "flag_for_bangladesh": "🇧🇩", + "flag_for_barbados": "🇧🇧", + "flag_for_belarus": "🇧🇾", + "flag_for_belgium": "🇧🇪", + "flag_for_belize": "🇧🇿", + "flag_for_benin": "🇧🇯", + "flag_for_bermuda": "🇧🇲", + "flag_for_bhutan": "🇧🇹", + "flag_for_bolivia": "🇧🇴", + "flag_for_bosnia_&_herzegovina": "🇧🇦", + "flag_for_botswana": "🇧🇼", + "flag_for_bouvet_island": "🇧🇻", + "flag_for_brazil": "🇧🇷", + "flag_for_british_indian_ocean_territory": "🇮🇴", + "flag_for_british_virgin_islands": "🇻🇬", + "flag_for_brunei": "🇧🇳", + "flag_for_bulgaria": "🇧🇬", + "flag_for_burkina_faso": "🇧🇫", + "flag_for_burundi": "🇧🇮", + "flag_for_cambodia": "🇰🇭", + "flag_for_cameroon": "🇨🇲", + "flag_for_canada": "🇨🇦", + "flag_for_canary_islands": "🇮🇨", + "flag_for_cape_verde": "🇨🇻", + "flag_for_caribbean_netherlands": "🇧🇶", + "flag_for_cayman_islands": "🇰🇾", + "flag_for_central_african_republic": "🇨🇫", + "flag_for_ceuta_&_melilla": "🇪🇦", + "flag_for_chad": "🇹🇩", + "flag_for_chile": "🇨🇱", + "flag_for_china": "🇨🇳", + "flag_for_christmas_island": "🇨🇽", + "flag_for_clipperton_island": "🇨🇵", + "flag_for_cocos__islands": "🇨🇨", + "flag_for_colombia": "🇨🇴", + "flag_for_comoros": "🇰🇲", + "flag_for_congo____brazzaville": "🇨🇬", + "flag_for_congo____kinshasa": "🇨🇩", + "flag_for_cook_islands": "🇨🇰", + "flag_for_costa_rica": "🇨🇷", + "flag_for_croatia": "🇭🇷", + "flag_for_cuba": "🇨🇺", + "flag_for_curaçao": "🇨🇼", + "flag_for_cyprus": "🇨🇾", + "flag_for_czech_republic": "🇨🇿", + "flag_for_côte_d’ivoire": "🇨🇮", + "flag_for_denmark": "🇩🇰", + "flag_for_diego_garcia": "🇩🇬", + "flag_for_djibouti": "🇩🇯", + "flag_for_dominica": "🇩🇲", + "flag_for_dominican_republic": "🇩🇴", + "flag_for_ecuador": "🇪🇨", + "flag_for_egypt": "🇪🇬", + "flag_for_el_salvador": "🇸🇻", + "flag_for_equatorial_guinea": "🇬🇶", + "flag_for_eritrea": "🇪🇷", + "flag_for_estonia": "🇪🇪", + "flag_for_ethiopia": "🇪🇹", + "flag_for_european_union": "🇪🇺", + "flag_for_falkland_islands": "🇫🇰", + "flag_for_faroe_islands": "🇫🇴", + "flag_for_fiji": "🇫🇯", + "flag_for_finland": "🇫🇮", + "flag_for_france": "🇫🇷", + "flag_for_french_guiana": "🇬🇫", + "flag_for_french_polynesia": "🇵🇫", + "flag_for_french_southern_territories": "🇹🇫", + "flag_for_gabon": "🇬🇦", + "flag_for_gambia": "🇬🇲", + "flag_for_georgia": "🇬🇪", + "flag_for_germany": "🇩🇪", + "flag_for_ghana": "🇬🇭", + "flag_for_gibraltar": "🇬🇮", + "flag_for_greece": "🇬🇷", + "flag_for_greenland": "🇬🇱", + "flag_for_grenada": "🇬🇩", + "flag_for_guadeloupe": "🇬🇵", + "flag_for_guam": "🇬🇺", + "flag_for_guatemala": "🇬🇹", + "flag_for_guernsey": "🇬🇬", + "flag_for_guinea": "🇬🇳", + "flag_for_guinea__bissau": "🇬🇼", + "flag_for_guyana": "🇬🇾", + "flag_for_haiti": "🇭🇹", + "flag_for_heard_&_mcdonald_islands": "🇭🇲", + "flag_for_honduras": "🇭🇳", + "flag_for_hong_kong": "🇭🇰", + "flag_for_hungary": "🇭🇺", + "flag_for_iceland": "🇮🇸", + "flag_for_india": "🇮🇳", + "flag_for_indonesia": "🇮🇩", + "flag_for_iran": "🇮🇷", + "flag_for_iraq": "🇮🇶", + "flag_for_ireland": "🇮🇪", + "flag_for_isle_of_man": "🇮🇲", + "flag_for_israel": "🇮🇱", + "flag_for_italy": "🇮🇹", + "flag_for_jamaica": "🇯🇲", + "flag_for_japan": "🇯🇵", + "flag_for_jersey": "🇯🇪", + "flag_for_jordan": "🇯🇴", + "flag_for_kazakhstan": "🇰🇿", + "flag_for_kenya": "🇰🇪", + "flag_for_kiribati": "🇰🇮", + "flag_for_kosovo": "🇽🇰", + "flag_for_kuwait": "🇰🇼", + "flag_for_kyrgyzstan": "🇰🇬", + "flag_for_laos": "🇱🇦", + "flag_for_latvia": "🇱🇻", + "flag_for_lebanon": "🇱🇧", + "flag_for_lesotho": "🇱🇸", + "flag_for_liberia": "🇱🇷", + "flag_for_libya": "🇱🇾", + "flag_for_liechtenstein": "🇱🇮", + "flag_for_lithuania": "🇱🇹", + "flag_for_luxembourg": "🇱🇺", + "flag_for_macau": "🇲🇴", + "flag_for_macedonia": "🇲🇰", + "flag_for_madagascar": "🇲🇬", + "flag_for_malawi": "🇲🇼", + "flag_for_malaysia": "🇲🇾", + "flag_for_maldives": "🇲🇻", + "flag_for_mali": "🇲🇱", + "flag_for_malta": "🇲🇹", + "flag_for_marshall_islands": "🇲🇭", + "flag_for_martinique": "🇲🇶", + "flag_for_mauritania": "🇲🇷", + "flag_for_mauritius": "🇲🇺", + "flag_for_mayotte": "🇾🇹", + "flag_for_mexico": "🇲🇽", + "flag_for_micronesia": "🇫🇲", + "flag_for_moldova": "🇲🇩", + "flag_for_monaco": "🇲🇨", + "flag_for_mongolia": "🇲🇳", + "flag_for_montenegro": "🇲🇪", + "flag_for_montserrat": "🇲🇸", + "flag_for_morocco": "🇲🇦", + "flag_for_mozambique": "🇲🇿", + "flag_for_myanmar": "🇲🇲", + "flag_for_namibia": "🇳🇦", + "flag_for_nauru": "🇳🇷", + "flag_for_nepal": "🇳🇵", + "flag_for_netherlands": "🇳🇱", + "flag_for_new_caledonia": "🇳🇨", + "flag_for_new_zealand": "🇳🇿", + "flag_for_nicaragua": "🇳🇮", + "flag_for_niger": "🇳🇪", + "flag_for_nigeria": "🇳🇬", + "flag_for_niue": "🇳🇺", + "flag_for_norfolk_island": "🇳🇫", + "flag_for_north_korea": "🇰🇵", + "flag_for_northern_mariana_islands": "🇲🇵", + "flag_for_norway": "🇳🇴", + "flag_for_oman": "🇴🇲", + "flag_for_pakistan": "🇵🇰", + "flag_for_palau": "🇵🇼", + "flag_for_palestinian_territories": "🇵🇸", + "flag_for_panama": "🇵🇦", + "flag_for_papua_new_guinea": "🇵🇬", + "flag_for_paraguay": "🇵🇾", + "flag_for_peru": "🇵🇪", + "flag_for_philippines": "🇵🇭", + "flag_for_pitcairn_islands": "🇵🇳", + "flag_for_poland": "🇵🇱", + "flag_for_portugal": "🇵🇹", + "flag_for_puerto_rico": "🇵🇷", + "flag_for_qatar": "🇶🇦", + "flag_for_romania": "🇷🇴", + "flag_for_russia": "🇷🇺", + "flag_for_rwanda": "🇷🇼", + "flag_for_réunion": "🇷🇪", + "flag_for_samoa": "🇼🇸", + "flag_for_san_marino": "🇸🇲", + "flag_for_saudi_arabia": "🇸🇦", + "flag_for_senegal": "🇸🇳", + "flag_for_serbia": "🇷🇸", + "flag_for_seychelles": "🇸🇨", + "flag_for_sierra_leone": "🇸🇱", + "flag_for_singapore": "🇸🇬", + "flag_for_sint_maarten": "🇸🇽", + "flag_for_slovakia": "🇸🇰", + "flag_for_slovenia": "🇸🇮", + "flag_for_solomon_islands": "🇸🇧", + "flag_for_somalia": "🇸🇴", + "flag_for_south_africa": "🇿🇦", + "flag_for_south_georgia_&_south_sandwich_islands": "🇬🇸", + "flag_for_south_korea": "🇰🇷", + "flag_for_south_sudan": "🇸🇸", + "flag_for_spain": "🇪🇸", + "flag_for_sri_lanka": "🇱🇰", + "flag_for_st._barthélemy": "🇧🇱", + "flag_for_st._helena": "🇸🇭", + "flag_for_st._kitts_&_nevis": "🇰🇳", + "flag_for_st._lucia": "🇱🇨", + "flag_for_st._martin": "🇲🇫", + "flag_for_st._pierre_&_miquelon": "🇵🇲", + "flag_for_st._vincent_&_grenadines": "🇻🇨", + "flag_for_sudan": "🇸🇩", + "flag_for_suriname": "🇸🇷", + "flag_for_svalbard_&_jan_mayen": "🇸🇯", + "flag_for_swaziland": "🇸🇿", + "flag_for_sweden": "🇸🇪", + "flag_for_switzerland": "🇨🇭", + "flag_for_syria": "🇸🇾", + "flag_for_são_tomé_&_príncipe": "🇸🇹", + "flag_for_taiwan": "🇹🇼", + "flag_for_tajikistan": "🇹🇯", + "flag_for_tanzania": "🇹🇿", + "flag_for_thailand": "🇹🇭", + "flag_for_timor__leste": "🇹🇱", + "flag_for_togo": "🇹🇬", + "flag_for_tokelau": "🇹🇰", + "flag_for_tonga": "🇹🇴", + "flag_for_trinidad_&_tobago": "🇹🇹", + "flag_for_tristan_da_cunha": "🇹🇦", + "flag_for_tunisia": "🇹🇳", + "flag_for_turkey": "🇹🇷", + "flag_for_turkmenistan": "🇹🇲", + "flag_for_turks_&_caicos_islands": "🇹🇨", + "flag_for_tuvalu": "🇹🇻", + "flag_for_u.s._outlying_islands": "🇺🇲", + "flag_for_u.s._virgin_islands": "🇻🇮", + "flag_for_uganda": "🇺🇬", + "flag_for_ukraine": "🇺🇦", + "flag_for_united_arab_emirates": "🇦🇪", + "flag_for_united_kingdom": "🇬🇧", + "flag_for_united_states": "🇺🇸", + "flag_for_uruguay": "🇺🇾", + "flag_for_uzbekistan": "🇺🇿", + "flag_for_vanuatu": "🇻🇺", + "flag_for_vatican_city": "🇻🇦", + "flag_for_venezuela": "🇻🇪", + "flag_for_vietnam": "🇻🇳", + "flag_for_wallis_&_futuna": "🇼🇫", + "flag_for_western_sahara": "🇪🇭", + "flag_for_yemen": "🇾🇪", + "flag_for_zambia": "🇿🇲", + "flag_for_zimbabwe": "🇿🇼", + "flag_for_åland_islands": "🇦🇽", + "golf": "⛳", + "fleur__de__lis": "⚜", + "muscle": "💪", + "flushed": "😳", + "frame_with_picture": "🖼", + "fries": "🍟", + "frog": "🐸", + "hatched_chick": "🐥", + "frowning": "😦", + "fuelpump": "⛽", + "full_moon_with_face": "🌝", + "gem": "💎", + "star2": "🌟", + "golfer": "🏌", + "mortar_board": "🎓", + "grimacing": "😬", + "smile_cat": "😸", + "grinning": "😀", + "grin": "😁", + "heartpulse": "💗", + "guardsman": "💂", + "haircut": "💇", + "hamster": "🐹", + "raising_hand": "🙋", + "headphones": "🎧", + "hear_no_evil": "🙉", + "cupid": "💘", + "gift_heart": "💝", + "heart": "❤", + "exclamation": "❗", + "heavy_exclamation_mark": "❗", + "heavy_heart_exclamation_mark_ornament": "❣", + "o": "⭕", + "helm_symbol": "⎈", + "helmet_with_white_cross": "⛑", + "high_heel": "👠", + "bullettrain_side": "🚄", + "bullettrain_front": "🚅", + "high_brightness": "🔆", + "zap": "⚡", + "hocho": "🔪", + "knife": "🔪", + "bee": "🐝", + "traffic_light": "🚥", + "racehorse": "🐎", + "coffee": "☕", + "hotsprings": "♨", + "hourglass": "⌛", + "hourglass_flowing_sand": "⏳", + "house_buildings": "🏘", + "100": "💯", + "hushed": "😯", + "ice_hockey_stick_and_puck": "🏒", + "imp": "👿", + "information_desk_person": "💁", + "information_source": "ℹ", + "capital_abcd": "🔠", + "abc": "🔤", + "abcd": "🔡", + "1234": "🔢", + "symbols": "🔣", + "izakaya_lantern": "🏮", + "lantern": "🏮", + "jack_o_lantern": "🎃", + "dolls": "🎎", + "japanese_goblin": "👺", + "japanese_ogre": "👹", + "beginner": "🔰", + "zero": "0️⃣", + "one": "1️⃣", + "ten": "🔟", + "two": "2️⃣", + "three": "3️⃣", + "four": "4️⃣", + "five": "5️⃣", + "six": "6️⃣", + "seven": "7️⃣", + "eight": "8️⃣", + "nine": "9️⃣", + "couplekiss": "💏", + "kissing_cat": "😽", + "kissing": "😗", + "kissing_closed_eyes": "😚", + "kissing_smiling_eyes": "😙", + "beetle": "🐞", + "large_blue_circle": "🔵", + "last_quarter_moon_with_face": "🌜", + "leaves": "🍃", + "mag": "🔍", + "left_right_arrow": "↔", + "leftwards_arrow_with_hook": "↩", + "arrow_left": "⬅", + "lock": "🔒", + "lock_with_ink_pen": "🔏", + "sob": "😭", + "low_brightness": "🔅", + "lower_left_ballpoint_pen": "🖊", + "lower_left_crayon": "🖍", + "lower_left_fountain_pen": "🖋", + "lower_left_paintbrush": "🖌", + "mahjong": "🀄", + "couple": "👫", + "man_in_business_suit_levitating": "🕴", + "man_with_gua_pi_mao": "👲", + "man_with_turban": "👳", + "mans_shoe": "👞", + "shoe": "👞", + "menorah_with_nine_branches": "🕎", + "mens": "🚹", + "minidisc": "💽", + "iphone": "📱", + "calling": "📲", + "money__mouth_face": "🤑", + "moneybag": "💰", + "rice_scene": "🎑", + "mountain_bicyclist": "🚵", + "mouse2": "🐁", + "lips": "👄", + "moyai": "🗿", + "notes": "🎶", + "nail_care": "💅", + "ab": "🆎", + "negative_squared_cross_mark": "❎", + "a": "🅰", + "b": "🅱", + "o2": "🅾", + "parking": "🅿", + "new_moon_with_face": "🌚", + "no_entry_sign": "🚫", + "underage": "🔞", + "non__potable_water": "🚱", + "arrow_upper_right": "↗", + "arrow_upper_left": "↖", + "office": "🏢", + "older_man": "👴", + "older_woman": "👵", + "om_symbol": "🕉", + "on": "🔛", + "book": "📖", + "unlock": "🔓", + "mailbox_with_no_mail": "📭", + "mailbox_with_mail": "📬", + "cd": "💿", + "tada": "🎉", + "feet": "🐾", + "walking": "🚶", + "pencil2": "✏", + "pensive": "😔", + "persevere": "😣", + "bow": "🙇", + "raised_hands": "🙌", + "person_with_ball": "⛹", + "person_with_blond_hair": "👱", + "pray": "🙏", + "person_with_pouting_face": "🙎", + "computer": "💻", + "pig2": "🐖", + "hankey": "💩", + "poop": "💩", + "shit": "💩", + "bamboo": "🎍", + "gun": "🔫", + "black_joker": "🃏", + "rotating_light": "🚨", + "cop": "👮", + "stew": "🍲", + "pouch": "👝", + "pouting_cat": "😾", + "rage": "😡", + "put_litter_in_its_place": "🚮", + "rabbit2": "🐇", + "racing_motorcycle": "🏍", + "radioactive_sign": "☢", + "fist": "✊", + "hand": "✋", + "raised_hand_with_fingers_splayed": "🖐", + "raised_hand_with_part_between_middle_and_ring_fingers": "🖖", + "blue_car": "🚙", + "apple": "🍎", + "relieved": "😌", + "reversed_hand_with_middle_finger_extended": "🖕", + "mag_right": "🔎", + "arrow_right_hook": "↪", + "sweet_potato": "🍠", + "robot": "🤖", + "rolled__up_newspaper": "🗞", + "rowboat": "🚣", + "runner": "🏃", + "running": "🏃", + "running_shirt_with_sash": "🎽", + "boat": "⛵", + "scales": "⚖", + "school_satchel": "🎒", + "scorpius": "♏", + "see_no_evil": "🙈", + "sheep": "🐑", + "stars": "🌠", + "cake": "🍰", + "six_pointed_star": "🔯", + "ski": "🎿", + "sleeping_accommodation": "🛌", + "sleeping": "😴", + "sleepy": "😪", + "sleuth_or_spy": "🕵", + "heart_eyes_cat": "😻", + "smiley_cat": "😺", + "innocent": "😇", + "heart_eyes": "😍", + "smiling_imp": "😈", + "smiley": "😃", + "sweat_smile": "😅", + "smile": "😄", + "laughing": "😆", + "satisfied": "😆", + "blush": "😊", + "smirk": "😏", + "smoking": "🚬", + "snow_capped_mountain": "🏔", + "soccer": "⚽", + "icecream": "🍦", + "soon": "🔜", + "arrow_lower_right": "↘", + "arrow_lower_left": "↙", + "speak_no_evil": "🙊", + "speaker": "🔈", + "mute": "🔇", + "sound": "🔉", + "loud_sound": "🔊", + "speaking_head_in_silhouette": "🗣", + "spiral_calendar_pad": "🗓", + "spiral_note_pad": "🗒", + "shell": "🐚", + "sweat_drops": "💦", + "u5272": "🈹", + "u5408": "🈴", + "u55b6": "🈺", + "u6307": "🈯", + "u6708": "🈷", + "u6709": "🈶", + "u6e80": "🈵", + "u7121": "🈚", + "u7533": "🈸", + "u7981": "🈲", + "u7a7a": "🈳", + "cl": "🆑", + "cool": "🆒", + "free": "🆓", + "id": "🆔", + "koko": "🈁", + "sa": "🈂", + "new": "🆕", + "ng": "🆖", + "ok": "🆗", + "sos": "🆘", + "up": "🆙", + "vs": "🆚", + "steam_locomotive": "🚂", + "ramen": "🍜", + "partly_sunny": "⛅", + "city_sunrise": "🌇", + "surfer": "🏄", + "swimmer": "🏊", + "shirt": "👕", + "tshirt": "👕", + "table_tennis_paddle_and_ball": "🏓", + "tea": "🍵", + "tv": "📺", + "three_button_mouse": "🖱", + "+1": "👍", + "thumbsup": "👍", + "__1": "👎", + "-1": "👎", + "thumbsdown": "👎", + "thunder_cloud_and_rain": "⛈", + "tiger2": "🐅", + "tophat": "🎩", + "top": "🔝", + "tm": "™", + "train2": "🚆", + "triangular_flag_on_post": "🚩", + "trident": "🔱", + "twisted_rightwards_arrows": "🔀", + "unamused": "😒", + "small_red_triangle": "🔺", + "arrow_up_small": "🔼", + "arrow_up_down": "↕", + "upside__down_face": "🙃", + "arrow_up": "⬆", + "v": "✌", + "vhs": "📼", + "wc": "🚾", + "ocean": "🌊", + "waving_black_flag": "🏴", + "wave": "👋", + "waving_white_flag": "🏳", + "moon": "🌔", + "scream_cat": "🙀", + "weary": "😩", + "weight_lifter": "🏋", + "whale2": "🐋", + "wheelchair": "♿", + "point_down": "👇", + "grey_exclamation": "❕", + "white_frowning_face": "☹", + "white_check_mark": "✅", + "point_left": "👈", + "white_medium_small_square": "◽", + "star": "⭐", + "grey_question": "❔", + "point_right": "👉", + "relaxed": "☺", + "white_sun_behind_cloud": "🌥", + "white_sun_behind_cloud_with_rain": "🌦", + "white_sun_with_small_cloud": "🌤", + "point_up_2": "👆", + "point_up": "☝", + "wind_blowing_face": "🌬", + "wink": "😉", + "wolf": "🐺", + "dancers": "👯", + "boot": "👢", + "womans_clothes": "👚", + "womans_hat": "👒", + "sandal": "👡", + "womens": "🚺", + "worried": "😟", + "gift": "🎁", + "zipper__mouth_face": "🤐", + "regional_indicator_a": "🇦", + "regional_indicator_b": "🇧", + "regional_indicator_c": "🇨", + "regional_indicator_d": "🇩", + "regional_indicator_e": "🇪", + "regional_indicator_f": "🇫", + "regional_indicator_g": "🇬", + "regional_indicator_h": "🇭", + "regional_indicator_i": "🇮", + "regional_indicator_j": "🇯", + "regional_indicator_k": "🇰", + "regional_indicator_l": "🇱", + "regional_indicator_m": "🇲", + "regional_indicator_n": "🇳", + "regional_indicator_o": "🇴", + "regional_indicator_p": "🇵", + "regional_indicator_q": "🇶", + "regional_indicator_r": "🇷", + "regional_indicator_s": "🇸", + "regional_indicator_t": "🇹", + "regional_indicator_u": "🇺", + "regional_indicator_v": "🇻", + "regional_indicator_w": "🇼", + "regional_indicator_x": "🇽", + "regional_indicator_y": "🇾", + "regional_indicator_z": "🇿", +} diff --git a/src/pip/_vendor/rich/_emoji_replace.py b/src/pip/_vendor/rich/_emoji_replace.py new file mode 100644 index 00000000000..bb2cafa1801 --- /dev/null +++ b/src/pip/_vendor/rich/_emoji_replace.py @@ -0,0 +1,32 @@ +from typing import Callable, Match, Optional +import re + +from ._emoji_codes import EMOJI + + +_ReStringMatch = Match[str] # regex match object +_ReSubCallable = Callable[[_ReStringMatch], str] # Callable invoked by re.sub +_EmojiSubMethod = Callable[[_ReSubCallable, str], str] # Sub method of a compiled re + + +def _emoji_replace( + text: str, + default_variant: Optional[str] = None, + _emoji_sub: _EmojiSubMethod = re.compile(r"(:(\S*?)(?:(?:\-)(emoji|text))?:)").sub, +) -> str: + """Replace emoji code in text.""" + get_emoji = EMOJI.__getitem__ + variants = {"text": "\uFE0E", "emoji": "\uFE0F"} + get_variant = variants.get + default_variant_code = variants.get(default_variant, "") if default_variant else "" + + def do_replace(match: Match[str]) -> str: + emoji_code, emoji_name, variant = match.groups() + try: + return get_emoji(emoji_name.lower()) + get_variant( + variant, default_variant_code + ) + except KeyError: + return emoji_code + + return _emoji_sub(do_replace, text) diff --git a/src/pip/_vendor/rich/_extension.py b/src/pip/_vendor/rich/_extension.py new file mode 100644 index 00000000000..cbd6da9be49 --- /dev/null +++ b/src/pip/_vendor/rich/_extension.py @@ -0,0 +1,10 @@ +from typing import Any + + +def load_ipython_extension(ip: Any) -> None: # pragma: no cover + # prevent circular import + from pip._vendor.rich.pretty import install + from pip._vendor.rich.traceback import install as tr_install + + install() + tr_install() diff --git a/src/pip/_vendor/rich/_inspect.py b/src/pip/_vendor/rich/_inspect.py new file mode 100644 index 00000000000..262695b1c47 --- /dev/null +++ b/src/pip/_vendor/rich/_inspect.py @@ -0,0 +1,210 @@ +from __future__ import absolute_import + +from inspect import cleandoc, getdoc, getfile, isclass, ismodule, signature +from typing import Any, Iterable, Optional, Tuple + +from .console import RenderableType, Group +from .highlighter import ReprHighlighter +from .jupyter import JupyterMixin +from .panel import Panel +from .pretty import Pretty +from .table import Table +from .text import Text, TextType + + +def _first_paragraph(doc: str) -> str: + """Get the first paragraph from a docstring.""" + paragraph, _, _ = doc.partition("\n\n") + return paragraph + + +def _reformat_doc(doc: str) -> str: + """Reformat docstring.""" + doc = cleandoc(doc).strip() + return doc + + +class Inspect(JupyterMixin): + """A renderable to inspect any Python Object. + + Args: + obj (Any): An object to inspect. + title (str, optional): Title to display over inspect result, or None use type. Defaults to None. + help (bool, optional): Show full help text rather than just first paragraph. Defaults to False. + methods (bool, optional): Enable inspection of callables. Defaults to False. + docs (bool, optional): Also render doc strings. Defaults to True. + private (bool, optional): Show private attributes (beginning with underscore). Defaults to False. + dunder (bool, optional): Show attributes starting with double underscore. Defaults to False. + sort (bool, optional): Sort attributes alphabetically. Defaults to True. + all (bool, optional): Show all attributes. Defaults to False. + value (bool, optional): Pretty print value of object. Defaults to True. + """ + + def __init__( + self, + obj: Any, + *, + title: Optional[TextType] = None, + help: bool = False, + methods: bool = False, + docs: bool = True, + private: bool = False, + dunder: bool = False, + sort: bool = True, + all: bool = True, + value: bool = True, + ) -> None: + self.highlighter = ReprHighlighter() + self.obj = obj + self.title = title or self._make_title(obj) + if all: + methods = private = dunder = True + self.help = help + self.methods = methods + self.docs = docs or help + self.private = private or dunder + self.dunder = dunder + self.sort = sort + self.value = value + + def _make_title(self, obj: Any) -> Text: + """Make a default title.""" + title_str = ( + str(obj) + if (isclass(obj) or callable(obj) or ismodule(obj)) + else str(type(obj)) + ) + title_text = self.highlighter(title_str) + return title_text + + def __rich__(self) -> Panel: + return Panel.fit( + Group(*self._render()), + title=self.title, + border_style="scope.border", + padding=(0, 1), + ) + + def _get_signature(self, name: str, obj: Any) -> Optional[Text]: + """Get a signature for a callable.""" + try: + _signature = str(signature(obj)) + ":" + except ValueError: + _signature = "(...)" + except TypeError: + return None + + source_filename: Optional[str] = None + try: + source_filename = getfile(obj) + except TypeError: + pass + + callable_name = Text(name, style="inspect.callable") + if source_filename: + callable_name.stylize(f"link file://{source_filename}") + signature_text = self.highlighter(_signature) + + qualname = name or getattr(obj, "__qualname__", name) + qual_signature = Text.assemble( + ("def ", "inspect.def"), (qualname, "inspect.callable"), signature_text + ) + + return qual_signature + + def _render(self) -> Iterable[RenderableType]: + """Render object.""" + + def sort_items(item: Tuple[str, Any]) -> Tuple[bool, str]: + key, (_error, value) = item + return (callable(value), key.strip("_").lower()) + + def safe_getattr(attr_name: str) -> Tuple[Any, Any]: + """Get attribute or any exception.""" + try: + return (None, getattr(obj, attr_name)) + except Exception as error: + return (error, None) + + obj = self.obj + keys = dir(obj) + total_items = len(keys) + if not self.dunder: + keys = [key for key in keys if not key.startswith("__")] + if not self.private: + keys = [key for key in keys if not key.startswith("_")] + not_shown_count = total_items - len(keys) + items = [(key, safe_getattr(key)) for key in keys] + if self.sort: + items.sort(key=sort_items) + + items_table = Table.grid(padding=(0, 1), expand=False) + items_table.add_column(justify="right") + add_row = items_table.add_row + highlighter = self.highlighter + + if callable(obj): + signature = self._get_signature("", obj) + if signature is not None: + yield signature + yield "" + + if self.docs: + _doc = getdoc(obj) + if _doc is not None: + if not self.help: + _doc = _first_paragraph(_doc) + doc_text = Text(_reformat_doc(_doc), style="inspect.help") + doc_text = highlighter(doc_text) + yield doc_text + yield "" + + if self.value and not (isclass(obj) or callable(obj) or ismodule(obj)): + yield Panel( + Pretty(obj, indent_guides=True, max_length=10, max_string=60), + border_style="inspect.value.border", + ) + yield "" + + for key, (error, value) in items: + key_text = Text.assemble( + ( + key, + "inspect.attr.dunder" if key.startswith("__") else "inspect.attr", + ), + (" =", "inspect.equals"), + ) + if error is not None: + warning = key_text.copy() + warning.stylize("inspect.error") + add_row(warning, highlighter(repr(error))) + continue + + if callable(value): + if not self.methods: + continue + + _signature_text = self._get_signature(key, value) + if _signature_text is None: + add_row(key_text, Pretty(value, highlighter=highlighter)) + else: + if self.docs: + docs = getdoc(value) + if docs is not None: + _doc = _reformat_doc(str(docs)) + if not self.help: + _doc = _first_paragraph(_doc) + _signature_text.append("\n" if "\n" in _doc else " ") + doc = highlighter(_doc) + doc.stylize("inspect.doc") + _signature_text.append(doc) + + add_row(key_text, _signature_text) + else: + add_row(key_text, Pretty(value, highlighter=highlighter)) + if items_table.row_count: + yield items_table + else: + yield Text.from_markup( + f"[b cyan]{not_shown_count}[/][i] attribute(s) not shown.[/i] Run [b][magenta]inspect[/]([not b]inspect[/])[/b] for options." + ) diff --git a/src/pip/_vendor/rich/_log_render.py b/src/pip/_vendor/rich/_log_render.py new file mode 100644 index 00000000000..fc16c84437a --- /dev/null +++ b/src/pip/_vendor/rich/_log_render.py @@ -0,0 +1,94 @@ +from datetime import datetime +from typing import Iterable, List, Optional, TYPE_CHECKING, Union, Callable + + +from .text import Text, TextType + +if TYPE_CHECKING: + from .console import Console, ConsoleRenderable, RenderableType + from .table import Table + +FormatTimeCallable = Callable[[datetime], Text] + + +class LogRender: + def __init__( + self, + show_time: bool = True, + show_level: bool = False, + show_path: bool = True, + time_format: Union[str, FormatTimeCallable] = "[%x %X]", + omit_repeated_times: bool = True, + level_width: Optional[int] = 8, + ) -> None: + self.show_time = show_time + self.show_level = show_level + self.show_path = show_path + self.time_format = time_format + self.omit_repeated_times = omit_repeated_times + self.level_width = level_width + self._last_time: Optional[Text] = None + + def __call__( + self, + console: "Console", + renderables: Iterable["ConsoleRenderable"], + log_time: Optional[datetime] = None, + time_format: Optional[Union[str, FormatTimeCallable]] = None, + level: TextType = "", + path: Optional[str] = None, + line_no: Optional[int] = None, + link_path: Optional[str] = None, + ) -> "Table": + from .containers import Renderables + from .table import Table + + output = Table.grid(padding=(0, 1)) + output.expand = True + if self.show_time: + output.add_column(style="log.time") + if self.show_level: + output.add_column(style="log.level", width=self.level_width) + output.add_column(ratio=1, style="log.message", overflow="fold") + if self.show_path and path: + output.add_column(style="log.path") + row: List["RenderableType"] = [] + if self.show_time: + log_time = log_time or console.get_datetime() + time_format = time_format or self.time_format + if callable(time_format): + log_time_display = time_format(log_time) + else: + log_time_display = Text(log_time.strftime(time_format)) + if log_time_display == self._last_time and self.omit_repeated_times: + row.append(Text(" " * len(log_time_display))) + else: + row.append(log_time_display) + self._last_time = log_time_display + if self.show_level: + row.append(level) + + row.append(Renderables(renderables)) + if self.show_path and path: + path_text = Text() + path_text.append( + path, style=f"link file://{link_path}" if link_path else "" + ) + if line_no: + path_text.append(":") + path_text.append( + f"{line_no}", + style=f"link file://{link_path}#{line_no}" if link_path else "", + ) + row.append(path_text) + + output.add_row(*row) + return output + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console + + c = Console() + c.print("[on blue]Hello", justify="right") + c.log("[on blue]hello", justify="right") diff --git a/src/pip/_vendor/rich/_loop.py b/src/pip/_vendor/rich/_loop.py new file mode 100644 index 00000000000..01c6cafbe53 --- /dev/null +++ b/src/pip/_vendor/rich/_loop.py @@ -0,0 +1,43 @@ +from typing import Iterable, Tuple, TypeVar + +T = TypeVar("T") + + +def loop_first(values: Iterable[T]) -> Iterable[Tuple[bool, T]]: + """Iterate and generate a tuple with a flag for first value.""" + iter_values = iter(values) + try: + value = next(iter_values) + except StopIteration: + return + yield True, value + for value in iter_values: + yield False, value + + +def loop_last(values: Iterable[T]) -> Iterable[Tuple[bool, T]]: + """Iterate and generate a tuple with a flag for last value.""" + iter_values = iter(values) + try: + previous_value = next(iter_values) + except StopIteration: + return + for value in iter_values: + yield False, previous_value + previous_value = value + yield True, previous_value + + +def loop_first_last(values: Iterable[T]) -> Iterable[Tuple[bool, bool, T]]: + """Iterate and generate a tuple with a flag for first and last value.""" + iter_values = iter(values) + try: + previous_value = next(iter_values) + except StopIteration: + return + first = True + for value in iter_values: + yield first, False, previous_value + first = False + previous_value = value + yield first, True, previous_value diff --git a/src/pip/_vendor/rich/_lru_cache.py b/src/pip/_vendor/rich/_lru_cache.py new file mode 100644 index 00000000000..b7bf2ce1ad7 --- /dev/null +++ b/src/pip/_vendor/rich/_lru_cache.py @@ -0,0 +1,34 @@ +from collections import OrderedDict +from typing import Dict, Generic, TypeVar + + +CacheKey = TypeVar("CacheKey") +CacheValue = TypeVar("CacheValue") + + +class LRUCache(Generic[CacheKey, CacheValue], OrderedDict): # type: ignore # https://github.com/python/mypy/issues/6904 + """ + A dictionary-like container that stores a given maximum items. + + If an additional item is added when the LRUCache is full, the least + recently used key is discarded to make room for the new item. + + """ + + def __init__(self, cache_size: int) -> None: + self.cache_size = cache_size + super(LRUCache, self).__init__() + + def __setitem__(self, key: CacheKey, value: CacheValue) -> None: + """Store a new views, potentially discarding an old value.""" + if key not in self: + if len(self) >= self.cache_size: + self.popitem(last=False) + OrderedDict.__setitem__(self, key, value) + + def __getitem__(self: Dict[CacheKey, CacheValue], key: CacheKey) -> CacheValue: + """Gets the item, but also makes it most recent.""" + value: CacheValue = OrderedDict.__getitem__(self, key) + OrderedDict.__delitem__(self, key) + OrderedDict.__setitem__(self, key, value) + return value diff --git a/src/pip/_vendor/rich/_palettes.py b/src/pip/_vendor/rich/_palettes.py new file mode 100644 index 00000000000..3c748d33e45 --- /dev/null +++ b/src/pip/_vendor/rich/_palettes.py @@ -0,0 +1,309 @@ +from .palette import Palette + + +# Taken from https://en.wikipedia.org/wiki/ANSI_escape_code (Windows 10 column) +WINDOWS_PALETTE = Palette( + [ + (12, 12, 12), + (197, 15, 31), + (19, 161, 14), + (193, 156, 0), + (0, 55, 218), + (136, 23, 152), + (58, 150, 221), + (204, 204, 204), + (118, 118, 118), + (231, 72, 86), + (22, 198, 12), + (249, 241, 165), + (59, 120, 255), + (180, 0, 158), + (97, 214, 214), + (242, 242, 242), + ] +) + +# # The standard ansi colors (including bright variants) +STANDARD_PALETTE = Palette( + [ + (0, 0, 0), + (170, 0, 0), + (0, 170, 0), + (170, 85, 0), + (0, 0, 170), + (170, 0, 170), + (0, 170, 170), + (170, 170, 170), + (85, 85, 85), + (255, 85, 85), + (85, 255, 85), + (255, 255, 85), + (85, 85, 255), + (255, 85, 255), + (85, 255, 255), + (255, 255, 255), + ] +) + + +# The 256 color palette +EIGHT_BIT_PALETTE = Palette( + [ + (0, 0, 0), + (128, 0, 0), + (0, 128, 0), + (128, 128, 0), + (0, 0, 128), + (128, 0, 128), + (0, 128, 128), + (192, 192, 192), + (128, 128, 128), + (255, 0, 0), + (0, 255, 0), + (255, 255, 0), + (0, 0, 255), + (255, 0, 255), + (0, 255, 255), + (255, 255, 255), + (0, 0, 0), + (0, 0, 95), + (0, 0, 135), + (0, 0, 175), + (0, 0, 215), + (0, 0, 255), + (0, 95, 0), + (0, 95, 95), + (0, 95, 135), + (0, 95, 175), + (0, 95, 215), + (0, 95, 255), + (0, 135, 0), + (0, 135, 95), + (0, 135, 135), + (0, 135, 175), + (0, 135, 215), + (0, 135, 255), + (0, 175, 0), + (0, 175, 95), + (0, 175, 135), + (0, 175, 175), + (0, 175, 215), + (0, 175, 255), + (0, 215, 0), + (0, 215, 95), + (0, 215, 135), + (0, 215, 175), + (0, 215, 215), + (0, 215, 255), + (0, 255, 0), + (0, 255, 95), + (0, 255, 135), + (0, 255, 175), + (0, 255, 215), + (0, 255, 255), + (95, 0, 0), + (95, 0, 95), + (95, 0, 135), + (95, 0, 175), + (95, 0, 215), + (95, 0, 255), + (95, 95, 0), + (95, 95, 95), + (95, 95, 135), + (95, 95, 175), + (95, 95, 215), + (95, 95, 255), + (95, 135, 0), + (95, 135, 95), + (95, 135, 135), + (95, 135, 175), + (95, 135, 215), + (95, 135, 255), + (95, 175, 0), + (95, 175, 95), + (95, 175, 135), + (95, 175, 175), + (95, 175, 215), + (95, 175, 255), + (95, 215, 0), + (95, 215, 95), + (95, 215, 135), + (95, 215, 175), + (95, 215, 215), + (95, 215, 255), + (95, 255, 0), + (95, 255, 95), + (95, 255, 135), + (95, 255, 175), + (95, 255, 215), + (95, 255, 255), + (135, 0, 0), + (135, 0, 95), + (135, 0, 135), + (135, 0, 175), + (135, 0, 215), + (135, 0, 255), + (135, 95, 0), + (135, 95, 95), + (135, 95, 135), + (135, 95, 175), + (135, 95, 215), + (135, 95, 255), + (135, 135, 0), + (135, 135, 95), + (135, 135, 135), + (135, 135, 175), + (135, 135, 215), + (135, 135, 255), + (135, 175, 0), + (135, 175, 95), + (135, 175, 135), + (135, 175, 175), + (135, 175, 215), + (135, 175, 255), + (135, 215, 0), + (135, 215, 95), + (135, 215, 135), + (135, 215, 175), + (135, 215, 215), + (135, 215, 255), + (135, 255, 0), + (135, 255, 95), + (135, 255, 135), + (135, 255, 175), + (135, 255, 215), + (135, 255, 255), + (175, 0, 0), + (175, 0, 95), + (175, 0, 135), + (175, 0, 175), + (175, 0, 215), + (175, 0, 255), + (175, 95, 0), + (175, 95, 95), + (175, 95, 135), + (175, 95, 175), + (175, 95, 215), + (175, 95, 255), + (175, 135, 0), + (175, 135, 95), + (175, 135, 135), + (175, 135, 175), + (175, 135, 215), + (175, 135, 255), + (175, 175, 0), + (175, 175, 95), + (175, 175, 135), + (175, 175, 175), + (175, 175, 215), + (175, 175, 255), + (175, 215, 0), + (175, 215, 95), + (175, 215, 135), + (175, 215, 175), + (175, 215, 215), + (175, 215, 255), + (175, 255, 0), + (175, 255, 95), + (175, 255, 135), + (175, 255, 175), + (175, 255, 215), + (175, 255, 255), + (215, 0, 0), + (215, 0, 95), + (215, 0, 135), + (215, 0, 175), + (215, 0, 215), + (215, 0, 255), + (215, 95, 0), + (215, 95, 95), + (215, 95, 135), + (215, 95, 175), + (215, 95, 215), + (215, 95, 255), + (215, 135, 0), + (215, 135, 95), + (215, 135, 135), + (215, 135, 175), + (215, 135, 215), + (215, 135, 255), + (215, 175, 0), + (215, 175, 95), + (215, 175, 135), + (215, 175, 175), + (215, 175, 215), + (215, 175, 255), + (215, 215, 0), + (215, 215, 95), + (215, 215, 135), + (215, 215, 175), + (215, 215, 215), + (215, 215, 255), + (215, 255, 0), + (215, 255, 95), + (215, 255, 135), + (215, 255, 175), + (215, 255, 215), + (215, 255, 255), + (255, 0, 0), + (255, 0, 95), + (255, 0, 135), + (255, 0, 175), + (255, 0, 215), + (255, 0, 255), + (255, 95, 0), + (255, 95, 95), + (255, 95, 135), + (255, 95, 175), + (255, 95, 215), + (255, 95, 255), + (255, 135, 0), + (255, 135, 95), + (255, 135, 135), + (255, 135, 175), + (255, 135, 215), + (255, 135, 255), + (255, 175, 0), + (255, 175, 95), + (255, 175, 135), + (255, 175, 175), + (255, 175, 215), + (255, 175, 255), + (255, 215, 0), + (255, 215, 95), + (255, 215, 135), + (255, 215, 175), + (255, 215, 215), + (255, 215, 255), + (255, 255, 0), + (255, 255, 95), + (255, 255, 135), + (255, 255, 175), + (255, 255, 215), + (255, 255, 255), + (8, 8, 8), + (18, 18, 18), + (28, 28, 28), + (38, 38, 38), + (48, 48, 48), + (58, 58, 58), + (68, 68, 68), + (78, 78, 78), + (88, 88, 88), + (98, 98, 98), + (108, 108, 108), + (118, 118, 118), + (128, 128, 128), + (138, 138, 138), + (148, 148, 148), + (158, 158, 158), + (168, 168, 168), + (178, 178, 178), + (188, 188, 188), + (198, 198, 198), + (208, 208, 208), + (218, 218, 218), + (228, 228, 228), + (238, 238, 238), + ] +) diff --git a/src/pip/_vendor/rich/_pick.py b/src/pip/_vendor/rich/_pick.py new file mode 100644 index 00000000000..4f6d8b2d794 --- /dev/null +++ b/src/pip/_vendor/rich/_pick.py @@ -0,0 +1,17 @@ +from typing import Optional + + +def pick_bool(*values: Optional[bool]) -> bool: + """Pick the first non-none bool or return the last value. + + Args: + *values (bool): Any number of boolean or None values. + + Returns: + bool: First non-none boolean. + """ + assert values, "1 or more values required" + for value in values: + if value is not None: + return value + return bool(value) diff --git a/src/pip/_vendor/rich/_ratio.py b/src/pip/_vendor/rich/_ratio.py new file mode 100644 index 00000000000..e8a3a674e00 --- /dev/null +++ b/src/pip/_vendor/rich/_ratio.py @@ -0,0 +1,160 @@ +import sys +from fractions import Fraction +from math import ceil +from typing import cast, List, Optional, Sequence + +if sys.version_info >= (3, 8): + from typing import Protocol +else: + from pip._vendor.typing_extensions import Protocol # pragma: no cover + + +class Edge(Protocol): + """Any object that defines an edge (such as Layout).""" + + size: Optional[int] = None + ratio: int = 1 + minimum_size: int = 1 + + +def ratio_resolve(total: int, edges: Sequence[Edge]) -> List[int]: + """Divide total space to satisfy size, ratio, and minimum_size, constraints. + + The returned list of integers should add up to total in most cases, unless it is + impossible to satisfy all the constraints. For instance, if there are two edges + with a minimum size of 20 each and `total` is 30 then the returned list will be + greater than total. In practice, this would mean that a Layout object would + clip the rows that would overflow the screen height. + + Args: + total (int): Total number of characters. + edges (List[Edge]): Edges within total space. + + Returns: + List[int]: Number of characters for each edge. + """ + # Size of edge or None for yet to be determined + sizes = [(edge.size or None) for edge in edges] + + _Fraction = Fraction + + # While any edges haven't been calculated + while None in sizes: + # Get flexible edges and index to map these back on to sizes list + flexible_edges = [ + (index, edge) + for index, (size, edge) in enumerate(zip(sizes, edges)) + if size is None + ] + # Remaining space in total + remaining = total - sum(size or 0 for size in sizes) + if remaining <= 0: + # No room for flexible edges + return [ + ((edge.minimum_size or 1) if size is None else size) + for size, edge in zip(sizes, edges) + ] + # Calculate number of characters in a ratio portion + portion = _Fraction( + remaining, sum((edge.ratio or 1) for _, edge in flexible_edges) + ) + + # If any edges will be less than their minimum, replace size with the minimum + for index, edge in flexible_edges: + if portion * edge.ratio <= edge.minimum_size: + sizes[index] = edge.minimum_size + # New fixed size will invalidate calculations, so we need to repeat the process + break + else: + # Distribute flexible space and compensate for rounding error + # Since edge sizes can only be integers we need to add the remainder + # to the following line + remainder = _Fraction(0) + for index, edge in flexible_edges: + size, remainder = divmod(portion * edge.ratio + remainder, 1) + sizes[index] = size + break + # Sizes now contains integers only + return cast(List[int], sizes) + + +def ratio_reduce( + total: int, ratios: List[int], maximums: List[int], values: List[int] +) -> List[int]: + """Divide an integer total in to parts based on ratios. + + Args: + total (int): The total to divide. + ratios (List[int]): A list of integer ratios. + maximums (List[int]): List of maximums values for each slot. + values (List[int]): List of values + + Returns: + List[int]: A list of integers guaranteed to sum to total. + """ + ratios = [ratio if _max else 0 for ratio, _max in zip(ratios, maximums)] + total_ratio = sum(ratios) + if not total_ratio: + return values[:] + total_remaining = total + result: List[int] = [] + append = result.append + for ratio, maximum, value in zip(ratios, maximums, values): + if ratio and total_ratio > 0: + distributed = min(maximum, round(ratio * total_remaining / total_ratio)) + append(value - distributed) + total_remaining -= distributed + total_ratio -= ratio + else: + append(value) + return result + + +def ratio_distribute( + total: int, ratios: List[int], minimums: Optional[List[int]] = None +) -> List[int]: + """Distribute an integer total in to parts based on ratios. + + Args: + total (int): The total to divide. + ratios (List[int]): A list of integer ratios. + minimums (List[int]): List of minimum values for each slot. + + Returns: + List[int]: A list of integers guaranteed to sum to total. + """ + if minimums: + ratios = [ratio if _min else 0 for ratio, _min in zip(ratios, minimums)] + total_ratio = sum(ratios) + assert total_ratio > 0, "Sum of ratios must be > 0" + + total_remaining = total + distributed_total: List[int] = [] + append = distributed_total.append + if minimums is None: + _minimums = [0] * len(ratios) + else: + _minimums = minimums + for ratio, minimum in zip(ratios, _minimums): + if total_ratio > 0: + distributed = max(minimum, ceil(ratio * total_remaining / total_ratio)) + else: + distributed = total_remaining + append(distributed) + total_ratio -= ratio + total_remaining -= distributed + return distributed_total + + +if __name__ == "__main__": + from dataclasses import dataclass + + @dataclass + class E: + + size: Optional[int] = None + ratio: int = 1 + minimum_size: int = 1 + + resolved = ratio_resolve(110, [E(None, 1, 1), E(None, 1, 1), E(None, 1, 1)]) + print(sum(resolved)) diff --git a/src/pip/_vendor/rich/_spinners.py b/src/pip/_vendor/rich/_spinners.py new file mode 100644 index 00000000000..dc1db0777ee --- /dev/null +++ b/src/pip/_vendor/rich/_spinners.py @@ -0,0 +1,848 @@ +""" +Spinners are from: +* cli-spinners: + MIT License + Copyright (c) Sindre Sorhus (sindresorhus.com) + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + IN THE SOFTWARE. +""" + +SPINNERS = { + "dots": { + "interval": 80, + "frames": ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"], + }, + "dots2": {"interval": 80, "frames": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"]}, + "dots3": { + "interval": 80, + "frames": ["⠋", "⠙", "⠚", "⠞", "⠖", "⠦", "⠴", "⠲", "⠳", "⠓"], + }, + "dots4": { + "interval": 80, + "frames": [ + "⠄", + "⠆", + "⠇", + "⠋", + "⠙", + "⠸", + "⠰", + "⠠", + "⠰", + "⠸", + "⠙", + "⠋", + "⠇", + "⠆", + ], + }, + "dots5": { + "interval": 80, + "frames": [ + "⠋", + "⠙", + "⠚", + "⠒", + "⠂", + "⠂", + "⠒", + "⠲", + "⠴", + "⠦", + "⠖", + "⠒", + "⠐", + "⠐", + "⠒", + "⠓", + "⠋", + ], + }, + "dots6": { + "interval": 80, + "frames": [ + "⠁", + "⠉", + "⠙", + "⠚", + "⠒", + "⠂", + "⠂", + "⠒", + "⠲", + "⠴", + "⠤", + "⠄", + "⠄", + "⠤", + "⠴", + "⠲", + "⠒", + "⠂", + "⠂", + "⠒", + "⠚", + "⠙", + "⠉", + "⠁", + ], + }, + "dots7": { + "interval": 80, + "frames": [ + "⠈", + "⠉", + "⠋", + "⠓", + "⠒", + "⠐", + "⠐", + "⠒", + "⠖", + "⠦", + "⠤", + "⠠", + "⠠", + "⠤", + "⠦", + "⠖", + "⠒", + "⠐", + "⠐", + "⠒", + "⠓", + "⠋", + "⠉", + "⠈", + ], + }, + "dots8": { + "interval": 80, + "frames": [ + "⠁", + "⠁", + "⠉", + "⠙", + "⠚", + "⠒", + "⠂", + "⠂", + "⠒", + "⠲", + "⠴", + "⠤", + "⠄", + "⠄", + "⠤", + "⠠", + "⠠", + "⠤", + "⠦", + "⠖", + "⠒", + "⠐", + "⠐", + "⠒", + "⠓", + "⠋", + "⠉", + "⠈", + "⠈", + ], + }, + "dots9": {"interval": 80, "frames": ["⢹", "⢺", "⢼", "⣸", "⣇", "⡧", "⡗", "⡏"]}, + "dots10": {"interval": 80, "frames": ["⢄", "⢂", "⢁", "⡁", "⡈", "⡐", "⡠"]}, + "dots11": {"interval": 100, "frames": ["⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"]}, + "dots12": { + "interval": 80, + "frames": [ + "⢀⠀", + "⡀⠀", + "⠄⠀", + "⢂⠀", + "⡂⠀", + "⠅⠀", + "⢃⠀", + "⡃⠀", + "⠍⠀", + "⢋⠀", + "⡋⠀", + "⠍⠁", + "⢋⠁", + "⡋⠁", + "⠍⠉", + "⠋⠉", + "⠋⠉", + "⠉⠙", + "⠉⠙", + "⠉⠩", + "⠈⢙", + "⠈⡙", + "⢈⠩", + "⡀⢙", + "⠄⡙", + "⢂⠩", + "⡂⢘", + "⠅⡘", + "⢃⠨", + "⡃⢐", + "⠍⡐", + "⢋⠠", + "⡋⢀", + "⠍⡁", + "⢋⠁", + "⡋⠁", + "⠍⠉", + "⠋⠉", + "⠋⠉", + "⠉⠙", + "⠉⠙", + "⠉⠩", + "⠈⢙", + "⠈⡙", + "⠈⠩", + "⠀⢙", + "⠀⡙", + "⠀⠩", + "⠀⢘", + "⠀⡘", + "⠀⠨", + "⠀⢐", + "⠀⡐", + "⠀⠠", + "⠀⢀", + "⠀⡀", + ], + }, + "dots8Bit": { + "interval": 80, + "frames": [ + "⠀", + "⠁", + "⠂", + "⠃", + "⠄", + "⠅", + "⠆", + "⠇", + "⡀", + "⡁", + "⡂", + "⡃", + "⡄", + "⡅", + "⡆", + "⡇", + "⠈", + "⠉", + "⠊", + "⠋", + "⠌", + "⠍", + "⠎", + "⠏", + "⡈", + "⡉", + "⡊", + "⡋", + "⡌", + "⡍", + "⡎", + "⡏", + "⠐", + "⠑", + "⠒", + "⠓", + "⠔", + "⠕", + "⠖", + "⠗", + "⡐", + "⡑", + "⡒", + "⡓", + "⡔", + "⡕", + "⡖", + "⡗", + "⠘", + "⠙", + "⠚", + "⠛", + "⠜", + "⠝", + "⠞", + "⠟", + "⡘", + "⡙", + "⡚", + "⡛", + "⡜", + "⡝", + "⡞", + "⡟", + "⠠", + "⠡", + "⠢", + "⠣", + "⠤", + "⠥", + "⠦", + "⠧", + "⡠", + "⡡", + "⡢", + "⡣", + "⡤", + "⡥", + "⡦", + "⡧", + "⠨", + "⠩", + "⠪", + "⠫", + "⠬", + "⠭", + "⠮", + "⠯", + "⡨", + "⡩", + "⡪", + "⡫", + "⡬", + "⡭", + "⡮", + "⡯", + "⠰", + "⠱", + "⠲", + "⠳", + "⠴", + "⠵", + "⠶", + "⠷", + "⡰", + "⡱", + "⡲", + "⡳", + "⡴", + "⡵", + "⡶", + "⡷", + "⠸", + "⠹", + "⠺", + "⠻", + "⠼", + "⠽", + "⠾", + "⠿", + "⡸", + "⡹", + "⡺", + "⡻", + "⡼", + "⡽", + "⡾", + "⡿", + "⢀", + "⢁", + "⢂", + "⢃", + "⢄", + "⢅", + "⢆", + "⢇", + "⣀", + "⣁", + "⣂", + "⣃", + "⣄", + "⣅", + "⣆", + "⣇", + "⢈", + "⢉", + "⢊", + "⢋", + "⢌", + "⢍", + "⢎", + "⢏", + "⣈", + "⣉", + "⣊", + "⣋", + "⣌", + "⣍", + "⣎", + "⣏", + "⢐", + "⢑", + "⢒", + "⢓", + "⢔", + "⢕", + "⢖", + "⢗", + "⣐", + "⣑", + "⣒", + "⣓", + "⣔", + "⣕", + "⣖", + "⣗", + "⢘", + "⢙", + "⢚", + "⢛", + "⢜", + "⢝", + "⢞", + "⢟", + "⣘", + "⣙", + "⣚", + "⣛", + "⣜", + "⣝", + "⣞", + "⣟", + "⢠", + "⢡", + "⢢", + "⢣", + "⢤", + "⢥", + "⢦", + "⢧", + "⣠", + "⣡", + "⣢", + "⣣", + "⣤", + "⣥", + "⣦", + "⣧", + "⢨", + "⢩", + "⢪", + "⢫", + "⢬", + "⢭", + "⢮", + "⢯", + "⣨", + "⣩", + "⣪", + "⣫", + "⣬", + "⣭", + "⣮", + "⣯", + "⢰", + "⢱", + "⢲", + "⢳", + "⢴", + "⢵", + "⢶", + "⢷", + "⣰", + "⣱", + "⣲", + "⣳", + "⣴", + "⣵", + "⣶", + "⣷", + "⢸", + "⢹", + "⢺", + "⢻", + "⢼", + "⢽", + "⢾", + "⢿", + "⣸", + "⣹", + "⣺", + "⣻", + "⣼", + "⣽", + "⣾", + "⣿", + ], + }, + "line": {"interval": 130, "frames": ["-", "\\", "|", "/"]}, + "line2": {"interval": 100, "frames": ["⠂", "-", "–", "—", "–", "-"]}, + "pipe": {"interval": 100, "frames": ["┤", "┘", "┴", "└", "├", "┌", "┬", "┐"]}, + "simpleDots": {"interval": 400, "frames": [". ", ".. ", "...", " "]}, + "simpleDotsScrolling": { + "interval": 200, + "frames": [". ", ".. ", "...", " ..", " .", " "], + }, + "star": {"interval": 70, "frames": ["✶", "✸", "✹", "✺", "✹", "✷"]}, + "star2": {"interval": 80, "frames": ["+", "x", "*"]}, + "flip": { + "interval": 70, + "frames": ["_", "_", "_", "-", "`", "`", "'", "´", "-", "_", "_", "_"], + }, + "hamburger": {"interval": 100, "frames": ["☱", "☲", "☴"]}, + "growVertical": { + "interval": 120, + "frames": ["▁", "▃", "▄", "▅", "▆", "▇", "▆", "▅", "▄", "▃"], + }, + "growHorizontal": { + "interval": 120, + "frames": ["▏", "▎", "▍", "▌", "▋", "▊", "▉", "▊", "▋", "▌", "▍", "▎"], + }, + "balloon": {"interval": 140, "frames": [" ", ".", "o", "O", "@", "*", " "]}, + "balloon2": {"interval": 120, "frames": [".", "o", "O", "°", "O", "o", "."]}, + "noise": {"interval": 100, "frames": ["▓", "▒", "░"]}, + "bounce": {"interval": 120, "frames": ["⠁", "⠂", "⠄", "⠂"]}, + "boxBounce": {"interval": 120, "frames": ["▖", "▘", "▝", "▗"]}, + "boxBounce2": {"interval": 100, "frames": ["▌", "▀", "▐", "▄"]}, + "triangle": {"interval": 50, "frames": ["◢", "◣", "◤", "◥"]}, + "arc": {"interval": 100, "frames": ["◜", "◠", "◝", "◞", "◡", "◟"]}, + "circle": {"interval": 120, "frames": ["◡", "⊙", "◠"]}, + "squareCorners": {"interval": 180, "frames": ["◰", "◳", "◲", "◱"]}, + "circleQuarters": {"interval": 120, "frames": ["◴", "◷", "◶", "◵"]}, + "circleHalves": {"interval": 50, "frames": ["◐", "◓", "◑", "◒"]}, + "squish": {"interval": 100, "frames": ["╫", "╪"]}, + "toggle": {"interval": 250, "frames": ["⊶", "⊷"]}, + "toggle2": {"interval": 80, "frames": ["▫", "▪"]}, + "toggle3": {"interval": 120, "frames": ["□", "■"]}, + "toggle4": {"interval": 100, "frames": ["■", "□", "▪", "▫"]}, + "toggle5": {"interval": 100, "frames": ["▮", "▯"]}, + "toggle6": {"interval": 300, "frames": ["ဝ", "၀"]}, + "toggle7": {"interval": 80, "frames": ["⦾", "⦿"]}, + "toggle8": {"interval": 100, "frames": ["◍", "◌"]}, + "toggle9": {"interval": 100, "frames": ["◉", "◎"]}, + "toggle10": {"interval": 100, "frames": ["㊂", "㊀", "㊁"]}, + "toggle11": {"interval": 50, "frames": ["⧇", "⧆"]}, + "toggle12": {"interval": 120, "frames": ["☗", "☖"]}, + "toggle13": {"interval": 80, "frames": ["=", "*", "-"]}, + "arrow": {"interval": 100, "frames": ["←", "↖", "↑", "↗", "→", "↘", "↓", "↙"]}, + "arrow2": { + "interval": 80, + "frames": ["⬆️ ", "↗️ ", "➡️ ", "↘️ ", "⬇️ ", "↙️ ", "⬅️ ", "↖️ "], + }, + "arrow3": { + "interval": 120, + "frames": ["▹▹▹▹▹", "▸▹▹▹▹", "▹▸▹▹▹", "▹▹▸▹▹", "▹▹▹▸▹", "▹▹▹▹▸"], + }, + "bouncingBar": { + "interval": 80, + "frames": [ + "[ ]", + "[= ]", + "[== ]", + "[=== ]", + "[ ===]", + "[ ==]", + "[ =]", + "[ ]", + "[ =]", + "[ ==]", + "[ ===]", + "[====]", + "[=== ]", + "[== ]", + "[= ]", + ], + }, + "bouncingBall": { + "interval": 80, + "frames": [ + "( ● )", + "( ● )", + "( ● )", + "( ● )", + "( ●)", + "( ● )", + "( ● )", + "( ● )", + "( ● )", + "(● )", + ], + }, + "smiley": {"interval": 200, "frames": ["😄 ", "😝 "]}, + "monkey": {"interval": 300, "frames": ["🙈 ", "🙈 ", "🙉 ", "🙊 "]}, + "hearts": {"interval": 100, "frames": ["💛 ", "💙 ", "💜 ", "💚 ", "❤️ "]}, + "clock": { + "interval": 100, + "frames": [ + "🕛 ", + "🕐 ", + "🕑 ", + "🕒 ", + "🕓 ", + "🕔 ", + "🕕 ", + "🕖 ", + "🕗 ", + "🕘 ", + "🕙 ", + "🕚 ", + ], + }, + "earth": {"interval": 180, "frames": ["🌍 ", "🌎 ", "🌏 "]}, + "material": { + "interval": 17, + "frames": [ + "█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "██▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "███▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "████▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "██████▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "██████▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "███████▁▁▁▁▁▁▁▁▁▁▁▁▁", + "████████▁▁▁▁▁▁▁▁▁▁▁▁", + "█████████▁▁▁▁▁▁▁▁▁▁▁", + "█████████▁▁▁▁▁▁▁▁▁▁▁", + "██████████▁▁▁▁▁▁▁▁▁▁", + "███████████▁▁▁▁▁▁▁▁▁", + "█████████████▁▁▁▁▁▁▁", + "██████████████▁▁▁▁▁▁", + "██████████████▁▁▁▁▁▁", + "▁██████████████▁▁▁▁▁", + "▁██████████████▁▁▁▁▁", + "▁██████████████▁▁▁▁▁", + "▁▁██████████████▁▁▁▁", + "▁▁▁██████████████▁▁▁", + "▁▁▁▁█████████████▁▁▁", + "▁▁▁▁██████████████▁▁", + "▁▁▁▁██████████████▁▁", + "▁▁▁▁▁██████████████▁", + "▁▁▁▁▁██████████████▁", + "▁▁▁▁▁██████████████▁", + "▁▁▁▁▁▁██████████████", + "▁▁▁▁▁▁██████████████", + "▁▁▁▁▁▁▁█████████████", + "▁▁▁▁▁▁▁█████████████", + "▁▁▁▁▁▁▁▁████████████", + "▁▁▁▁▁▁▁▁████████████", + "▁▁▁▁▁▁▁▁▁███████████", + "▁▁▁▁▁▁▁▁▁███████████", + "▁▁▁▁▁▁▁▁▁▁██████████", + "▁▁▁▁▁▁▁▁▁▁██████████", + "▁▁▁▁▁▁▁▁▁▁▁▁████████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁███████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁██████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█████", + "█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁████", + "██▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁███", + "██▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁███", + "███▁▁▁▁▁▁▁▁▁▁▁▁▁▁███", + "████▁▁▁▁▁▁▁▁▁▁▁▁▁▁██", + "█████▁▁▁▁▁▁▁▁▁▁▁▁▁▁█", + "█████▁▁▁▁▁▁▁▁▁▁▁▁▁▁█", + "██████▁▁▁▁▁▁▁▁▁▁▁▁▁█", + "████████▁▁▁▁▁▁▁▁▁▁▁▁", + "█████████▁▁▁▁▁▁▁▁▁▁▁", + "█████████▁▁▁▁▁▁▁▁▁▁▁", + "█████████▁▁▁▁▁▁▁▁▁▁▁", + "█████████▁▁▁▁▁▁▁▁▁▁▁", + "███████████▁▁▁▁▁▁▁▁▁", + "████████████▁▁▁▁▁▁▁▁", + "████████████▁▁▁▁▁▁▁▁", + "██████████████▁▁▁▁▁▁", + "██████████████▁▁▁▁▁▁", + "▁██████████████▁▁▁▁▁", + "▁██████████████▁▁▁▁▁", + "▁▁▁█████████████▁▁▁▁", + "▁▁▁▁▁████████████▁▁▁", + "▁▁▁▁▁████████████▁▁▁", + "▁▁▁▁▁▁███████████▁▁▁", + "▁▁▁▁▁▁▁▁█████████▁▁▁", + "▁▁▁▁▁▁▁▁█████████▁▁▁", + "▁▁▁▁▁▁▁▁▁█████████▁▁", + "▁▁▁▁▁▁▁▁▁█████████▁▁", + "▁▁▁▁▁▁▁▁▁▁█████████▁", + "▁▁▁▁▁▁▁▁▁▁▁████████▁", + "▁▁▁▁▁▁▁▁▁▁▁████████▁", + "▁▁▁▁▁▁▁▁▁▁▁▁███████▁", + "▁▁▁▁▁▁▁▁▁▁▁▁███████▁", + "▁▁▁▁▁▁▁▁▁▁▁▁▁███████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁███████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁████", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁███", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁███", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁██", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁██", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁██", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + ], + }, + "moon": { + "interval": 80, + "frames": ["🌑 ", "🌒 ", "🌓 ", "🌔 ", "🌕 ", "🌖 ", "🌗 ", "🌘 "], + }, + "runner": {"interval": 140, "frames": ["🚶 ", "🏃 "]}, + "pong": { + "interval": 80, + "frames": [ + "▐⠂ ▌", + "▐⠈ ▌", + "▐ ⠂ ▌", + "▐ ⠠ ▌", + "▐ ⡀ ▌", + "▐ ⠠ ▌", + "▐ ⠂ ▌", + "▐ ⠈ ▌", + "▐ ⠂ ▌", + "▐ ⠠ ▌", + "▐ ⡀ ▌", + "▐ ⠠ ▌", + "▐ ⠂ ▌", + "▐ ⠈ ▌", + "▐ ⠂▌", + "▐ ⠠▌", + "▐ ⡀▌", + "▐ ⠠ ▌", + "▐ ⠂ ▌", + "▐ ⠈ ▌", + "▐ ⠂ ▌", + "▐ ⠠ ▌", + "▐ ⡀ ▌", + "▐ ⠠ ▌", + "▐ ⠂ ▌", + "▐ ⠈ ▌", + "▐ ⠂ ▌", + "▐ ⠠ ▌", + "▐ ⡀ ▌", + "▐⠠ ▌", + ], + }, + "shark": { + "interval": 120, + "frames": [ + "▐|\\____________▌", + "▐_|\\___________▌", + "▐__|\\__________▌", + "▐___|\\_________▌", + "▐____|\\________▌", + "▐_____|\\_______▌", + "▐______|\\______▌", + "▐_______|\\_____▌", + "▐________|\\____▌", + "▐_________|\\___▌", + "▐__________|\\__▌", + "▐___________|\\_▌", + "▐____________|\\▌", + "▐____________/|▌", + "▐___________/|_▌", + "▐__________/|__▌", + "▐_________/|___▌", + "▐________/|____▌", + "▐_______/|_____▌", + "▐______/|______▌", + "▐_____/|_______▌", + "▐____/|________▌", + "▐___/|_________▌", + "▐__/|__________▌", + "▐_/|___________▌", + "▐/|____________▌", + ], + }, + "dqpb": {"interval": 100, "frames": ["d", "q", "p", "b"]}, + "weather": { + "interval": 100, + "frames": [ + "☀️ ", + "☀️ ", + "☀️ ", + "🌤 ", + "⛅️ ", + "🌥 ", + "☁️ ", + "🌧 ", + "🌨 ", + "🌧 ", + "🌨 ", + "🌧 ", + "🌨 ", + "⛈ ", + "🌨 ", + "🌧 ", + "🌨 ", + "☁️ ", + "🌥 ", + "⛅️ ", + "🌤 ", + "☀️ ", + "☀️ ", + ], + }, + "christmas": {"interval": 400, "frames": ["🌲", "🎄"]}, + "grenade": { + "interval": 80, + "frames": [ + "، ", + "′ ", + " ´ ", + " ‾ ", + " ⸌", + " ⸊", + " |", + " ⁎", + " ⁕", + " ෴ ", + " ⁓", + " ", + " ", + " ", + ], + }, + "point": {"interval": 125, "frames": ["∙∙∙", "●∙∙", "∙●∙", "∙∙●", "∙∙∙"]}, + "layer": {"interval": 150, "frames": ["-", "=", "≡"]}, + "betaWave": { + "interval": 80, + "frames": [ + "ρββββββ", + "βρβββββ", + "ββρββββ", + "βββρβββ", + "ββββρββ", + "βββββρβ", + "ββββββρ", + ], + }, + "aesthetic": { + "interval": 80, + "frames": [ + "▰▱▱▱▱▱▱", + "▰▰▱▱▱▱▱", + "▰▰▰▱▱▱▱", + "▰▰▰▰▱▱▱", + "▰▰▰▰▰▱▱", + "▰▰▰▰▰▰▱", + "▰▰▰▰▰▰▰", + "▰▱▱▱▱▱▱", + ], + }, +} diff --git a/src/pip/_vendor/rich/_stack.py b/src/pip/_vendor/rich/_stack.py new file mode 100644 index 00000000000..194564e761d --- /dev/null +++ b/src/pip/_vendor/rich/_stack.py @@ -0,0 +1,16 @@ +from typing import List, TypeVar + +T = TypeVar("T") + + +class Stack(List[T]): + """A small shim over builtin list.""" + + @property + def top(self) -> T: + """Get top of stack.""" + return self[-1] + + def push(self, item: T) -> None: + """Push an item on to the stack (append in stack nomenclature).""" + self.append(item) diff --git a/src/pip/_vendor/rich/_timer.py b/src/pip/_vendor/rich/_timer.py new file mode 100644 index 00000000000..a2ca6be03c4 --- /dev/null +++ b/src/pip/_vendor/rich/_timer.py @@ -0,0 +1,19 @@ +""" +Timer context manager, only used in debug. + +""" + +from time import time + +import contextlib +from typing import Generator + + +@contextlib.contextmanager +def timer(subject: str = "time") -> Generator[None, None, None]: + """print the elapsed time. (only used in debugging)""" + start = time() + yield + elapsed = time() - start + elapsed_ms = elapsed * 1000 + print(f"{subject} elapsed {elapsed_ms:.1f}ms") diff --git a/src/pip/_vendor/rich/_windows.py b/src/pip/_vendor/rich/_windows.py new file mode 100644 index 00000000000..ca3a680d354 --- /dev/null +++ b/src/pip/_vendor/rich/_windows.py @@ -0,0 +1,72 @@ +import sys +from dataclasses import dataclass + + +@dataclass +class WindowsConsoleFeatures: + """Windows features available.""" + + vt: bool = False + """The console supports VT codes.""" + truecolor: bool = False + """The console supports truecolor.""" + + +try: + import ctypes + from ctypes import LibraryLoader, wintypes + + if sys.platform == "win32": + windll = LibraryLoader(ctypes.WinDLL) + else: + windll = None + raise ImportError("Not windows") +except (AttributeError, ImportError, ValueError): + + # Fallback if we can't load the Windows DLL + def get_windows_console_features() -> WindowsConsoleFeatures: + features = WindowsConsoleFeatures() + return features + +else: + + STDOUT = -11 + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 + _GetConsoleMode = windll.kernel32.GetConsoleMode + _GetConsoleMode.argtypes = [wintypes.HANDLE, wintypes.LPDWORD] + _GetConsoleMode.restype = wintypes.BOOL + + _GetStdHandle = windll.kernel32.GetStdHandle + _GetStdHandle.argtypes = [ + wintypes.DWORD, + ] + _GetStdHandle.restype = wintypes.HANDLE + + def get_windows_console_features() -> WindowsConsoleFeatures: + """Get windows console features. + + Returns: + WindowsConsoleFeatures: An instance of WindowsConsoleFeatures. + """ + handle = _GetStdHandle(STDOUT) + console_mode = wintypes.DWORD() + result = _GetConsoleMode(handle, console_mode) + vt = bool(result and console_mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING) + truecolor = False + if vt: + win_version = sys.getwindowsversion() + truecolor = win_version.major > 10 or ( + win_version.major == 10 and win_version.build >= 15063 + ) + features = WindowsConsoleFeatures(vt=vt, truecolor=truecolor) + return features + + +if __name__ == "__main__": + import platform + + features = get_windows_console_features() + from pip._vendor.rich import print + + print(f'platform="{platform.system()}"') + print(repr(features)) diff --git a/src/pip/_vendor/rich/_wrap.py b/src/pip/_vendor/rich/_wrap.py new file mode 100644 index 00000000000..b537757a573 --- /dev/null +++ b/src/pip/_vendor/rich/_wrap.py @@ -0,0 +1,55 @@ +import re +from typing import Iterable, List, Tuple + +from .cells import cell_len, chop_cells +from ._loop import loop_last + +re_word = re.compile(r"\s*\S+\s*") + + +def words(text: str) -> Iterable[Tuple[int, int, str]]: + position = 0 + word_match = re_word.match(text, position) + while word_match is not None: + start, end = word_match.span() + word = word_match.group(0) + yield start, end, word + word_match = re_word.match(text, end) + + +def divide_line(text: str, width: int, fold: bool = True) -> List[int]: + divides: List[int] = [] + append = divides.append + line_position = 0 + _cell_len = cell_len + for start, _end, word in words(text): + word_length = _cell_len(word.rstrip()) + if line_position + word_length > width: + if word_length > width: + if fold: + for last, line in loop_last( + chop_cells(word, width, position=line_position) + ): + if last: + line_position = _cell_len(line) + else: + start += len(line) + append(start) + else: + if start: + append(start) + line_position = _cell_len(word) + elif line_position and start: + append(start) + line_position = _cell_len(word) + else: + line_position += _cell_len(word) + return divides + + +if __name__ == "__main__": # pragma: no cover + from .console import Console + + console = Console(width=10) + console.print("12345 abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ 12345") + print(chop_cells("abcdefghijklmnopqrstuvwxyz", 10, position=2)) diff --git a/src/pip/_vendor/rich/abc.py b/src/pip/_vendor/rich/abc.py new file mode 100644 index 00000000000..e6e498efabf --- /dev/null +++ b/src/pip/_vendor/rich/abc.py @@ -0,0 +1,33 @@ +from abc import ABC + + +class RichRenderable(ABC): + """An abstract base class for Rich renderables. + + Note that there is no need to extend this class, the intended use is to check if an + object supports the Rich renderable protocol. For example:: + + if isinstance(my_object, RichRenderable): + console.print(my_object) + + """ + + @classmethod + def __subclasshook__(cls, other: type) -> bool: + """Check if this class supports the rich render protocol.""" + return hasattr(other, "__rich_console__") or hasattr(other, "__rich__") + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.text import Text + + t = Text() + print(isinstance(Text, RichRenderable)) + print(isinstance(t, RichRenderable)) + + class Foo: + pass + + f = Foo() + print(isinstance(f, RichRenderable)) + print(isinstance("", RichRenderable)) diff --git a/src/pip/_vendor/rich/align.py b/src/pip/_vendor/rich/align.py new file mode 100644 index 00000000000..4344ae141c5 --- /dev/null +++ b/src/pip/_vendor/rich/align.py @@ -0,0 +1,312 @@ +import sys +from itertools import chain +from typing import TYPE_CHECKING, Iterable, Optional + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from pip._vendor.typing_extensions import Literal # pragma: no cover + +from .constrain import Constrain +from .jupyter import JupyterMixin +from .measure import Measurement +from .segment import Segment +from .style import StyleType + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderableType, RenderResult + +AlignMethod = Literal["left", "center", "right"] +VerticalAlignMethod = Literal["top", "middle", "bottom"] +AlignValues = AlignMethod # TODO: deprecate AlignValues + + +class Align(JupyterMixin): + """Align a renderable by adding spaces if necessary. + + Args: + renderable (RenderableType): A console renderable. + align (AlignMethod): One of "left", "center", or "right"" + style (StyleType, optional): An optional style to apply to the background. + vertical (Optional[VerticalAlginMethod], optional): Optional vertical align, one of "top", "middle", or "bottom". Defaults to None. + pad (bool, optional): Pad the right with spaces. Defaults to True. + width (int, optional): Restrict contents to given width, or None to use default width. Defaults to None. + height (int, optional): Set height of align renderable, or None to fit to contents. Defaults to None. + + Raises: + ValueError: if ``align`` is not one of the expected values. + """ + + def __init__( + self, + renderable: "RenderableType", + align: AlignMethod = "left", + style: Optional[StyleType] = None, + *, + vertical: Optional[VerticalAlignMethod] = None, + pad: bool = True, + width: Optional[int] = None, + height: Optional[int] = None, + ) -> None: + if align not in ("left", "center", "right"): + raise ValueError( + f'invalid value for align, expected "left", "center", or "right" (not {align!r})' + ) + if vertical is not None and vertical not in ("top", "middle", "bottom"): + raise ValueError( + f'invalid value for vertical, expected "top", "middle", or "bottom" (not {vertical!r})' + ) + self.renderable = renderable + self.align = align + self.style = style + self.vertical = vertical + self.pad = pad + self.width = width + self.height = height + + def __repr__(self) -> str: + return f"Align({self.renderable!r}, {self.align!r})" + + @classmethod + def left( + cls, + renderable: "RenderableType", + style: Optional[StyleType] = None, + *, + vertical: Optional[VerticalAlignMethod] = None, + pad: bool = True, + width: Optional[int] = None, + height: Optional[int] = None, + ) -> "Align": + """Align a renderable to the left.""" + return cls( + renderable, + "left", + style=style, + vertical=vertical, + pad=pad, + width=width, + height=height, + ) + + @classmethod + def center( + cls, + renderable: "RenderableType", + style: Optional[StyleType] = None, + *, + vertical: Optional[VerticalAlignMethod] = None, + pad: bool = True, + width: Optional[int] = None, + height: Optional[int] = None, + ) -> "Align": + """Align a renderable to the center.""" + return cls( + renderable, + "center", + style=style, + vertical=vertical, + pad=pad, + width=width, + height=height, + ) + + @classmethod + def right( + cls, + renderable: "RenderableType", + style: Optional[StyleType] = None, + *, + vertical: Optional[VerticalAlignMethod] = None, + pad: bool = True, + width: Optional[int] = None, + height: Optional[int] = None, + ) -> "Align": + """Align a renderable to the right.""" + return cls( + renderable, + "right", + style=style, + vertical=vertical, + pad=pad, + width=width, + height=height, + ) + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + align = self.align + width = console.measure(self.renderable, options=options).maximum + rendered = console.render( + Constrain( + self.renderable, width if self.width is None else min(width, self.width) + ), + options.update(height=None), + ) + lines = list(Segment.split_lines(rendered)) + width, height = Segment.get_shape(lines) + lines = Segment.set_shape(lines, width, height) + new_line = Segment.line() + excess_space = options.max_width - width + style = console.get_style(self.style) if self.style is not None else None + + def generate_segments() -> Iterable[Segment]: + if excess_space <= 0: + # Exact fit + for line in lines: + yield from line + yield new_line + + elif align == "left": + # Pad on the right + pad = Segment(" " * excess_space, style) if self.pad else None + for line in lines: + yield from line + if pad: + yield pad + yield new_line + + elif align == "center": + # Pad left and right + left = excess_space // 2 + pad = Segment(" " * left, style) + pad_right = ( + Segment(" " * (excess_space - left), style) if self.pad else None + ) + for line in lines: + if left: + yield pad + yield from line + if pad_right: + yield pad_right + yield new_line + + elif align == "right": + # Padding on left + pad = Segment(" " * excess_space, style) + for line in lines: + yield pad + yield from line + yield new_line + + blank_line = ( + Segment(f"{' ' * (self.width or options.max_width)}\n", style) + if self.pad + else Segment("\n") + ) + + def blank_lines(count: int) -> Iterable[Segment]: + if count > 0: + for _ in range(count): + yield blank_line + + vertical_height = self.height or options.height + iter_segments: Iterable[Segment] + if self.vertical and vertical_height is not None: + if self.vertical == "top": + bottom_space = vertical_height - height + iter_segments = chain(generate_segments(), blank_lines(bottom_space)) + elif self.vertical == "middle": + top_space = (vertical_height - height) // 2 + bottom_space = vertical_height - top_space - height + iter_segments = chain( + blank_lines(top_space), + generate_segments(), + blank_lines(bottom_space), + ) + else: # self.vertical == "bottom": + top_space = vertical_height - height + iter_segments = chain(blank_lines(top_space), generate_segments()) + else: + iter_segments = generate_segments() + if self.style: + style = console.get_style(self.style) + iter_segments = Segment.apply_style(iter_segments, style) + yield from iter_segments + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> Measurement: + measurement = Measurement.get(console, options, self.renderable) + return measurement + + +class VerticalCenter(JupyterMixin): + """Vertically aligns a renderable. + + Warn: + This class is deprecated and may be removed in a future version. Use Align class with + `vertical="middle"`. + + Args: + renderable (RenderableType): A renderable object. + """ + + def __init__( + self, + renderable: "RenderableType", + style: Optional[StyleType] = None, + ) -> None: + self.renderable = renderable + self.style = style + + def __repr__(self) -> str: + return f"VerticalCenter({self.renderable!r})" + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + style = console.get_style(self.style) if self.style is not None else None + lines = console.render_lines( + self.renderable, options.update(height=None), pad=False + ) + width, _height = Segment.get_shape(lines) + new_line = Segment.line() + height = options.height or options.size.height + top_space = (height - len(lines)) // 2 + bottom_space = height - top_space - len(lines) + blank_line = Segment(f"{' ' * width}", style) + + def blank_lines(count: int) -> Iterable[Segment]: + for _ in range(count): + yield blank_line + yield new_line + + if top_space > 0: + yield from blank_lines(top_space) + for line in lines: + yield from line + yield new_line + if bottom_space > 0: + yield from blank_lines(bottom_space) + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> Measurement: + measurement = Measurement.get(console, options, self.renderable) + return measurement + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console, Group + from pip._vendor.rich.highlighter import ReprHighlighter + from pip._vendor.rich.panel import Panel + + highlighter = ReprHighlighter() + console = Console() + + panel = Panel( + Group( + Align.left(highlighter("align='left'")), + Align.center(highlighter("align='center'")), + Align.right(highlighter("align='right'")), + ), + width=60, + style="on dark_blue", + title="Algin", + ) + + console.print( + Align.center(panel, vertical="middle", style="on red", height=console.height) + ) diff --git a/src/pip/_vendor/rich/ansi.py b/src/pip/_vendor/rich/ansi.py new file mode 100644 index 00000000000..92e4772eddf --- /dev/null +++ b/src/pip/_vendor/rich/ansi.py @@ -0,0 +1,228 @@ +from contextlib import suppress +import re +from typing import Iterable, NamedTuple + +from .color import Color +from .style import Style +from .text import Text + +re_ansi = re.compile(r"(?:\x1b\[(.*?)m)|(?:\x1b\](.*?)\x1b\\)") +re_csi = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + + +class _AnsiToken(NamedTuple): + """Result of ansi tokenized string.""" + + plain: str = "" + sgr: str = "" + osc: str = "" + + +def _ansi_tokenize(ansi_text: str) -> Iterable[_AnsiToken]: + """Tokenize a string in to plain text and ANSI codes. + + Args: + ansi_text (str): A String containing ANSI codes. + + Yields: + AnsiToken: A named tuple of (plain, sgr, osc) + """ + + def remove_csi(ansi_text: str) -> str: + """Remove unknown CSI sequences.""" + return re_csi.sub("", ansi_text) + + position = 0 + for match in re_ansi.finditer(ansi_text): + start, end = match.span(0) + sgr, osc = match.groups() + if start > position: + yield _AnsiToken(remove_csi(ansi_text[position:start])) + yield _AnsiToken("", sgr, osc) + position = end + if position < len(ansi_text): + yield _AnsiToken(remove_csi(ansi_text[position:])) + + +SGR_STYLE_MAP = { + 1: "bold", + 2: "dim", + 3: "italic", + 4: "underline", + 5: "blink", + 6: "blink2", + 7: "reverse", + 8: "conceal", + 9: "strike", + 21: "underline2", + 22: "not dim not bold", + 23: "not italic", + 24: "not underline", + 25: "not blink", + 26: "not blink2", + 27: "not reverse", + 28: "not conceal", + 29: "not strike", + 30: "color(0)", + 31: "color(1)", + 32: "color(2)", + 33: "color(3)", + 34: "color(4)", + 35: "color(5)", + 36: "color(6)", + 37: "color(7)", + 39: "default", + 40: "on color(0)", + 41: "on color(1)", + 42: "on color(2)", + 43: "on color(3)", + 44: "on color(4)", + 45: "on color(5)", + 46: "on color(6)", + 47: "on color(7)", + 49: "on default", + 51: "frame", + 52: "encircle", + 53: "overline", + 54: "not frame not encircle", + 55: "not overline", + 90: "color(8)", + 91: "color(9)", + 92: "color(10)", + 93: "color(11)", + 94: "color(12)", + 95: "color(13)", + 96: "color(14)", + 97: "color(15)", + 100: "on color(8)", + 101: "on color(9)", + 102: "on color(10)", + 103: "on color(11)", + 104: "on color(12)", + 105: "on color(13)", + 106: "on color(14)", + 107: "on color(15)", +} + + +class AnsiDecoder: + """Translate ANSI code in to styled Text.""" + + def __init__(self) -> None: + self.style = Style.null() + + def decode(self, terminal_text: str) -> Iterable[Text]: + """Decode ANSI codes in an interable of lines. + + Args: + lines (Iterable[str]): An iterable of lines of terminal output. + + Yields: + Text: Marked up Text. + """ + for line in terminal_text.splitlines(): + yield self.decode_line(line) + + def decode_line(self, line: str) -> Text: + """Decode a line containing ansi codes. + + Args: + line (str): A line of terminal output. + + Returns: + Text: A Text instance marked up according to ansi codes. + """ + from_ansi = Color.from_ansi + from_rgb = Color.from_rgb + _Style = Style + text = Text() + append = text.append + line = line.rsplit("\r", 1)[-1] + for token in _ansi_tokenize(line): + plain_text, sgr, osc = token + if plain_text: + append(plain_text, self.style or None) + elif osc: + if osc.startswith("8;"): + _params, semicolon, link = osc[2:].partition(";") + if semicolon: + self.style = self.style.update_link(link or None) + elif sgr: + # Translate in to semi-colon separated codes + # Ignore invalid codes, because we want to be lenient + codes = [ + min(255, int(_code)) for _code in sgr.split(";") if _code.isdigit() + ] + iter_codes = iter(codes) + for code in iter_codes: + if code == 0: + # reset + self.style = _Style.null() + elif code in SGR_STYLE_MAP: + # styles + self.style += _Style.parse(SGR_STYLE_MAP[code]) + elif code == 38: + #  Foreground + with suppress(StopIteration): + color_type = next(iter_codes) + if color_type == 5: + self.style += _Style.from_color( + from_ansi(next(iter_codes)) + ) + elif color_type == 2: + self.style += _Style.from_color( + from_rgb( + next(iter_codes), + next(iter_codes), + next(iter_codes), + ) + ) + elif code == 48: + # Background + with suppress(StopIteration): + color_type = next(iter_codes) + if color_type == 5: + self.style += _Style.from_color( + None, from_ansi(next(iter_codes)) + ) + elif color_type == 2: + self.style += _Style.from_color( + None, + from_rgb( + next(iter_codes), + next(iter_codes), + next(iter_codes), + ), + ) + + return text + + +if __name__ == "__main__": # pragma: no cover + import pty + import io + import os + import sys + + decoder = AnsiDecoder() + + stdout = io.BytesIO() + + def read(fd: int) -> bytes: + data = os.read(fd, 1024) + stdout.write(data) + return data + + pty.spawn(sys.argv[1:], read) + + from .console import Console + + console = Console(record=True) + + stdout_result = stdout.getvalue().decode("utf-8") + print(stdout_result) + + for line in decoder.decode(stdout_result): + console.print(line) + + console.save_html("stdout.html") diff --git a/src/pip/_vendor/rich/bar.py b/src/pip/_vendor/rich/bar.py new file mode 100644 index 00000000000..ed86a552d1c --- /dev/null +++ b/src/pip/_vendor/rich/bar.py @@ -0,0 +1,94 @@ +from typing import Optional, Union + +from .color import Color +from .console import Console, ConsoleOptions, RenderResult +from .jupyter import JupyterMixin +from .measure import Measurement +from .segment import Segment +from .style import Style + +# There are left-aligned characters for 1/8 to 7/8, but +# the right-aligned characters exist only for 1/8 and 4/8. +BEGIN_BLOCK_ELEMENTS = ["█", "█", "█", "▐", "▐", "▐", "▕", "▕"] +END_BLOCK_ELEMENTS = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉"] +FULL_BLOCK = "█" + + +class Bar(JupyterMixin): + """Renders a solid block bar. + + Args: + size (float): Value for the end of the bar. + begin (float): Begin point (between 0 and size, inclusive). + end (float): End point (between 0 and size, inclusive). + width (int, optional): Width of the bar, or ``None`` for maximum width. Defaults to None. + color (Union[Color, str], optional): Color of the bar. Defaults to "default". + bgcolor (Union[Color, str], optional): Color of bar background. Defaults to "default". + """ + + def __init__( + self, + size: float, + begin: float, + end: float, + *, + width: Optional[int] = None, + color: Union[Color, str] = "default", + bgcolor: Union[Color, str] = "default", + ): + self.size = size + self.begin = max(begin, 0) + self.end = min(end, size) + self.width = width + self.style = Style(color=color, bgcolor=bgcolor) + + def __repr__(self) -> str: + return f"Bar({self.size}, {self.begin}, {self.end})" + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + + width = min( + self.width if self.width is not None else options.max_width, + options.max_width, + ) + + if self.begin >= self.end: + yield Segment(" " * width, self.style) + yield Segment.line() + return + + prefix_complete_eights = int(width * 8 * self.begin / self.size) + prefix_bar_count = prefix_complete_eights // 8 + prefix_eights_count = prefix_complete_eights % 8 + + body_complete_eights = int(width * 8 * self.end / self.size) + body_bar_count = body_complete_eights // 8 + body_eights_count = body_complete_eights % 8 + + # When start and end fall into the same cell, we ideally should render + # a symbol that's "center-aligned", but there is no good symbol in Unicode. + # In this case, we fall back to right-aligned block symbol for simplicity. + + prefix = " " * prefix_bar_count + if prefix_eights_count: + prefix += BEGIN_BLOCK_ELEMENTS[prefix_eights_count] + + body = FULL_BLOCK * body_bar_count + if body_eights_count: + body += END_BLOCK_ELEMENTS[body_eights_count] + + suffix = " " * (width - len(body)) + + yield Segment(prefix + body[len(prefix) :] + suffix, self.style) + yield Segment.line() + + def __rich_measure__( + self, console: Console, options: ConsoleOptions + ) -> Measurement: + return ( + Measurement(self.width, self.width) + if self.width is not None + else Measurement(4, options.max_width) + ) diff --git a/src/pip/_vendor/rich/box.py b/src/pip/_vendor/rich/box.py new file mode 100644 index 00000000000..aec2926bea2 --- /dev/null +++ b/src/pip/_vendor/rich/box.py @@ -0,0 +1,483 @@ +import sys +from typing import TYPE_CHECKING, Iterable, List + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from pip._vendor.typing_extensions import Literal # pragma: no cover + + +from ._loop import loop_last + +if TYPE_CHECKING: + from pip._vendor.rich.console import ConsoleOptions + + +class Box: + """Defines characters to render boxes. + + ┌─┬┐ top + │ ││ head + ├─┼┤ head_row + │ ││ mid + ├─┼┤ row + ├─┼┤ foot_row + │ ││ foot + └─┴┘ bottom + + Args: + box (str): Characters making up box. + ascii (bool, optional): True if this box uses ascii characters only. Default is False. + """ + + def __init__(self, box: str, *, ascii: bool = False) -> None: + self._box = box + self.ascii = ascii + line1, line2, line3, line4, line5, line6, line7, line8 = box.splitlines() + # top + self.top_left, self.top, self.top_divider, self.top_right = iter(line1) + # head + self.head_left, _, self.head_vertical, self.head_right = iter(line2) + # head_row + ( + self.head_row_left, + self.head_row_horizontal, + self.head_row_cross, + self.head_row_right, + ) = iter(line3) + + # mid + self.mid_left, _, self.mid_vertical, self.mid_right = iter(line4) + # row + self.row_left, self.row_horizontal, self.row_cross, self.row_right = iter(line5) + # foot_row + ( + self.foot_row_left, + self.foot_row_horizontal, + self.foot_row_cross, + self.foot_row_right, + ) = iter(line6) + # foot + self.foot_left, _, self.foot_vertical, self.foot_right = iter(line7) + # bottom + self.bottom_left, self.bottom, self.bottom_divider, self.bottom_right = iter( + line8 + ) + + def __repr__(self) -> str: + return "Box(...)" + + def __str__(self) -> str: + return self._box + + def substitute(self, options: "ConsoleOptions", safe: bool = True) -> "Box": + """Substitute this box for another if it won't render due to platform issues. + + Args: + options (ConsoleOptions): Console options used in rendering. + safe (bool, optional): Substitute this for another Box if there are known problems + displaying on the platform (currently only relevant on Windows). Default is True. + + Returns: + Box: A different Box or the same Box. + """ + box = self + if options.legacy_windows and safe: + box = LEGACY_WINDOWS_SUBSTITUTIONS.get(box, box) + if options.ascii_only and not box.ascii: + box = ASCII + return box + + def get_top(self, widths: Iterable[int]) -> str: + """Get the top of a simple box. + + Args: + widths (List[int]): Widths of columns. + + Returns: + str: A string of box characters. + """ + + parts: List[str] = [] + append = parts.append + append(self.top_left) + for last, width in loop_last(widths): + append(self.top * width) + if not last: + append(self.top_divider) + append(self.top_right) + return "".join(parts) + + def get_row( + self, + widths: Iterable[int], + level: Literal["head", "row", "foot", "mid"] = "row", + edge: bool = True, + ) -> str: + """Get the top of a simple box. + + Args: + width (List[int]): Widths of columns. + + Returns: + str: A string of box characters. + """ + if level == "head": + left = self.head_row_left + horizontal = self.head_row_horizontal + cross = self.head_row_cross + right = self.head_row_right + elif level == "row": + left = self.row_left + horizontal = self.row_horizontal + cross = self.row_cross + right = self.row_right + elif level == "mid": + left = self.mid_left + horizontal = " " + cross = self.mid_vertical + right = self.mid_right + elif level == "foot": + left = self.foot_row_left + horizontal = self.foot_row_horizontal + cross = self.foot_row_cross + right = self.foot_row_right + else: + raise ValueError("level must be 'head', 'row' or 'foot'") + + parts: List[str] = [] + append = parts.append + if edge: + append(left) + for last, width in loop_last(widths): + append(horizontal * width) + if not last: + append(cross) + if edge: + append(right) + return "".join(parts) + + def get_bottom(self, widths: Iterable[int]) -> str: + """Get the bottom of a simple box. + + Args: + widths (List[int]): Widths of columns. + + Returns: + str: A string of box characters. + """ + + parts: List[str] = [] + append = parts.append + append(self.bottom_left) + for last, width in loop_last(widths): + append(self.bottom * width) + if not last: + append(self.bottom_divider) + append(self.bottom_right) + return "".join(parts) + + +ASCII: Box = Box( + """\ ++--+ +| || +|-+| +| || +|-+| +|-+| +| || ++--+ +""", + ascii=True, +) + +ASCII2: Box = Box( + """\ ++-++ +| || ++-++ +| || ++-++ ++-++ +| || ++-++ +""", + ascii=True, +) + +ASCII_DOUBLE_HEAD: Box = Box( + """\ ++-++ +| || ++=++ +| || ++-++ ++-++ +| || ++-++ +""", + ascii=True, +) + +SQUARE: Box = Box( + """\ +┌─┬┐ +│ ││ +├─┼┤ +│ ││ +├─┼┤ +├─┼┤ +│ ││ +└─┴┘ +""" +) + +SQUARE_DOUBLE_HEAD: Box = Box( + """\ +┌─┬┐ +│ ││ +╞═╪╡ +│ ││ +├─┼┤ +├─┼┤ +│ ││ +└─┴┘ +""" +) + +MINIMAL: Box = Box( + """\ + ╷ + │ +╶─┼╴ + │ +╶─┼╴ +╶─┼╴ + │ + ╵ +""" +) + + +MINIMAL_HEAVY_HEAD: Box = Box( + """\ + ╷ + │ +╺━┿╸ + │ +╶─┼╴ +╶─┼╴ + │ + ╵ +""" +) + +MINIMAL_DOUBLE_HEAD: Box = Box( + """\ + ╷ + │ + ═╪ + │ + ─┼ + ─┼ + │ + ╵ +""" +) + + +SIMPLE: Box = Box( + """\ + + + ── + + + ── + + +""" +) + +SIMPLE_HEAD: Box = Box( + """\ + + + ── + + + + + +""" +) + + +SIMPLE_HEAVY: Box = Box( + """\ + + + ━━ + + + ━━ + + +""" +) + + +HORIZONTALS: Box = Box( + """\ + ── + + ── + + ── + ── + + ── +""" +) + +ROUNDED: Box = Box( + """\ +╭─┬╮ +│ ││ +├─┼┤ +│ ││ +├─┼┤ +├─┼┤ +│ ││ +╰─┴╯ +""" +) + +HEAVY: Box = Box( + """\ +┏━┳┓ +┃ ┃┃ +┣━╋┫ +┃ ┃┃ +┣━╋┫ +┣━╋┫ +┃ ┃┃ +┗━┻┛ +""" +) + +HEAVY_EDGE: Box = Box( + """\ +┏━┯┓ +┃ │┃ +┠─┼┨ +┃ │┃ +┠─┼┨ +┠─┼┨ +┃ │┃ +┗━┷┛ +""" +) + +HEAVY_HEAD: Box = Box( + """\ +┏━┳┓ +┃ ┃┃ +┡━╇┩ +│ ││ +├─┼┤ +├─┼┤ +│ ││ +└─┴┘ +""" +) + +DOUBLE: Box = Box( + """\ +╔═╦╗ +║ ║║ +╠═╬╣ +║ ║║ +╠═╬╣ +╠═╬╣ +║ ║║ +╚═╩╝ +""" +) + +DOUBLE_EDGE: Box = Box( + """\ +╔═╤╗ +║ │║ +╟─┼╢ +║ │║ +╟─┼╢ +╟─┼╢ +║ │║ +╚═╧╝ +""" +) + +# Map Boxes that don't render with raster fonts on to equivalent that do +LEGACY_WINDOWS_SUBSTITUTIONS = { + ROUNDED: SQUARE, + MINIMAL_HEAVY_HEAD: MINIMAL, + SIMPLE_HEAVY: SIMPLE, + HEAVY: SQUARE, + HEAVY_EDGE: SQUARE, + HEAVY_HEAD: SQUARE, +} + + +if __name__ == "__main__": # pragma: no cover + + from pip._vendor.rich.columns import Columns + from pip._vendor.rich.panel import Panel + + from . import box as box + from .console import Console + from .table import Table + from .text import Text + + console = Console(record=True) + + BOXES = [ + "ASCII", + "ASCII2", + "ASCII_DOUBLE_HEAD", + "SQUARE", + "SQUARE_DOUBLE_HEAD", + "MINIMAL", + "MINIMAL_HEAVY_HEAD", + "MINIMAL_DOUBLE_HEAD", + "SIMPLE", + "SIMPLE_HEAD", + "SIMPLE_HEAVY", + "HORIZONTALS", + "ROUNDED", + "HEAVY", + "HEAVY_EDGE", + "HEAVY_HEAD", + "DOUBLE", + "DOUBLE_EDGE", + ] + + console.print(Panel("[bold green]Box Constants", style="green"), justify="center") + console.print() + + columns = Columns(expand=True, padding=2) + for box_name in sorted(BOXES): + table = Table( + show_footer=True, style="dim", border_style="not dim", expand=True + ) + table.add_column("Header 1", "Footer 1") + table.add_column("Header 2", "Footer 2") + table.add_row("Cell", "Cell") + table.add_row("Cell", "Cell") + table.box = getattr(box, box_name) + table.title = Text(f"box.{box_name}", style="magenta") + columns.add_renderable(table) + console.print(columns) + + # console.save_html("box.html", inline_styles=True) diff --git a/src/pip/_vendor/rich/cells.py b/src/pip/_vendor/rich/cells.py new file mode 100644 index 00000000000..e824ea2a6df --- /dev/null +++ b/src/pip/_vendor/rich/cells.py @@ -0,0 +1,147 @@ +from functools import lru_cache +import re +from typing import Dict, List + +from ._cell_widths import CELL_WIDTHS +from ._lru_cache import LRUCache + +# Regex to match sequence of the most common character ranges +_is_single_cell_widths = re.compile("^[\u0020-\u006f\u00a0\u02ff\u0370-\u0482]*$").match + + +def cell_len(text: str, _cache: Dict[str, int] = LRUCache(1024 * 4)) -> int: + """Get the number of cells required to display text. + + Args: + text (str): Text to display. + + Returns: + int: Get the number of cells required to display text. + """ + + if _is_single_cell_widths(text): + return len(text) + else: + cached_result = _cache.get(text, None) + if cached_result is not None: + return cached_result + _get_size = get_character_cell_size + total_size = sum(_get_size(character) for character in text) + if len(text) <= 64: + _cache[text] = total_size + return total_size + + +@lru_cache(maxsize=4096) +def get_character_cell_size(character: str) -> int: + """Get the cell size of a character. + + Args: + character (str): A single character. + + Returns: + int: Number of cells (0, 1 or 2) occupied by that character. + """ + if _is_single_cell_widths(character): + return 1 + + return _get_codepoint_cell_size(ord(character)) + + +@lru_cache(maxsize=4096) +def _get_codepoint_cell_size(codepoint: int) -> int: + """Get the cell size of a character. + + Args: + character (str): A single character. + + Returns: + int: Number of cells (0, 1 or 2) occupied by that character. + """ + + _table = CELL_WIDTHS + lower_bound = 0 + upper_bound = len(_table) - 1 + index = (lower_bound + upper_bound) // 2 + while True: + start, end, width = _table[index] + if codepoint < start: + upper_bound = index - 1 + elif codepoint > end: + lower_bound = index + 1 + else: + return 0 if width == -1 else width + if upper_bound < lower_bound: + break + index = (lower_bound + upper_bound) // 2 + return 1 + + +def set_cell_size(text: str, total: int) -> str: + """Set the length of a string to fit within given number of cells.""" + + if _is_single_cell_widths(text): + size = len(text) + if size < total: + return text + " " * (total - size) + return text[:total] + + if not total: + return "" + cell_size = cell_len(text) + if cell_size == total: + return text + if cell_size < total: + return text + " " * (total - cell_size) + + start = 0 + end = len(text) + + # Binary search until we find the right size + while True: + pos = (start + end) // 2 + before = text[: pos + 1] + before_len = cell_len(before) + if before_len == total + 1 and cell_len(before[-1]) == 2: + return before[:-1] + " " + if before_len == total: + return before + if before_len > total: + end = pos + else: + start = pos + + +# TODO: This is inefficient +# TODO: This might not work with CWJ type characters +def chop_cells(text: str, max_size: int, position: int = 0) -> List[str]: + """Break text in to equal (cell) length strings.""" + _get_character_cell_size = get_character_cell_size + characters = [ + (character, _get_character_cell_size(character)) for character in text + ][::-1] + total_size = position + lines: List[List[str]] = [[]] + append = lines[-1].append + + pop = characters.pop + while characters: + character, size = pop() + if total_size + size > max_size: + lines.append([character]) + append = lines[-1].append + total_size = size + else: + total_size += size + append(character) + return ["".join(line) for line in lines] + + +if __name__ == "__main__": # pragma: no cover + + print(get_character_cell_size("😽")) + for line in chop_cells("""这是对亚洲语言支持的测试。面对模棱两可的想法,拒绝猜测的诱惑。""", 8): + print(line) + for n in range(80, 1, -1): + print(set_cell_size("""这是对亚洲语言支持的测试。面对模棱两可的想法,拒绝猜测的诱惑。""", n) + "|") + print("x" * n) diff --git a/src/pip/_vendor/rich/color.py b/src/pip/_vendor/rich/color.py new file mode 100644 index 00000000000..f0fa026d646 --- /dev/null +++ b/src/pip/_vendor/rich/color.py @@ -0,0 +1,581 @@ +import platform +import re +from colorsys import rgb_to_hls +from enum import IntEnum +from functools import lru_cache +from typing import TYPE_CHECKING, NamedTuple, Optional, Tuple + +from ._palettes import EIGHT_BIT_PALETTE, STANDARD_PALETTE, WINDOWS_PALETTE +from .color_triplet import ColorTriplet +from .repr import rich_repr, Result +from .terminal_theme import DEFAULT_TERMINAL_THEME + +if TYPE_CHECKING: # pragma: no cover + from .terminal_theme import TerminalTheme + from .text import Text + + +WINDOWS = platform.system() == "Windows" + + +class ColorSystem(IntEnum): + """One of the 3 color system supported by terminals.""" + + STANDARD = 1 + EIGHT_BIT = 2 + TRUECOLOR = 3 + WINDOWS = 4 + + def __repr__(self) -> str: + return f"ColorSystem.{self.name}" + + +class ColorType(IntEnum): + """Type of color stored in Color class.""" + + DEFAULT = 0 + STANDARD = 1 + EIGHT_BIT = 2 + TRUECOLOR = 3 + WINDOWS = 4 + + def __repr__(self) -> str: + return f"ColorType.{self.name}" + + +ANSI_COLOR_NAMES = { + "black": 0, + "red": 1, + "green": 2, + "yellow": 3, + "blue": 4, + "magenta": 5, + "cyan": 6, + "white": 7, + "bright_black": 8, + "bright_red": 9, + "bright_green": 10, + "bright_yellow": 11, + "bright_blue": 12, + "bright_magenta": 13, + "bright_cyan": 14, + "bright_white": 15, + "grey0": 16, + "navy_blue": 17, + "dark_blue": 18, + "blue3": 20, + "blue1": 21, + "dark_green": 22, + "deep_sky_blue4": 25, + "dodger_blue3": 26, + "dodger_blue2": 27, + "green4": 28, + "spring_green4": 29, + "turquoise4": 30, + "deep_sky_blue3": 32, + "dodger_blue1": 33, + "green3": 40, + "spring_green3": 41, + "dark_cyan": 36, + "light_sea_green": 37, + "deep_sky_blue2": 38, + "deep_sky_blue1": 39, + "spring_green2": 47, + "cyan3": 43, + "dark_turquoise": 44, + "turquoise2": 45, + "green1": 46, + "spring_green1": 48, + "medium_spring_green": 49, + "cyan2": 50, + "cyan1": 51, + "dark_red": 88, + "deep_pink4": 125, + "purple4": 55, + "purple3": 56, + "blue_violet": 57, + "orange4": 94, + "grey37": 59, + "medium_purple4": 60, + "slate_blue3": 62, + "royal_blue1": 63, + "chartreuse4": 64, + "dark_sea_green4": 71, + "pale_turquoise4": 66, + "steel_blue": 67, + "steel_blue3": 68, + "cornflower_blue": 69, + "chartreuse3": 76, + "cadet_blue": 73, + "sky_blue3": 74, + "steel_blue1": 81, + "pale_green3": 114, + "sea_green3": 78, + "aquamarine3": 79, + "medium_turquoise": 80, + "chartreuse2": 112, + "sea_green2": 83, + "sea_green1": 85, + "aquamarine1": 122, + "dark_slate_gray2": 87, + "dark_magenta": 91, + "dark_violet": 128, + "purple": 129, + "light_pink4": 95, + "plum4": 96, + "medium_purple3": 98, + "slate_blue1": 99, + "yellow4": 106, + "wheat4": 101, + "grey53": 102, + "light_slate_grey": 103, + "medium_purple": 104, + "light_slate_blue": 105, + "dark_olive_green3": 149, + "dark_sea_green": 108, + "light_sky_blue3": 110, + "sky_blue2": 111, + "dark_sea_green3": 150, + "dark_slate_gray3": 116, + "sky_blue1": 117, + "chartreuse1": 118, + "light_green": 120, + "pale_green1": 156, + "dark_slate_gray1": 123, + "red3": 160, + "medium_violet_red": 126, + "magenta3": 164, + "dark_orange3": 166, + "indian_red": 167, + "hot_pink3": 168, + "medium_orchid3": 133, + "medium_orchid": 134, + "medium_purple2": 140, + "dark_goldenrod": 136, + "light_salmon3": 173, + "rosy_brown": 138, + "grey63": 139, + "medium_purple1": 141, + "gold3": 178, + "dark_khaki": 143, + "navajo_white3": 144, + "grey69": 145, + "light_steel_blue3": 146, + "light_steel_blue": 147, + "yellow3": 184, + "dark_sea_green2": 157, + "light_cyan3": 152, + "light_sky_blue1": 153, + "green_yellow": 154, + "dark_olive_green2": 155, + "dark_sea_green1": 193, + "pale_turquoise1": 159, + "deep_pink3": 162, + "magenta2": 200, + "hot_pink2": 169, + "orchid": 170, + "medium_orchid1": 207, + "orange3": 172, + "light_pink3": 174, + "pink3": 175, + "plum3": 176, + "violet": 177, + "light_goldenrod3": 179, + "tan": 180, + "misty_rose3": 181, + "thistle3": 182, + "plum2": 183, + "khaki3": 185, + "light_goldenrod2": 222, + "light_yellow3": 187, + "grey84": 188, + "light_steel_blue1": 189, + "yellow2": 190, + "dark_olive_green1": 192, + "honeydew2": 194, + "light_cyan1": 195, + "red1": 196, + "deep_pink2": 197, + "deep_pink1": 199, + "magenta1": 201, + "orange_red1": 202, + "indian_red1": 204, + "hot_pink": 206, + "dark_orange": 208, + "salmon1": 209, + "light_coral": 210, + "pale_violet_red1": 211, + "orchid2": 212, + "orchid1": 213, + "orange1": 214, + "sandy_brown": 215, + "light_salmon1": 216, + "light_pink1": 217, + "pink1": 218, + "plum1": 219, + "gold1": 220, + "navajo_white1": 223, + "misty_rose1": 224, + "thistle1": 225, + "yellow1": 226, + "light_goldenrod1": 227, + "khaki1": 228, + "wheat1": 229, + "cornsilk1": 230, + "grey100": 231, + "grey3": 232, + "grey7": 233, + "grey11": 234, + "grey15": 235, + "grey19": 236, + "grey23": 237, + "grey27": 238, + "grey30": 239, + "grey35": 240, + "grey39": 241, + "grey42": 242, + "grey46": 243, + "grey50": 244, + "grey54": 245, + "grey58": 246, + "grey62": 247, + "grey66": 248, + "grey70": 249, + "grey74": 250, + "grey78": 251, + "grey82": 252, + "grey85": 253, + "grey89": 254, + "grey93": 255, +} + + +class ColorParseError(Exception): + """The color could not be parsed.""" + + +RE_COLOR = re.compile( + r"""^ +\#([0-9a-f]{6})$| +color\(([0-9]{1,3})\)$| +rgb\(([\d\s,]+)\)$ +""", + re.VERBOSE, +) + + +@rich_repr +class Color(NamedTuple): + """Terminal color definition.""" + + name: str + """The name of the color (typically the input to Color.parse).""" + type: ColorType + """The type of the color.""" + number: Optional[int] = None + """The color number, if a standard color, or None.""" + triplet: Optional[ColorTriplet] = None + """A triplet of color components, if an RGB color.""" + + def __rich__(self) -> "Text": + """Dispays the actual color if Rich printed.""" + from .text import Text + from .style import Style + + return Text.assemble( + f"", + ) + + def __rich_repr__(self) -> Result: + yield self.name + yield self.type + yield "number", self.number, None + yield "triplet", self.triplet, None + + @property + def system(self) -> ColorSystem: + """Get the native color system for this color.""" + if self.type == ColorType.DEFAULT: + return ColorSystem.STANDARD + return ColorSystem(int(self.type)) + + @property + def is_system_defined(self) -> bool: + """Check if the color is ultimately defined by the system.""" + return self.system not in (ColorSystem.EIGHT_BIT, ColorSystem.TRUECOLOR) + + @property + def is_default(self) -> bool: + """Check if the color is a default color.""" + return self.type == ColorType.DEFAULT + + def get_truecolor( + self, theme: Optional["TerminalTheme"] = None, foreground: bool = True + ) -> ColorTriplet: + """Get an equivalent color triplet for this color. + + Args: + theme (TerminalTheme, optional): Optional terminal theme, or None to use default. Defaults to None. + foreground (bool, optional): True for a foreground color, or False for background. Defaults to True. + + Returns: + ColorTriplet: A color triplet containing RGB components. + """ + + if theme is None: + theme = DEFAULT_TERMINAL_THEME + if self.type == ColorType.TRUECOLOR: + assert self.triplet is not None + return self.triplet + elif self.type == ColorType.EIGHT_BIT: + assert self.number is not None + return EIGHT_BIT_PALETTE[self.number] + elif self.type == ColorType.STANDARD: + assert self.number is not None + return theme.ansi_colors[self.number] + elif self.type == ColorType.WINDOWS: + assert self.number is not None + return WINDOWS_PALETTE[self.number] + else: # self.type == ColorType.DEFAULT: + assert self.number is None + return theme.foreground_color if foreground else theme.background_color + + @classmethod + def from_ansi(cls, number: int) -> "Color": + """Create a Color number from it's 8-bit ansi number. + + Args: + number (int): A number between 0-255 inclusive. + + Returns: + Color: A new Color instance. + """ + return cls( + name=f"color({number})", + type=(ColorType.STANDARD if number < 16 else ColorType.EIGHT_BIT), + number=number, + ) + + @classmethod + def from_triplet(cls, triplet: "ColorTriplet") -> "Color": + """Create a truecolor RGB color from a triplet of values. + + Args: + triplet (ColorTriplet): A color triplet containing red, green and blue components. + + Returns: + Color: A new color object. + """ + return cls(name=triplet.hex, type=ColorType.TRUECOLOR, triplet=triplet) + + @classmethod + def from_rgb(cls, red: float, green: float, blue: float) -> "Color": + """Create a truecolor from three color components in the range(0->255). + + Args: + red (float): Red component in range 0-255. + green (float): Green component in range 0-255. + blue (float): Blue component in range 0-255. + + Returns: + Color: A new color object. + """ + return cls.from_triplet(ColorTriplet(int(red), int(green), int(blue))) + + @classmethod + def default(cls) -> "Color": + """Get a Color instance representing the default color. + + Returns: + Color: Default color. + """ + return cls(name="default", type=ColorType.DEFAULT) + + @classmethod + @lru_cache(maxsize=1024) + def parse(cls, color: str) -> "Color": + """Parse a color definition.""" + original_color = color + color = color.lower().strip() + + if color == "default": + return cls(color, type=ColorType.DEFAULT) + + color_number = ANSI_COLOR_NAMES.get(color) + if color_number is not None: + return cls( + color, + type=(ColorType.STANDARD if color_number < 16 else ColorType.EIGHT_BIT), + number=color_number, + ) + + color_match = RE_COLOR.match(color) + if color_match is None: + raise ColorParseError(f"{original_color!r} is not a valid color") + + color_24, color_8, color_rgb = color_match.groups() + if color_24: + triplet = ColorTriplet( + int(color_24[0:2], 16), int(color_24[2:4], 16), int(color_24[4:6], 16) + ) + return cls(color, ColorType.TRUECOLOR, triplet=triplet) + + elif color_8: + number = int(color_8) + if number > 255: + raise ColorParseError(f"color number must be <= 255 in {color!r}") + return cls( + color, + type=(ColorType.STANDARD if number < 16 else ColorType.EIGHT_BIT), + number=number, + ) + + else: # color_rgb: + components = color_rgb.split(",") + if len(components) != 3: + raise ColorParseError( + f"expected three components in {original_color!r}" + ) + red, green, blue = components + triplet = ColorTriplet(int(red), int(green), int(blue)) + if not all(component <= 255 for component in triplet): + raise ColorParseError( + f"color components must be <= 255 in {original_color!r}" + ) + return cls(color, ColorType.TRUECOLOR, triplet=triplet) + + @lru_cache(maxsize=1024) + def get_ansi_codes(self, foreground: bool = True) -> Tuple[str, ...]: + """Get the ANSI escape codes for this color.""" + _type = self.type + if _type == ColorType.DEFAULT: + return ("39" if foreground else "49",) + + elif _type == ColorType.WINDOWS: + number = self.number + assert number is not None + fore, back = (30, 40) if number < 8 else (82, 92) + return (str(fore + number if foreground else back + number),) + + elif _type == ColorType.STANDARD: + number = self.number + assert number is not None + fore, back = (30, 40) if number < 8 else (82, 92) + return (str(fore + number if foreground else back + number),) + + elif _type == ColorType.EIGHT_BIT: + assert self.number is not None + return ("38" if foreground else "48", "5", str(self.number)) + + else: # self.standard == ColorStandard.TRUECOLOR: + assert self.triplet is not None + red, green, blue = self.triplet + return ("38" if foreground else "48", "2", str(red), str(green), str(blue)) + + @lru_cache(maxsize=1024) + def downgrade(self, system: ColorSystem) -> "Color": + """Downgrade a color system to a system with fewer colors.""" + + if self.type in [ColorType.DEFAULT, system]: + return self + # Convert to 8-bit color from truecolor color + if system == ColorSystem.EIGHT_BIT and self.system == ColorSystem.TRUECOLOR: + assert self.triplet is not None + red, green, blue = self.triplet.normalized + _h, l, s = rgb_to_hls(red, green, blue) + # If saturation is under 10% assume it is grayscale + if s < 0.1: + gray = round(l * 25.0) + if gray == 0: + color_number = 16 + elif gray == 25: + color_number = 231 + else: + color_number = 231 + gray + return Color(self.name, ColorType.EIGHT_BIT, number=color_number) + + color_number = ( + 16 + 36 * round(red * 5.0) + 6 * round(green * 5.0) + round(blue * 5.0) + ) + return Color(self.name, ColorType.EIGHT_BIT, number=color_number) + + # Convert to standard from truecolor or 8-bit + elif system == ColorSystem.STANDARD: + if self.system == ColorSystem.TRUECOLOR: + assert self.triplet is not None + triplet = self.triplet + else: # self.system == ColorSystem.EIGHT_BIT + assert self.number is not None + triplet = ColorTriplet(*EIGHT_BIT_PALETTE[self.number]) + + color_number = STANDARD_PALETTE.match(triplet) + return Color(self.name, ColorType.STANDARD, number=color_number) + + elif system == ColorSystem.WINDOWS: + if self.system == ColorSystem.TRUECOLOR: + assert self.triplet is not None + triplet = self.triplet + else: # self.system == ColorSystem.EIGHT_BIT + assert self.number is not None + if self.number < 16: + return Color(self.name, ColorType.WINDOWS, number=self.number) + triplet = ColorTriplet(*EIGHT_BIT_PALETTE[self.number]) + + color_number = WINDOWS_PALETTE.match(triplet) + return Color(self.name, ColorType.WINDOWS, number=color_number) + + return self + + +def parse_rgb_hex(hex_color: str) -> ColorTriplet: + """Parse six hex characters in to RGB triplet.""" + assert len(hex_color) == 6, "must be 6 characters" + color = ColorTriplet( + int(hex_color[0:2], 16), int(hex_color[2:4], 16), int(hex_color[4:6], 16) + ) + return color + + +def blend_rgb( + color1: ColorTriplet, color2: ColorTriplet, cross_fade: float = 0.5 +) -> ColorTriplet: + """Blend one RGB color in to another.""" + r1, g1, b1 = color1 + r2, g2, b2 = color2 + new_color = ColorTriplet( + int(r1 + (r2 - r1) * cross_fade), + int(g1 + (g2 - g1) * cross_fade), + int(b1 + (b2 - b1) * cross_fade), + ) + return new_color + + +if __name__ == "__main__": # pragma: no cover + + from .console import Console + from .table import Table + from .text import Text + + console = Console() + + table = Table(show_footer=False, show_edge=True) + table.add_column("Color", width=10, overflow="ellipsis") + table.add_column("Number", justify="right", style="yellow") + table.add_column("Name", style="green") + table.add_column("Hex", style="blue") + table.add_column("RGB", style="magenta") + + colors = sorted((v, k) for k, v in ANSI_COLOR_NAMES.items()) + for color_number, name in colors: + color_cell = Text(" " * 10, style=f"on {name}") + if color_number < 16: + table.add_row(color_cell, f"{color_number}", Text(f'"{name}"')) + else: + color = EIGHT_BIT_PALETTE[color_number] # type: ignore + table.add_row( + color_cell, str(color_number), Text(f'"{name}"'), color.hex, color.rgb + ) + + console.print(table) diff --git a/src/pip/_vendor/rich/color_triplet.py b/src/pip/_vendor/rich/color_triplet.py new file mode 100644 index 00000000000..02cab328251 --- /dev/null +++ b/src/pip/_vendor/rich/color_triplet.py @@ -0,0 +1,38 @@ +from typing import NamedTuple, Tuple + + +class ColorTriplet(NamedTuple): + """The red, green, and blue components of a color.""" + + red: int + """Red component in 0 to 255 range.""" + green: int + """Green component in 0 to 255 range.""" + blue: int + """Blue component in 0 to 255 range.""" + + @property + def hex(self) -> str: + """get the color triplet in CSS style.""" + red, green, blue = self + return f"#{red:02x}{green:02x}{blue:02x}" + + @property + def rgb(self) -> str: + """The color in RGB format. + + Returns: + str: An rgb color, e.g. ``"rgb(100,23,255)"``. + """ + red, green, blue = self + return f"rgb({red},{green},{blue})" + + @property + def normalized(self) -> Tuple[float, float, float]: + """Convert components into floats between 0 and 1. + + Returns: + Tuple[float, float, float]: A tuple of three normalized colour components. + """ + red, green, blue = self + return red / 255.0, green / 255.0, blue / 255.0 diff --git a/src/pip/_vendor/rich/columns.py b/src/pip/_vendor/rich/columns.py new file mode 100644 index 00000000000..669a3a7074f --- /dev/null +++ b/src/pip/_vendor/rich/columns.py @@ -0,0 +1,187 @@ +from collections import defaultdict +from itertools import chain +from operator import itemgetter +from typing import Dict, Iterable, List, Optional, Tuple + +from .align import Align, AlignMethod +from .console import Console, ConsoleOptions, RenderableType, RenderResult +from .constrain import Constrain +from .measure import Measurement +from .padding import Padding, PaddingDimensions +from .table import Table +from .text import TextType +from .jupyter import JupyterMixin + + +class Columns(JupyterMixin): + """Display renderables in neat columns. + + Args: + renderables (Iterable[RenderableType]): Any number of Rich renderables (including str). + width (int, optional): The desired width of the columns, or None to auto detect. Defaults to None. + padding (PaddingDimensions, optional): Optional padding around cells. Defaults to (0, 1). + expand (bool, optional): Expand columns to full width. Defaults to False. + equal (bool, optional): Arrange in to equal sized columns. Defaults to False. + column_first (bool, optional): Align items from top to bottom (rather than left to right). Defaults to False. + right_to_left (bool, optional): Start column from right hand side. Defaults to False. + align (str, optional): Align value ("left", "right", or "center") or None for default. Defaults to None. + title (TextType, optional): Optional title for Columns. + """ + + def __init__( + self, + renderables: Optional[Iterable[RenderableType]] = None, + padding: PaddingDimensions = (0, 1), + *, + width: Optional[int] = None, + expand: bool = False, + equal: bool = False, + column_first: bool = False, + right_to_left: bool = False, + align: Optional[AlignMethod] = None, + title: Optional[TextType] = None, + ) -> None: + self.renderables = list(renderables or []) + self.width = width + self.padding = padding + self.expand = expand + self.equal = equal + self.column_first = column_first + self.right_to_left = right_to_left + self.align: Optional[AlignMethod] = align + self.title = title + + def add_renderable(self, renderable: RenderableType) -> None: + """Add a renderable to the columns. + + Args: + renderable (RenderableType): Any renderable object. + """ + self.renderables.append(renderable) + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + render_str = console.render_str + renderables = [ + render_str(renderable) if isinstance(renderable, str) else renderable + for renderable in self.renderables + ] + if not renderables: + return + _top, right, _bottom, left = Padding.unpack(self.padding) + width_padding = max(left, right) + max_width = options.max_width + widths: Dict[int, int] = defaultdict(int) + column_count = len(renderables) + + get_measurement = Measurement.get + renderable_widths = [ + get_measurement(console, options, renderable).maximum + for renderable in renderables + ] + if self.equal: + renderable_widths = [max(renderable_widths)] * len(renderable_widths) + + def iter_renderables( + column_count: int, + ) -> Iterable[Tuple[int, Optional[RenderableType]]]: + item_count = len(renderables) + if self.column_first: + width_renderables = list(zip(renderable_widths, renderables)) + + column_lengths: List[int] = [item_count // column_count] * column_count + for col_no in range(item_count % column_count): + column_lengths[col_no] += 1 + + row_count = (item_count + column_count - 1) // column_count + cells = [[-1] * column_count for _ in range(row_count)] + row = col = 0 + for index in range(item_count): + cells[row][col] = index + column_lengths[col] -= 1 + if column_lengths[col]: + row += 1 + else: + col += 1 + row = 0 + for index in chain.from_iterable(cells): + if index == -1: + break + yield width_renderables[index] + else: + yield from zip(renderable_widths, renderables) + # Pad odd elements with spaces + if item_count % column_count: + for _ in range(column_count - (item_count % column_count)): + yield 0, None + + table = Table.grid(padding=self.padding, collapse_padding=True, pad_edge=False) + table.expand = self.expand + table.title = self.title + + if self.width is not None: + column_count = (max_width) // (self.width + width_padding) + for _ in range(column_count): + table.add_column(width=self.width) + else: + while column_count > 1: + widths.clear() + column_no = 0 + for renderable_width, _ in iter_renderables(column_count): + widths[column_no] = max(widths[column_no], renderable_width) + total_width = sum(widths.values()) + width_padding * ( + len(widths) - 1 + ) + if total_width > max_width: + column_count = len(widths) - 1 + break + else: + column_no = (column_no + 1) % column_count + else: + break + + get_renderable = itemgetter(1) + _renderables = [ + get_renderable(_renderable) + for _renderable in iter_renderables(column_count) + ] + if self.equal: + _renderables = [ + None + if renderable is None + else Constrain(renderable, renderable_widths[0]) + for renderable in _renderables + ] + if self.align: + align = self.align + _Align = Align + _renderables = [ + None if renderable is None else _Align(renderable, align) + for renderable in _renderables + ] + + right_to_left = self.right_to_left + add_row = table.add_row + for start in range(0, len(_renderables), column_count): + row = _renderables[start : start + column_count] + if right_to_left: + row = row[::-1] + add_row(*row) + yield table + + +if __name__ == "__main__": # pragma: no cover + import os + + console = Console() + + files = [f"{i} {s}" for i, s in enumerate(sorted(os.listdir()))] + columns = Columns(files, padding=(0, 1), expand=False, equal=False) + console.print(columns) + console.rule() + columns.column_first = True + console.print(columns) + columns.right_to_left = True + console.rule() + console.print(columns) diff --git a/src/pip/_vendor/rich/console.py b/src/pip/_vendor/rich/console.py new file mode 100644 index 00000000000..27e722760ff --- /dev/null +++ b/src/pip/_vendor/rich/console.py @@ -0,0 +1,2211 @@ +import inspect +import os +import platform +import sys +import threading +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from functools import wraps +from getpass import getpass +from html import escape +from inspect import isclass +from itertools import islice +from threading import RLock +from time import monotonic +from types import FrameType, ModuleType, TracebackType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + NamedTuple, + Optional, + TextIO, + Tuple, + Type, + Union, + cast, +) + +if sys.version_info >= (3, 8): + from typing import Literal, Protocol, runtime_checkable +else: + from pip._vendor.typing_extensions import ( + Literal, + Protocol, + runtime_checkable, + ) # pragma: no cover + +from . import errors, themes +from ._emoji_replace import _emoji_replace +from ._log_render import FormatTimeCallable, LogRender +from .align import Align, AlignMethod +from .color import ColorSystem +from .control import Control +from .emoji import EmojiVariant +from .highlighter import NullHighlighter, ReprHighlighter +from .markup import render as render_markup +from .measure import Measurement, measure_renderables +from .pager import Pager, SystemPager +from .pretty import Pretty, is_expandable +from .protocol import rich_cast +from .region import Region +from .scope import render_scope +from .screen import Screen +from .segment import Segment +from .style import Style, StyleType +from .styled import Styled +from .terminal_theme import DEFAULT_TERMINAL_THEME, TerminalTheme +from .text import Text, TextType +from .theme import Theme, ThemeStack + +if TYPE_CHECKING: + from ._windows import WindowsConsoleFeatures + from .live import Live + from .status import Status + +WINDOWS = platform.system() == "Windows" + +HighlighterType = Callable[[Union[str, "Text"]], "Text"] +JustifyMethod = Literal["default", "left", "center", "right", "full"] +OverflowMethod = Literal["fold", "crop", "ellipsis", "ignore"] + + +class NoChange: + pass + + +NO_CHANGE = NoChange() + + +CONSOLE_HTML_FORMAT = """\ + + + + + + + + +
{code}
+
+ + +""" + +_TERM_COLORS = {"256color": ColorSystem.EIGHT_BIT, "16color": ColorSystem.STANDARD} + + +class ConsoleDimensions(NamedTuple): + """Size of the terminal.""" + + width: int + """The width of the console in 'cells'.""" + height: int + """The height of the console in lines.""" + + +@dataclass +class ConsoleOptions: + """Options for __rich_console__ method.""" + + size: ConsoleDimensions + """Size of console.""" + legacy_windows: bool + """legacy_windows: flag for legacy windows.""" + min_width: int + """Minimum width of renderable.""" + max_width: int + """Maximum width of renderable.""" + is_terminal: bool + """True if the target is a terminal, otherwise False.""" + encoding: str + """Encoding of terminal.""" + max_height: int + """Height of container (starts as terminal)""" + justify: Optional[JustifyMethod] = None + """Justify value override for renderable.""" + overflow: Optional[OverflowMethod] = None + """Overflow value override for renderable.""" + no_wrap: Optional[bool] = False + """Disable wrapping for text.""" + highlight: Optional[bool] = None + """Highlight override for render_str.""" + markup: Optional[bool] = None + """Enable markup when rendering strings.""" + height: Optional[int] = None + + @property + def ascii_only(self) -> bool: + """Check if renderables should use ascii only.""" + return not self.encoding.startswith("utf") + + def copy(self) -> "ConsoleOptions": + """Return a copy of the options. + + Returns: + ConsoleOptions: a copy of self. + """ + options: ConsoleOptions = ConsoleOptions.__new__(ConsoleOptions) + options.__dict__ = self.__dict__.copy() + return options + + def update( + self, + *, + width: Union[int, NoChange] = NO_CHANGE, + min_width: Union[int, NoChange] = NO_CHANGE, + max_width: Union[int, NoChange] = NO_CHANGE, + justify: Union[Optional[JustifyMethod], NoChange] = NO_CHANGE, + overflow: Union[Optional[OverflowMethod], NoChange] = NO_CHANGE, + no_wrap: Union[Optional[bool], NoChange] = NO_CHANGE, + highlight: Union[Optional[bool], NoChange] = NO_CHANGE, + markup: Union[Optional[bool], NoChange] = NO_CHANGE, + height: Union[Optional[int], NoChange] = NO_CHANGE, + ) -> "ConsoleOptions": + """Update values, return a copy.""" + options = self.copy() + if not isinstance(width, NoChange): + options.min_width = options.max_width = max(0, width) + if not isinstance(min_width, NoChange): + options.min_width = min_width + if not isinstance(max_width, NoChange): + options.max_width = max_width + if not isinstance(justify, NoChange): + options.justify = justify + if not isinstance(overflow, NoChange): + options.overflow = overflow + if not isinstance(no_wrap, NoChange): + options.no_wrap = no_wrap + if not isinstance(highlight, NoChange): + options.highlight = highlight + if not isinstance(markup, NoChange): + options.markup = markup + if not isinstance(height, NoChange): + if height is not None: + options.max_height = height + options.height = None if height is None else max(0, height) + return options + + def update_width(self, width: int) -> "ConsoleOptions": + """Update just the width, return a copy. + + Args: + width (int): New width (sets both min_width and max_width) + + Returns: + ~ConsoleOptions: New console options instance. + """ + options = self.copy() + options.min_width = options.max_width = max(0, width) + return options + + def update_height(self, height: int) -> "ConsoleOptions": + """Update the height, and return a copy. + + Args: + height (int): New height + + Returns: + ~ConsoleOptions: New Console options instance. + """ + options = self.copy() + options.max_height = options.height = height + return options + + def update_dimensions(self, width: int, height: int) -> "ConsoleOptions": + """Update the width and height, and return a copy. + + Args: + width (int): New width (sets both min_width and max_width). + height (int): New height. + + Returns: + ~ConsoleOptions: New console options instance. + """ + options = self.copy() + options.min_width = options.max_width = max(0, width) + options.height = options.max_height = height + return options + + +@runtime_checkable +class RichCast(Protocol): + """An object that may be 'cast' to a console renderable.""" + + def __rich__(self) -> Union["ConsoleRenderable", str]: # pragma: no cover + ... + + +@runtime_checkable +class ConsoleRenderable(Protocol): + """An object that supports the console protocol.""" + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": # pragma: no cover + ... + + +# A type that may be rendered by Console. +RenderableType = Union[ConsoleRenderable, RichCast, str] + + +# The result of calling a __rich_console__ method. +RenderResult = Iterable[Union[RenderableType, Segment]] + + +_null_highlighter = NullHighlighter() + + +class CaptureError(Exception): + """An error in the Capture context manager.""" + + +class NewLine: + """A renderable to generate new line(s)""" + + def __init__(self, count: int = 1) -> None: + self.count = count + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> Iterable[Segment]: + yield Segment("\n" * self.count) + + +class ScreenUpdate: + """Render a list of lines at a given offset.""" + + def __init__(self, lines: List[List[Segment]], x: int, y: int) -> None: + self._lines = lines + self.x = x + self.y = y + + def __rich_console__( + self, console: "Console", options: ConsoleOptions + ) -> RenderResult: + x = self.x + move_to = Control.move_to + for offset, line in enumerate(self._lines, self.y): + yield move_to(x, offset) + yield from line + + +class Capture: + """Context manager to capture the result of printing to the console. + See :meth:`~rich.console.Console.capture` for how to use. + + Args: + console (Console): A console instance to capture output. + """ + + def __init__(self, console: "Console") -> None: + self._console = console + self._result: Optional[str] = None + + def __enter__(self) -> "Capture": + self._console.begin_capture() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self._result = self._console.end_capture() + + def get(self) -> str: + """Get the result of the capture.""" + if self._result is None: + raise CaptureError( + "Capture result is not available until context manager exits." + ) + return self._result + + +class ThemeContext: + """A context manager to use a temporary theme. See :meth:`~rich.console.Console.use_theme` for usage.""" + + def __init__(self, console: "Console", theme: Theme, inherit: bool = True) -> None: + self.console = console + self.theme = theme + self.inherit = inherit + + def __enter__(self) -> "ThemeContext": + self.console.push_theme(self.theme) + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.console.pop_theme() + + +class PagerContext: + """A context manager that 'pages' content. See :meth:`~rich.console.Console.pager` for usage.""" + + def __init__( + self, + console: "Console", + pager: Optional[Pager] = None, + styles: bool = False, + links: bool = False, + ) -> None: + self._console = console + self.pager = SystemPager() if pager is None else pager + self.styles = styles + self.links = links + + def __enter__(self) -> "PagerContext": + self._console._enter_buffer() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + if exc_type is None: + with self._console._lock: + buffer: List[Segment] = self._console._buffer[:] + del self._console._buffer[:] + segments: Iterable[Segment] = buffer + if not self.styles: + segments = Segment.strip_styles(segments) + elif not self.links: + segments = Segment.strip_links(segments) + content = self._console._render_buffer(segments) + self.pager.show(content) + self._console._exit_buffer() + + +class ScreenContext: + """A context manager that enables an alternative screen. See :meth:`~rich.console.Console.screen` for usage.""" + + def __init__( + self, console: "Console", hide_cursor: bool, style: StyleType = "" + ) -> None: + self.console = console + self.hide_cursor = hide_cursor + self.screen = Screen(style=style) + self._changed = False + + def update( + self, *renderables: RenderableType, style: Optional[StyleType] = None + ) -> None: + """Update the screen. + + Args: + renderable (RenderableType, optional): Optional renderable to replace current renderable, + or None for no change. Defaults to None. + style: (Style, optional): Replacement style, or None for no change. Defaults to None. + """ + if renderables: + self.screen.renderable = ( + Group(*renderables) if len(renderables) > 1 else renderables[0] + ) + if style is not None: + self.screen.style = style + self.console.print(self.screen, end="") + + def __enter__(self) -> "ScreenContext": + self._changed = self.console.set_alt_screen(True) + if self._changed and self.hide_cursor: + self.console.show_cursor(False) + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + if self._changed: + self.console.set_alt_screen(False) + if self.hide_cursor: + self.console.show_cursor(True) + + +class Group: + """Takes a group of renderables and returns a renderable object that renders the group. + + Args: + renderables (Iterable[RenderableType]): An iterable of renderable objects. + fit (bool, optional): Fit dimension of group to contents, or fill available space. Defaults to True. + """ + + def __init__(self, *renderables: "RenderableType", fit: bool = True) -> None: + self._renderables = renderables + self.fit = fit + self._render: Optional[List[RenderableType]] = None + + @property + def renderables(self) -> List["RenderableType"]: + if self._render is None: + self._render = list(self._renderables) + return self._render + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + if self.fit: + return measure_renderables(console, options, self.renderables) + else: + return Measurement(options.max_width, options.max_width) + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> RenderResult: + yield from self.renderables + + +def group(fit: bool = True) -> Callable[..., Callable[..., Group]]: + """A decorator that turns an iterable of renderables in to a group. + + Args: + fit (bool, optional): Fit dimension of group to contents, or fill available space. Defaults to True. + """ + + def decorator( + method: Callable[..., Iterable[RenderableType]] + ) -> Callable[..., Group]: + """Convert a method that returns an iterable of renderables in to a Group.""" + + @wraps(method) + def _replace(*args: Any, **kwargs: Any) -> Group: + renderables = method(*args, **kwargs) + return Group(*renderables, fit=fit) + + return _replace + + return decorator + + +def _is_jupyter() -> bool: # pragma: no cover + """Check if we're running in a Jupyter notebook.""" + try: + get_ipython # type: ignore + except NameError: + return False + ipython = get_ipython() # type: ignore + shell = ipython.__class__.__name__ + if "google.colab" in str(ipython.__class__) or shell == "ZMQInteractiveShell": + return True # Jupyter notebook or qtconsole + elif shell == "TerminalInteractiveShell": + return False # Terminal running IPython + else: + return False # Other type (?) + + +COLOR_SYSTEMS = { + "standard": ColorSystem.STANDARD, + "256": ColorSystem.EIGHT_BIT, + "truecolor": ColorSystem.TRUECOLOR, + "windows": ColorSystem.WINDOWS, +} + + +_COLOR_SYSTEMS_NAMES = {system: name for name, system in COLOR_SYSTEMS.items()} + + +@dataclass +class ConsoleThreadLocals(threading.local): + """Thread local values for Console context.""" + + theme_stack: ThemeStack + buffer: List[Segment] = field(default_factory=list) + buffer_index: int = 0 + + +class RenderHook(ABC): + """Provides hooks in to the render process.""" + + @abstractmethod + def process_renderables( + self, renderables: List[ConsoleRenderable] + ) -> List[ConsoleRenderable]: + """Called with a list of objects to render. + + This method can return a new list of renderables, or modify and return the same list. + + Args: + renderables (List[ConsoleRenderable]): A number of renderable objects. + + Returns: + List[ConsoleRenderable]: A replacement list of renderables. + """ + + +_windows_console_features: Optional["WindowsConsoleFeatures"] = None + + +def get_windows_console_features() -> "WindowsConsoleFeatures": # pragma: no cover + global _windows_console_features + if _windows_console_features is not None: + return _windows_console_features + from ._windows import get_windows_console_features + + _windows_console_features = get_windows_console_features() + return _windows_console_features + + +def detect_legacy_windows() -> bool: + """Detect legacy Windows.""" + return WINDOWS and not get_windows_console_features().vt + + +if detect_legacy_windows(): # pragma: no cover + from pip._vendor.colorama import init + + init(strip=False) + + +class Console: + """A high level console interface. + + Args: + color_system (str, optional): The color system supported by your terminal, + either ``"standard"``, ``"256"`` or ``"truecolor"``. Leave as ``"auto"`` to autodetect. + force_terminal (Optional[bool], optional): Enable/disable terminal control codes, or None to auto-detect terminal. Defaults to None. + force_jupyter (Optional[bool], optional): Enable/disable Jupyter rendering, or None to auto-detect Jupyter. Defaults to None. + force_interactive (Optional[bool], optional): Enable/disable interactive mode, or None to auto detect. Defaults to None. + soft_wrap (Optional[bool], optional): Set soft wrap default on print method. Defaults to False. + theme (Theme, optional): An optional style theme object, or ``None`` for default theme. + stderr (bool, optional): Use stderr rather than stdout if ``file`` is not specified. Defaults to False. + file (IO, optional): A file object where the console should write to. Defaults to stdout. + quiet (bool, Optional): Boolean to suppress all output. Defaults to False. + width (int, optional): The width of the terminal. Leave as default to auto-detect width. + height (int, optional): The height of the terminal. Leave as default to auto-detect height. + style (StyleType, optional): Style to apply to all output, or None for no style. Defaults to None. + no_color (Optional[bool], optional): Enabled no color mode, or None to auto detect. Defaults to None. + tab_size (int, optional): Number of spaces used to replace a tab character. Defaults to 8. + record (bool, optional): Boolean to enable recording of terminal output, + required to call :meth:`export_html` and :meth:`export_text`. Defaults to False. + markup (bool, optional): Boolean to enable :ref:`console_markup`. Defaults to True. + emoji (bool, optional): Enable emoji code. Defaults to True. + emoji_variant (str, optional): Optional emoji variant, either "text" or "emoji". Defaults to None. + highlight (bool, optional): Enable automatic highlighting. Defaults to True. + log_time (bool, optional): Boolean to enable logging of time by :meth:`log` methods. Defaults to True. + log_path (bool, optional): Boolean to enable the logging of the caller by :meth:`log`. Defaults to True. + log_time_format (Union[str, TimeFormatterCallable], optional): If ``log_time`` is enabled, either string for strftime or callable that formats the time. Defaults to "[%X] ". + highlighter (HighlighterType, optional): Default highlighter. + legacy_windows (bool, optional): Enable legacy Windows mode, or ``None`` to auto detect. Defaults to ``None``. + safe_box (bool, optional): Restrict box options that don't render on legacy Windows. + get_datetime (Callable[[], datetime], optional): Callable that gets the current time as a datetime.datetime object (used by Console.log), + or None for datetime.now. + get_time (Callable[[], time], optional): Callable that gets the current time in seconds, default uses time.monotonic. + """ + + _environ: Mapping[str, str] = os.environ + + def __init__( + self, + *, + color_system: Optional[ + Literal["auto", "standard", "256", "truecolor", "windows"] + ] = "auto", + force_terminal: Optional[bool] = None, + force_jupyter: Optional[bool] = None, + force_interactive: Optional[bool] = None, + soft_wrap: bool = False, + theme: Optional[Theme] = None, + stderr: bool = False, + file: Optional[IO[str]] = None, + quiet: bool = False, + width: Optional[int] = None, + height: Optional[int] = None, + style: Optional[StyleType] = None, + no_color: Optional[bool] = None, + tab_size: int = 8, + record: bool = False, + markup: bool = True, + emoji: bool = True, + emoji_variant: Optional[EmojiVariant] = None, + highlight: bool = True, + log_time: bool = True, + log_path: bool = True, + log_time_format: Union[str, FormatTimeCallable] = "[%X]", + highlighter: Optional["HighlighterType"] = ReprHighlighter(), + legacy_windows: Optional[bool] = None, + safe_box: bool = True, + get_datetime: Optional[Callable[[], datetime]] = None, + get_time: Optional[Callable[[], float]] = None, + _environ: Optional[Mapping[str, str]] = None, + ): + # Copy of os.environ allows us to replace it for testing + if _environ is not None: + self._environ = _environ + + self.is_jupyter = _is_jupyter() if force_jupyter is None else force_jupyter + if self.is_jupyter: + width = width or 93 + height = height or 100 + + self.soft_wrap = soft_wrap + self._width = width + self._height = height + self.tab_size = tab_size + self.record = record + self._markup = markup + self._emoji = emoji + self._emoji_variant: Optional[EmojiVariant] = emoji_variant + self._highlight = highlight + self.legacy_windows: bool = ( + (detect_legacy_windows() and not self.is_jupyter) + if legacy_windows is None + else legacy_windows + ) + if width is None: + columns = self._environ.get("COLUMNS") + if columns is not None and columns.isdigit(): + width = int(columns) - self.legacy_windows + if height is None: + lines = self._environ.get("LINES") + if lines is not None and lines.isdigit(): + height = int(lines) + + self.soft_wrap = soft_wrap + self._width = width + self._height = height + + self._color_system: Optional[ColorSystem] + self._force_terminal = force_terminal + self._file = file + self.quiet = quiet + self.stderr = stderr + + if color_system is None: + self._color_system = None + elif color_system == "auto": + self._color_system = self._detect_color_system() + else: + self._color_system = COLOR_SYSTEMS[color_system] + + self._lock = threading.RLock() + self._log_render = LogRender( + show_time=log_time, + show_path=log_path, + time_format=log_time_format, + ) + self.highlighter: HighlighterType = highlighter or _null_highlighter + self.safe_box = safe_box + self.get_datetime = get_datetime or datetime.now + self.get_time = get_time or monotonic + self.style = style + self.no_color = ( + no_color if no_color is not None else "NO_COLOR" in self._environ + ) + self.is_interactive = ( + (self.is_terminal and not self.is_dumb_terminal) + if force_interactive is None + else force_interactive + ) + + self._record_buffer_lock = threading.RLock() + self._thread_locals = ConsoleThreadLocals( + theme_stack=ThemeStack(themes.DEFAULT if theme is None else theme) + ) + self._record_buffer: List[Segment] = [] + self._render_hooks: List[RenderHook] = [] + self._live: Optional["Live"] = None + self._is_alt_screen = False + + def __repr__(self) -> str: + return f"" + + @property + def file(self) -> IO[str]: + """Get the file object to write to.""" + file = self._file or (sys.stderr if self.stderr else sys.stdout) + file = getattr(file, "rich_proxied_file", file) + return file + + @file.setter + def file(self, new_file: IO[str]) -> None: + """Set a new file object.""" + self._file = new_file + + @property + def _buffer(self) -> List[Segment]: + """Get a thread local buffer.""" + return self._thread_locals.buffer + + @property + def _buffer_index(self) -> int: + """Get a thread local buffer.""" + return self._thread_locals.buffer_index + + @_buffer_index.setter + def _buffer_index(self, value: int) -> None: + self._thread_locals.buffer_index = value + + @property + def _theme_stack(self) -> ThemeStack: + """Get the thread local theme stack.""" + return self._thread_locals.theme_stack + + def _detect_color_system(self) -> Optional[ColorSystem]: + """Detect color system from env vars.""" + if self.is_jupyter: + return ColorSystem.TRUECOLOR + if not self.is_terminal or self.is_dumb_terminal: + return None + if WINDOWS: # pragma: no cover + if self.legacy_windows: # pragma: no cover + return ColorSystem.WINDOWS + windows_console_features = get_windows_console_features() + return ( + ColorSystem.TRUECOLOR + if windows_console_features.truecolor + else ColorSystem.EIGHT_BIT + ) + else: + color_term = self._environ.get("COLORTERM", "").strip().lower() + if color_term in ("truecolor", "24bit"): + return ColorSystem.TRUECOLOR + term = self._environ.get("TERM", "").strip().lower() + _term_name, _hyphen, colors = term.rpartition("-") + color_system = _TERM_COLORS.get(colors, ColorSystem.STANDARD) + return color_system + + def _enter_buffer(self) -> None: + """Enter in to a buffer context, and buffer all output.""" + self._buffer_index += 1 + + def _exit_buffer(self) -> None: + """Leave buffer context, and render content if required.""" + self._buffer_index -= 1 + self._check_buffer() + + def set_live(self, live: "Live") -> None: + """Set Live instance. Used by Live context manager. + + Args: + live (Live): Live instance using this Console. + + Raises: + errors.LiveError: If this Console has a Live context currently active. + """ + with self._lock: + if self._live is not None: + raise errors.LiveError("Only one live display may be active at once") + self._live = live + + def clear_live(self) -> None: + """Clear the Live instance.""" + with self._lock: + self._live = None + + def push_render_hook(self, hook: RenderHook) -> None: + """Add a new render hook to the stack. + + Args: + hook (RenderHook): Render hook instance. + """ + with self._lock: + self._render_hooks.append(hook) + + def pop_render_hook(self) -> None: + """Pop the last renderhook from the stack.""" + with self._lock: + self._render_hooks.pop() + + def __enter__(self) -> "Console": + """Own context manager to enter buffer context.""" + self._enter_buffer() + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + """Exit buffer context.""" + self._exit_buffer() + + def begin_capture(self) -> None: + """Begin capturing console output. Call :meth:`end_capture` to exit capture mode and return output.""" + self._enter_buffer() + + def end_capture(self) -> str: + """End capture mode and return captured string. + + Returns: + str: Console output. + """ + render_result = self._render_buffer(self._buffer) + del self._buffer[:] + self._exit_buffer() + return render_result + + def push_theme(self, theme: Theme, *, inherit: bool = True) -> None: + """Push a new theme on to the top of the stack, replacing the styles from the previous theme. + Generally speaking, you should call :meth:`~rich.console.Console.use_theme` to get a context manager, rather + than calling this method directly. + + Args: + theme (Theme): A theme instance. + inherit (bool, optional): Inherit existing styles. Defaults to True. + """ + self._theme_stack.push_theme(theme, inherit=inherit) + + def pop_theme(self) -> None: + """Remove theme from top of stack, restoring previous theme.""" + self._theme_stack.pop_theme() + + def use_theme(self, theme: Theme, *, inherit: bool = True) -> ThemeContext: + """Use a different theme for the duration of the context manager. + + Args: + theme (Theme): Theme instance to user. + inherit (bool, optional): Inherit existing console styles. Defaults to True. + + Returns: + ThemeContext: [description] + """ + return ThemeContext(self, theme, inherit) + + @property + def color_system(self) -> Optional[str]: + """Get color system string. + + Returns: + Optional[str]: "standard", "256" or "truecolor". + """ + + if self._color_system is not None: + return _COLOR_SYSTEMS_NAMES[self._color_system] + else: + return None + + @property + def encoding(self) -> str: + """Get the encoding of the console file, e.g. ``"utf-8"``. + + Returns: + str: A standard encoding string. + """ + return (getattr(self.file, "encoding", "utf-8") or "utf-8").lower() + + @property + def is_terminal(self) -> bool: + """Check if the console is writing to a terminal. + + Returns: + bool: True if the console writing to a device capable of + understanding terminal codes, otherwise False. + """ + if self._force_terminal is not None: + return self._force_terminal + isatty: Optional[Callable[[], bool]] = getattr(self.file, "isatty", None) + try: + return False if isatty is None else isatty() + except ValueError: + # in some situation (at the end of a pytest run for example) isatty() can raise + # ValueError: I/O operation on closed file + # return False because we aren't in a terminal anymore + return False + + @property + def is_dumb_terminal(self) -> bool: + """Detect dumb terminal. + + Returns: + bool: True if writing to a dumb terminal, otherwise False. + + """ + _term = self._environ.get("TERM", "") + is_dumb = _term.lower() in ("dumb", "unknown") + return self.is_terminal and is_dumb + + @property + def options(self) -> ConsoleOptions: + """Get default console options.""" + return ConsoleOptions( + max_height=self.size.height, + size=self.size, + legacy_windows=self.legacy_windows, + min_width=1, + max_width=self.width, + encoding=self.encoding, + is_terminal=self.is_terminal, + ) + + @property + def size(self) -> ConsoleDimensions: + """Get the size of the console. + + Returns: + ConsoleDimensions: A named tuple containing the dimensions. + """ + + if self._width is not None and self._height is not None: + return ConsoleDimensions(self._width - self.legacy_windows, self._height) + + if self.is_dumb_terminal: + return ConsoleDimensions(80, 25) + + width: Optional[int] = None + height: Optional[int] = None + + if WINDOWS: # pragma: no cover + try: + width, height = os.get_terminal_size() + except OSError: # Probably not a terminal + pass + else: + try: + width, height = os.get_terminal_size(sys.__stdin__.fileno()) + except (AttributeError, ValueError, OSError): + try: + width, height = os.get_terminal_size(sys.__stdout__.fileno()) + except (AttributeError, ValueError, OSError): + pass + + columns = self._environ.get("COLUMNS") + if columns is not None and columns.isdigit(): + width = int(columns) + lines = self._environ.get("LINES") + if lines is not None and lines.isdigit(): + height = int(lines) + + # get_terminal_size can report 0, 0 if run from pseudo-terminal + width = width or 80 + height = height or 25 + return ConsoleDimensions( + width - self.legacy_windows if self._width is None else self._width, + height if self._height is None else self._height, + ) + + @size.setter + def size(self, new_size: Tuple[int, int]) -> None: + """Set a new size for the terminal. + + Args: + new_size (Tuple[int, int]): New width and height. + """ + width, height = new_size + self._width = width + self._height = height + + @property + def width(self) -> int: + """Get the width of the console. + + Returns: + int: The width (in characters) of the console. + """ + return self.size.width + + @width.setter + def width(self, width: int) -> None: + """Set width. + + Args: + width (int): New width. + """ + self._width = width + + @property + def height(self) -> int: + """Get the height of the console. + + Returns: + int: The height (in lines) of the console. + """ + return self.size.height + + @height.setter + def height(self, height: int) -> None: + """Set height. + + Args: + height (int): new height. + """ + self._height = height + + def bell(self) -> None: + """Play a 'bell' sound (if supported by the terminal).""" + self.control(Control.bell()) + + def capture(self) -> Capture: + """A context manager to *capture* the result of print() or log() in a string, + rather than writing it to the console. + + Example: + >>> from rich.console import Console + >>> console = Console() + >>> with console.capture() as capture: + ... console.print("[bold magenta]Hello World[/]") + >>> print(capture.get()) + + Returns: + Capture: Context manager with disables writing to the terminal. + """ + capture = Capture(self) + return capture + + def pager( + self, pager: Optional[Pager] = None, styles: bool = False, links: bool = False + ) -> PagerContext: + """A context manager to display anything printed within a "pager". The pager application + is defined by the system and will typically support at least pressing a key to scroll. + + Args: + pager (Pager, optional): A pager object, or None to use :class:`~rich.pager.SystemPager`. Defaults to None. + styles (bool, optional): Show styles in pager. Defaults to False. + links (bool, optional): Show links in pager. Defaults to False. + + Example: + >>> from rich.console import Console + >>> from rich.__main__ import make_test_card + >>> console = Console() + >>> with console.pager(): + console.print(make_test_card()) + + Returns: + PagerContext: A context manager. + """ + return PagerContext(self, pager=pager, styles=styles, links=links) + + def line(self, count: int = 1) -> None: + """Write new line(s). + + Args: + count (int, optional): Number of new lines. Defaults to 1. + """ + + assert count >= 0, "count must be >= 0" + self.print(NewLine(count)) + + def clear(self, home: bool = True) -> None: + """Clear the screen. + + Args: + home (bool, optional): Also move the cursor to 'home' position. Defaults to True. + """ + if home: + self.control(Control.clear(), Control.home()) + else: + self.control(Control.clear()) + + def status( + self, + status: RenderableType, + *, + spinner: str = "dots", + spinner_style: str = "status.spinner", + speed: float = 1.0, + refresh_per_second: float = 12.5, + ) -> "Status": + """Display a status and spinner. + + Args: + status (RenderableType): A status renderable (str or Text typically). + spinner (str, optional): Name of spinner animation (see python -m rich.spinner). Defaults to "dots". + spinner_style (StyleType, optional): Style of spinner. Defaults to "status.spinner". + speed (float, optional): Speed factor for spinner animation. Defaults to 1.0. + refresh_per_second (float, optional): Number of refreshes per second. Defaults to 12.5. + + Returns: + Status: A Status object that may be used as a context manager. + """ + from .status import Status + + status_renderable = Status( + status, + console=self, + spinner=spinner, + spinner_style=spinner_style, + speed=speed, + refresh_per_second=refresh_per_second, + ) + return status_renderable + + def show_cursor(self, show: bool = True) -> bool: + """Show or hide the cursor. + + Args: + show (bool, optional): Set visibility of the cursor. + """ + if self.is_terminal and not self.legacy_windows: + self.control(Control.show_cursor(show)) + return True + return False + + def set_alt_screen(self, enable: bool = True) -> bool: + """Enables alternative screen mode. + + Note, if you enable this mode, you should ensure that is disabled before + the application exits. See :meth:`~rich.Console.screen` for a context manager + that handles this for you. + + Args: + enable (bool, optional): Enable (True) or disable (False) alternate screen. Defaults to True. + + Returns: + bool: True if the control codes were written. + + """ + changed = False + if self.is_terminal and not self.legacy_windows: + self.control(Control.alt_screen(enable)) + changed = True + self._is_alt_screen = enable + return changed + + @property + def is_alt_screen(self) -> bool: + """Check if the alt screen was enabled. + + Returns: + bool: True if the alt screen was enabled, otherwise False. + """ + return self._is_alt_screen + + def screen( + self, hide_cursor: bool = True, style: Optional[StyleType] = None + ) -> "ScreenContext": + """Context manager to enable and disable 'alternative screen' mode. + + Args: + hide_cursor (bool, optional): Also hide the cursor. Defaults to False. + style (Style, optional): Optional style for screen. Defaults to None. + + Returns: + ~ScreenContext: Context which enables alternate screen on enter, and disables it on exit. + """ + return ScreenContext(self, hide_cursor=hide_cursor, style=style or "") + + def measure( + self, renderable: RenderableType, *, options: Optional[ConsoleOptions] = None + ) -> Measurement: + """Measure a renderable. Returns a :class:`~rich.measure.Measurement` object which contains + information regarding the number of characters required to print the renderable. + + Args: + renderable (RenderableType): Any renderable or string. + options (Optional[ConsoleOptions], optional): Options to use when measuring, or None + to use default options. Defaults to None. + + Returns: + Measurement: A measurement of the renderable. + """ + measurement = Measurement.get(self, options or self.options, renderable) + return measurement + + def render( + self, renderable: RenderableType, options: Optional[ConsoleOptions] = None + ) -> Iterable[Segment]: + """Render an object in to an iterable of `Segment` instances. + + This method contains the logic for rendering objects with the console protocol. + You are unlikely to need to use it directly, unless you are extending the library. + + Args: + renderable (RenderableType): An object supporting the console protocol, or + an object that may be converted to a string. + options (ConsoleOptions, optional): An options object, or None to use self.options. Defaults to None. + + Returns: + Iterable[Segment]: An iterable of segments that may be rendered. + """ + + _options = options or self.options + if _options.max_width < 1: + # No space to render anything. This prevents potential recursion errors. + return + render_iterable: RenderResult + + renderable = rich_cast(renderable) + if hasattr(renderable, "__rich_console__") and not isclass(renderable): + render_iterable = renderable.__rich_console__(self, _options) # type: ignore + elif isinstance(renderable, str): + text_renderable = self.render_str( + renderable, highlight=_options.highlight, markup=_options.markup + ) + render_iterable = text_renderable.__rich_console__(self, _options) + else: + raise errors.NotRenderableError( + f"Unable to render {renderable!r}; " + "A str, Segment or object with __rich_console__ method is required" + ) + + try: + iter_render = iter(render_iterable) + except TypeError: + raise errors.NotRenderableError( + f"object {render_iterable!r} is not renderable" + ) + _Segment = Segment + for render_output in iter_render: + if isinstance(render_output, _Segment): + yield render_output + else: + yield from self.render(render_output, _options) + + def render_lines( + self, + renderable: RenderableType, + options: Optional[ConsoleOptions] = None, + *, + style: Optional[Style] = None, + pad: bool = True, + new_lines: bool = False, + ) -> List[List[Segment]]: + """Render objects in to a list of lines. + + The output of render_lines is useful when further formatting of rendered console text + is required, such as the Panel class which draws a border around any renderable object. + + Args: + renderable (RenderableType): Any object renderable in the console. + options (Optional[ConsoleOptions], optional): Console options, or None to use self.options. Default to ``None``. + style (Style, optional): Optional style to apply to renderables. Defaults to ``None``. + pad (bool, optional): Pad lines shorter than render width. Defaults to ``True``. + new_lines (bool, optional): Include "\n" characters at end of lines. + + Returns: + List[List[Segment]]: A list of lines, where a line is a list of Segment objects. + """ + with self._lock: + render_options = options or self.options + _rendered = self.render(renderable, render_options) + if style: + _rendered = Segment.apply_style(_rendered, style) + lines = list( + islice( + Segment.split_and_crop_lines( + _rendered, + render_options.max_width, + include_new_lines=new_lines, + pad=pad, + ), + None, + render_options.height, + ) + ) + if render_options.height is not None: + extra_lines = render_options.height - len(lines) + if extra_lines > 0: + pad_line = [ + [Segment(" " * render_options.max_width, style), Segment("\n")] + if new_lines + else [Segment(" " * render_options.max_width, style)] + ] + lines.extend(pad_line * extra_lines) + + return lines + + def render_str( + self, + text: str, + *, + style: Union[str, Style] = "", + justify: Optional[JustifyMethod] = None, + overflow: Optional[OverflowMethod] = None, + emoji: Optional[bool] = None, + markup: Optional[bool] = None, + highlight: Optional[bool] = None, + highlighter: Optional[HighlighterType] = None, + ) -> "Text": + """Convert a string to a Text instance. This is is called automatically if + you print or log a string. + + Args: + text (str): Text to render. + style (Union[str, Style], optional): Style to apply to rendered text. + justify (str, optional): Justify method: "default", "left", "center", "full", or "right". Defaults to ``None``. + overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to ``None``. + emoji (Optional[bool], optional): Enable emoji, or ``None`` to use Console default. + markup (Optional[bool], optional): Enable markup, or ``None`` to use Console default. + highlight (Optional[bool], optional): Enable highlighting, or ``None`` to use Console default. + highlighter (HighlighterType, optional): Optional highlighter to apply. + Returns: + ConsoleRenderable: Renderable object. + + """ + emoji_enabled = emoji or (emoji is None and self._emoji) + markup_enabled = markup or (markup is None and self._markup) + highlight_enabled = highlight or (highlight is None and self._highlight) + + if markup_enabled: + rich_text = render_markup( + text, + style=style, + emoji=emoji_enabled, + emoji_variant=self._emoji_variant, + ) + rich_text.justify = justify + rich_text.overflow = overflow + else: + rich_text = Text( + _emoji_replace(text, default_variant=self._emoji_variant) + if emoji_enabled + else text, + justify=justify, + overflow=overflow, + style=style, + ) + + _highlighter = (highlighter or self.highlighter) if highlight_enabled else None + if _highlighter is not None: + highlight_text = _highlighter(str(rich_text)) + highlight_text.copy_styles(rich_text) + return highlight_text + + return rich_text + + def get_style( + self, name: Union[str, Style], *, default: Optional[Union[Style, str]] = None + ) -> Style: + """Get a Style instance by it's theme name or parse a definition. + + Args: + name (str): The name of a style or a style definition. + + Returns: + Style: A Style object. + + Raises: + MissingStyle: If no style could be parsed from name. + + """ + if isinstance(name, Style): + return name + + try: + style = self._theme_stack.get(name) + if style is None: + style = Style.parse(name) + return style.copy() if style.link else style + except errors.StyleSyntaxError as error: + if default is not None: + return self.get_style(default) + raise errors.MissingStyle( + f"Failed to get style {name!r}; {error}" + ) from None + + def _collect_renderables( + self, + objects: Iterable[Any], + sep: str, + end: str, + *, + justify: Optional[JustifyMethod] = None, + emoji: Optional[bool] = None, + markup: Optional[bool] = None, + highlight: Optional[bool] = None, + ) -> List[ConsoleRenderable]: + """Combine a number of renderables and text into one renderable. + + Args: + objects (Iterable[Any]): Anything that Rich can render. + sep (str): String to write between print data. + end (str): String to write at end of print data. + justify (str, optional): One of "left", "right", "center", or "full". Defaults to ``None``. + emoji (Optional[bool], optional): Enable emoji code, or ``None`` to use console default. + markup (Optional[bool], optional): Enable markup, or ``None`` to use console default. + highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use console default. + + Returns: + List[ConsoleRenderable]: A list of things to render. + """ + renderables: List[ConsoleRenderable] = [] + _append = renderables.append + text: List[Text] = [] + append_text = text.append + + append = _append + if justify in ("left", "center", "right"): + + def align_append(renderable: RenderableType) -> None: + _append(Align(renderable, cast(AlignMethod, justify))) + + append = align_append + + _highlighter: HighlighterType = _null_highlighter + if highlight or (highlight is None and self._highlight): + _highlighter = self.highlighter + + def check_text() -> None: + if text: + sep_text = Text(sep, justify=justify, end=end) + append(sep_text.join(text)) + del text[:] + + for renderable in objects: + renderable = rich_cast(renderable) + if isinstance(renderable, str): + append_text( + self.render_str( + renderable, emoji=emoji, markup=markup, highlighter=_highlighter + ) + ) + elif isinstance(renderable, Text): + append_text(renderable) + elif isinstance(renderable, ConsoleRenderable): + check_text() + append(renderable) + elif is_expandable(renderable): + check_text() + append(Pretty(renderable, highlighter=_highlighter)) + else: + append_text(_highlighter(str(renderable))) + + check_text() + + if self.style is not None: + style = self.get_style(self.style) + renderables = [Styled(renderable, style) for renderable in renderables] + + return renderables + + def rule( + self, + title: TextType = "", + *, + characters: str = "─", + style: Union[str, Style] = "rule.line", + align: AlignMethod = "center", + ) -> None: + """Draw a line with optional centered title. + + Args: + title (str, optional): Text to render over the rule. Defaults to "". + characters (str, optional): Character(s) to form the line. Defaults to "─". + style (str, optional): Style of line. Defaults to "rule.line". + align (str, optional): How to align the title, one of "left", "center", or "right". Defaults to "center". + """ + from .rule import Rule + + rule = Rule(title=title, characters=characters, style=style, align=align) + self.print(rule) + + def control(self, *control: Control) -> None: + """Insert non-printing control codes. + + Args: + control_codes (str): Control codes, such as those that may move the cursor. + """ + if not self.is_dumb_terminal: + with self: + self._buffer.extend(_control.segment for _control in control) + + def out( + self, + *objects: Any, + sep: str = " ", + end: str = "\n", + style: Optional[Union[str, Style]] = None, + highlight: Optional[bool] = None, + ) -> None: + """Output to the terminal. This is a low-level way of writing to the terminal which unlike + :meth:`~rich.console.Console.print` won't pretty print, wrap text, or apply markup, but will + optionally apply highlighting and a basic style. + + Args: + sep (str, optional): String to write between print data. Defaults to " ". + end (str, optional): String to write at end of print data. Defaults to "\\\\n". + style (Union[str, Style], optional): A style to apply to output. Defaults to None. + highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use + console default. Defaults to ``None``. + """ + raw_output: str = sep.join(str(_object) for _object in objects) + self.print( + raw_output, + style=style, + highlight=highlight, + emoji=False, + markup=False, + no_wrap=True, + overflow="ignore", + crop=False, + end=end, + ) + + def print( + self, + *objects: Any, + sep: str = " ", + end: str = "\n", + style: Optional[Union[str, Style]] = None, + justify: Optional[JustifyMethod] = None, + overflow: Optional[OverflowMethod] = None, + no_wrap: Optional[bool] = None, + emoji: Optional[bool] = None, + markup: Optional[bool] = None, + highlight: Optional[bool] = None, + width: Optional[int] = None, + height: Optional[int] = None, + crop: bool = True, + soft_wrap: Optional[bool] = None, + new_line_start: bool = False, + ) -> None: + """Print to the console. + + Args: + objects (positional args): Objects to log to the terminal. + sep (str, optional): String to write between print data. Defaults to " ". + end (str, optional): String to write at end of print data. Defaults to "\\\\n". + style (Union[str, Style], optional): A style to apply to output. Defaults to None. + justify (str, optional): Justify method: "default", "left", "right", "center", or "full". Defaults to ``None``. + overflow (str, optional): Overflow method: "ignore", "crop", "fold", or "ellipsis". Defaults to None. + no_wrap (Optional[bool], optional): Disable word wrapping. Defaults to None. + emoji (Optional[bool], optional): Enable emoji code, or ``None`` to use console default. Defaults to ``None``. + markup (Optional[bool], optional): Enable markup, or ``None`` to use console default. Defaults to ``None``. + highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use console default. Defaults to ``None``. + width (Optional[int], optional): Width of output, or ``None`` to auto-detect. Defaults to ``None``. + crop (Optional[bool], optional): Crop output to width of terminal. Defaults to True. + soft_wrap (bool, optional): Enable soft wrap mode which disables word wrapping and cropping of text or ``None`` for + Console default. Defaults to ``None``. + new_line_start (bool, False): Insert a new line at the start if the output contains more than one line. Defaults to ``False``. + """ + if not objects: + objects = (NewLine(),) + + if soft_wrap is None: + soft_wrap = self.soft_wrap + if soft_wrap: + if no_wrap is None: + no_wrap = True + if overflow is None: + overflow = "ignore" + crop = False + render_hooks = self._render_hooks[:] + with self: + renderables = self._collect_renderables( + objects, + sep, + end, + justify=justify, + emoji=emoji, + markup=markup, + highlight=highlight, + ) + for hook in render_hooks: + renderables = hook.process_renderables(renderables) + render_options = self.options.update( + justify=justify, + overflow=overflow, + width=min(width, self.width) if width is not None else NO_CHANGE, + height=height, + no_wrap=no_wrap, + markup=markup, + highlight=highlight, + ) + + new_segments: List[Segment] = [] + extend = new_segments.extend + render = self.render + if style is None: + for renderable in renderables: + extend(render(renderable, render_options)) + else: + for renderable in renderables: + extend( + Segment.apply_style( + render(renderable, render_options), self.get_style(style) + ) + ) + if new_line_start: + if ( + len("".join(segment.text for segment in new_segments).splitlines()) + > 1 + ): + new_segments.insert(0, Segment.line()) + if crop: + buffer_extend = self._buffer.extend + for line in Segment.split_and_crop_lines( + new_segments, self.width, pad=False + ): + buffer_extend(line) + else: + self._buffer.extend(new_segments) + + def print_json( + self, + json: Optional[str] = None, + *, + data: Any = None, + indent: Union[None, int, str] = 2, + highlight: bool = True, + skip_keys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + default: Optional[Callable[[Any], Any]] = None, + sort_keys: bool = False, + ) -> None: + """Pretty prints JSON. Output will be valid JSON. + + Args: + json (Optional[str]): A string containing JSON. + data (Any): If json is not supplied, then encode this data. + indent (Union[None, int, str], optional): Number of spaces to indent. Defaults to 2. + highlight (bool, optional): Enable highlighting of output: Defaults to True. + skip_keys (bool, optional): Skip keys not of a basic type. Defaults to False. + ensure_ascii (bool, optional): Escape all non-ascii characters. Defaults to False. + check_circular (bool, optional): Check for circular references. Defaults to True. + allow_nan (bool, optional): Allow NaN and Infinity values. Defaults to True. + default (Callable, optional): A callable that converts values that can not be encoded + in to something that can be JSON encoded. Defaults to None. + sort_keys (bool, optional): Sort dictionary keys. Defaults to False. + """ + from pip._vendor.rich.json import JSON + + if json is None: + json_renderable = JSON.from_data( + data, + indent=indent, + highlight=highlight, + skip_keys=skip_keys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + default=default, + sort_keys=sort_keys, + ) + else: + if not isinstance(json, str): + raise TypeError( + f"json must be str. Did you mean print_json(data={json!r}) ?" + ) + json_renderable = JSON( + json, + indent=indent, + highlight=highlight, + skip_keys=skip_keys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + default=default, + sort_keys=sort_keys, + ) + self.print(json_renderable, soft_wrap=True) + + def update_screen( + self, + renderable: RenderableType, + *, + region: Optional[Region] = None, + options: Optional[ConsoleOptions] = None, + ) -> None: + """Update the screen at a given offset. + + Args: + renderable (RenderableType): A Rich renderable. + region (Region, optional): Region of screen to update, or None for entire screen. Defaults to None. + x (int, optional): x offset. Defaults to 0. + y (int, optional): y offset. Defaults to 0. + + Raises: + errors.NoAltScreen: If the Console isn't in alt screen mode. + + """ + if not self.is_alt_screen: + raise errors.NoAltScreen("Alt screen must be enabled to call update_screen") + render_options = options or self.options + if region is None: + x = y = 0 + render_options = render_options.update_dimensions( + render_options.max_width, render_options.height or self.height + ) + else: + x, y, width, height = region + render_options = render_options.update_dimensions(width, height) + + lines = self.render_lines(renderable, options=render_options) + self.update_screen_lines(lines, x, y) + + def update_screen_lines( + self, lines: List[List[Segment]], x: int = 0, y: int = 0 + ) -> None: + """Update lines of the screen at a given offset. + + Args: + lines (List[List[Segment]]): Rendered lines (as produced by :meth:`~rich.Console.render_lines`). + x (int, optional): x offset (column no). Defaults to 0. + y (int, optional): y offset (column no). Defaults to 0. + + Raises: + errors.NoAltScreen: If the Console isn't in alt screen mode. + """ + if not self.is_alt_screen: + raise errors.NoAltScreen("Alt screen must be enabled to call update_screen") + screen_update = ScreenUpdate(lines, x, y) + segments = self.render(screen_update) + self._buffer.extend(segments) + self._check_buffer() + + def print_exception( + self, + *, + width: Optional[int] = 100, + extra_lines: int = 3, + theme: Optional[str] = None, + word_wrap: bool = False, + show_locals: bool = False, + suppress: Iterable[Union[str, ModuleType]] = (), + max_frames: int = 100, + ) -> None: + """Prints a rich render of the last exception and traceback. + + Args: + width (Optional[int], optional): Number of characters used to render code. Defaults to 88. + extra_lines (int, optional): Additional lines of code to render. Defaults to 3. + theme (str, optional): Override pygments theme used in traceback + word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False. + show_locals (bool, optional): Enable display of local variables. Defaults to False. + suppress (Iterable[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback. + max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100. + """ + from .traceback import Traceback + + traceback = Traceback( + width=width, + extra_lines=extra_lines, + theme=theme, + word_wrap=word_wrap, + show_locals=show_locals, + suppress=suppress, + max_frames=max_frames, + ) + self.print(traceback) + + @staticmethod + def _caller_frame_info( + offset: int, + currentframe: Callable[[], Optional[FrameType]] = inspect.currentframe, + ) -> Tuple[str, int, Dict[str, Any]]: + """Get caller frame information. + + Args: + offset (int): the caller offset within the current frame stack. + currentframe (Callable[[], Optional[FrameType]], optional): the callable to use to + retrieve the current frame. Defaults to ``inspect.currentframe``. + + Returns: + Tuple[str, int, Dict[str, Any]]: A tuple containing the filename, the line number and + the dictionary of local variables associated with the caller frame. + + Raises: + RuntimeError: If the stack offset is invalid. + """ + # Ignore the frame of this local helper + offset += 1 + + frame = currentframe() + if frame is not None: + # Use the faster currentframe where implemented + while offset and frame: + frame = frame.f_back + offset -= 1 + assert frame is not None + return frame.f_code.co_filename, frame.f_lineno, frame.f_locals + else: + # Fallback to the slower stack + frame_info = inspect.stack()[offset] + return frame_info.filename, frame_info.lineno, frame_info.frame.f_locals + + def log( + self, + *objects: Any, + sep: str = " ", + end: str = "\n", + style: Optional[Union[str, Style]] = None, + justify: Optional[JustifyMethod] = None, + emoji: Optional[bool] = None, + markup: Optional[bool] = None, + highlight: Optional[bool] = None, + log_locals: bool = False, + _stack_offset: int = 1, + ) -> None: + """Log rich content to the terminal. + + Args: + objects (positional args): Objects to log to the terminal. + sep (str, optional): String to write between print data. Defaults to " ". + end (str, optional): String to write at end of print data. Defaults to "\\\\n". + style (Union[str, Style], optional): A style to apply to output. Defaults to None. + justify (str, optional): One of "left", "right", "center", or "full". Defaults to ``None``. + overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None. + emoji (Optional[bool], optional): Enable emoji code, or ``None`` to use console default. Defaults to None. + markup (Optional[bool], optional): Enable markup, or ``None`` to use console default. Defaults to None. + highlight (Optional[bool], optional): Enable automatic highlighting, or ``None`` to use console default. Defaults to None. + log_locals (bool, optional): Boolean to enable logging of locals where ``log()`` + was called. Defaults to False. + _stack_offset (int, optional): Offset of caller from end of call stack. Defaults to 1. + """ + if not objects: + objects = (NewLine(),) + + render_hooks = self._render_hooks[:] + + with self: + renderables = self._collect_renderables( + objects, + sep, + end, + justify=justify, + emoji=emoji, + markup=markup, + highlight=highlight, + ) + if style is not None: + renderables = [Styled(renderable, style) for renderable in renderables] + + filename, line_no, locals = self._caller_frame_info(_stack_offset) + link_path = None if filename.startswith("<") else os.path.abspath(filename) + path = filename.rpartition(os.sep)[-1] + if log_locals: + locals_map = { + key: value + for key, value in locals.items() + if not key.startswith("__") + } + renderables.append(render_scope(locals_map, title="[i]locals")) + + renderables = [ + self._log_render( + self, + renderables, + log_time=self.get_datetime(), + path=path, + line_no=line_no, + link_path=link_path, + ) + ] + for hook in render_hooks: + renderables = hook.process_renderables(renderables) + new_segments: List[Segment] = [] + extend = new_segments.extend + render = self.render + render_options = self.options + for renderable in renderables: + extend(render(renderable, render_options)) + buffer_extend = self._buffer.extend + for line in Segment.split_and_crop_lines( + new_segments, self.width, pad=False + ): + buffer_extend(line) + + def _check_buffer(self) -> None: + """Check if the buffer may be rendered.""" + if self.quiet: + del self._buffer[:] + return + with self._lock: + if self._buffer_index == 0: + if self.is_jupyter: # pragma: no cover + from .jupyter import display + + display(self._buffer, self._render_buffer(self._buffer[:])) + del self._buffer[:] + else: + text = self._render_buffer(self._buffer[:]) + del self._buffer[:] + if text: + try: + if WINDOWS: # pragma: no cover + # https://bugs.python.org/issue37871 + write = self.file.write + for line in text.splitlines(True): + write(line) + else: + self.file.write(text) + self.file.flush() + except UnicodeEncodeError as error: + error.reason = f"{error.reason}\n*** You may need to add PYTHONIOENCODING=utf-8 to your environment ***" + raise + + def _render_buffer(self, buffer: Iterable[Segment]) -> str: + """Render buffered output, and clear buffer.""" + output: List[str] = [] + append = output.append + color_system = self._color_system + legacy_windows = self.legacy_windows + if self.record: + with self._record_buffer_lock: + self._record_buffer.extend(buffer) + not_terminal = not self.is_terminal + if self.no_color and color_system: + buffer = Segment.remove_color(buffer) + for text, style, control in buffer: + if style: + append( + style.render( + text, + color_system=color_system, + legacy_windows=legacy_windows, + ) + ) + elif not (not_terminal and control): + append(text) + + rendered = "".join(output) + return rendered + + def input( + self, + prompt: TextType = "", + *, + markup: bool = True, + emoji: bool = True, + password: bool = False, + stream: Optional[TextIO] = None, + ) -> str: + """Displays a prompt and waits for input from the user. The prompt may contain color / style. + + It works in the same way as Python's builtin :func:`input` function and provides elaborate line editing and history features if Python's builtin :mod:`readline` module is previously loaded. + + Args: + prompt (Union[str, Text]): Text to render in the prompt. + markup (bool, optional): Enable console markup (requires a str prompt). Defaults to True. + emoji (bool, optional): Enable emoji (requires a str prompt). Defaults to True. + password: (bool, optional): Hide typed text. Defaults to False. + stream: (TextIO, optional): Optional file to read input from (rather than stdin). Defaults to None. + + Returns: + str: Text read from stdin. + """ + prompt_str = "" + if prompt: + with self.capture() as capture: + self.print(prompt, markup=markup, emoji=emoji, end="") + prompt_str = capture.get() + if self.legacy_windows: + # Legacy windows doesn't like ANSI codes in getpass or input (colorama bug)? + self.file.write(prompt_str) + prompt_str = "" + if password: + result = getpass(prompt_str, stream=stream) + else: + if stream: + self.file.write(prompt_str) + result = stream.readline() + else: + result = input(prompt_str) + return result + + def export_text(self, *, clear: bool = True, styles: bool = False) -> str: + """Generate text from console contents (requires record=True argument in constructor). + + Args: + clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``. + styles (bool, optional): If ``True``, ansi escape codes will be included. ``False`` for plain text. + Defaults to ``False``. + + Returns: + str: String containing console contents. + + """ + assert ( + self.record + ), "To export console contents set record=True in the constructor or instance" + + with self._record_buffer_lock: + if styles: + text = "".join( + (style.render(text) if style else text) + for text, style, _ in self._record_buffer + ) + else: + text = "".join( + segment.text + for segment in self._record_buffer + if not segment.control + ) + if clear: + del self._record_buffer[:] + return text + + def save_text(self, path: str, *, clear: bool = True, styles: bool = False) -> None: + """Generate text from console and save to a given location (requires record=True argument in constructor). + + Args: + path (str): Path to write text files. + clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``. + styles (bool, optional): If ``True``, ansi style codes will be included. ``False`` for plain text. + Defaults to ``False``. + + """ + text = self.export_text(clear=clear, styles=styles) + with open(path, "wt", encoding="utf-8") as write_file: + write_file.write(text) + + def export_html( + self, + *, + theme: Optional[TerminalTheme] = None, + clear: bool = True, + code_format: Optional[str] = None, + inline_styles: bool = False, + ) -> str: + """Generate HTML from console contents (requires record=True argument in constructor). + + Args: + theme (TerminalTheme, optional): TerminalTheme object containing console colors. + clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``. + code_format (str, optional): Format string to render HTML, should contain {foreground} + {background} and {code}. + inline_styles (bool, optional): If ``True`` styles will be inlined in to spans, which makes files + larger but easier to cut and paste markup. If ``False``, styles will be embedded in a style tag. + Defaults to False. + + Returns: + str: String containing console contents as HTML. + """ + assert ( + self.record + ), "To export console contents set record=True in the constructor or instance" + fragments: List[str] = [] + append = fragments.append + _theme = theme or DEFAULT_TERMINAL_THEME + stylesheet = "" + + render_code_format = CONSOLE_HTML_FORMAT if code_format is None else code_format + + with self._record_buffer_lock: + if inline_styles: + for text, style, _ in Segment.filter_control( + Segment.simplify(self._record_buffer) + ): + text = escape(text) + if style: + rule = style.get_html_style(_theme) + if style.link: + text = f'{text}' + text = f'{text}' if rule else text + append(text) + else: + styles: Dict[str, int] = {} + for text, style, _ in Segment.filter_control( + Segment.simplify(self._record_buffer) + ): + text = escape(text) + if style: + rule = style.get_html_style(_theme) + style_number = styles.setdefault(rule, len(styles) + 1) + if style.link: + text = f'{text}' + else: + text = f'{text}' + append(text) + stylesheet_rules: List[str] = [] + stylesheet_append = stylesheet_rules.append + for style_rule, style_number in styles.items(): + if style_rule: + stylesheet_append(f".r{style_number} {{{style_rule}}}") + stylesheet = "\n".join(stylesheet_rules) + + rendered_code = render_code_format.format( + code="".join(fragments), + stylesheet=stylesheet, + foreground=_theme.foreground_color.hex, + background=_theme.background_color.hex, + ) + if clear: + del self._record_buffer[:] + return rendered_code + + def save_html( + self, + path: str, + *, + theme: Optional[TerminalTheme] = None, + clear: bool = True, + code_format: str = CONSOLE_HTML_FORMAT, + inline_styles: bool = False, + ) -> None: + """Generate HTML from console contents and write to a file (requires record=True argument in constructor). + + Args: + path (str): Path to write html file. + theme (TerminalTheme, optional): TerminalTheme object containing console colors. + clear (bool, optional): Clear record buffer after exporting. Defaults to ``True``. + code_format (str, optional): Format string to render HTML, should contain {foreground} + {background} and {code}. + inline_styles (bool, optional): If ``True`` styles will be inlined in to spans, which makes files + larger but easier to cut and paste markup. If ``False``, styles will be embedded in a style tag. + Defaults to False. + + """ + html = self.export_html( + theme=theme, + clear=clear, + code_format=code_format, + inline_styles=inline_styles, + ) + with open(path, "wt", encoding="utf-8") as write_file: + write_file.write(html) + + +if __name__ == "__main__": # pragma: no cover + console = Console() + + console.log( + "JSONRPC [i]request[/i]", + 5, + 1.3, + True, + False, + None, + { + "jsonrpc": "2.0", + "method": "subtract", + "params": {"minuend": 42, "subtrahend": 23}, + "id": 3, + }, + ) + + console.log("Hello, World!", "{'a': 1}", repr(console)) + + console.print( + { + "name": None, + "empty": [], + "quiz": { + "sport": { + "answered": True, + "q1": { + "question": "Which one is correct team name in NBA?", + "options": [ + "New York Bulls", + "Los Angeles Kings", + "Golden State Warriors", + "Huston Rocket", + ], + "answer": "Huston Rocket", + }, + }, + "maths": { + "answered": False, + "q1": { + "question": "5 + 7 = ?", + "options": [10, 11, 12, 13], + "answer": 12, + }, + "q2": { + "question": "12 - 8 = ?", + "options": [1, 2, 3, 4], + "answer": 4, + }, + }, + }, + } + ) + console.log("foo") diff --git a/src/pip/_vendor/rich/constrain.py b/src/pip/_vendor/rich/constrain.py new file mode 100644 index 00000000000..65fdf56342e --- /dev/null +++ b/src/pip/_vendor/rich/constrain.py @@ -0,0 +1,37 @@ +from typing import Optional, TYPE_CHECKING + +from .jupyter import JupyterMixin +from .measure import Measurement + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderableType, RenderResult + + +class Constrain(JupyterMixin): + """Constrain the width of a renderable to a given number of characters. + + Args: + renderable (RenderableType): A renderable object. + width (int, optional): The maximum width (in characters) to render. Defaults to 80. + """ + + def __init__(self, renderable: "RenderableType", width: Optional[int] = 80) -> None: + self.renderable = renderable + self.width = width + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + if self.width is None: + yield self.renderable + else: + child_options = options.update_width(min(self.width, options.max_width)) + yield from console.render(self.renderable, child_options) + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + if self.width is not None: + options = options.update_width(self.width) + measurement = Measurement.get(console, options, self.renderable) + return measurement diff --git a/src/pip/_vendor/rich/containers.py b/src/pip/_vendor/rich/containers.py new file mode 100644 index 00000000000..e29cf368991 --- /dev/null +++ b/src/pip/_vendor/rich/containers.py @@ -0,0 +1,167 @@ +from itertools import zip_longest +from typing import ( + Iterator, + Iterable, + List, + Optional, + Union, + overload, + TypeVar, + TYPE_CHECKING, +) + +if TYPE_CHECKING: + from .console import ( + Console, + ConsoleOptions, + JustifyMethod, + OverflowMethod, + RenderResult, + RenderableType, + ) + from .text import Text + +from .cells import cell_len +from .measure import Measurement + +T = TypeVar("T") + + +class Renderables: + """A list subclass which renders its contents to the console.""" + + def __init__( + self, renderables: Optional[Iterable["RenderableType"]] = None + ) -> None: + self._renderables: List["RenderableType"] = ( + list(renderables) if renderables is not None else [] + ) + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + """Console render method to insert line-breaks.""" + yield from self._renderables + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + dimensions = [ + Measurement.get(console, options, renderable) + for renderable in self._renderables + ] + if not dimensions: + return Measurement(1, 1) + _min = max(dimension.minimum for dimension in dimensions) + _max = max(dimension.maximum for dimension in dimensions) + return Measurement(_min, _max) + + def append(self, renderable: "RenderableType") -> None: + self._renderables.append(renderable) + + def __iter__(self) -> Iterable["RenderableType"]: + return iter(self._renderables) + + +class Lines: + """A list subclass which can render to the console.""" + + def __init__(self, lines: Iterable["Text"] = ()) -> None: + self._lines: List["Text"] = list(lines) + + def __repr__(self) -> str: + return f"Lines({self._lines!r})" + + def __iter__(self) -> Iterator["Text"]: + return iter(self._lines) + + @overload + def __getitem__(self, index: int) -> "Text": + ... + + @overload + def __getitem__(self, index: slice) -> List["Text"]: + ... + + def __getitem__(self, index: Union[slice, int]) -> Union["Text", List["Text"]]: + return self._lines[index] + + def __setitem__(self, index: int, value: "Text") -> "Lines": + self._lines[index] = value + return self + + def __len__(self) -> int: + return self._lines.__len__() + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + """Console render method to insert line-breaks.""" + yield from self._lines + + def append(self, line: "Text") -> None: + self._lines.append(line) + + def extend(self, lines: Iterable["Text"]) -> None: + self._lines.extend(lines) + + def pop(self, index: int = -1) -> "Text": + return self._lines.pop(index) + + def justify( + self, + console: "Console", + width: int, + justify: "JustifyMethod" = "left", + overflow: "OverflowMethod" = "fold", + ) -> None: + """Justify and overflow text to a given width. + + Args: + console (Console): Console instance. + width (int): Number of characters per line. + justify (str, optional): Default justify method for text: "left", "center", "full" or "right". Defaults to "left". + overflow (str, optional): Default overflow for text: "crop", "fold", or "ellipsis". Defaults to "fold". + + """ + from .text import Text + + if justify == "left": + for line in self._lines: + line.truncate(width, overflow=overflow, pad=True) + elif justify == "center": + for line in self._lines: + line.rstrip() + line.truncate(width, overflow=overflow) + line.pad_left((width - cell_len(line.plain)) // 2) + line.pad_right(width - cell_len(line.plain)) + elif justify == "right": + for line in self._lines: + line.rstrip() + line.truncate(width, overflow=overflow) + line.pad_left(width - cell_len(line.plain)) + elif justify == "full": + for line_index, line in enumerate(self._lines): + if line_index == len(self._lines) - 1: + break + words = line.split(" ") + words_size = sum(cell_len(word.plain) for word in words) + num_spaces = len(words) - 1 + spaces = [1 for _ in range(num_spaces)] + index = 0 + if spaces: + while words_size + num_spaces < width: + spaces[len(spaces) - index - 1] += 1 + num_spaces += 1 + index = (index + 1) % len(spaces) + tokens: List[Text] = [] + for index, (word, next_word) in enumerate( + zip_longest(words, words[1:]) + ): + tokens.append(word) + if index < len(spaces): + style = word.get_style_at_offset(console, -1) + next_style = next_word.get_style_at_offset(console, 0) + space_style = style if style == next_style else line.style + tokens.append(Text(" " * spaces[index], style=space_style)) + self[line_index] = Text("").join(tokens) diff --git a/src/pip/_vendor/rich/control.py b/src/pip/_vendor/rich/control.py new file mode 100644 index 00000000000..c98d0d7d98b --- /dev/null +++ b/src/pip/_vendor/rich/control.py @@ -0,0 +1,175 @@ +from typing import Any, Callable, Dict, Iterable, List, TYPE_CHECKING, Union + +from .segment import ControlCode, ControlType, Segment + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderResult + +STRIP_CONTROL_CODES = [ + 8, # Backspace + 11, # Vertical tab + 12, # Form feed + 13, # Carriage return +] +_CONTROL_TRANSLATE = {_codepoint: None for _codepoint in STRIP_CONTROL_CODES} + + +CONTROL_CODES_FORMAT: Dict[int, Callable[..., str]] = { + ControlType.BELL: lambda: "\x07", + ControlType.CARRIAGE_RETURN: lambda: "\r", + ControlType.HOME: lambda: "\x1b[H", + ControlType.CLEAR: lambda: "\x1b[2J", + ControlType.ENABLE_ALT_SCREEN: lambda: "\x1b[?1049h", + ControlType.DISABLE_ALT_SCREEN: lambda: "\x1b[?1049l", + ControlType.SHOW_CURSOR: lambda: "\x1b[?25h", + ControlType.HIDE_CURSOR: lambda: "\x1b[?25l", + ControlType.CURSOR_UP: lambda param: f"\x1b[{param}A", + ControlType.CURSOR_DOWN: lambda param: f"\x1b[{param}B", + ControlType.CURSOR_FORWARD: lambda param: f"\x1b[{param}C", + ControlType.CURSOR_BACKWARD: lambda param: f"\x1b[{param}D", + ControlType.CURSOR_MOVE_TO_COLUMN: lambda param: f"\x1b[{param+1}G", + ControlType.ERASE_IN_LINE: lambda param: f"\x1b[{param}K", + ControlType.CURSOR_MOVE_TO: lambda x, y: f"\x1b[{y+1};{x+1}H", +} + + +class Control: + """A renderable that inserts a control code (non printable but may move cursor). + + Args: + *codes (str): Positional arguments are either a :class:`~rich.segment.ControlType` enum or a + tuple of ControlType and an integer parameter + """ + + __slots__ = ["segment"] + + def __init__(self, *codes: Union[ControlType, ControlCode]) -> None: + control_codes: List[ControlCode] = [ + (code,) if isinstance(code, ControlType) else code for code in codes + ] + _format_map = CONTROL_CODES_FORMAT + rendered_codes = "".join( + _format_map[code](*parameters) for code, *parameters in control_codes + ) + self.segment = Segment(rendered_codes, None, control_codes) + + @classmethod + def bell(cls) -> "Control": + """Ring the 'bell'.""" + return cls(ControlType.BELL) + + @classmethod + def home(cls) -> "Control": + """Move cursor to 'home' position.""" + return cls(ControlType.HOME) + + @classmethod + def move(cls, x: int = 0, y: int = 0) -> "Control": + """Move cursor relative to current position. + + Args: + x (int): X offset. + y (int): Y offset. + + Returns: + ~Control: Control object. + + """ + + def get_codes() -> Iterable[ControlCode]: + control = ControlType + if x: + yield ( + control.CURSOR_FORWARD if x > 0 else control.CURSOR_BACKWARD, + abs(x), + ) + if y: + yield ( + control.CURSOR_DOWN if y > 0 else control.CURSOR_UP, + abs(y), + ) + + control = cls(*get_codes()) + return control + + @classmethod + def move_to_column(cls, x: int, y: int = 0) -> "Control": + """Move to the given column, optionally add offset to row. + + Returns: + x (int): absolute x (column) + y (int): optional y offset (row) + + Returns: + ~Control: Control object. + """ + + return ( + cls( + (ControlType.CURSOR_MOVE_TO_COLUMN, x), + ( + ControlType.CURSOR_DOWN if y > 0 else ControlType.CURSOR_UP, + abs(y), + ), + ) + if y + else cls((ControlType.CURSOR_MOVE_TO_COLUMN, x)) + ) + + @classmethod + def move_to(cls, x: int, y: int) -> "Control": + """Move cursor to absolute position. + + Args: + x (int): x offset (column) + y (int): y offset (row) + + Returns: + ~Control: Control object. + """ + return cls((ControlType.CURSOR_MOVE_TO, x, y)) + + @classmethod + def clear(cls) -> "Control": + """Clear the screen.""" + return cls(ControlType.CLEAR) + + @classmethod + def show_cursor(cls, show: bool) -> "Control": + """Show or hide the cursor.""" + return cls(ControlType.SHOW_CURSOR if show else ControlType.HIDE_CURSOR) + + @classmethod + def alt_screen(cls, enable: bool) -> "Control": + """Enable or disable alt screen.""" + if enable: + return cls(ControlType.ENABLE_ALT_SCREEN, ControlType.HOME) + else: + return cls(ControlType.DISABLE_ALT_SCREEN) + + def __str__(self) -> str: + return self.segment.text + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + if self.segment.text: + yield self.segment + + +def strip_control_codes( + text: str, _translate_table: Dict[int, None] = _CONTROL_TRANSLATE +) -> str: + """Remove control codes from text. + + Args: + text (str): A string possibly contain control codes. + + Returns: + str: String with control codes removed. + """ + return text.translate(_translate_table) + + +if __name__ == "__main__": # pragma: no cover + print(strip_control_codes("hello\rWorld")) diff --git a/src/pip/_vendor/rich/default_styles.py b/src/pip/_vendor/rich/default_styles.py new file mode 100644 index 00000000000..91ab232d31d --- /dev/null +++ b/src/pip/_vendor/rich/default_styles.py @@ -0,0 +1,183 @@ +from typing import Dict + +from .style import Style + + +DEFAULT_STYLES: Dict[str, Style] = { + "none": Style.null(), + "reset": Style( + color="default", + bgcolor="default", + dim=False, + bold=False, + italic=False, + underline=False, + blink=False, + blink2=False, + reverse=False, + conceal=False, + strike=False, + ), + "dim": Style(dim=True), + "bright": Style(dim=False), + "bold": Style(bold=True), + "strong": Style(bold=True), + "code": Style(reverse=True, bold=True), + "italic": Style(italic=True), + "emphasize": Style(italic=True), + "underline": Style(underline=True), + "blink": Style(blink=True), + "blink2": Style(blink2=True), + "reverse": Style(reverse=True), + "strike": Style(strike=True), + "black": Style(color="black"), + "red": Style(color="red"), + "green": Style(color="green"), + "yellow": Style(color="yellow"), + "magenta": Style(color="magenta"), + "cyan": Style(color="cyan"), + "white": Style(color="white"), + "inspect.attr": Style(color="yellow", italic=True), + "inspect.attr.dunder": Style(color="yellow", italic=True, dim=True), + "inspect.callable": Style(bold=True, color="red"), + "inspect.def": Style(italic=True, color="bright_cyan"), + "inspect.error": Style(bold=True, color="red"), + "inspect.equals": Style(), + "inspect.help": Style(color="cyan"), + "inspect.doc": Style(dim=True), + "inspect.value.border": Style(color="green"), + "live.ellipsis": Style(bold=True, color="red"), + "layout.tree.row": Style(dim=False, color="red"), + "layout.tree.column": Style(dim=False, color="blue"), + "logging.keyword": Style(bold=True, color="yellow"), + "logging.level.notset": Style(dim=True), + "logging.level.debug": Style(color="green"), + "logging.level.info": Style(color="blue"), + "logging.level.warning": Style(color="red"), + "logging.level.error": Style(color="red", bold=True), + "logging.level.critical": Style(color="red", bold=True, reverse=True), + "log.level": Style.null(), + "log.time": Style(color="cyan", dim=True), + "log.message": Style.null(), + "log.path": Style(dim=True), + "repr.ellipsis": Style(color="yellow"), + "repr.indent": Style(color="green", dim=True), + "repr.error": Style(color="red", bold=True), + "repr.str": Style(color="green", italic=False, bold=False), + "repr.brace": Style(bold=True), + "repr.comma": Style(bold=True), + "repr.ipv4": Style(bold=True, color="bright_green"), + "repr.ipv6": Style(bold=True, color="bright_green"), + "repr.eui48": Style(bold=True, color="bright_green"), + "repr.eui64": Style(bold=True, color="bright_green"), + "repr.tag_start": Style(bold=True), + "repr.tag_name": Style(color="bright_magenta", bold=True), + "repr.tag_contents": Style(color="default"), + "repr.tag_end": Style(bold=True), + "repr.attrib_name": Style(color="yellow", italic=False), + "repr.attrib_equal": Style(bold=True), + "repr.attrib_value": Style(color="magenta", italic=False), + "repr.number": Style(color="cyan", bold=True, italic=False), + "repr.bool_true": Style(color="bright_green", italic=True), + "repr.bool_false": Style(color="bright_red", italic=True), + "repr.none": Style(color="magenta", italic=True), + "repr.url": Style(underline=True, color="bright_blue", italic=False, bold=False), + "repr.uuid": Style(color="bright_yellow", bold=False), + "repr.call": Style(color="magenta", bold=True), + "repr.path": Style(color="magenta"), + "repr.filename": Style(color="bright_magenta"), + "rule.line": Style(color="bright_green"), + "rule.text": Style.null(), + "json.brace": Style(bold=True), + "json.bool_true": Style(color="bright_green", italic=True), + "json.bool_false": Style(color="bright_red", italic=True), + "json.null": Style(color="magenta", italic=True), + "json.number": Style(color="cyan", bold=True, italic=False), + "json.str": Style(color="green", italic=False, bold=False), + "json.key": Style(color="blue", bold=True), + "prompt": Style.null(), + "prompt.choices": Style(color="magenta", bold=True), + "prompt.default": Style(color="cyan", bold=True), + "prompt.invalid": Style(color="red"), + "prompt.invalid.choice": Style(color="red"), + "pretty": Style.null(), + "scope.border": Style(color="blue"), + "scope.key": Style(color="yellow", italic=True), + "scope.key.special": Style(color="yellow", italic=True, dim=True), + "scope.equals": Style(color="red"), + "table.header": Style(bold=True), + "table.footer": Style(bold=True), + "table.cell": Style.null(), + "table.title": Style(italic=True), + "table.caption": Style(italic=True, dim=True), + "traceback.error": Style(color="red", italic=True), + "traceback.border.syntax_error": Style(color="bright_red"), + "traceback.border": Style(color="red"), + "traceback.text": Style.null(), + "traceback.title": Style(color="red", bold=True), + "traceback.exc_type": Style(color="bright_red", bold=True), + "traceback.exc_value": Style.null(), + "traceback.offset": Style(color="bright_red", bold=True), + "bar.back": Style(color="grey23"), + "bar.complete": Style(color="rgb(249,38,114)"), + "bar.finished": Style(color="rgb(114,156,31)"), + "bar.pulse": Style(color="rgb(249,38,114)"), + "progress.description": Style.null(), + "progress.filesize": Style(color="green"), + "progress.filesize.total": Style(color="green"), + "progress.download": Style(color="green"), + "progress.elapsed": Style(color="yellow"), + "progress.percentage": Style(color="magenta"), + "progress.remaining": Style(color="cyan"), + "progress.data.speed": Style(color="red"), + "progress.spinner": Style(color="green"), + "status.spinner": Style(color="green"), + "tree": Style(), + "tree.line": Style(), + "markdown.paragraph": Style(), + "markdown.text": Style(), + "markdown.emph": Style(italic=True), + "markdown.strong": Style(bold=True), + "markdown.code": Style(bgcolor="black", color="bright_white"), + "markdown.code_block": Style(dim=True, color="cyan", bgcolor="black"), + "markdown.block_quote": Style(color="magenta"), + "markdown.list": Style(color="cyan"), + "markdown.item": Style(), + "markdown.item.bullet": Style(color="yellow", bold=True), + "markdown.item.number": Style(color="yellow", bold=True), + "markdown.hr": Style(color="yellow"), + "markdown.h1.border": Style(), + "markdown.h1": Style(bold=True), + "markdown.h2": Style(bold=True, underline=True), + "markdown.h3": Style(bold=True), + "markdown.h4": Style(bold=True, dim=True), + "markdown.h5": Style(underline=True), + "markdown.h6": Style(italic=True), + "markdown.h7": Style(italic=True, dim=True), + "markdown.link": Style(color="bright_blue"), + "markdown.link_url": Style(color="blue"), +} + + +if __name__ == "__main__": # pragma: no cover + import argparse + import io + + from pip._vendor.rich.console import Console + from pip._vendor.rich.table import Table + from pip._vendor.rich.text import Text + + parser = argparse.ArgumentParser() + parser.add_argument("--html", action="store_true", help="Export as HTML table") + args = parser.parse_args() + html: bool = args.html + console = Console(record=True, width=70, file=io.StringIO()) if html else Console() + + table = Table("Name", "Styling") + + for style_name, style in DEFAULT_STYLES.items(): + table.add_row(Text(style_name, style=style), str(style)) + + console.print(table) + if html: + print(console.export_html(inline_styles=True)) diff --git a/src/pip/_vendor/rich/diagnose.py b/src/pip/_vendor/rich/diagnose.py new file mode 100644 index 00000000000..38728da2ae2 --- /dev/null +++ b/src/pip/_vendor/rich/diagnose.py @@ -0,0 +1,6 @@ +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console + from pip._vendor.rich import inspect + + console = Console() + inspect(console) diff --git a/src/pip/_vendor/rich/emoji.py b/src/pip/_vendor/rich/emoji.py new file mode 100644 index 00000000000..791f0465de1 --- /dev/null +++ b/src/pip/_vendor/rich/emoji.py @@ -0,0 +1,96 @@ +import sys +from typing import TYPE_CHECKING, Optional, Union + +from .jupyter import JupyterMixin +from .segment import Segment +from .style import Style +from ._emoji_codes import EMOJI +from ._emoji_replace import _emoji_replace + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from pip._vendor.typing_extensions import Literal # pragma: no cover + + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderResult + + +EmojiVariant = Literal["emoji", "text"] + + +class NoEmoji(Exception): + """No emoji by that name.""" + + +class Emoji(JupyterMixin): + __slots__ = ["name", "style", "_char", "variant"] + + VARIANTS = {"text": "\uFE0E", "emoji": "\uFE0F"} + + def __init__( + self, + name: str, + style: Union[str, Style] = "none", + variant: Optional[EmojiVariant] = None, + ) -> None: + """A single emoji character. + + Args: + name (str): Name of emoji. + style (Union[str, Style], optional): Optional style. Defaults to None. + + Raises: + NoEmoji: If the emoji doesn't exist. + """ + self.name = name + self.style = style + self.variant = variant + try: + self._char = EMOJI[name] + except KeyError: + raise NoEmoji(f"No emoji called {name!r}") + if variant is not None: + self._char += self.VARIANTS.get(variant, "") + + @classmethod + def replace(cls, text: str) -> str: + """Replace emoji markup with corresponding unicode characters. + + Args: + text (str): A string with emojis codes, e.g. "Hello :smiley:!" + + Returns: + str: A string with emoji codes replaces with actual emoji. + """ + return _emoji_replace(text) + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return self._char + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + yield Segment(self._char, console.get_style(self.style)) + + +if __name__ == "__main__": # pragma: no cover + import sys + + from pip._vendor.rich.columns import Columns + from pip._vendor.rich.console import Console + + console = Console(record=True) + + columns = Columns( + (f":{name}: {name}" for name in sorted(EMOJI.keys()) if "\u200D" not in name), + column_first=True, + ) + + console.print(columns) + if len(sys.argv) > 1: + console.save_html(sys.argv[1]) diff --git a/src/pip/_vendor/rich/errors.py b/src/pip/_vendor/rich/errors.py new file mode 100644 index 00000000000..0bcbe53ef59 --- /dev/null +++ b/src/pip/_vendor/rich/errors.py @@ -0,0 +1,34 @@ +class ConsoleError(Exception): + """An error in console operation.""" + + +class StyleError(Exception): + """An error in styles.""" + + +class StyleSyntaxError(ConsoleError): + """Style was badly formatted.""" + + +class MissingStyle(StyleError): + """No such style.""" + + +class StyleStackError(ConsoleError): + """Style stack is invalid.""" + + +class NotRenderableError(ConsoleError): + """Object is not renderable.""" + + +class MarkupError(ConsoleError): + """Markup was badly formatted.""" + + +class LiveError(ConsoleError): + """Error related to Live display.""" + + +class NoAltScreen(ConsoleError): + """Alt screen mode was required.""" diff --git a/src/pip/_vendor/rich/file_proxy.py b/src/pip/_vendor/rich/file_proxy.py new file mode 100644 index 00000000000..3ec593a5a48 --- /dev/null +++ b/src/pip/_vendor/rich/file_proxy.py @@ -0,0 +1,54 @@ +import io +from typing import List, Any, IO, TYPE_CHECKING + +from .ansi import AnsiDecoder +from .text import Text + +if TYPE_CHECKING: + from .console import Console + + +class FileProxy(io.TextIOBase): + """Wraps a file (e.g. sys.stdout) and redirects writes to a console.""" + + def __init__(self, console: "Console", file: IO[str]) -> None: + self.__console = console + self.__file = file + self.__buffer: List[str] = [] + self.__ansi_decoder = AnsiDecoder() + + @property + def rich_proxied_file(self) -> IO[str]: + """Get proxied file.""" + return self.__file + + def __getattr__(self, name: str) -> Any: + return getattr(self.__file, name) + + def write(self, text: str) -> int: + if not isinstance(text, str): + raise TypeError(f"write() argument must be str, not {type(text).__name__}") + buffer = self.__buffer + lines: List[str] = [] + while text: + line, new_line, text = text.partition("\n") + if new_line: + lines.append("".join(buffer) + line) + del buffer[:] + else: + buffer.append(line) + break + if lines: + console = self.__console + with console: + output = Text("\n").join( + self.__ansi_decoder.decode_line(line) for line in lines + ) + console.print(output) + return len(text) + + def flush(self) -> None: + buffer = self.__buffer + if buffer: + self.__console.print("".join(buffer)) + del buffer[:] diff --git a/src/pip/_vendor/rich/filesize.py b/src/pip/_vendor/rich/filesize.py new file mode 100644 index 00000000000..b3a0996b05e --- /dev/null +++ b/src/pip/_vendor/rich/filesize.py @@ -0,0 +1,89 @@ +# coding: utf-8 +"""Functions for reporting filesizes. Borrowed from https://github.com/PyFilesystem/pyfilesystem2 + +The functions declared in this module should cover the different +usecases needed to generate a string representation of a file size +using several different units. Since there are many standards regarding +file size units, three different functions have been implemented. + +See Also: + * `Wikipedia: Binary prefix `_ + +""" + +__all__ = ["decimal"] + +from typing import Iterable, List, Tuple, Optional + + +def _to_str( + size: int, + suffixes: Iterable[str], + base: int, + *, + precision: Optional[int] = 1, + separator: Optional[str] = " ", +) -> str: + if size == 1: + return "1 byte" + elif size < base: + return "{:,} bytes".format(size) + + for i, suffix in enumerate(suffixes, 2): # noqa: B007 + unit = base ** i + if size < unit: + break + return "{:,.{precision}f}{separator}{}".format( + (base * size / unit), + suffix, + precision=precision, + separator=separator, + ) + + +def pick_unit_and_suffix(size: int, suffixes: List[str], base: int) -> Tuple[int, str]: + """Pick a suffix and base for the given size.""" + for i, suffix in enumerate(suffixes): + unit = base ** i + if size < unit * base: + break + return unit, suffix + + +def decimal( + size: int, + *, + precision: Optional[int] = 1, + separator: Optional[str] = " ", +) -> str: + """Convert a filesize in to a string (powers of 1000, SI prefixes). + + In this convention, ``1000 B = 1 kB``. + + This is typically the format used to advertise the storage + capacity of USB flash drives and the like (*256 MB* meaning + actually a storage capacity of more than *256 000 000 B*), + or used by **Mac OS X** since v10.6 to report file sizes. + + Arguments: + int (size): A file size. + int (precision): The number of decimal places to include (default = 1). + str (separator): The string to separate the value from the units (default = " "). + + Returns: + `str`: A string containing a abbreviated file size and units. + + Example: + >>> filesize.decimal(30000) + '30.0 kB' + >>> filesize.decimal(30000, precision=2, separator="") + '30.00kB' + + """ + return _to_str( + size, + ("kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"), + 1000, + precision=precision, + separator=separator, + ) diff --git a/src/pip/_vendor/rich/highlighter.py b/src/pip/_vendor/rich/highlighter.py new file mode 100644 index 00000000000..8afdd017b6e --- /dev/null +++ b/src/pip/_vendor/rich/highlighter.py @@ -0,0 +1,147 @@ +from abc import ABC, abstractmethod +from typing import List, Union + +from .text import Text + + +def _combine_regex(*regexes: str) -> str: + """Combine a number of regexes in to a single regex. + + Returns: + str: New regex with all regexes ORed together. + """ + return "|".join(regexes) + + +class Highlighter(ABC): + """Abstract base class for highlighters.""" + + def __call__(self, text: Union[str, Text]) -> Text: + """Highlight a str or Text instance. + + Args: + text (Union[str, ~Text]): Text to highlight. + + Raises: + TypeError: If not called with text or str. + + Returns: + Text: A test instance with highlighting applied. + """ + if isinstance(text, str): + highlight_text = Text(text) + elif isinstance(text, Text): + highlight_text = text.copy() + else: + raise TypeError(f"str or Text instance required, not {text!r}") + self.highlight(highlight_text) + return highlight_text + + @abstractmethod + def highlight(self, text: Text) -> None: + """Apply highlighting in place to text. + + Args: + text (~Text): A text object highlight. + """ + + +class NullHighlighter(Highlighter): + """A highlighter object that doesn't highlight. + + May be used to disable highlighting entirely. + + """ + + def highlight(self, text: Text) -> None: + """Nothing to do""" + + +class RegexHighlighter(Highlighter): + """Applies highlighting from a list of regular expressions.""" + + highlights: List[str] = [] + base_style: str = "" + + def highlight(self, text: Text) -> None: + """Highlight :class:`rich.text.Text` using regular expressions. + + Args: + text (~Text): Text to highlighted. + + """ + + highlight_regex = text.highlight_regex + for re_highlight in self.highlights: + highlight_regex(re_highlight, style_prefix=self.base_style) + + +class ReprHighlighter(RegexHighlighter): + """Highlights the text typically produced from ``__repr__`` methods.""" + + base_style = "repr." + highlights = [ + r"(?P\<)(?P[\w\-\.\:]*)(?P[\w\W]*?)(?P\>)", + r"(?P[\w_]{1,50})=(?P\"?[\w_]+\"?)?", + r"(?P[\{\[\(\)\]\}])", + _combine_regex( + r"(?P[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})", + r"(?P([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})", + r"(?P(?:[0-9A-Fa-f]{1,2}-){7}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{1,2}:){7}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{4}\.){3}[0-9A-Fa-f]{4})", + r"(?P(?:[0-9A-Fa-f]{1,2}-){5}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{1,2}:){5}[0-9A-Fa-f]{1,2}|(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})", + r"(?P[\w\.]*?)\(", + r"\b(?PTrue)\b|\b(?PFalse)\b|\b(?PNone)\b", + r"(?P\.\.\.)", + r"(?P(?\B(\/[\w\.\-\_\+]+)*\/)(?P[\w\.\-\_\+]*)?", + r"(?b?\'\'\'.*?(?[a-fA-F0-9]{8}\-[a-fA-F0-9]{4}\-[a-fA-F0-9]{4}\-[a-fA-F0-9]{4}\-[a-fA-F0-9]{12})", + r"(?P(file|https|http|ws|wss):\/\/[0-9a-zA-Z\$\-\_\+\!`\(\)\,\.\?\/\;\:\&\=\%\#]*)", + ), + ] + + +class JSONHighlighter(RegexHighlighter): + """Highlights JSON""" + + base_style = "json." + highlights = [ + _combine_regex( + r"(?P[\{\[\(\)\]\}])", + r"\b(?Ptrue)\b|\b(?Pfalse)\b|\b(?Pnull)\b", + r"(?P(?b?\".*?(?b?\".*?(? None: + data = loads(json) + json = dumps( + data, + indent=indent, + skipkeys=skip_keys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + default=default, + sort_keys=sort_keys, + ) + highlighter = JSONHighlighter() if highlight else NullHighlighter() + self.text = highlighter(json) + self.text.no_wrap = True + self.text.overflow = None + + @classmethod + def from_data( + cls, + data: Any, + indent: Union[None, int, str] = 2, + highlight: bool = True, + skip_keys: bool = False, + ensure_ascii: bool = True, + check_circular: bool = True, + allow_nan: bool = True, + default: Optional[Callable[[Any], Any]] = None, + sort_keys: bool = False, + ) -> "JSON": + """Encodes a JSON object from arbitrary data. + + Args: + data (Any): An object that may be encoded in to JSON + indent (Union[None, int, str], optional): Number of characters to indent by. Defaults to 2. + highlight (bool, optional): Enable highlighting. Defaults to True. + default (Callable, optional): Optional callable which will be called for objects that cannot be serialized. Defaults to None. + skip_keys (bool, optional): Skip keys not of a basic type. Defaults to False. + ensure_ascii (bool, optional): Escape all non-ascii characters. Defaults to False. + check_circular (bool, optional): Check for circular references. Defaults to True. + allow_nan (bool, optional): Allow NaN and Infinity values. Defaults to True. + default (Callable, optional): A callable that converts values that can not be encoded + in to something that can be JSON encoded. Defaults to None. + sort_keys (bool, optional): Sort dictionary keys. Defaults to False. + + Returns: + JSON: New JSON object from the given data. + """ + json_instance: "JSON" = cls.__new__(cls) + json = dumps( + data, + indent=indent, + skipkeys=skip_keys, + ensure_ascii=ensure_ascii, + check_circular=check_circular, + allow_nan=allow_nan, + default=default, + sort_keys=sort_keys, + ) + highlighter = JSONHighlighter() if highlight else NullHighlighter() + json_instance.text = highlighter(json) + json_instance.text.no_wrap = True + json_instance.text.overflow = None + return json_instance + + def __rich__(self) -> Text: + return self.text + + +if __name__ == "__main__": + + import argparse + import sys + + parser = argparse.ArgumentParser(description="Pretty print json") + parser.add_argument( + "path", + metavar="PATH", + help="path to file, or - for stdin", + ) + parser.add_argument( + "-i", + "--indent", + metavar="SPACES", + type=int, + help="Number of spaces in an indent", + default=2, + ) + args = parser.parse_args() + + from pip._vendor.rich.console import Console + + console = Console() + error_console = Console(stderr=True) + + try: + if args.path == "-": + json_data = sys.stdin.read() + else: + with open(args.path, "rt") as json_file: + json_data = json_file.read() + except Exception as error: + error_console.print(f"Unable to read {args.path!r}; {error}") + sys.exit(-1) + + console.print(JSON(json_data, indent=args.indent), soft_wrap=True) diff --git a/src/pip/_vendor/rich/jupyter.py b/src/pip/_vendor/rich/jupyter.py new file mode 100644 index 00000000000..bedf5cb19a3 --- /dev/null +++ b/src/pip/_vendor/rich/jupyter.py @@ -0,0 +1,92 @@ +from typing import Any, Dict, Iterable, List + +from . import get_console +from .segment import Segment +from .terminal_theme import DEFAULT_TERMINAL_THEME + +JUPYTER_HTML_FORMAT = """\ +
{code}
+""" + + +class JupyterRenderable: + """A shim to write html to Jupyter notebook.""" + + def __init__(self, html: str, text: str) -> None: + self.html = html + self.text = text + + def _repr_mimebundle_( + self, include: Iterable[str], exclude: Iterable[str], **kwargs: Any + ) -> Dict[str, str]: + data = {"text/plain": self.text, "text/html": self.html} + if include: + data = {k: v for (k, v) in data.items() if k in include} + if exclude: + data = {k: v for (k, v) in data.items() if k not in exclude} + return data + + +class JupyterMixin: + """Add to an Rich renderable to make it render in Jupyter notebook.""" + + __slots__ = () + + def _repr_mimebundle_( + self, include: Iterable[str], exclude: Iterable[str], **kwargs: Any + ) -> Dict[str, str]: + console = get_console() + segments = list(console.render(self, console.options)) # type: ignore + html = _render_segments(segments) + text = console._render_buffer(segments) + data = {"text/plain": text, "text/html": html} + if include: + data = {k: v for (k, v) in data.items() if k in include} + if exclude: + data = {k: v for (k, v) in data.items() if k not in exclude} + return data + + +def _render_segments(segments: Iterable[Segment]) -> str: + def escape(text: str) -> str: + """Escape html.""" + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + fragments: List[str] = [] + append_fragment = fragments.append + theme = DEFAULT_TERMINAL_THEME + for text, style, control in Segment.simplify(segments): + if control: + continue + text = escape(text) + if style: + rule = style.get_html_style(theme) + text = f'{text}' if rule else text + if style.link: + text = f'{text}' + append_fragment(text) + + code = "".join(fragments) + html = JUPYTER_HTML_FORMAT.format(code=code) + + return html + + +def display(segments: Iterable[Segment], text: str) -> None: + """Render segments to Jupyter.""" + html = _render_segments(segments) + jupyter_renderable = JupyterRenderable(html, text) + try: + from IPython.display import display as ipython_display + + ipython_display(jupyter_renderable) + except ModuleNotFoundError: + # Handle the case where the Console has force_jupyter=True, + # but IPython is not installed. + pass + + +def print(*args: Any, **kwargs: Any) -> None: + """Proxy for Console print.""" + console = get_console() + return console.print(*args, **kwargs) diff --git a/src/pip/_vendor/rich/layout.py b/src/pip/_vendor/rich/layout.py new file mode 100644 index 00000000000..22a4c54786d --- /dev/null +++ b/src/pip/_vendor/rich/layout.py @@ -0,0 +1,444 @@ +from abc import ABC, abstractmethod +from itertools import islice +from operator import itemgetter +from threading import RLock +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Sequence, + Tuple, + Union, +) + +from ._ratio import ratio_resolve +from .align import Align +from .console import Console, ConsoleOptions, RenderableType, RenderResult +from .highlighter import ReprHighlighter +from .panel import Panel +from .pretty import Pretty +from .repr import rich_repr, Result +from .region import Region +from .segment import Segment +from .style import StyleType + +if TYPE_CHECKING: + from pip._vendor.rich.tree import Tree + + +class LayoutRender(NamedTuple): + """An individual layout render.""" + + region: Region + render: List[List[Segment]] + + +RegionMap = Dict["Layout", Region] +RenderMap = Dict["Layout", LayoutRender] + + +class LayoutError(Exception): + """Layout related error.""" + + +class NoSplitter(LayoutError): + """Requested splitter does not exist.""" + + +class _Placeholder: + """An internal renderable used as a Layout placeholder.""" + + highlighter = ReprHighlighter() + + def __init__(self, layout: "Layout", style: StyleType = "") -> None: + self.layout = layout + self.style = style + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + width = options.max_width + height = options.height or options.size.height + layout = self.layout + title = ( + f"{layout.name!r} ({width} x {height})" + if layout.name + else f"({width} x {height})" + ) + yield Panel( + Align.center(Pretty(layout), vertical="middle"), + style=self.style, + title=self.highlighter(title), + border_style="blue", + ) + + +class Splitter(ABC): + """Base class for a splitter.""" + + name: str = "" + + @abstractmethod + def get_tree_icon(self) -> str: + """Get the icon (emoji) used in layout.tree""" + + @abstractmethod + def divide( + self, children: Sequence["Layout"], region: Region + ) -> Iterable[Tuple["Layout", Region]]: + """Divide a region amongst several child layouts. + + Args: + children (Sequence(Layout)): A number of child layouts. + region (Region): A rectangular region to divide. + """ + + +class RowSplitter(Splitter): + """Split a layout region in to rows.""" + + name = "row" + + def get_tree_icon(self) -> str: + return "[layout.tree.row]⬌" + + def divide( + self, children: Sequence["Layout"], region: Region + ) -> Iterable[Tuple["Layout", Region]]: + x, y, width, height = region + render_widths = ratio_resolve(width, children) + offset = 0 + _Region = Region + for child, child_width in zip(children, render_widths): + yield child, _Region(x + offset, y, child_width, height) + offset += child_width + + +class ColumnSplitter(Splitter): + """Split a layout region in to columns.""" + + name = "column" + + def get_tree_icon(self) -> str: + return "[layout.tree.column]⬍" + + def divide( + self, children: Sequence["Layout"], region: Region + ) -> Iterable[Tuple["Layout", Region]]: + x, y, width, height = region + render_heights = ratio_resolve(height, children) + offset = 0 + _Region = Region + for child, child_height in zip(children, render_heights): + yield child, _Region(x, y + offset, width, child_height) + offset += child_height + + +@rich_repr +class Layout: + """A renderable to divide a fixed height in to rows or columns. + + Args: + renderable (RenderableType, optional): Renderable content, or None for placeholder. Defaults to None. + name (str, optional): Optional identifier for Layout. Defaults to None. + size (int, optional): Optional fixed size of layout. Defaults to None. + minimum_size (int, optional): Minimum size of layout. Defaults to 1. + ratio (int, optional): Optional ratio for flexible layout. Defaults to 1. + visible (bool, optional): Visibility of layout. Defaults to True. + """ + + splitters = {"row": RowSplitter, "column": ColumnSplitter} + + def __init__( + self, + renderable: Optional[RenderableType] = None, + *, + name: Optional[str] = None, + size: Optional[int] = None, + minimum_size: int = 1, + ratio: int = 1, + visible: bool = True, + height: Optional[int] = None, + ) -> None: + self._renderable = renderable or _Placeholder(self) + self.size = size + self.minimum_size = minimum_size + self.ratio = ratio + self.name = name + self.visible = visible + self.height = height + self.splitter: Splitter = self.splitters["column"]() + self._children: List[Layout] = [] + self._render_map: RenderMap = {} + self._lock = RLock() + + def __rich_repr__(self) -> Result: + yield "name", self.name, None + yield "size", self.size, None + yield "minimum_size", self.minimum_size, 1 + yield "ratio", self.ratio, 1 + + @property + def renderable(self) -> RenderableType: + """Layout renderable.""" + return self if self._children else self._renderable + + @property + def children(self) -> List["Layout"]: + """Gets (visible) layout children.""" + return [child for child in self._children if child.visible] + + @property + def map(self) -> RenderMap: + """Get a map of the last render.""" + return self._render_map + + def get(self, name: str) -> Optional["Layout"]: + """Get a named layout, or None if it doesn't exist. + + Args: + name (str): Name of layout. + + Returns: + Optional[Layout]: Layout instance or None if no layout was found. + """ + if self.name == name: + return self + else: + for child in self._children: + named_layout = child.get(name) + if named_layout is not None: + return named_layout + return None + + def __getitem__(self, name: str) -> "Layout": + layout = self.get(name) + if layout is None: + raise KeyError(f"No layout with name {name!r}") + return layout + + @property + def tree(self) -> "Tree": + """Get a tree renderable to show layout structure.""" + from pip._vendor.rich.styled import Styled + from pip._vendor.rich.table import Table + from pip._vendor.rich.tree import Tree + + def summary(layout: "Layout") -> Table: + + icon = layout.splitter.get_tree_icon() + + table = Table.grid(padding=(0, 1, 0, 0)) + + text: RenderableType = ( + Pretty(layout) if layout.visible else Styled(Pretty(layout), "dim") + ) + table.add_row(icon, text) + _summary = table + return _summary + + layout = self + tree = Tree( + summary(layout), + guide_style=f"layout.tree.{layout.splitter.name}", + highlight=True, + ) + + def recurse(tree: "Tree", layout: "Layout") -> None: + for child in layout._children: + recurse( + tree.add( + summary(child), + guide_style=f"layout.tree.{child.splitter.name}", + ), + child, + ) + + recurse(tree, self) + return tree + + def split( + self, + *layouts: Union["Layout", RenderableType], + splitter: Union[Splitter, str] = "column", + ) -> None: + """Split the layout in to multiple sub-layouts. + + Args: + *layouts (Layout): Positional arguments should be (sub) Layout instances. + splitter (Union[Splitter, str]): Splitter instance or name of splitter. + """ + _layouts = [ + layout if isinstance(layout, Layout) else Layout(layout) + for layout in layouts + ] + try: + self.splitter = ( + splitter + if isinstance(splitter, Splitter) + else self.splitters[splitter]() + ) + except KeyError: + raise NoSplitter(f"No splitter called {splitter!r}") + self._children[:] = _layouts + + def add_split(self, *layouts: Union["Layout", RenderableType]) -> None: + """Add a new layout(s) to existing split. + + Args: + *layouts (Union[Layout, RenderableType]): Positional arguments should be renderables or (sub) Layout instances. + + """ + _layouts = ( + layout if isinstance(layout, Layout) else Layout(layout) + for layout in layouts + ) + self._children.extend(_layouts) + + def split_row(self, *layouts: Union["Layout", RenderableType]) -> None: + """Split the layout in tow a row (Layouts side by side). + + Args: + *layouts (Layout): Positional arguments should be (sub) Layout instances. + """ + self.split(*layouts, splitter="row") + + def split_column(self, *layouts: Union["Layout", RenderableType]) -> None: + """Split the layout in to a column (layouts stacked on top of each other). + + Args: + *layouts (Layout): Positional arguments should be (sub) Layout instances. + """ + self.split(*layouts, splitter="column") + + def unsplit(self) -> None: + """Reset splits to initial state.""" + del self._children[:] + + def update(self, renderable: RenderableType) -> None: + """Update renderable. + + Args: + renderable (RenderableType): New renderable object. + """ + with self._lock: + self._renderable = renderable + + def refresh_screen(self, console: "Console", layout_name: str) -> None: + """Refresh a sub-layout. + + Args: + console (Console): Console instance where Layout is to be rendered. + layout_name (str): Name of layout. + """ + with self._lock: + layout = self[layout_name] + region, _lines = self._render_map[layout] + (x, y, width, height) = region + lines = console.render_lines( + layout, console.options.update_dimensions(width, height) + ) + self._render_map[layout] = LayoutRender(region, lines) + console.update_screen_lines(lines, x, y) + + def _make_region_map(self, width: int, height: int) -> RegionMap: + """Create a dict that maps layout on to Region.""" + stack: List[Tuple[Layout, Region]] = [(self, Region(0, 0, width, height))] + push = stack.append + pop = stack.pop + layout_regions: List[Tuple[Layout, Region]] = [] + append_layout_region = layout_regions.append + while stack: + append_layout_region(pop()) + layout, region = layout_regions[-1] + children = layout.children + if children: + for child_and_region in layout.splitter.divide(children, region): + push(child_and_region) + + region_map = { + layout: region + for layout, region in sorted(layout_regions, key=itemgetter(1)) + } + return region_map + + def render(self, console: Console, options: ConsoleOptions) -> RenderMap: + """Render the sub_layouts. + + Args: + console (Console): Console instance. + options (ConsoleOptions): Console options. + + Returns: + RenderMap: A dict that maps Layout on to a tuple of Region, lines + """ + render_width = options.max_width + render_height = options.height or console.height + region_map = self._make_region_map(render_width, render_height) + layout_regions = [ + (layout, region) + for layout, region in region_map.items() + if not layout.children + ] + render_map: Dict["Layout", "LayoutRender"] = {} + render_lines = console.render_lines + update_dimensions = options.update_dimensions + + for layout, region in layout_regions: + lines = render_lines( + layout.renderable, update_dimensions(region.width, region.height) + ) + render_map[layout] = LayoutRender(region, lines) + return render_map + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + with self._lock: + width = options.max_width or console.width + height = options.height or console.height + render_map = self.render(console, options.update_dimensions(width, height)) + self._render_map = render_map + layout_lines: List[List[Segment]] = [[] for _ in range(height)] + _islice = islice + for (region, lines) in render_map.values(): + _x, y, _layout_width, layout_height = region + for row, line in zip( + _islice(layout_lines, y, y + layout_height), lines + ): + row.extend(line) + + new_line = Segment.line() + for layout_row in layout_lines: + yield from layout_row + yield new_line + + +if __name__ == "__main__": + from pip._vendor.rich.console import Console + + console = Console() + layout = Layout() + + layout.split_column( + Layout(name="header", size=3), + Layout(ratio=1, name="main"), + Layout(size=10, name="footer"), + ) + + layout["main"].split_row(Layout(name="side"), Layout(name="body", ratio=2)) + + layout["body"].split_row(Layout(name="content", ratio=2), Layout(name="s2")) + + layout["s2"].split_column( + Layout(name="top"), Layout(name="middle"), Layout(name="bottom") + ) + + layout["side"].split_column(Layout(layout.tree, name="left1"), Layout(name="left2")) + + layout["content"].update("foo") + + console.print(layout) diff --git a/src/pip/_vendor/rich/live.py b/src/pip/_vendor/rich/live.py new file mode 100644 index 00000000000..6db5b605f90 --- /dev/null +++ b/src/pip/_vendor/rich/live.py @@ -0,0 +1,365 @@ +import sys +from threading import Event, RLock, Thread +from types import TracebackType +from typing import IO, Any, Callable, List, Optional, TextIO, Type, cast + +from . import get_console +from .console import Console, ConsoleRenderable, RenderableType, RenderHook +from .control import Control +from .file_proxy import FileProxy +from .jupyter import JupyterMixin +from .live_render import LiveRender, VerticalOverflowMethod +from .screen import Screen +from .text import Text + + +class _RefreshThread(Thread): + """A thread that calls refresh() at regular intervals.""" + + def __init__(self, live: "Live", refresh_per_second: float) -> None: + self.live = live + self.refresh_per_second = refresh_per_second + self.done = Event() + super().__init__(daemon=True) + + def stop(self) -> None: + self.done.set() + + def run(self) -> None: + while not self.done.wait(1 / self.refresh_per_second): + with self.live._lock: + if not self.done.is_set(): + self.live.refresh() + + +class Live(JupyterMixin, RenderHook): + """Renders an auto-updating live display of any given renderable. + + Args: + renderable (RenderableType, optional): The renderable to live display. Defaults to displaying nothing. + console (Console, optional): Optional Console instance. Default will an internal Console instance writing to stdout. + screen (bool, optional): Enable alternate screen mode. Defaults to False. + auto_refresh (bool, optional): Enable auto refresh. If disabled, you will need to call `refresh()` or `update()` with refresh flag. Defaults to True + refresh_per_second (float, optional): Number of times per second to refresh the live display. Defaults to 4. + transient (bool, optional): Clear the renderable on exit (has no effect when screen=True). Defaults to False. + redirect_stdout (bool, optional): Enable redirection of stdout, so ``print`` may be used. Defaults to True. + redirect_stderr (bool, optional): Enable redirection of stderr. Defaults to True. + vertical_overflow (VerticalOverflowMethod, optional): How to handle renderable when it is too tall for the console. Defaults to "ellipsis". + get_renderable (Callable[[], RenderableType], optional): Optional callable to get renderable. Defaults to None. + """ + + def __init__( + self, + renderable: Optional[RenderableType] = None, + *, + console: Optional[Console] = None, + screen: bool = False, + auto_refresh: bool = True, + refresh_per_second: float = 4, + transient: bool = False, + redirect_stdout: bool = True, + redirect_stderr: bool = True, + vertical_overflow: VerticalOverflowMethod = "ellipsis", + get_renderable: Optional[Callable[[], RenderableType]] = None, + ) -> None: + assert refresh_per_second > 0, "refresh_per_second must be > 0" + self._renderable = renderable + self.console = console if console is not None else get_console() + self._screen = screen + self._alt_screen = False + + self._redirect_stdout = redirect_stdout + self._redirect_stderr = redirect_stderr + self._restore_stdout: Optional[IO[str]] = None + self._restore_stderr: Optional[IO[str]] = None + + self._lock = RLock() + self.ipy_widget: Optional[Any] = None + self.auto_refresh = auto_refresh + self._started: bool = False + self.transient = True if screen else transient + + self._refresh_thread: Optional[_RefreshThread] = None + self.refresh_per_second = refresh_per_second + + self.vertical_overflow = vertical_overflow + self._get_renderable = get_renderable + self._live_render = LiveRender( + self.get_renderable(), vertical_overflow=vertical_overflow + ) + + @property + def is_started(self) -> bool: + """Check if live display has been started.""" + return self._started + + def get_renderable(self) -> RenderableType: + renderable = ( + self._get_renderable() + if self._get_renderable is not None + else self._renderable + ) + return renderable or "" + + def start(self, refresh: bool = False) -> None: + """Start live rendering display. + + Args: + refresh (bool, optional): Also refresh. Defaults to False. + """ + with self._lock: + if self._started: + return + self.console.set_live(self) + self._started = True + if self._screen: + self._alt_screen = self.console.set_alt_screen(True) + self.console.show_cursor(False) + self._enable_redirect_io() + self.console.push_render_hook(self) + if refresh: + self.refresh() + if self.auto_refresh: + self._refresh_thread = _RefreshThread(self, self.refresh_per_second) + self._refresh_thread.start() + + def stop(self) -> None: + """Stop live rendering display.""" + with self._lock: + if not self._started: + return + self.console.clear_live() + self._started = False + + if self.auto_refresh and self._refresh_thread is not None: + self._refresh_thread.stop() + self._refresh_thread = None + # allow it to fully render on the last even if overflow + self.vertical_overflow = "visible" + with self.console: + try: + if not self._alt_screen and not self.console.is_jupyter: + self.refresh() + finally: + self._disable_redirect_io() + self.console.pop_render_hook() + if not self._alt_screen and self.console.is_terminal: + self.console.line() + self.console.show_cursor(True) + if self._alt_screen: + self.console.set_alt_screen(False) + + if self.transient and not self._alt_screen: + self.console.control(self._live_render.restore_cursor()) + if self.ipy_widget is not None and self.transient: + self.ipy_widget.close() # pragma: no cover + + def __enter__(self) -> "Live": + self.start(refresh=self._renderable is not None) + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.stop() + + def _enable_redirect_io(self) -> None: + """Enable redirecting of stdout / stderr.""" + if self.console.is_terminal or self.console.is_jupyter: + if self._redirect_stdout and not isinstance(sys.stdout, FileProxy): + self._restore_stdout = sys.stdout + sys.stdout = cast("TextIO", FileProxy(self.console, sys.stdout)) + if self._redirect_stderr and not isinstance(sys.stderr, FileProxy): + self._restore_stderr = sys.stderr + sys.stderr = cast("TextIO", FileProxy(self.console, sys.stderr)) + + def _disable_redirect_io(self) -> None: + """Disable redirecting of stdout / stderr.""" + if self._restore_stdout: + sys.stdout = cast("TextIO", self._restore_stdout) + self._restore_stdout = None + if self._restore_stderr: + sys.stderr = cast("TextIO", self._restore_stderr) + self._restore_stderr = None + + @property + def renderable(self) -> RenderableType: + """Get the renderable that is being displayed + + Returns: + RenderableType: Displayed renderable. + """ + renderable = self.get_renderable() + return Screen(renderable) if self._alt_screen else renderable + + def update(self, renderable: RenderableType, *, refresh: bool = False) -> None: + """Update the renderable that is being displayed + + Args: + renderable (RenderableType): New renderable to use. + refresh (bool, optional): Refresh the display. Defaults to False. + """ + with self._lock: + self._renderable = renderable + if refresh: + self.refresh() + + def refresh(self) -> None: + """Update the display of the Live Render.""" + with self._lock: + self._live_render.set_renderable(self.renderable) + if self.console.is_jupyter: # pragma: no cover + try: + from IPython.display import display + from ipywidgets import Output + except ImportError: + import warnings + + warnings.warn('install "ipywidgets" for Jupyter support') + else: + if self.ipy_widget is None: + self.ipy_widget = Output() + display(self.ipy_widget) + + with self.ipy_widget: + self.ipy_widget.clear_output(wait=True) + self.console.print(self._live_render.renderable) + elif self.console.is_terminal and not self.console.is_dumb_terminal: + with self.console: + self.console.print(Control()) + elif ( + not self._started and not self.transient + ): # if it is finished allow files or dumb-terminals to see final result + with self.console: + self.console.print(Control()) + + def process_renderables( + self, renderables: List[ConsoleRenderable] + ) -> List[ConsoleRenderable]: + """Process renderables to restore cursor and display progress.""" + self._live_render.vertical_overflow = self.vertical_overflow + if self.console.is_interactive: + # lock needs acquiring as user can modify live_render renderable at any time unlike in Progress. + with self._lock: + reset = ( + Control.home() + if self._alt_screen + else self._live_render.position_cursor() + ) + renderables = [reset, *renderables, self._live_render] + elif ( + not self._started and not self.transient + ): # if it is finished render the final output for files or dumb_terminals + renderables = [*renderables, self._live_render] + + return renderables + + +if __name__ == "__main__": # pragma: no cover + import random + import time + from itertools import cycle + from typing import Dict, List, Tuple + + from .align import Align + from .console import Console + from .live import Live as Live + from .panel import Panel + from .rule import Rule + from .syntax import Syntax + from .table import Table + + console = Console() + + syntax = Syntax( + '''def loop_last(values: Iterable[T]) -> Iterable[Tuple[bool, T]]: + """Iterate and generate a tuple with a flag for last value.""" + iter_values = iter(values) + try: + previous_value = next(iter_values) + except StopIteration: + return + for value in iter_values: + yield False, previous_value + previous_value = value + yield True, previous_value''', + "python", + line_numbers=True, + ) + + table = Table("foo", "bar", "baz") + table.add_row("1", "2", "3") + + progress_renderables = [ + "You can make the terminal shorter and taller to see the live table hide" + "Text may be printed while the progress bars are rendering.", + Panel("In fact, [i]any[/i] renderable will work"), + "Such as [magenta]tables[/]...", + table, + "Pretty printed structures...", + {"type": "example", "text": "Pretty printed"}, + "Syntax...", + syntax, + Rule("Give it a try!"), + ] + + examples = cycle(progress_renderables) + + exchanges = [ + "SGD", + "MYR", + "EUR", + "USD", + "AUD", + "JPY", + "CNH", + "HKD", + "CAD", + "INR", + "DKK", + "GBP", + "RUB", + "NZD", + "MXN", + "IDR", + "TWD", + "THB", + "VND", + ] + with Live(console=console) as live_table: + exchange_rate_dict: Dict[Tuple[str, str], float] = {} + + for index in range(100): + select_exchange = exchanges[index % len(exchanges)] + + for exchange in exchanges: + if exchange == select_exchange: + continue + time.sleep(0.4) + if random.randint(0, 10) < 1: + console.log(next(examples)) + exchange_rate_dict[(select_exchange, exchange)] = 200 / ( + (random.random() * 320) + 1 + ) + if len(exchange_rate_dict) > len(exchanges) - 1: + exchange_rate_dict.pop(list(exchange_rate_dict.keys())[0]) + table = Table(title="Exchange Rates") + + table.add_column("Source Currency") + table.add_column("Destination Currency") + table.add_column("Exchange Rate") + + for ((source, dest), exchange_rate) in exchange_rate_dict.items(): + table.add_row( + source, + dest, + Text( + f"{exchange_rate:.4f}", + style="red" if exchange_rate < 1.0 else "green", + ), + ) + + live_table.update(Align.center(table)) diff --git a/src/pip/_vendor/rich/live_render.py b/src/pip/_vendor/rich/live_render.py new file mode 100644 index 00000000000..b90fbf7f350 --- /dev/null +++ b/src/pip/_vendor/rich/live_render.py @@ -0,0 +1,113 @@ +import sys +from typing import Optional, Tuple + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from pip._vendor.typing_extensions import Literal # pragma: no cover + + +from ._loop import loop_last +from .console import Console, ConsoleOptions, RenderableType, RenderResult +from .control import Control +from .segment import ControlType, Segment +from .style import StyleType +from .text import Text + +VerticalOverflowMethod = Literal["crop", "ellipsis", "visible"] + + +class LiveRender: + """Creates a renderable that may be updated. + + Args: + renderable (RenderableType): Any renderable object. + style (StyleType, optional): An optional style to apply to the renderable. Defaults to "". + """ + + def __init__( + self, + renderable: RenderableType, + style: StyleType = "", + vertical_overflow: VerticalOverflowMethod = "ellipsis", + ) -> None: + self.renderable = renderable + self.style = style + self.vertical_overflow = vertical_overflow + self._shape: Optional[Tuple[int, int]] = None + + def set_renderable(self, renderable: RenderableType) -> None: + """Set a new renderable. + + Args: + renderable (RenderableType): Any renderable object, including str. + """ + self.renderable = renderable + + def position_cursor(self) -> Control: + """Get control codes to move cursor to beginning of live render. + + Returns: + Control: A control instance that may be printed. + """ + if self._shape is not None: + _, height = self._shape + return Control( + ControlType.CARRIAGE_RETURN, + (ControlType.ERASE_IN_LINE, 2), + *( + ( + (ControlType.CURSOR_UP, 1), + (ControlType.ERASE_IN_LINE, 2), + ) + * (height - 1) + ) + ) + return Control() + + def restore_cursor(self) -> Control: + """Get control codes to clear the render and restore the cursor to its previous position. + + Returns: + Control: A Control instance that may be printed. + """ + if self._shape is not None: + _, height = self._shape + return Control( + ControlType.CARRIAGE_RETURN, + *((ControlType.CURSOR_UP, 1), (ControlType.ERASE_IN_LINE, 2)) * height + ) + return Control() + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + + renderable = self.renderable + style = console.get_style(self.style) + lines = console.render_lines(renderable, options, style=style, pad=False) + shape = Segment.get_shape(lines) + + _, height = shape + if height > options.size.height: + if self.vertical_overflow == "crop": + lines = lines[: options.size.height] + shape = Segment.get_shape(lines) + elif self.vertical_overflow == "ellipsis": + lines = lines[: (options.size.height - 1)] + overflow_text = Text( + "...", + overflow="crop", + justify="center", + end="", + style="live.ellipsis", + ) + lines.append(list(console.render(overflow_text))) + shape = Segment.get_shape(lines) + self._shape = shape + + new_line = Segment.line() + for last, line in loop_last(lines): + yield from line + if not last: + yield new_line diff --git a/src/pip/_vendor/rich/logging.py b/src/pip/_vendor/rich/logging.py new file mode 100644 index 00000000000..002f1f7bf1c --- /dev/null +++ b/src/pip/_vendor/rich/logging.py @@ -0,0 +1,268 @@ +import logging +from datetime import datetime +from logging import Handler, LogRecord +from pathlib import Path +from typing import ClassVar, List, Optional, Type, Union + +from . import get_console +from ._log_render import LogRender, FormatTimeCallable +from .console import Console, ConsoleRenderable +from .highlighter import Highlighter, ReprHighlighter +from .text import Text +from .traceback import Traceback + + +class RichHandler(Handler): + """A logging handler that renders output with Rich. The time / level / message and file are displayed in columns. + The level is color coded, and the message is syntax highlighted. + + Note: + Be careful when enabling console markup in log messages if you have configured logging for libraries not + under your control. If a dependency writes messages containing square brackets, it may not produce the intended output. + + Args: + level (Union[int, str], optional): Log level. Defaults to logging.NOTSET. + console (:class:`~rich.console.Console`, optional): Optional console instance to write logs. + Default will use a global console instance writing to stdout. + show_time (bool, optional): Show a column for the time. Defaults to True. + omit_repeated_times (bool, optional): Omit repetition of the same time. Defaults to True. + show_level (bool, optional): Show a column for the level. Defaults to True. + show_path (bool, optional): Show the path to the original log call. Defaults to True. + enable_link_path (bool, optional): Enable terminal link of path column to file. Defaults to True. + highlighter (Highlighter, optional): Highlighter to style log messages, or None to use ReprHighlighter. Defaults to None. + markup (bool, optional): Enable console markup in log messages. Defaults to False. + rich_tracebacks (bool, optional): Enable rich tracebacks with syntax highlighting and formatting. Defaults to False. + tracebacks_width (Optional[int], optional): Number of characters used to render tracebacks, or None for full width. Defaults to None. + tracebacks_extra_lines (int, optional): Additional lines of code to render tracebacks, or None for full width. Defaults to None. + tracebacks_theme (str, optional): Override pygments theme used in traceback. + tracebacks_word_wrap (bool, optional): Enable word wrapping of long tracebacks lines. Defaults to True. + tracebacks_show_locals (bool, optional): Enable display of locals in tracebacks. Defaults to False. + locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to 10. + locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80. + log_time_format (Union[str, TimeFormatterCallable], optional): If ``log_time`` is enabled, either string for strftime or callable that formats the time. Defaults to "[%x %X] ". + """ + + KEYWORDS: ClassVar[Optional[List[str]]] = [ + "GET", + "POST", + "HEAD", + "PUT", + "DELETE", + "OPTIONS", + "TRACE", + "PATCH", + ] + HIGHLIGHTER_CLASS: ClassVar[Type[Highlighter]] = ReprHighlighter + + def __init__( + self, + level: Union[int, str] = logging.NOTSET, + console: Optional[Console] = None, + *, + show_time: bool = True, + omit_repeated_times: bool = True, + show_level: bool = True, + show_path: bool = True, + enable_link_path: bool = True, + highlighter: Optional[Highlighter] = None, + markup: bool = False, + rich_tracebacks: bool = False, + tracebacks_width: Optional[int] = None, + tracebacks_extra_lines: int = 3, + tracebacks_theme: Optional[str] = None, + tracebacks_word_wrap: bool = True, + tracebacks_show_locals: bool = False, + locals_max_length: int = 10, + locals_max_string: int = 80, + log_time_format: Union[str, FormatTimeCallable] = "[%x %X]", + ) -> None: + super().__init__(level=level) + self.console = console or get_console() + self.highlighter = highlighter or self.HIGHLIGHTER_CLASS() + self._log_render = LogRender( + show_time=show_time, + show_level=show_level, + show_path=show_path, + time_format=log_time_format, + omit_repeated_times=omit_repeated_times, + level_width=None, + ) + self.enable_link_path = enable_link_path + self.markup = markup + self.rich_tracebacks = rich_tracebacks + self.tracebacks_width = tracebacks_width + self.tracebacks_extra_lines = tracebacks_extra_lines + self.tracebacks_theme = tracebacks_theme + self.tracebacks_word_wrap = tracebacks_word_wrap + self.tracebacks_show_locals = tracebacks_show_locals + self.locals_max_length = locals_max_length + self.locals_max_string = locals_max_string + + def get_level_text(self, record: LogRecord) -> Text: + """Get the level name from the record. + + Args: + record (LogRecord): LogRecord instance. + + Returns: + Text: A tuple of the style and level name. + """ + level_name = record.levelname + level_text = Text.styled( + level_name.ljust(8), f"logging.level.{level_name.lower()}" + ) + return level_text + + def emit(self, record: LogRecord) -> None: + """Invoked by logging.""" + message = self.format(record) + traceback = None + if ( + self.rich_tracebacks + and record.exc_info + and record.exc_info != (None, None, None) + ): + exc_type, exc_value, exc_traceback = record.exc_info + assert exc_type is not None + assert exc_value is not None + traceback = Traceback.from_exception( + exc_type, + exc_value, + exc_traceback, + width=self.tracebacks_width, + extra_lines=self.tracebacks_extra_lines, + theme=self.tracebacks_theme, + word_wrap=self.tracebacks_word_wrap, + show_locals=self.tracebacks_show_locals, + locals_max_length=self.locals_max_length, + locals_max_string=self.locals_max_string, + ) + message = record.getMessage() + if self.formatter: + record.message = record.getMessage() + formatter = self.formatter + if hasattr(formatter, "usesTime") and formatter.usesTime(): + record.asctime = formatter.formatTime(record, formatter.datefmt) + message = formatter.formatMessage(record) + + message_renderable = self.render_message(record, message) + log_renderable = self.render( + record=record, traceback=traceback, message_renderable=message_renderable + ) + try: + self.console.print(log_renderable) + except Exception: + self.handleError(record) + + def render_message(self, record: LogRecord, message: str) -> "ConsoleRenderable": + """Render message text in to Text. + + record (LogRecord): logging Record. + message (str): String containing log message. + + Returns: + ConsoleRenderable: Renderable to display log message. + """ + use_markup = getattr(record, "markup", self.markup) + message_text = Text.from_markup(message) if use_markup else Text(message) + + highlighter = getattr(record, "highlighter", self.highlighter) + if highlighter: + message_text = highlighter(message_text) + + if self.KEYWORDS: + message_text.highlight_words(self.KEYWORDS, "logging.keyword") + return message_text + + def render( + self, + *, + record: LogRecord, + traceback: Optional[Traceback], + message_renderable: "ConsoleRenderable", + ) -> "ConsoleRenderable": + """Render log for display. + + Args: + record (LogRecord): logging Record. + traceback (Optional[Traceback]): Traceback instance or None for no Traceback. + message_renderable (ConsoleRenderable): Renderable (typically Text) containing log message contents. + + Returns: + ConsoleRenderable: Renderable to display log. + """ + path = Path(record.pathname).name + level = self.get_level_text(record) + time_format = None if self.formatter is None else self.formatter.datefmt + log_time = datetime.fromtimestamp(record.created) + + log_renderable = self._log_render( + self.console, + [message_renderable] if not traceback else [message_renderable, traceback], + log_time=log_time, + time_format=time_format, + level=level, + path=path, + line_no=record.lineno, + link_path=record.pathname if self.enable_link_path else None, + ) + return log_renderable + + +if __name__ == "__main__": # pragma: no cover + from time import sleep + + FORMAT = "%(message)s" + # FORMAT = "%(asctime)-15s - %(levelname)s - %(message)s" + logging.basicConfig( + level="NOTSET", + format=FORMAT, + datefmt="[%X]", + handlers=[RichHandler(rich_tracebacks=True, tracebacks_show_locals=True)], + ) + log = logging.getLogger("rich") + + log.info("Server starting...") + log.info("Listening on http://127.0.0.1:8080") + sleep(1) + + log.info("GET /index.html 200 1298") + log.info("GET /imgs/backgrounds/back1.jpg 200 54386") + log.info("GET /css/styles.css 200 54386") + log.warning("GET /favicon.ico 404 242") + sleep(1) + + log.debug( + "JSONRPC request\n--> %r\n<-- %r", + { + "version": "1.1", + "method": "confirmFruitPurchase", + "params": [["apple", "orange", "mangoes", "pomelo"], 1.123], + "id": "194521489", + }, + {"version": "1.1", "result": True, "error": None, "id": "194521489"}, + ) + log.debug( + "Loading configuration file /adasd/asdasd/qeqwe/qwrqwrqwr/sdgsdgsdg/werwerwer/dfgerert/ertertert/ertetert/werwerwer" + ) + log.error("Unable to find 'pomelo' in database!") + log.info("POST /jsonrpc/ 200 65532") + log.info("POST /admin/ 401 42234") + log.warning("password was rejected for admin site.") + + def divide() -> None: + number = 1 + divisor = 0 + foos = ["foo"] * 100 + log.debug("in divide") + try: + number / divisor + except: + log.exception("An error of some kind occurred!") + + divide() + sleep(1) + log.critical("Out of memory!") + log.info("Server exited with code=-1") + log.info("[bold]EXITING...[/bold]", extra=dict(markup=True)) diff --git a/src/pip/_vendor/rich/markup.py b/src/pip/_vendor/rich/markup.py new file mode 100644 index 00000000000..619540202cb --- /dev/null +++ b/src/pip/_vendor/rich/markup.py @@ -0,0 +1,244 @@ +from ast import literal_eval +from operator import attrgetter +import re +from typing import Callable, Iterable, List, Match, NamedTuple, Optional, Tuple, Union + +from .errors import MarkupError +from .style import Style +from .text import Span, Text +from .emoji import EmojiVariant +from ._emoji_replace import _emoji_replace + + +RE_TAGS = re.compile( + r"""((\\*)\[([a-z#\/@].*?)\])""", + re.VERBOSE, +) + +RE_HANDLER = re.compile(r"^([\w\.]*?)(\(.*?\))?$") + + +class Tag(NamedTuple): + """A tag in console markup.""" + + name: str + """The tag name. e.g. 'bold'.""" + parameters: Optional[str] + """Any additional parameters after the name.""" + + def __str__(self) -> str: + return ( + self.name if self.parameters is None else f"{self.name} {self.parameters}" + ) + + @property + def markup(self) -> str: + """Get the string representation of this tag.""" + return ( + f"[{self.name}]" + if self.parameters is None + else f"[{self.name}={self.parameters}]" + ) + + +_ReStringMatch = Match[str] # regex match object +_ReSubCallable = Callable[[_ReStringMatch], str] # Callable invoked by re.sub +_EscapeSubMethod = Callable[[_ReSubCallable, str], str] # Sub method of a compiled re + + +def escape( + markup: str, _escape: _EscapeSubMethod = re.compile(r"(\\*)(\[[a-z#\/@].*?\])").sub +) -> str: + """Escapes text so that it won't be interpreted as markup. + + Args: + markup (str): Content to be inserted in to markup. + + Returns: + str: Markup with square brackets escaped. + """ + + def escape_backslashes(match: Match[str]) -> str: + """Called by re.sub replace matches.""" + backslashes, text = match.groups() + return f"{backslashes}{backslashes}\\{text}" + + markup = _escape(escape_backslashes, markup) + return markup + + +def _parse(markup: str) -> Iterable[Tuple[int, Optional[str], Optional[Tag]]]: + """Parse markup in to an iterable of tuples of (position, text, tag). + + Args: + markup (str): A string containing console markup + + """ + position = 0 + _divmod = divmod + _Tag = Tag + for match in RE_TAGS.finditer(markup): + full_text, escapes, tag_text = match.groups() + start, end = match.span() + if start > position: + yield start, markup[position:start], None + if escapes: + backslashes, escaped = _divmod(len(escapes), 2) + if backslashes: + # Literal backslashes + yield start, "\\" * backslashes, None + start += backslashes * 2 + if escaped: + # Escape of tag + yield start, full_text[len(escapes) :], None + position = end + continue + text, equals, parameters = tag_text.partition("=") + yield start, None, _Tag(text, parameters if equals else None) + position = end + if position < len(markup): + yield position, markup[position:], None + + +def render( + markup: str, + style: Union[str, Style] = "", + emoji: bool = True, + emoji_variant: Optional[EmojiVariant] = None, +) -> Text: + """Render console markup in to a Text instance. + + Args: + markup (str): A string containing console markup. + emoji (bool, optional): Also render emoji code. Defaults to True. + + Raises: + MarkupError: If there is a syntax error in the markup. + + Returns: + Text: A test instance. + """ + emoji_replace = _emoji_replace + if "[" not in markup: + return Text( + emoji_replace(markup, default_variant=emoji_variant) if emoji else markup, + style=style, + ) + text = Text(style=style) + append = text.append + normalize = Style.normalize + + style_stack: List[Tuple[int, Tag]] = [] + pop = style_stack.pop + + spans: List[Span] = [] + append_span = spans.append + + _Span = Span + _Tag = Tag + + def pop_style(style_name: str) -> Tuple[int, Tag]: + """Pop tag matching given style name.""" + for index, (_, tag) in enumerate(reversed(style_stack), 1): + if tag.name == style_name: + return pop(-index) + raise KeyError(style_name) + + for position, plain_text, tag in _parse(markup): + if plain_text is not None: + append(emoji_replace(plain_text) if emoji else plain_text) + elif tag is not None: + if tag.name.startswith("/"): # Closing tag + style_name = tag.name[1:].strip() + + if style_name: # explicit close + style_name = normalize(style_name) + try: + start, open_tag = pop_style(style_name) + except KeyError: + raise MarkupError( + f"closing tag '{tag.markup}' at position {position} doesn't match any open tag" + ) from None + else: # implicit close + try: + start, open_tag = pop() + except IndexError: + raise MarkupError( + f"closing tag '[/]' at position {position} has nothing to close" + ) from None + + if open_tag.name.startswith("@"): + if open_tag.parameters: + handler_name = "" + parameters = open_tag.parameters.strip() + handler_match = RE_HANDLER.match(parameters) + if handler_match is not None: + handler_name, match_parameters = handler_match.groups() + parameters = ( + "()" if match_parameters is None else match_parameters + ) + + try: + meta_params = literal_eval(parameters) + except SyntaxError as error: + raise MarkupError( + f"error parsing {parameters!r} in {open_tag.parameters!r}; {error.msg}" + ) + except Exception as error: + raise MarkupError( + f"error parsing {open_tag.parameters!r}; {error}" + ) from None + + if handler_name: + meta_params = ( + handler_name, + meta_params + if isinstance(meta_params, tuple) + else (meta_params,), + ) + + else: + meta_params = () + + append_span( + _Span( + start, len(text), Style(meta={open_tag.name: meta_params}) + ) + ) + else: + append_span(_Span(start, len(text), str(open_tag))) + + else: # Opening tag + normalized_tag = _Tag(normalize(tag.name), tag.parameters) + style_stack.append((len(text), normalized_tag)) + + text_length = len(text) + while style_stack: + start, tag = style_stack.pop() + style = str(tag) + if style: + append_span(_Span(start, text_length, style)) + + text.spans = sorted(spans[::-1], key=attrgetter("start")) + return text + + +if __name__ == "__main__": # pragma: no cover + + MARKUP = [ + "[red]Hello World[/red]", + "[magenta]Hello [b]World[/b]", + "[bold]Bold[italic] bold and italic [/bold]italic[/italic]", + "Click [link=https://www.willmcgugan.com]here[/link] to visit my Blog", + ":warning-emoji: [bold red blink] DANGER![/]", + ] + + from pip._vendor.rich.table import Table + from pip._vendor.rich import print + + grid = Table("Markup", "Result", padding=(0, 1)) + + for markup in MARKUP: + grid.add_row(Text(markup), markup) + + print(grid) diff --git a/src/pip/_vendor/rich/measure.py b/src/pip/_vendor/rich/measure.py new file mode 100644 index 00000000000..aea238df938 --- /dev/null +++ b/src/pip/_vendor/rich/measure.py @@ -0,0 +1,149 @@ +from operator import itemgetter +from typing import Callable, Iterable, NamedTuple, Optional, TYPE_CHECKING + +from . import errors +from .protocol import is_renderable, rich_cast + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderableType + + +class Measurement(NamedTuple): + """Stores the minimum and maximum widths (in characters) required to render an object.""" + + minimum: int + """Minimum number of cells required to render.""" + maximum: int + """Maximum number of cells required to render.""" + + @property + def span(self) -> int: + """Get difference between maximum and minimum.""" + return self.maximum - self.minimum + + def normalize(self) -> "Measurement": + """Get measurement that ensures that minimum <= maximum and minimum >= 0 + + Returns: + Measurement: A normalized measurement. + """ + minimum, maximum = self + minimum = min(max(0, minimum), maximum) + return Measurement(max(0, minimum), max(0, max(minimum, maximum))) + + def with_maximum(self, width: int) -> "Measurement": + """Get a RenderableWith where the widths are <= width. + + Args: + width (int): Maximum desired width. + + Returns: + Measurement: New Measurement object. + """ + minimum, maximum = self + return Measurement(min(minimum, width), min(maximum, width)) + + def with_minimum(self, width: int) -> "Measurement": + """Get a RenderableWith where the widths are >= width. + + Args: + width (int): Minimum desired width. + + Returns: + Measurement: New Measurement object. + """ + minimum, maximum = self + width = max(0, width) + return Measurement(max(minimum, width), max(maximum, width)) + + def clamp( + self, min_width: Optional[int] = None, max_width: Optional[int] = None + ) -> "Measurement": + """Clamp a measurement within the specified range. + + Args: + min_width (int): Minimum desired width, or ``None`` for no minimum. Defaults to None. + max_width (int): Maximum desired width, or ``None`` for no maximum. Defaults to None. + + Returns: + Measurement: New Measurement object. + """ + measurement = self + if min_width is not None: + measurement = measurement.with_minimum(min_width) + if max_width is not None: + measurement = measurement.with_maximum(max_width) + return measurement + + @classmethod + def get( + cls, console: "Console", options: "ConsoleOptions", renderable: "RenderableType" + ) -> "Measurement": + """Get a measurement for a renderable. + + Args: + console (~rich.console.Console): Console instance. + options (~rich.console.ConsoleOptions): Console options. + renderable (RenderableType): An object that may be rendered with Rich. + + Raises: + errors.NotRenderableError: If the object is not renderable. + + Returns: + Measurement: Measurement object containing range of character widths required to render the object. + """ + _max_width = options.max_width + if _max_width < 1: + return Measurement(0, 0) + if isinstance(renderable, str): + renderable = console.render_str(renderable, markup=options.markup) + renderable = rich_cast(renderable) + if is_renderable(renderable): + get_console_width: Optional[ + Callable[["Console", "ConsoleOptions"], "Measurement"] + ] = getattr(renderable, "__rich_measure__", None) + if get_console_width is not None: + render_width = ( + get_console_width(console, options) + .normalize() + .with_maximum(_max_width) + ) + if render_width.maximum < 1: + return Measurement(0, 0) + return render_width.normalize() + else: + return Measurement(0, _max_width) + else: + raise errors.NotRenderableError( + f"Unable to get render width for {renderable!r}; " + "a str, Segment, or object with __rich_console__ method is required" + ) + + +def measure_renderables( + console: "Console", + options: "ConsoleOptions", + renderables: Iterable["RenderableType"], +) -> "Measurement": + """Get a measurement that would fit a number of renderables. + + Args: + console (~rich.console.Console): Console instance. + options (~rich.console.ConsoleOptions): Console options. + renderables (Iterable[RenderableType]): One or more renderable objects. + + Returns: + Measurement: Measurement object containing range of character widths required to + contain all given renderables. + """ + if not renderables: + return Measurement(0, 0) + get_measurement = Measurement.get + measurements = [ + get_measurement(console, options, renderable) for renderable in renderables + ] + measured_width = Measurement( + max(measurements, key=itemgetter(0)).minimum, + max(measurements, key=itemgetter(1)).maximum, + ) + return measured_width diff --git a/src/pip/_vendor/rich/padding.py b/src/pip/_vendor/rich/padding.py new file mode 100644 index 00000000000..1b2204f59f2 --- /dev/null +++ b/src/pip/_vendor/rich/padding.py @@ -0,0 +1,141 @@ +from typing import cast, List, Optional, Tuple, TYPE_CHECKING, Union + +if TYPE_CHECKING: + from .console import ( + Console, + ConsoleOptions, + RenderableType, + RenderResult, + ) +from .jupyter import JupyterMixin +from .measure import Measurement +from .style import Style +from .segment import Segment + + +PaddingDimensions = Union[int, Tuple[int], Tuple[int, int], Tuple[int, int, int, int]] + + +class Padding(JupyterMixin): + """Draw space around content. + + Example: + >>> print(Padding("Hello", (2, 4), style="on blue")) + + Args: + renderable (RenderableType): String or other renderable. + pad (Union[int, Tuple[int]]): Padding for top, right, bottom, and left borders. + May be specified with 1, 2, or 4 integers (CSS style). + style (Union[str, Style], optional): Style for padding characters. Defaults to "none". + expand (bool, optional): Expand padding to fit available width. Defaults to True. + """ + + def __init__( + self, + renderable: "RenderableType", + pad: "PaddingDimensions" = (0, 0, 0, 0), + *, + style: Union[str, Style] = "none", + expand: bool = True, + ): + self.renderable = renderable + self.top, self.right, self.bottom, self.left = self.unpack(pad) + self.style = style + self.expand = expand + + @classmethod + def indent(cls, renderable: "RenderableType", level: int) -> "Padding": + """Make padding instance to render an indent. + + Args: + renderable (RenderableType): String or other renderable. + level (int): Number of characters to indent. + + Returns: + Padding: A Padding instance. + """ + + return Padding(renderable, pad=(0, 0, 0, level), expand=False) + + @staticmethod + def unpack(pad: "PaddingDimensions") -> Tuple[int, int, int, int]: + """Unpack padding specified in CSS style.""" + if isinstance(pad, int): + return (pad, pad, pad, pad) + if len(pad) == 1: + _pad = pad[0] + return (_pad, _pad, _pad, _pad) + if len(pad) == 2: + pad_top, pad_right = cast(Tuple[int, int], pad) + return (pad_top, pad_right, pad_top, pad_right) + if len(pad) == 4: + top, right, bottom, left = cast(Tuple[int, int, int, int], pad) + return (top, right, bottom, left) + raise ValueError(f"1, 2 or 4 integers required for padding; {len(pad)} given") + + def __repr__(self) -> str: + return f"Padding({self.renderable!r}, ({self.top},{self.right},{self.bottom},{self.left}))" + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + style = console.get_style(self.style) + if self.expand: + width = options.max_width + else: + width = min( + Measurement.get(console, options, self.renderable).maximum + + self.left + + self.right, + options.max_width, + ) + render_options = options.update_width(width - self.left - self.right) + if render_options.height is not None: + render_options = render_options.update_height( + height=render_options.height - self.top - self.bottom + ) + lines = console.render_lines( + self.renderable, render_options, style=style, pad=True + ) + _Segment = Segment + + left = _Segment(" " * self.left, style) if self.left else None + right = ( + [_Segment(f'{" " * self.right}', style), _Segment.line()] + if self.right + else [_Segment.line()] + ) + blank_line: Optional[List[Segment]] = None + if self.top: + blank_line = [_Segment(f'{" " * width}\n', style)] + yield from blank_line * self.top + if left: + for line in lines: + yield left + yield from line + yield from right + else: + for line in lines: + yield from line + yield from right + if self.bottom: + blank_line = blank_line or [_Segment(f'{" " * width}\n', style)] + yield from blank_line * self.bottom + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + max_width = options.max_width + extra_width = self.left + self.right + if max_width - extra_width < 1: + return Measurement(max_width, max_width) + measure_min, measure_max = Measurement.get(console, options, self.renderable) + measurement = Measurement(measure_min + extra_width, measure_max + extra_width) + measurement = measurement.with_maximum(max_width) + return measurement + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich import print + + print(Padding("Hello, World", (2, 4), style="on blue")) diff --git a/src/pip/_vendor/rich/pager.py b/src/pip/_vendor/rich/pager.py new file mode 100644 index 00000000000..dbfb973e368 --- /dev/null +++ b/src/pip/_vendor/rich/pager.py @@ -0,0 +1,34 @@ +from abc import ABC, abstractmethod +from typing import Any, Callable + + +class Pager(ABC): + """Base class for a pager.""" + + @abstractmethod + def show(self, content: str) -> None: + """Show content in pager. + + Args: + content (str): Content to be displayed. + """ + + +class SystemPager(Pager): + """Uses the pager installed on the system.""" + + def _pager(self, content: str) -> Any: #  pragma: no cover + return __import__("pydoc").pager(content) + + def show(self, content: str) -> None: + """Use the same pager used by pydoc.""" + self._pager(content) + + +if __name__ == "__main__": # pragma: no cover + from .__main__ import make_test_card + from .console import Console + + console = Console() + with console.pager(styles=True): + console.print(make_test_card()) diff --git a/src/pip/_vendor/rich/palette.py b/src/pip/_vendor/rich/palette.py new file mode 100644 index 00000000000..fa0c4dd4038 --- /dev/null +++ b/src/pip/_vendor/rich/palette.py @@ -0,0 +1,100 @@ +from math import sqrt +from functools import lru_cache +from typing import Sequence, Tuple, TYPE_CHECKING + +from .color_triplet import ColorTriplet + +if TYPE_CHECKING: + from pip._vendor.rich.table import Table + + +class Palette: + """A palette of available colors.""" + + def __init__(self, colors: Sequence[Tuple[int, int, int]]): + self._colors = colors + + def __getitem__(self, number: int) -> ColorTriplet: + return ColorTriplet(*self._colors[number]) + + def __rich__(self) -> "Table": + from pip._vendor.rich.color import Color + from pip._vendor.rich.style import Style + from pip._vendor.rich.text import Text + from pip._vendor.rich.table import Table + + table = Table( + "index", + "RGB", + "Color", + title="Palette", + caption=f"{len(self._colors)} colors", + highlight=True, + caption_justify="right", + ) + for index, color in enumerate(self._colors): + table.add_row( + str(index), + repr(color), + Text(" " * 16, style=Style(bgcolor=Color.from_rgb(*color))), + ) + return table + + # This is somewhat inefficient and needs caching + @lru_cache(maxsize=1024) + def match(self, color: Tuple[int, int, int]) -> int: + """Find a color from a palette that most closely matches a given color. + + Args: + color (Tuple[int, int, int]): RGB components in range 0 > 255. + + Returns: + int: Index of closes matching color. + """ + red1, green1, blue1 = color + _sqrt = sqrt + get_color = self._colors.__getitem__ + + def get_color_distance(index: int) -> float: + """Get the distance to a color.""" + red2, green2, blue2 = get_color(index) + red_mean = (red1 + red2) // 2 + red = red1 - red2 + green = green1 - green2 + blue = blue1 - blue2 + return _sqrt( + (((512 + red_mean) * red * red) >> 8) + + 4 * green * green + + (((767 - red_mean) * blue * blue) >> 8) + ) + + min_index = min(range(len(self._colors)), key=get_color_distance) + return min_index + + +if __name__ == "__main__": # pragma: no cover + import colorsys + from typing import Iterable + from pip._vendor.rich.color import Color + from pip._vendor.rich.console import Console, ConsoleOptions + from pip._vendor.rich.segment import Segment + from pip._vendor.rich.style import Style + + class ColorBox: + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> Iterable[Segment]: + height = console.size.height - 3 + for y in range(0, height): + for x in range(options.max_width): + h = x / options.max_width + l = y / (height + 1) + r1, g1, b1 = colorsys.hls_to_rgb(h, l, 1.0) + r2, g2, b2 = colorsys.hls_to_rgb(h, l + (1 / height / 2), 1.0) + bgcolor = Color.from_rgb(r1 * 255, g1 * 255, b1 * 255) + color = Color.from_rgb(r2 * 255, g2 * 255, b2 * 255) + yield Segment("▄", Style(color=color, bgcolor=bgcolor)) + yield Segment.line() + + console = Console() + console.print(ColorBox()) diff --git a/src/pip/_vendor/rich/panel.py b/src/pip/_vendor/rich/panel.py new file mode 100644 index 00000000000..151fe5f017f --- /dev/null +++ b/src/pip/_vendor/rich/panel.py @@ -0,0 +1,250 @@ +from typing import Optional, TYPE_CHECKING + +from .box import Box, ROUNDED + +from .align import AlignMethod +from .jupyter import JupyterMixin +from .measure import Measurement, measure_renderables +from .padding import Padding, PaddingDimensions +from .style import StyleType +from .text import Text, TextType +from .segment import Segment + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderableType, RenderResult + + +class Panel(JupyterMixin): + """A console renderable that draws a border around its contents. + + Example: + >>> console.print(Panel("Hello, World!")) + + Args: + renderable (RenderableType): A console renderable object. + box (Box, optional): A Box instance that defines the look of the border (see :ref:`appendix_box`. + Defaults to box.ROUNDED. + safe_box (bool, optional): Disable box characters that don't display on windows legacy terminal with *raster* fonts. Defaults to True. + expand (bool, optional): If True the panel will stretch to fill the console + width, otherwise it will be sized to fit the contents. Defaults to True. + style (str, optional): The style of the panel (border and contents). Defaults to "none". + border_style (str, optional): The style of the border. Defaults to "none". + width (Optional[int], optional): Optional width of panel. Defaults to None to auto-detect. + height (Optional[int], optional): Optional height of panel. Defaults to None to auto-detect. + padding (Optional[PaddingDimensions]): Optional padding around renderable. Defaults to 0. + highlight (bool, optional): Enable automatic highlighting of panel title (if str). Defaults to False. + """ + + def __init__( + self, + renderable: "RenderableType", + box: Box = ROUNDED, + *, + title: Optional[TextType] = None, + title_align: AlignMethod = "center", + subtitle: Optional[TextType] = None, + subtitle_align: AlignMethod = "center", + safe_box: Optional[bool] = None, + expand: bool = True, + style: StyleType = "none", + border_style: StyleType = "none", + width: Optional[int] = None, + height: Optional[int] = None, + padding: PaddingDimensions = (0, 1), + highlight: bool = False, + ) -> None: + self.renderable = renderable + self.box = box + self.title = title + self.title_align: AlignMethod = title_align + self.subtitle = subtitle + self.subtitle_align = subtitle_align + self.safe_box = safe_box + self.expand = expand + self.style = style + self.border_style = border_style + self.width = width + self.height = height + self.padding = padding + self.highlight = highlight + + @classmethod + def fit( + cls, + renderable: "RenderableType", + box: Box = ROUNDED, + *, + title: Optional[TextType] = None, + title_align: AlignMethod = "center", + subtitle: Optional[TextType] = None, + subtitle_align: AlignMethod = "center", + safe_box: Optional[bool] = None, + style: StyleType = "none", + border_style: StyleType = "none", + width: Optional[int] = None, + padding: PaddingDimensions = (0, 1), + ) -> "Panel": + """An alternative constructor that sets expand=False.""" + return cls( + renderable, + box, + title=title, + title_align=title_align, + subtitle=subtitle, + subtitle_align=subtitle_align, + safe_box=safe_box, + style=style, + border_style=border_style, + width=width, + padding=padding, + expand=False, + ) + + @property + def _title(self) -> Optional[Text]: + if self.title: + title_text = ( + Text.from_markup(self.title) + if isinstance(self.title, str) + else self.title.copy() + ) + title_text.end = "" + title_text.plain = title_text.plain.replace("\n", " ") + title_text.no_wrap = True + title_text.expand_tabs() + title_text.pad(1) + return title_text + return None + + @property + def _subtitle(self) -> Optional[Text]: + if self.subtitle: + subtitle_text = ( + Text.from_markup(self.subtitle) + if isinstance(self.subtitle, str) + else self.subtitle.copy() + ) + subtitle_text.end = "" + subtitle_text.plain = subtitle_text.plain.replace("\n", " ") + subtitle_text.no_wrap = True + subtitle_text.expand_tabs() + subtitle_text.pad(1) + return subtitle_text + return None + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + _padding = Padding.unpack(self.padding) + renderable = ( + Padding(self.renderable, _padding) if any(_padding) else self.renderable + ) + style = console.get_style(self.style) + border_style = style + console.get_style(self.border_style) + width = ( + options.max_width + if self.width is None + else min(options.max_width, self.width) + ) + + safe_box: bool = console.safe_box if self.safe_box is None else self.safe_box + box = self.box.substitute(options, safe=safe_box) + + title_text = self._title + if title_text is not None: + title_text.style = border_style + + child_width = ( + width - 2 + if self.expand + else console.measure( + renderable, options=options.update_width(width - 2) + ).maximum + ) + child_height = self.height or options.height or None + if child_height: + child_height -= 2 + if title_text is not None: + child_width = min( + options.max_width - 2, max(child_width, title_text.cell_len + 2) + ) + + width = child_width + 2 + child_options = options.update( + width=child_width, height=child_height, highlight=self.highlight + ) + lines = console.render_lines(renderable, child_options, style=style) + + line_start = Segment(box.mid_left, border_style) + line_end = Segment(f"{box.mid_right}", border_style) + new_line = Segment.line() + if title_text is None or width <= 4: + yield Segment(box.get_top([width - 2]), border_style) + else: + title_text.align(self.title_align, width - 4, character=box.top) + yield Segment(box.top_left + box.top, border_style) + yield from console.render(title_text) + yield Segment(box.top + box.top_right, border_style) + + yield new_line + for line in lines: + yield line_start + yield from line + yield line_end + yield new_line + + subtitle_text = self._subtitle + if subtitle_text is not None: + subtitle_text.style = border_style + + if subtitle_text is None or width <= 4: + yield Segment(box.get_bottom([width - 2]), border_style) + else: + subtitle_text.align(self.subtitle_align, width - 4, character=box.bottom) + yield Segment(box.bottom_left + box.bottom, border_style) + yield from console.render(subtitle_text) + yield Segment(box.bottom + box.bottom_right, border_style) + + yield new_line + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + _title = self._title + _, right, _, left = Padding.unpack(self.padding) + padding = left + right + renderables = [self.renderable, _title] if _title else [self.renderable] + + if self.width is None: + width = ( + measure_renderables( + console, + options.update_width(options.max_width - padding - 2), + renderables, + ).maximum + + padding + + 2 + ) + else: + width = self.width + return Measurement(width, width) + + +if __name__ == "__main__": # pragma: no cover + from .console import Console + + c = Console() + + from .padding import Padding + from .box import ROUNDED, DOUBLE + + p = Panel( + "Hello, World!", + title="rich.Panel", + style="white on blue", + box=DOUBLE, + padding=1, + ) + + c.print() + c.print(p) diff --git a/src/pip/_vendor/rich/pretty.py b/src/pip/_vendor/rich/pretty.py new file mode 100644 index 00000000000..606ee33822a --- /dev/null +++ b/src/pip/_vendor/rich/pretty.py @@ -0,0 +1,903 @@ +import builtins +import dataclasses +import inspect +import os +import re +import sys +from array import array +from collections import Counter, UserDict, UserList, defaultdict, deque +from dataclasses import dataclass, fields, is_dataclass +from inspect import isclass +from itertools import islice +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + Union, +) + +from pip._vendor.rich.repr import RichReprResult + +try: + import attr as _attr_module +except ImportError: # pragma: no cover + _attr_module = None # type: ignore + + +from . import get_console +from ._loop import loop_last +from ._pick import pick_bool +from .abc import RichRenderable +from .cells import cell_len +from .highlighter import ReprHighlighter +from .jupyter import JupyterMixin, JupyterRenderable +from .measure import Measurement +from .text import Text + +if TYPE_CHECKING: + from .console import ( + Console, + ConsoleOptions, + HighlighterType, + JustifyMethod, + OverflowMethod, + RenderResult, + ) + + +def _is_attr_object(obj: Any) -> bool: + """Check if an object was created with attrs module.""" + return _attr_module is not None and _attr_module.has(type(obj)) + + +def _get_attr_fields(obj: Any) -> Iterable["_attr_module.Attribute[Any]"]: + """Get fields for an attrs object.""" + return _attr_module.fields(type(obj)) if _attr_module is not None else [] + + +def _is_dataclass_repr(obj: object) -> bool: + """Check if an instance of a dataclass contains the default repr. + + Args: + obj (object): A dataclass instance. + + Returns: + bool: True if the default repr is used, False if there is a custom repr. + """ + # Digging in to a lot of internals here + # Catching all exceptions in case something is missing on a non CPython implementation + try: + return obj.__repr__.__code__.co_filename == dataclasses.__file__ + except Exception: # pragma: no coverage + return False + + +def _ipy_display_hook( + value: Any, + console: Optional["Console"] = None, + overflow: "OverflowMethod" = "ignore", + crop: bool = False, + indent_guides: bool = False, + max_length: Optional[int] = None, + max_string: Optional[int] = None, + expand_all: bool = False, +) -> None: + from .console import ConsoleRenderable # needed here to prevent circular import + + # always skip rich generated jupyter renderables or None values + if isinstance(value, JupyterRenderable) or value is None: + return + + console = console or get_console() + if console.is_jupyter: + # Delegate rendering to IPython if the object (and IPython) supports it + # https://ipython.readthedocs.io/en/stable/config/integrating.html#rich-display + ipython_repr_methods = [ + "_repr_html_", + "_repr_markdown_", + "_repr_json_", + "_repr_latex_", + "_repr_jpeg_", + "_repr_png_", + "_repr_svg_", + "_repr_mimebundle_", + ] + for repr_method in ipython_repr_methods: + method = getattr(value, repr_method, None) + if inspect.ismethod(method): + # Calling the method ourselves isn't ideal. The interface for the `_repr_*_` methods + # specifies that if they return None, then they should not be rendered + # by the notebook. + try: + repr_result = method() + except Exception: + continue # If the method raises, treat it as if it doesn't exist, try any others + if repr_result is not None: + return # Delegate rendering to IPython + + # certain renderables should start on a new line + if isinstance(value, ConsoleRenderable): + console.line() + + console.print( + value + if isinstance(value, RichRenderable) + else Pretty( + value, + overflow=overflow, + indent_guides=indent_guides, + max_length=max_length, + max_string=max_string, + expand_all=expand_all, + margin=12, + ), + crop=crop, + new_line_start=True, + ) + + +def install( + console: Optional["Console"] = None, + overflow: "OverflowMethod" = "ignore", + crop: bool = False, + indent_guides: bool = False, + max_length: Optional[int] = None, + max_string: Optional[int] = None, + expand_all: bool = False, +) -> None: + """Install automatic pretty printing in the Python REPL. + + Args: + console (Console, optional): Console instance or ``None`` to use global console. Defaults to None. + overflow (Optional[OverflowMethod], optional): Overflow method. Defaults to "ignore". + crop (Optional[bool], optional): Enable cropping of long lines. Defaults to False. + indent_guides (bool, optional): Enable indentation guides. Defaults to False. + max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to None. + max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to None. + expand_all (bool, optional): Expand all containers. Defaults to False. + max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100. + """ + from pip._vendor.rich import get_console + + console = console or get_console() + assert console is not None + + def display_hook(value: Any) -> None: + """Replacement sys.displayhook which prettifies objects with Rich.""" + if value is not None: + assert console is not None + builtins._ = None # type: ignore + console.print( + value + if isinstance(value, RichRenderable) + else Pretty( + value, + overflow=overflow, + indent_guides=indent_guides, + max_length=max_length, + max_string=max_string, + expand_all=expand_all, + ), + crop=crop, + ) + builtins._ = value # type: ignore + + try: # pragma: no cover + ip = get_ipython() # type: ignore + from IPython.core.formatters import BaseFormatter + + class RichFormatter(BaseFormatter): # type: ignore + pprint: bool = True + + def __call__(self, value: Any) -> Any: + if self.pprint: + return _ipy_display_hook( + value, + console=get_console(), + overflow=overflow, + indent_guides=indent_guides, + max_length=max_length, + max_string=max_string, + expand_all=expand_all, + ) + else: + return repr(value) + + # replace plain text formatter with rich formatter + rich_formatter = RichFormatter() + ip.display_formatter.formatters["text/plain"] = rich_formatter + except Exception: + sys.displayhook = display_hook + + +class Pretty(JupyterMixin): + """A rich renderable that pretty prints an object. + + Args: + _object (Any): An object to pretty print. + highlighter (HighlighterType, optional): Highlighter object to apply to result, or None for ReprHighlighter. Defaults to None. + indent_size (int, optional): Number of spaces in indent. Defaults to 4. + justify (JustifyMethod, optional): Justify method, or None for default. Defaults to None. + overflow (OverflowMethod, optional): Overflow method, or None for default. Defaults to None. + no_wrap (Optional[bool], optional): Disable word wrapping. Defaults to False. + indent_guides (bool, optional): Enable indentation guides. Defaults to False. + max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to None. + max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to None. + max_depth (int, optional): Maximum depth of nested data structures, or None for no maximum. Defaults to None. + expand_all (bool, optional): Expand all containers. Defaults to False. + margin (int, optional): Subtrace a margin from width to force containers to expand earlier. Defaults to 0. + insert_line (bool, optional): Insert a new line if the output has multiple new lines. Defaults to False. + """ + + def __init__( + self, + _object: Any, + highlighter: Optional["HighlighterType"] = None, + *, + indent_size: int = 4, + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + no_wrap: Optional[bool] = False, + indent_guides: bool = False, + max_length: Optional[int] = None, + max_string: Optional[int] = None, + max_depth: Optional[int] = None, + expand_all: bool = False, + margin: int = 0, + insert_line: bool = False, + ) -> None: + self._object = _object + self.highlighter = highlighter or ReprHighlighter() + self.indent_size = indent_size + self.justify: Optional["JustifyMethod"] = justify + self.overflow: Optional["OverflowMethod"] = overflow + self.no_wrap = no_wrap + self.indent_guides = indent_guides + self.max_length = max_length + self.max_string = max_string + self.max_depth = max_depth + self.expand_all = expand_all + self.margin = margin + self.insert_line = insert_line + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + pretty_str = pretty_repr( + self._object, + max_width=options.max_width - self.margin, + indent_size=self.indent_size, + max_length=self.max_length, + max_string=self.max_string, + max_depth=self.max_depth, + expand_all=self.expand_all, + ) + pretty_text = Text( + pretty_str, + justify=self.justify or options.justify, + overflow=self.overflow or options.overflow, + no_wrap=pick_bool(self.no_wrap, options.no_wrap), + style="pretty", + ) + pretty_text = ( + self.highlighter(pretty_text) + if pretty_text + else Text( + f"{type(self._object)}.__repr__ returned empty string", + style="dim italic", + ) + ) + if self.indent_guides and not options.ascii_only: + pretty_text = pretty_text.with_indent_guides( + self.indent_size, style="repr.indent" + ) + if self.insert_line and "\n" in pretty_text: + yield "" + yield pretty_text + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + pretty_str = pretty_repr( + self._object, + max_width=options.max_width, + indent_size=self.indent_size, + max_length=self.max_length, + max_string=self.max_string, + ) + text_width = ( + max(cell_len(line) for line in pretty_str.splitlines()) if pretty_str else 0 + ) + return Measurement(text_width, text_width) + + +def _get_braces_for_defaultdict(_object: DefaultDict[Any, Any]) -> Tuple[str, str, str]: + return ( + f"defaultdict({_object.default_factory!r}, {{", + "})", + f"defaultdict({_object.default_factory!r}, {{}})", + ) + + +def _get_braces_for_array(_object: "array[Any]") -> Tuple[str, str, str]: + return (f"array({_object.typecode!r}, [", "])", "array({_object.typecode!r})") + + +_BRACES: Dict[type, Callable[[Any], Tuple[str, str, str]]] = { + os._Environ: lambda _object: ("environ({", "})", "environ({})"), + array: _get_braces_for_array, + defaultdict: _get_braces_for_defaultdict, + Counter: lambda _object: ("Counter({", "})", "Counter()"), + deque: lambda _object: ("deque([", "])", "deque()"), + dict: lambda _object: ("{", "}", "{}"), + UserDict: lambda _object: ("{", "}", "{}"), + frozenset: lambda _object: ("frozenset({", "})", "frozenset()"), + list: lambda _object: ("[", "]", "[]"), + UserList: lambda _object: ("[", "]", "[]"), + set: lambda _object: ("{", "}", "set()"), + tuple: lambda _object: ("(", ")", "()"), + MappingProxyType: lambda _object: ("mappingproxy({", "})", "mappingproxy({})"), +} +_CONTAINERS = tuple(_BRACES.keys()) +_MAPPING_CONTAINERS = (dict, os._Environ, MappingProxyType, UserDict) + + +def is_expandable(obj: Any) -> bool: + """Check if an object may be expanded by pretty print.""" + return ( + isinstance(obj, _CONTAINERS) + or (is_dataclass(obj)) + or (hasattr(obj, "__rich_repr__")) + or _is_attr_object(obj) + ) and not isclass(obj) + + +@dataclass +class Node: + """A node in a repr tree. May be atomic or a container.""" + + key_repr: str = "" + value_repr: str = "" + open_brace: str = "" + close_brace: str = "" + empty: str = "" + last: bool = False + is_tuple: bool = False + children: Optional[List["Node"]] = None + key_separator = ": " + separator: str = ", " + + def iter_tokens(self) -> Iterable[str]: + """Generate tokens for this node.""" + if self.key_repr: + yield self.key_repr + yield self.key_separator + if self.value_repr: + yield self.value_repr + elif self.children is not None: + if self.children: + yield self.open_brace + if self.is_tuple and len(self.children) == 1: + yield from self.children[0].iter_tokens() + yield "," + else: + for child in self.children: + yield from child.iter_tokens() + if not child.last: + yield self.separator + yield self.close_brace + else: + yield self.empty + + def check_length(self, start_length: int, max_length: int) -> bool: + """Check the length fits within a limit. + + Args: + start_length (int): Starting length of the line (indent, prefix, suffix). + max_length (int): Maximum length. + + Returns: + bool: True if the node can be rendered within max length, otherwise False. + """ + total_length = start_length + for token in self.iter_tokens(): + total_length += cell_len(token) + if total_length > max_length: + return False + return True + + def __str__(self) -> str: + repr_text = "".join(self.iter_tokens()) + return repr_text + + def render( + self, max_width: int = 80, indent_size: int = 4, expand_all: bool = False + ) -> str: + """Render the node to a pretty repr. + + Args: + max_width (int, optional): Maximum width of the repr. Defaults to 80. + indent_size (int, optional): Size of indents. Defaults to 4. + expand_all (bool, optional): Expand all levels. Defaults to False. + + Returns: + str: A repr string of the original object. + """ + lines = [_Line(node=self, is_root=True)] + line_no = 0 + while line_no < len(lines): + line = lines[line_no] + if line.expandable and not line.expanded: + if expand_all or not line.check_length(max_width): + lines[line_no : line_no + 1] = line.expand(indent_size) + line_no += 1 + + repr_str = "\n".join(str(line) for line in lines) + return repr_str + + +@dataclass +class _Line: + """A line in repr output.""" + + parent: Optional["_Line"] = None + is_root: bool = False + node: Optional[Node] = None + text: str = "" + suffix: str = "" + whitespace: str = "" + expanded: bool = False + last: bool = False + + @property + def expandable(self) -> bool: + """Check if the line may be expanded.""" + return bool(self.node is not None and self.node.children) + + def check_length(self, max_length: int) -> bool: + """Check this line fits within a given number of cells.""" + start_length = ( + len(self.whitespace) + cell_len(self.text) + cell_len(self.suffix) + ) + assert self.node is not None + return self.node.check_length(start_length, max_length) + + def expand(self, indent_size: int) -> Iterable["_Line"]: + """Expand this line by adding children on their own line.""" + node = self.node + assert node is not None + whitespace = self.whitespace + assert node.children + if node.key_repr: + new_line = yield _Line( + text=f"{node.key_repr}{node.key_separator}{node.open_brace}", + whitespace=whitespace, + ) + else: + new_line = yield _Line(text=node.open_brace, whitespace=whitespace) + child_whitespace = self.whitespace + " " * indent_size + tuple_of_one = node.is_tuple and len(node.children) == 1 + for last, child in loop_last(node.children): + separator = "," if tuple_of_one else node.separator + line = _Line( + parent=new_line, + node=child, + whitespace=child_whitespace, + suffix=separator, + last=last and not tuple_of_one, + ) + yield line + + yield _Line( + text=node.close_brace, + whitespace=whitespace, + suffix=self.suffix, + last=self.last, + ) + + def __str__(self) -> str: + if self.last: + return f"{self.whitespace}{self.text}{self.node or ''}" + else: + return ( + f"{self.whitespace}{self.text}{self.node or ''}{self.suffix.rstrip()}" + ) + + +def traverse( + _object: Any, + max_length: Optional[int] = None, + max_string: Optional[int] = None, + max_depth: Optional[int] = None, +) -> Node: + """Traverse object and generate a tree. + + Args: + _object (Any): Object to be traversed. + max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to None. + max_string (int, optional): Maximum length of string before truncating, or None to disable truncating. + Defaults to None. + max_depth (int, optional): Maximum depth of data structures, or None for no maximum. + Defaults to None. + + Returns: + Node: The root of a tree structure which can be used to render a pretty repr. + """ + + def to_repr(obj: Any) -> str: + """Get repr string for an object, but catch errors.""" + if ( + max_string is not None + and isinstance(obj, (bytes, str)) + and len(obj) > max_string + ): + truncated = len(obj) - max_string + obj_repr = f"{obj[:max_string]!r}+{truncated}" + else: + try: + obj_repr = repr(obj) + except Exception as error: + obj_repr = f"" + return obj_repr + + visited_ids: Set[int] = set() + push_visited = visited_ids.add + pop_visited = visited_ids.remove + + def _traverse(obj: Any, root: bool = False, depth: int = 0) -> Node: + """Walk the object depth first.""" + + obj_type = type(obj) + py_version = (sys.version_info.major, sys.version_info.minor) + children: List[Node] + reached_max_depth = max_depth is not None and depth >= max_depth + + def iter_rich_args(rich_args: Any) -> Iterable[Union[Any, Tuple[str, Any]]]: + for arg in rich_args: + if isinstance(arg, tuple): + if len(arg) == 3: + key, child, default = arg + if default == child: + continue + yield key, child + elif len(arg) == 2: + key, child = arg + yield key, child + elif len(arg) == 1: + yield arg[0] + else: + yield arg + + try: + fake_attributes = hasattr( + obj, "awehoi234_wdfjwljet234_234wdfoijsdfmmnxpi492" + ) + except Exception: + fake_attributes = False + + rich_repr_result: Optional[RichReprResult] = None + if not fake_attributes: + try: + if hasattr(obj, "__rich_repr__") and not isclass(obj): + rich_repr_result = obj.__rich_repr__() + except Exception: + pass + + if rich_repr_result is not None: + angular = getattr(obj.__rich_repr__, "angular", False) + args = list(iter_rich_args(rich_repr_result)) + class_name = obj.__class__.__name__ + + if args: + children = [] + append = children.append + + if reached_max_depth: + node = Node(value_repr=f"...") + else: + if angular: + node = Node( + open_brace=f"<{class_name} ", + close_brace=">", + children=children, + last=root, + separator=" ", + ) + else: + node = Node( + open_brace=f"{class_name}(", + close_brace=")", + children=children, + last=root, + ) + for last, arg in loop_last(args): + if isinstance(arg, tuple): + key, child = arg + child_node = _traverse(child, depth=depth + 1) + child_node.last = last + child_node.key_repr = key + child_node.key_separator = "=" + append(child_node) + else: + child_node = _traverse(arg, depth=depth + 1) + child_node.last = last + append(child_node) + else: + node = Node( + value_repr=f"<{class_name}>" if angular else f"{class_name}()", + children=[], + last=root, + ) + elif _is_attr_object(obj) and not fake_attributes: + children = [] + append = children.append + + attr_fields = _get_attr_fields(obj) + if attr_fields: + if reached_max_depth: + node = Node(value_repr=f"...") + else: + node = Node( + open_brace=f"{obj.__class__.__name__}(", + close_brace=")", + children=children, + last=root, + ) + + def iter_attrs() -> Iterable[ + Tuple[str, Any, Optional[Callable[[Any], str]]] + ]: + """Iterate over attr fields and values.""" + for attr in attr_fields: + if attr.repr: + try: + value = getattr(obj, attr.name) + except Exception as error: + # Can happen, albeit rarely + yield (attr.name, error, None) + else: + yield ( + attr.name, + value, + attr.repr if callable(attr.repr) else None, + ) + + for last, (name, value, repr_callable) in loop_last(iter_attrs()): + if repr_callable: + child_node = Node(value_repr=str(repr_callable(value))) + else: + child_node = _traverse(value, depth=depth + 1) + child_node.last = last + child_node.key_repr = name + child_node.key_separator = "=" + append(child_node) + else: + node = Node( + value_repr=f"{obj.__class__.__name__}()", children=[], last=root + ) + + elif ( + is_dataclass(obj) + and not isinstance(obj, type) + and not fake_attributes + and (_is_dataclass_repr(obj) or py_version == (3, 6)) + ): + obj_id = id(obj) + if obj_id in visited_ids: + # Recursion detected + return Node(value_repr="...") + push_visited(obj_id) + + children = [] + append = children.append + if reached_max_depth: + node = Node(value_repr=f"...") + else: + node = Node( + open_brace=f"{obj.__class__.__name__}(", + close_brace=")", + children=children, + last=root, + ) + + for last, field in loop_last( + field for field in fields(obj) if field.repr + ): + child_node = _traverse(getattr(obj, field.name), depth=depth + 1) + child_node.key_repr = field.name + child_node.last = last + child_node.key_separator = "=" + append(child_node) + + pop_visited(obj_id) + + elif isinstance(obj, _CONTAINERS): + for container_type in _CONTAINERS: + if isinstance(obj, container_type): + obj_type = container_type + break + + obj_id = id(obj) + if obj_id in visited_ids: + # Recursion detected + return Node(value_repr="...") + push_visited(obj_id) + + open_brace, close_brace, empty = _BRACES[obj_type](obj) + + if reached_max_depth: + node = Node(value_repr=f"...", last=root) + elif obj_type.__repr__ != type(obj).__repr__: + node = Node(value_repr=to_repr(obj), last=root) + elif obj: + children = [] + node = Node( + open_brace=open_brace, + close_brace=close_brace, + children=children, + last=root, + ) + append = children.append + num_items = len(obj) + last_item_index = num_items - 1 + + if isinstance(obj, _MAPPING_CONTAINERS): + iter_items = iter(obj.items()) + if max_length is not None: + iter_items = islice(iter_items, max_length) + for index, (key, child) in enumerate(iter_items): + child_node = _traverse(child, depth=depth + 1) + child_node.key_repr = to_repr(key) + child_node.last = index == last_item_index + append(child_node) + else: + iter_values = iter(obj) + if max_length is not None: + iter_values = islice(iter_values, max_length) + for index, child in enumerate(iter_values): + child_node = _traverse(child, depth=depth + 1) + child_node.last = index == last_item_index + append(child_node) + if max_length is not None and num_items > max_length: + append(Node(value_repr=f"... +{num_items-max_length}", last=True)) + else: + node = Node(empty=empty, children=[], last=root) + + pop_visited(obj_id) + else: + node = Node(value_repr=to_repr(obj), last=root) + node.is_tuple = isinstance(obj, tuple) + return node + + node = _traverse(_object, root=True) + return node + + +def pretty_repr( + _object: Any, + *, + max_width: int = 80, + indent_size: int = 4, + max_length: Optional[int] = None, + max_string: Optional[int] = None, + max_depth: Optional[int] = None, + expand_all: bool = False, +) -> str: + """Prettify repr string by expanding on to new lines to fit within a given width. + + Args: + _object (Any): Object to repr. + max_width (int, optional): Desired maximum width of repr string. Defaults to 80. + indent_size (int, optional): Number of spaces to indent. Defaults to 4. + max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to None. + max_string (int, optional): Maximum length of string before truncating, or None to disable truncating. + Defaults to None. + max_depth (int, optional): Maximum depth of nested data structure, or None for no depth. + Defaults to None. + expand_all (bool, optional): Expand all containers regardless of available width. Defaults to False. + + Returns: + str: A possibly multi-line representation of the object. + """ + + if isinstance(_object, Node): + node = _object + else: + node = traverse( + _object, max_length=max_length, max_string=max_string, max_depth=max_depth + ) + repr_str = node.render( + max_width=max_width, indent_size=indent_size, expand_all=expand_all + ) + return repr_str + + +def pprint( + _object: Any, + *, + console: Optional["Console"] = None, + indent_guides: bool = True, + max_length: Optional[int] = None, + max_string: Optional[int] = None, + max_depth: Optional[int] = None, + expand_all: bool = False, +) -> None: + """A convenience function for pretty printing. + + Args: + _object (Any): Object to pretty print. + console (Console, optional): Console instance, or None to use default. Defaults to None. + max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to None. + max_string (int, optional): Maximum length of strings before truncating, or None to disable. Defaults to None. + max_depth (int, optional): Maximum depth for nested data structures, or None for unlimited depth. Defaults to None. + indent_guides (bool, optional): Enable indentation guides. Defaults to True. + expand_all (bool, optional): Expand all containers. Defaults to False. + """ + _console = get_console() if console is None else console + _console.print( + Pretty( + _object, + max_length=max_length, + max_string=max_string, + max_depth=max_depth, + indent_guides=indent_guides, + expand_all=expand_all, + overflow="ignore", + ), + soft_wrap=True, + ) + + +if __name__ == "__main__": # pragma: no cover + + class BrokenRepr: + def __repr__(self) -> str: + 1 / 0 + return "this will fail" + + d = defaultdict(int) + d["foo"] = 5 + data = { + "foo": [ + 1, + "Hello World!", + 100.123, + 323.232, + 432324.0, + {5, 6, 7, (1, 2, 3, 4), 8}, + ], + "bar": frozenset({1, 2, 3}), + "defaultdict": defaultdict( + list, {"crumble": ["apple", "rhubarb", "butter", "sugar", "flour"]} + ), + "counter": Counter( + [ + "apple", + "orange", + "pear", + "kumquat", + "kumquat", + "durian" * 100, + ] + ), + "atomic": (False, True, None), + "Broken": BrokenRepr(), + } + data["foo"].append(data) # type: ignore + + from pip._vendor.rich import print + + print(Pretty(data, indent_guides=True, max_string=20)) diff --git a/src/pip/_vendor/rich/progress.py b/src/pip/_vendor/rich/progress.py new file mode 100644 index 00000000000..1f670db4385 --- /dev/null +++ b/src/pip/_vendor/rich/progress.py @@ -0,0 +1,1036 @@ +from abc import ABC, abstractmethod +from collections import deque +from collections.abc import Sized +from dataclasses import dataclass, field +from datetime import timedelta +from math import ceil +from threading import Event, RLock, Thread +from types import TracebackType +from typing import ( + Any, + Callable, + Deque, + Dict, + Iterable, + List, + NamedTuple, + NewType, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, +) + +from . import filesize, get_console +from .console import Console, JustifyMethod, RenderableType, Group +from .highlighter import Highlighter +from .jupyter import JupyterMixin +from .live import Live +from .progress_bar import ProgressBar +from .spinner import Spinner +from .style import StyleType +from .table import Column, Table +from .text import Text, TextType + +TaskID = NewType("TaskID", int) + +ProgressType = TypeVar("ProgressType") + +GetTimeCallable = Callable[[], float] + + +class _TrackThread(Thread): + """A thread to periodically update progress.""" + + def __init__(self, progress: "Progress", task_id: "TaskID", update_period: float): + self.progress = progress + self.task_id = task_id + self.update_period = update_period + self.done = Event() + + self.completed = 0 + super().__init__() + + def run(self) -> None: + task_id = self.task_id + advance = self.progress.advance + update_period = self.update_period + last_completed = 0 + wait = self.done.wait + while not wait(update_period): + completed = self.completed + if last_completed != completed: + advance(task_id, completed - last_completed) + last_completed = completed + + self.progress.update(self.task_id, completed=self.completed, refresh=True) + + def __enter__(self) -> "_TrackThread": + self.start() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.done.set() + self.join() + + +def track( + sequence: Union[Sequence[ProgressType], Iterable[ProgressType]], + description: str = "Working...", + total: Optional[float] = None, + auto_refresh: bool = True, + console: Optional[Console] = None, + transient: bool = False, + get_time: Optional[Callable[[], float]] = None, + refresh_per_second: float = 10, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + update_period: float = 0.1, + disable: bool = False, +) -> Iterable[ProgressType]: + """Track progress by iterating over a sequence. + + Args: + sequence (Iterable[ProgressType]): A sequence (must support "len") you wish to iterate over. + description (str, optional): Description of task show next to progress bar. Defaults to "Working". + total: (float, optional): Total number of steps. Default is len(sequence). + auto_refresh (bool, optional): Automatic refresh, disable to force a refresh after each iteration. Default is True. + transient: (bool, optional): Clear the progress on exit. Defaults to False. + console (Console, optional): Console to write to. Default creates internal Console instance. + refresh_per_second (float): Number of times per second to refresh the progress information. Defaults to 10. + style (StyleType, optional): Style for the bar background. Defaults to "bar.back". + complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". + finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". + pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". + update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. + disable (bool, optional): Disable display of progress. + Returns: + Iterable[ProgressType]: An iterable of the values in the sequence. + + """ + + columns: List["ProgressColumn"] = ( + [TextColumn("[progress.description]{task.description}")] if description else [] + ) + columns.extend( + ( + BarColumn( + style=style, + complete_style=complete_style, + finished_style=finished_style, + pulse_style=pulse_style, + ), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeRemainingColumn(), + ) + ) + progress = Progress( + *columns, + auto_refresh=auto_refresh, + console=console, + transient=transient, + get_time=get_time, + refresh_per_second=refresh_per_second or 10, + disable=disable, + ) + + with progress: + yield from progress.track( + sequence, total=total, description=description, update_period=update_period + ) + + +class ProgressColumn(ABC): + """Base class for a widget to use in progress display.""" + + max_refresh: Optional[float] = None + + def __init__(self, table_column: Optional[Column] = None) -> None: + self._table_column = table_column + self._renderable_cache: Dict[TaskID, Tuple[float, RenderableType]] = {} + self._update_time: Optional[float] = None + + def get_table_column(self) -> Column: + """Get a table column, used to build tasks table.""" + return self._table_column or Column() + + def __call__(self, task: "Task") -> RenderableType: + """Called by the Progress object to return a renderable for the given task. + + Args: + task (Task): An object containing information regarding the task. + + Returns: + RenderableType: Anything renderable (including str). + """ + current_time = task.get_time() + if self.max_refresh is not None and not task.completed: + try: + timestamp, renderable = self._renderable_cache[task.id] + except KeyError: + pass + else: + if timestamp + self.max_refresh > current_time: + return renderable + + renderable = self.render(task) + self._renderable_cache[task.id] = (current_time, renderable) + return renderable + + @abstractmethod + def render(self, task: "Task") -> RenderableType: + """Should return a renderable object.""" + + +class RenderableColumn(ProgressColumn): + """A column to insert an arbitrary column. + + Args: + renderable (RenderableType, optional): Any renderable. Defaults to empty string. + """ + + def __init__( + self, renderable: RenderableType = "", *, table_column: Optional[Column] = None + ): + self.renderable = renderable + super().__init__(table_column=table_column) + + def render(self, task: "Task") -> RenderableType: + return self.renderable + + +class SpinnerColumn(ProgressColumn): + """A column with a 'spinner' animation. + + Args: + spinner_name (str, optional): Name of spinner animation. Defaults to "dots". + style (StyleType, optional): Style of spinner. Defaults to "progress.spinner". + speed (float, optional): Speed factor of spinner. Defaults to 1.0. + finished_text (TextType, optional): Text used when task is finished. Defaults to " ". + """ + + def __init__( + self, + spinner_name: str = "dots", + style: Optional[StyleType] = "progress.spinner", + speed: float = 1.0, + finished_text: TextType = " ", + table_column: Optional[Column] = None, + ): + self.spinner = Spinner(spinner_name, style=style, speed=speed) + self.finished_text = ( + Text.from_markup(finished_text) + if isinstance(finished_text, str) + else finished_text + ) + super().__init__(table_column=table_column) + + def set_spinner( + self, + spinner_name: str, + spinner_style: Optional[StyleType] = "progress.spinner", + speed: float = 1.0, + ) -> None: + """Set a new spinner. + + Args: + spinner_name (str): Spinner name, see python -m rich.spinner. + spinner_style (Optional[StyleType], optional): Spinner style. Defaults to "progress.spinner". + speed (float, optional): Speed factor of spinner. Defaults to 1.0. + """ + self.spinner = Spinner(spinner_name, style=spinner_style, speed=speed) + + def render(self, task: "Task") -> RenderableType: + text = ( + self.finished_text + if task.finished + else self.spinner.render(task.get_time()) + ) + return text + + +class TextColumn(ProgressColumn): + """A column containing text.""" + + def __init__( + self, + text_format: str, + style: StyleType = "none", + justify: JustifyMethod = "left", + markup: bool = True, + highlighter: Optional[Highlighter] = None, + table_column: Optional[Column] = None, + ) -> None: + self.text_format = text_format + self.justify: JustifyMethod = justify + self.style = style + self.markup = markup + self.highlighter = highlighter + super().__init__(table_column=table_column or Column(no_wrap=True)) + + def render(self, task: "Task") -> Text: + _text = self.text_format.format(task=task) + if self.markup: + text = Text.from_markup(_text, style=self.style, justify=self.justify) + else: + text = Text(_text, style=self.style, justify=self.justify) + if self.highlighter: + self.highlighter.highlight(text) + return text + + +class BarColumn(ProgressColumn): + """Renders a visual progress bar. + + Args: + bar_width (Optional[int], optional): Width of bar or None for full width. Defaults to 40. + style (StyleType, optional): Style for the bar background. Defaults to "bar.back". + complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". + finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". + pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". + """ + + def __init__( + self, + bar_width: Optional[int] = 40, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + table_column: Optional[Column] = None, + ) -> None: + self.bar_width = bar_width + self.style = style + self.complete_style = complete_style + self.finished_style = finished_style + self.pulse_style = pulse_style + super().__init__(table_column=table_column) + + def render(self, task: "Task") -> ProgressBar: + """Gets a progress bar widget for a task.""" + return ProgressBar( + total=max(0, task.total), + completed=max(0, task.completed), + width=None if self.bar_width is None else max(1, self.bar_width), + pulse=not task.started, + animation_time=task.get_time(), + style=self.style, + complete_style=self.complete_style, + finished_style=self.finished_style, + pulse_style=self.pulse_style, + ) + + +class TimeElapsedColumn(ProgressColumn): + """Renders time elapsed.""" + + def render(self, task: "Task") -> Text: + """Show time remaining.""" + elapsed = task.finished_time if task.finished else task.elapsed + if elapsed is None: + return Text("-:--:--", style="progress.elapsed") + delta = timedelta(seconds=int(elapsed)) + return Text(str(delta), style="progress.elapsed") + + +class TimeRemainingColumn(ProgressColumn): + """Renders estimated time remaining.""" + + # Only refresh twice a second to prevent jitter + max_refresh = 0.5 + + def render(self, task: "Task") -> Text: + """Show time remaining.""" + remaining = task.time_remaining + if remaining is None: + return Text("-:--:--", style="progress.remaining") + remaining_delta = timedelta(seconds=int(remaining)) + return Text(str(remaining_delta), style="progress.remaining") + + +class FileSizeColumn(ProgressColumn): + """Renders completed filesize.""" + + def render(self, task: "Task") -> Text: + """Show data completed.""" + data_size = filesize.decimal(int(task.completed)) + return Text(data_size, style="progress.filesize") + + +class TotalFileSizeColumn(ProgressColumn): + """Renders total filesize.""" + + def render(self, task: "Task") -> Text: + """Show data completed.""" + data_size = filesize.decimal(int(task.total)) + return Text(data_size, style="progress.filesize.total") + + +class DownloadColumn(ProgressColumn): + """Renders file size downloaded and total, e.g. '0.5/2.3 GB'. + + Args: + binary_units (bool, optional): Use binary units, KiB, MiB etc. Defaults to False. + """ + + def __init__( + self, binary_units: bool = False, table_column: Optional[Column] = None + ) -> None: + self.binary_units = binary_units + super().__init__(table_column=table_column) + + def render(self, task: "Task") -> Text: + """Calculate common unit for completed and total.""" + completed = int(task.completed) + total = int(task.total) + if self.binary_units: + unit, suffix = filesize.pick_unit_and_suffix( + total, + ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"], + 1024, + ) + else: + unit, suffix = filesize.pick_unit_and_suffix( + total, ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"], 1000 + ) + completed_ratio = completed / unit + total_ratio = total / unit + precision = 0 if unit == 1 else 1 + completed_str = f"{completed_ratio:,.{precision}f}" + total_str = f"{total_ratio:,.{precision}f}" + download_status = f"{completed_str}/{total_str} {suffix}" + download_text = Text(download_status, style="progress.download") + return download_text + + +class TransferSpeedColumn(ProgressColumn): + """Renders human readable transfer speed.""" + + def render(self, task: "Task") -> Text: + """Show data transfer speed.""" + speed = task.finished_speed or task.speed + if speed is None: + return Text("?", style="progress.data.speed") + data_speed = filesize.decimal(int(speed)) + return Text(f"{data_speed}/s", style="progress.data.speed") + + +class ProgressSample(NamedTuple): + """Sample of progress for a given time.""" + + timestamp: float + """Timestamp of sample.""" + completed: float + """Number of steps completed.""" + + +@dataclass +class Task: + """Information regarding a progress task. + + This object should be considered read-only outside of the :class:`~Progress` class. + + """ + + id: TaskID + """Task ID associated with this task (used in Progress methods).""" + + description: str + """str: Description of the task.""" + + total: float + """str: Total number of steps in this task.""" + + completed: float + """float: Number of steps completed""" + + _get_time: GetTimeCallable + """Callable to get the current time.""" + + finished_time: Optional[float] = None + """float: Time task was finished.""" + + visible: bool = True + """bool: Indicates if this task is visible in the progress display.""" + + fields: Dict[str, Any] = field(default_factory=dict) + """dict: Arbitrary fields passed in via Progress.update.""" + + start_time: Optional[float] = field(default=None, init=False, repr=False) + """Optional[float]: Time this task was started, or None if not started.""" + + stop_time: Optional[float] = field(default=None, init=False, repr=False) + """Optional[float]: Time this task was stopped, or None if not stopped.""" + + finished_speed: Optional[float] = None + """Optional[float]: The last speed for a finished task.""" + + _progress: Deque[ProgressSample] = field( + default_factory=deque, init=False, repr=False + ) + + _lock: RLock = field(repr=False, default_factory=RLock) + """Thread lock.""" + + def get_time(self) -> float: + """float: Get the current time, in seconds.""" + return self._get_time() + + @property + def started(self) -> bool: + """bool: Check if the task as started.""" + return self.start_time is not None + + @property + def remaining(self) -> float: + """float: Get the number of steps remaining.""" + return self.total - self.completed + + @property + def elapsed(self) -> Optional[float]: + """Optional[float]: Time elapsed since task was started, or ``None`` if the task hasn't started.""" + if self.start_time is None: + return None + if self.stop_time is not None: + return self.stop_time - self.start_time + return self.get_time() - self.start_time + + @property + def finished(self) -> bool: + """Check if the task has finished.""" + return self.finished_time is not None + + @property + def percentage(self) -> float: + """float: Get progress of task as a percentage.""" + if not self.total: + return 0.0 + completed = (self.completed / self.total) * 100.0 + completed = min(100.0, max(0.0, completed)) + return completed + + @property + def speed(self) -> Optional[float]: + """Optional[float]: Get the estimated speed in steps per second.""" + if self.start_time is None: + return None + with self._lock: + progress = self._progress + if not progress: + return None + total_time = progress[-1].timestamp - progress[0].timestamp + if total_time == 0: + return None + iter_progress = iter(progress) + next(iter_progress) + total_completed = sum(sample.completed for sample in iter_progress) + speed = total_completed / total_time + return speed + + @property + def time_remaining(self) -> Optional[float]: + """Optional[float]: Get estimated time to completion, or ``None`` if no data.""" + if self.finished: + return 0.0 + speed = self.speed + if not speed: + return None + estimate = ceil(self.remaining / speed) + return estimate + + def _reset(self) -> None: + """Reset progress.""" + self._progress.clear() + self.finished_time = None + self.finished_speed = None + + +class Progress(JupyterMixin): + """Renders an auto-updating progress bar(s). + + Args: + console (Console, optional): Optional Console instance. Default will an internal Console instance writing to stdout. + auto_refresh (bool, optional): Enable auto refresh. If disabled, you will need to call `refresh()`. + refresh_per_second (Optional[float], optional): Number of times per second to refresh the progress information or None to use default (10). Defaults to None. + speed_estimate_period: (float, optional): Period (in seconds) used to calculate the speed estimate. Defaults to 30. + transient: (bool, optional): Clear the progress on exit. Defaults to False. + redirect_stdout: (bool, optional): Enable redirection of stdout, so ``print`` may be used. Defaults to True. + redirect_stderr: (bool, optional): Enable redirection of stderr. Defaults to True. + get_time: (Callable, optional): A callable that gets the current time, or None to use Console.get_time. Defaults to None. + disable (bool, optional): Disable progress display. Defaults to False + expand (bool, optional): Expand tasks table to fit width. Defaults to False. + """ + + def __init__( + self, + *columns: Union[str, ProgressColumn], + console: Optional[Console] = None, + auto_refresh: bool = True, + refresh_per_second: float = 10, + speed_estimate_period: float = 30.0, + transient: bool = False, + redirect_stdout: bool = True, + redirect_stderr: bool = True, + get_time: Optional[GetTimeCallable] = None, + disable: bool = False, + expand: bool = False, + ) -> None: + assert ( + refresh_per_second is None or refresh_per_second > 0 + ), "refresh_per_second must be > 0" + self._lock = RLock() + self.columns = columns or ( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeRemainingColumn(), + ) + self.speed_estimate_period = speed_estimate_period + + self.disable = disable + self.expand = expand + self._tasks: Dict[TaskID, Task] = {} + self._task_index: TaskID = TaskID(0) + self.live = Live( + console=console or get_console(), + auto_refresh=auto_refresh, + refresh_per_second=refresh_per_second, + transient=transient, + redirect_stdout=redirect_stdout, + redirect_stderr=redirect_stderr, + get_renderable=self.get_renderable, + ) + self.get_time = get_time or self.console.get_time + self.print = self.console.print + self.log = self.console.log + + @property + def console(self) -> Console: + return self.live.console + + @property + def tasks(self) -> List[Task]: + """Get a list of Task instances.""" + with self._lock: + return list(self._tasks.values()) + + @property + def task_ids(self) -> List[TaskID]: + """A list of task IDs.""" + with self._lock: + return list(self._tasks.keys()) + + @property + def finished(self) -> bool: + """Check if all tasks have been completed.""" + with self._lock: + if not self._tasks: + return True + return all(task.finished for task in self._tasks.values()) + + def start(self) -> None: + """Start the progress display.""" + if not self.disable: + self.live.start(refresh=True) + + def stop(self) -> None: + """Stop the progress display.""" + self.live.stop() + if not self.console.is_interactive: + self.console.print() + + def __enter__(self) -> "Progress": + self.start() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.stop() + + def track( + self, + sequence: Union[Iterable[ProgressType], Sequence[ProgressType]], + total: Optional[float] = None, + task_id: Optional[TaskID] = None, + description: str = "Working...", + update_period: float = 0.1, + ) -> Iterable[ProgressType]: + """Track progress by iterating over a sequence. + + Args: + sequence (Sequence[ProgressType]): A sequence of values you want to iterate over and track progress. + total: (float, optional): Total number of steps. Default is len(sequence). + task_id: (TaskID): Task to track. Default is new task. + description: (str, optional): Description of task, if new task is created. + update_period (float, optional): Minimum time (in seconds) between calls to update(). Defaults to 0.1. + + Returns: + Iterable[ProgressType]: An iterable of values taken from the provided sequence. + """ + + if total is None: + if isinstance(sequence, Sized): + task_total = float(len(sequence)) + else: + raise ValueError( + f"unable to get size of {sequence!r}, please specify 'total'" + ) + else: + task_total = total + + if task_id is None: + task_id = self.add_task(description, total=task_total) + else: + self.update(task_id, total=task_total) + + if self.live.auto_refresh: + with _TrackThread(self, task_id, update_period) as track_thread: + for value in sequence: + yield value + track_thread.completed += 1 + else: + advance = self.advance + refresh = self.refresh + for value in sequence: + yield value + advance(task_id, 1) + refresh() + + def start_task(self, task_id: TaskID) -> None: + """Start a task. + + Starts a task (used when calculating elapsed time). You may need to call this manually, + if you called ``add_task`` with ``start=False``. + + Args: + task_id (TaskID): ID of task. + """ + with self._lock: + task = self._tasks[task_id] + if task.start_time is None: + task.start_time = self.get_time() + + def stop_task(self, task_id: TaskID) -> None: + """Stop a task. + + This will freeze the elapsed time on the task. + + Args: + task_id (TaskID): ID of task. + """ + with self._lock: + task = self._tasks[task_id] + current_time = self.get_time() + if task.start_time is None: + task.start_time = current_time + task.stop_time = current_time + + def update( + self, + task_id: TaskID, + *, + total: Optional[float] = None, + completed: Optional[float] = None, + advance: Optional[float] = None, + description: Optional[str] = None, + visible: Optional[bool] = None, + refresh: bool = False, + **fields: Any, + ) -> None: + """Update information associated with a task. + + Args: + task_id (TaskID): Task id (returned by add_task). + total (float, optional): Updates task.total if not None. + completed (float, optional): Updates task.completed if not None. + advance (float, optional): Add a value to task.completed if not None. + description (str, optional): Change task description if not None. + visible (bool, optional): Set visible flag if not None. + refresh (bool): Force a refresh of progress information. Default is False. + **fields (Any): Additional data fields required for rendering. + """ + with self._lock: + task = self._tasks[task_id] + completed_start = task.completed + + if total is not None and total != task.total: + task.total = total + task._reset() + if advance is not None: + task.completed += advance + if completed is not None: + task.completed = completed + if description is not None: + task.description = description + if visible is not None: + task.visible = visible + task.fields.update(fields) + update_completed = task.completed - completed_start + + current_time = self.get_time() + old_sample_time = current_time - self.speed_estimate_period + _progress = task._progress + + popleft = _progress.popleft + while _progress and _progress[0].timestamp < old_sample_time: + popleft() + while len(_progress) > 1000: + popleft() + if update_completed > 0: + _progress.append(ProgressSample(current_time, update_completed)) + if task.completed >= task.total and task.finished_time is None: + task.finished_time = task.elapsed + + if refresh: + self.refresh() + + def reset( + self, + task_id: TaskID, + *, + start: bool = True, + total: Optional[float] = None, + completed: int = 0, + visible: Optional[bool] = None, + description: Optional[str] = None, + **fields: Any, + ) -> None: + """Reset a task so completed is 0 and the clock is reset. + + Args: + task_id (TaskID): ID of task. + start (bool, optional): Start the task after reset. Defaults to True. + total (float, optional): New total steps in task, or None to use current total. Defaults to None. + completed (int, optional): Number of steps completed. Defaults to 0. + **fields (str): Additional data fields required for rendering. + """ + current_time = self.get_time() + with self._lock: + task = self._tasks[task_id] + task._reset() + task.start_time = current_time if start else None + if total is not None: + task.total = total + task.completed = completed + if visible is not None: + task.visible = visible + if fields: + task.fields = fields + if description is not None: + task.description = description + task.finished_time = None + self.refresh() + + def advance(self, task_id: TaskID, advance: float = 1) -> None: + """Advance task by a number of steps. + + Args: + task_id (TaskID): ID of task. + advance (float): Number of steps to advance. Default is 1. + """ + current_time = self.get_time() + with self._lock: + task = self._tasks[task_id] + completed_start = task.completed + task.completed += advance + update_completed = task.completed - completed_start + old_sample_time = current_time - self.speed_estimate_period + _progress = task._progress + + popleft = _progress.popleft + while _progress and _progress[0].timestamp < old_sample_time: + popleft() + while len(_progress) > 1000: + popleft() + _progress.append(ProgressSample(current_time, update_completed)) + if task.completed >= task.total and task.finished_time is None: + task.finished_time = task.elapsed + task.finished_speed = task.speed + + def refresh(self) -> None: + """Refresh (render) the progress information.""" + if not self.disable and self.live.is_started: + self.live.refresh() + + def get_renderable(self) -> RenderableType: + """Get a renderable for the progress display.""" + renderable = Group(*self.get_renderables()) + return renderable + + def get_renderables(self) -> Iterable[RenderableType]: + """Get a number of renderables for the progress display.""" + table = self.make_tasks_table(self.tasks) + yield table + + def make_tasks_table(self, tasks: Iterable[Task]) -> Table: + """Get a table to render the Progress display. + + Args: + tasks (Iterable[Task]): An iterable of Task instances, one per row of the table. + + Returns: + Table: A table instance. + """ + table_columns = ( + ( + Column(no_wrap=True) + if isinstance(_column, str) + else _column.get_table_column().copy() + ) + for _column in self.columns + ) + table = Table.grid(*table_columns, padding=(0, 1), expand=self.expand) + + for task in tasks: + if task.visible: + table.add_row( + *( + ( + column.format(task=task) + if isinstance(column, str) + else column(task) + ) + for column in self.columns + ) + ) + return table + + def __rich__(self) -> RenderableType: + """Makes the Progress class itself renderable.""" + with self._lock: + return self.get_renderable() + + def add_task( + self, + description: str, + start: bool = True, + total: float = 100.0, + completed: int = 0, + visible: bool = True, + **fields: Any, + ) -> TaskID: + """Add a new 'task' to the Progress display. + + Args: + description (str): A description of the task. + start (bool, optional): Start the task immediately (to calculate elapsed time). If set to False, + you will need to call `start` manually. Defaults to True. + total (float, optional): Number of total steps in the progress if know. Defaults to 100. + completed (int, optional): Number of steps completed so far.. Defaults to 0. + visible (bool, optional): Enable display of the task. Defaults to True. + **fields (str): Additional data fields required for rendering. + + Returns: + TaskID: An ID you can use when calling `update`. + """ + with self._lock: + task = Task( + self._task_index, + description, + total, + completed, + visible=visible, + fields=fields, + _get_time=self.get_time, + _lock=self._lock, + ) + self._tasks[self._task_index] = task + if start: + self.start_task(self._task_index) + new_task_index = self._task_index + self._task_index = TaskID(int(self._task_index) + 1) + self.refresh() + return new_task_index + + def remove_task(self, task_id: TaskID) -> None: + """Delete a task if it exists. + + Args: + task_id (TaskID): A task ID. + + """ + with self._lock: + del self._tasks[task_id] + + +if __name__ == "__main__": # pragma: no coverage + + import random + import time + + from .panel import Panel + from .rule import Rule + from .syntax import Syntax + from .table import Table + + syntax = Syntax( + '''def loop_last(values: Iterable[T]) -> Iterable[Tuple[bool, T]]: + """Iterate and generate a tuple with a flag for last value.""" + iter_values = iter(values) + try: + previous_value = next(iter_values) + except StopIteration: + return + for value in iter_values: + yield False, previous_value + previous_value = value + yield True, previous_value''', + "python", + line_numbers=True, + ) + + table = Table("foo", "bar", "baz") + table.add_row("1", "2", "3") + + progress_renderables = [ + "Text may be printed while the progress bars are rendering.", + Panel("In fact, [i]any[/i] renderable will work"), + "Such as [magenta]tables[/]...", + table, + "Pretty printed structures...", + {"type": "example", "text": "Pretty printed"}, + "Syntax...", + syntax, + Rule("Give it a try!"), + ] + + from itertools import cycle + + examples = cycle(progress_renderables) + + console = Console(record=True) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TimeRemainingColumn(), + TimeElapsedColumn(), + console=console, + transient=True, + ) as progress: + + task1 = progress.add_task("[red]Downloading", total=1000) + task2 = progress.add_task("[green]Processing", total=1000) + task3 = progress.add_task("[yellow]Thinking", total=1000, start=False) + + while not progress.finished: + progress.update(task1, advance=0.5) + progress.update(task2, advance=0.3) + time.sleep(0.01) + if random.randint(0, 100) < 1: + progress.log(next(examples)) diff --git a/src/pip/_vendor/rich/progress_bar.py b/src/pip/_vendor/rich/progress_bar.py new file mode 100644 index 00000000000..1797b5f786e --- /dev/null +++ b/src/pip/_vendor/rich/progress_bar.py @@ -0,0 +1,216 @@ +import math +from functools import lru_cache +from time import monotonic +from typing import Iterable, List, Optional + +from .color import Color, blend_rgb +from .color_triplet import ColorTriplet +from .console import Console, ConsoleOptions, RenderResult +from .jupyter import JupyterMixin +from .measure import Measurement +from .segment import Segment +from .style import Style, StyleType + +# Number of characters before 'pulse' animation repeats +PULSE_SIZE = 20 + + +class ProgressBar(JupyterMixin): + """Renders a (progress) bar. Used by rich.progress. + + Args: + total (float, optional): Number of steps in the bar. Defaults to 100. + completed (float, optional): Number of steps completed. Defaults to 0. + width (int, optional): Width of the bar, or ``None`` for maximum width. Defaults to None. + pulse (bool, optional): Enable pulse effect. Defaults to False. + style (StyleType, optional): Style for the bar background. Defaults to "bar.back". + complete_style (StyleType, optional): Style for the completed bar. Defaults to "bar.complete". + finished_style (StyleType, optional): Style for a finished bar. Defaults to "bar.done". + pulse_style (StyleType, optional): Style for pulsing bars. Defaults to "bar.pulse". + animation_time (Optional[float], optional): Time in seconds to use for animation, or None to use system time. + """ + + def __init__( + self, + total: float = 100.0, + completed: float = 0, + width: Optional[int] = None, + pulse: bool = False, + style: StyleType = "bar.back", + complete_style: StyleType = "bar.complete", + finished_style: StyleType = "bar.finished", + pulse_style: StyleType = "bar.pulse", + animation_time: Optional[float] = None, + ): + self.total = total + self.completed = completed + self.width = width + self.pulse = pulse + self.style = style + self.complete_style = complete_style + self.finished_style = finished_style + self.pulse_style = pulse_style + self.animation_time = animation_time + + self._pulse_segments: Optional[List[Segment]] = None + + def __repr__(self) -> str: + return f"" + + @property + def percentage_completed(self) -> float: + """Calculate percentage complete.""" + completed = (self.completed / self.total) * 100.0 + completed = min(100, max(0.0, completed)) + return completed + + @lru_cache(maxsize=16) + def _get_pulse_segments( + self, + fore_style: Style, + back_style: Style, + color_system: str, + no_color: bool, + ascii: bool = False, + ) -> List[Segment]: + """Get a list of segments to render a pulse animation. + + Returns: + List[Segment]: A list of segments, one segment per character. + """ + bar = "-" if ascii else "━" + segments: List[Segment] = [] + if color_system not in ("standard", "eight_bit", "truecolor") or no_color: + segments += [Segment(bar, fore_style)] * (PULSE_SIZE // 2) + segments += [Segment(" " if no_color else bar, back_style)] * ( + PULSE_SIZE - (PULSE_SIZE // 2) + ) + return segments + + append = segments.append + fore_color = ( + fore_style.color.get_truecolor() + if fore_style.color + else ColorTriplet(255, 0, 255) + ) + back_color = ( + back_style.color.get_truecolor() + if back_style.color + else ColorTriplet(0, 0, 0) + ) + cos = math.cos + pi = math.pi + _Segment = Segment + _Style = Style + from_triplet = Color.from_triplet + + for index in range(PULSE_SIZE): + position = index / PULSE_SIZE + fade = 0.5 + cos((position * pi * 2)) / 2.0 + color = blend_rgb(fore_color, back_color, cross_fade=fade) + append(_Segment(bar, _Style(color=from_triplet(color)))) + return segments + + def update(self, completed: float, total: Optional[float] = None) -> None: + """Update progress with new values. + + Args: + completed (float): Number of steps completed. + total (float, optional): Total number of steps, or ``None`` to not change. Defaults to None. + """ + self.completed = completed + self.total = total if total is not None else self.total + + def _render_pulse( + self, console: Console, width: int, ascii: bool = False + ) -> Iterable[Segment]: + """Renders the pulse animation. + + Args: + console (Console): Console instance. + width (int): Width in characters of pulse animation. + + Returns: + RenderResult: [description] + + Yields: + Iterator[Segment]: Segments to render pulse + """ + fore_style = console.get_style(self.pulse_style, default="white") + back_style = console.get_style(self.style, default="black") + + pulse_segments = self._get_pulse_segments( + fore_style, back_style, console.color_system, console.no_color, ascii=ascii + ) + segment_count = len(pulse_segments) + current_time = ( + monotonic() if self.animation_time is None else self.animation_time + ) + segments = pulse_segments * (int(width / segment_count) + 2) + offset = int(-current_time * 15) % segment_count + segments = segments[offset : offset + width] + yield from segments + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + + width = min(self.width or options.max_width, options.max_width) + ascii = options.legacy_windows or options.ascii_only + if self.pulse: + yield from self._render_pulse(console, width, ascii=ascii) + return + + completed = min(self.total, max(0, self.completed)) + + bar = "-" if ascii else "━" + half_bar_right = " " if ascii else "╸" + half_bar_left = " " if ascii else "╺" + complete_halves = ( + int(width * 2 * completed / self.total) if self.total else width * 2 + ) + bar_count = complete_halves // 2 + half_bar_count = complete_halves % 2 + style = console.get_style(self.style) + complete_style = console.get_style( + self.complete_style if self.completed < self.total else self.finished_style + ) + _Segment = Segment + if bar_count: + yield _Segment(bar * bar_count, complete_style) + if half_bar_count: + yield _Segment(half_bar_right * half_bar_count, complete_style) + + if not console.no_color: + remaining_bars = width - bar_count - half_bar_count + if remaining_bars and console.color_system is not None: + if not half_bar_count and bar_count: + yield _Segment(half_bar_left, style) + remaining_bars -= 1 + if remaining_bars: + yield _Segment(bar * remaining_bars, style) + + def __rich_measure__( + self, console: Console, options: ConsoleOptions + ) -> Measurement: + return ( + Measurement(self.width, self.width) + if self.width is not None + else Measurement(4, options.max_width) + ) + + +if __name__ == "__main__": # pragma: no cover + console = Console() + bar = ProgressBar(width=50, total=100) + + import time + + console.show_cursor(False) + for n in range(0, 101, 1): + bar.update(n) + console.print(bar) + console.file.write("\r") + time.sleep(0.05) + console.show_cursor(True) + console.print() diff --git a/src/pip/_vendor/rich/prompt.py b/src/pip/_vendor/rich/prompt.py new file mode 100644 index 00000000000..b2cea2b529a --- /dev/null +++ b/src/pip/_vendor/rich/prompt.py @@ -0,0 +1,376 @@ +from typing import Any, Generic, List, Optional, TextIO, TypeVar, Union, overload + +from . import get_console +from .console import Console +from .text import Text, TextType + +PromptType = TypeVar("PromptType") +DefaultType = TypeVar("DefaultType") + + +class PromptError(Exception): + """Exception base class for prompt related errors.""" + + +class InvalidResponse(PromptError): + """Exception to indicate a response was invalid. Raise this within process_response() to indicate an error + and provide an error message. + + Args: + message (Union[str, Text]): Error message. + """ + + def __init__(self, message: TextType) -> None: + self.message = message + + def __rich__(self) -> TextType: + return self.message + + +class PromptBase(Generic[PromptType]): + """Ask the user for input until a valid response is received. This is the base class, see one of + the concrete classes for examples. + + Args: + prompt (TextType, optional): Prompt text. Defaults to "". + console (Console, optional): A Console instance or None to use global console. Defaults to None. + password (bool, optional): Enable password input. Defaults to False. + choices (List[str], optional): A list of valid choices. Defaults to None. + show_default (bool, optional): Show default in prompt. Defaults to True. + show_choices (bool, optional): Show choices in prompt. Defaults to True. + """ + + response_type: type = str + + validate_error_message = "[prompt.invalid]Please enter a valid value" + illegal_choice_message = ( + "[prompt.invalid.choice]Please select one of the available options" + ) + prompt_suffix = ": " + + choices: Optional[List[str]] = None + + def __init__( + self, + prompt: TextType = "", + *, + console: Optional[Console] = None, + password: bool = False, + choices: Optional[List[str]] = None, + show_default: bool = True, + show_choices: bool = True, + ) -> None: + self.console = console or get_console() + self.prompt = ( + Text.from_markup(prompt, style="prompt") + if isinstance(prompt, str) + else prompt + ) + self.password = password + if choices is not None: + self.choices = choices + self.show_default = show_default + self.show_choices = show_choices + + @classmethod + @overload + def ask( + cls, + prompt: TextType = "", + *, + console: Optional[Console] = None, + password: bool = False, + choices: Optional[List[str]] = None, + show_default: bool = True, + show_choices: bool = True, + default: DefaultType, + stream: Optional[TextIO] = None, + ) -> Union[DefaultType, PromptType]: + ... + + @classmethod + @overload + def ask( + cls, + prompt: TextType = "", + *, + console: Optional[Console] = None, + password: bool = False, + choices: Optional[List[str]] = None, + show_default: bool = True, + show_choices: bool = True, + stream: Optional[TextIO] = None, + ) -> PromptType: + ... + + @classmethod + def ask( + cls, + prompt: TextType = "", + *, + console: Optional[Console] = None, + password: bool = False, + choices: Optional[List[str]] = None, + show_default: bool = True, + show_choices: bool = True, + default: Any = ..., + stream: Optional[TextIO] = None, + ) -> Any: + """Shortcut to construct and run a prompt loop and return the result. + + Example: + >>> filename = Prompt.ask("Enter a filename") + + Args: + prompt (TextType, optional): Prompt text. Defaults to "". + console (Console, optional): A Console instance or None to use global console. Defaults to None. + password (bool, optional): Enable password input. Defaults to False. + choices (List[str], optional): A list of valid choices. Defaults to None. + show_default (bool, optional): Show default in prompt. Defaults to True. + show_choices (bool, optional): Show choices in prompt. Defaults to True. + stream (TextIO, optional): Optional text file open for reading to get input. Defaults to None. + """ + _prompt = cls( + prompt, + console=console, + password=password, + choices=choices, + show_default=show_default, + show_choices=show_choices, + ) + return _prompt(default=default, stream=stream) + + def render_default(self, default: DefaultType) -> Text: + """Turn the supplied default in to a Text instance. + + Args: + default (DefaultType): Default value. + + Returns: + Text: Text containing rendering of default value. + """ + return Text(f"({default})", "prompt.default") + + def make_prompt(self, default: DefaultType) -> Text: + """Make prompt text. + + Args: + default (DefaultType): Default value. + + Returns: + Text: Text to display in prompt. + """ + prompt = self.prompt.copy() + prompt.end = "" + + if self.show_choices and self.choices: + _choices = "/".join(self.choices) + choices = f"[{_choices}]" + prompt.append(" ") + prompt.append(choices, "prompt.choices") + + if ( + default != ... + and self.show_default + and isinstance(default, (str, self.response_type)) + ): + prompt.append(" ") + _default = self.render_default(default) + prompt.append(_default) + + prompt.append(self.prompt_suffix) + + return prompt + + @classmethod + def get_input( + cls, + console: Console, + prompt: TextType, + password: bool, + stream: Optional[TextIO] = None, + ) -> str: + """Get input from user. + + Args: + console (Console): Console instance. + prompt (TextType): Prompt text. + password (bool): Enable password entry. + + Returns: + str: String from user. + """ + return console.input(prompt, password=password, stream=stream) + + def check_choice(self, value: str) -> bool: + """Check value is in the list of valid choices. + + Args: + value (str): Value entered by user. + + Returns: + bool: True if choice was valid, otherwise False. + """ + assert self.choices is not None + return value.strip() in self.choices + + def process_response(self, value: str) -> PromptType: + """Process response from user, convert to prompt type. + + Args: + value (str): String typed by user. + + Raises: + InvalidResponse: If ``value`` is invalid. + + Returns: + PromptType: The value to be returned from ask method. + """ + value = value.strip() + try: + return_value = self.response_type(value) + except ValueError: + raise InvalidResponse(self.validate_error_message) + + if self.choices is not None and not self.check_choice(value): + raise InvalidResponse(self.illegal_choice_message) + + return return_value # type: ignore + + def on_validate_error(self, value: str, error: InvalidResponse) -> None: + """Called to handle validation error. + + Args: + value (str): String entered by user. + error (InvalidResponse): Exception instance the initiated the error. + """ + self.console.print(error) + + def pre_prompt(self) -> None: + """Hook to display something before the prompt.""" + + @overload + def __call__(self, *, stream: Optional[TextIO] = None) -> PromptType: + ... + + @overload + def __call__( + self, *, default: DefaultType, stream: Optional[TextIO] = None + ) -> Union[PromptType, DefaultType]: + ... + + def __call__(self, *, default: Any = ..., stream: Optional[TextIO] = None) -> Any: + """Run the prompt loop. + + Args: + default (Any, optional): Optional default value. + + Returns: + PromptType: Processed value. + """ + while True: + self.pre_prompt() + prompt = self.make_prompt(default) + value = self.get_input(self.console, prompt, self.password, stream=stream) + if value == "" and default != ...: + return default + try: + return_value = self.process_response(value) + except InvalidResponse as error: + self.on_validate_error(value, error) + continue + else: + return return_value + + +class Prompt(PromptBase[str]): + """A prompt that returns a str. + + Example: + >>> name = Prompt.ask("Enter your name") + + + """ + + response_type = str + + +class IntPrompt(PromptBase[int]): + """A prompt that returns an integer. + + Example: + >>> burrito_count = IntPrompt.ask("How many burritos do you want to order") + + """ + + response_type = int + validate_error_message = "[prompt.invalid]Please enter a valid integer number" + + +class FloatPrompt(PromptBase[int]): + """A prompt that returns a float. + + Example: + >>> temperature = FloatPrompt.ask("Enter desired temperature") + + """ + + response_type = float + validate_error_message = "[prompt.invalid]Please enter a number" + + +class Confirm(PromptBase[bool]): + """A yes / no confirmation prompt. + + Example: + >>> if Confirm.ask("Continue"): + run_job() + + """ + + response_type = bool + validate_error_message = "[prompt.invalid]Please enter Y or N" + choices: List[str] = ["y", "n"] + + def render_default(self, default: DefaultType) -> Text: + """Render the default as (y) or (n) rather than True/False.""" + yes, no = self.choices + return Text(f"({yes})" if default else f"({no})", style="prompt.default") + + def process_response(self, value: str) -> bool: + """Convert choices to a bool.""" + value = value.strip().lower() + if value not in self.choices: + raise InvalidResponse(self.validate_error_message) + return value == self.choices[0] + + +if __name__ == "__main__": # pragma: no cover + + from pip._vendor.rich import print + + if Confirm.ask("Run [i]prompt[/i] tests?", default=True): + while True: + result = IntPrompt.ask( + ":rocket: Enter a number between [b]1[/b] and [b]10[/b]", default=5 + ) + if result >= 1 and result <= 10: + break + print(":pile_of_poo: [prompt.invalid]Number must be between 1 and 10") + print(f"number={result}") + + while True: + password = Prompt.ask( + "Please enter a password [cyan](must be at least 5 characters)", + password=True, + ) + if len(password) >= 5: + break + print("[prompt.invalid]password too short") + print(f"password={password!r}") + + fruit = Prompt.ask("Enter a fruit", choices=["apple", "orange", "pear"]) + print(f"fruit={fruit!r}") + + else: + print("[b]OK :loudly_crying_face:") diff --git a/src/pip/_vendor/rich/protocol.py b/src/pip/_vendor/rich/protocol.py new file mode 100644 index 00000000000..6248052119a --- /dev/null +++ b/src/pip/_vendor/rich/protocol.py @@ -0,0 +1,42 @@ +from typing import Any, Callable, cast, Set, TYPE_CHECKING +from inspect import isclass + +if TYPE_CHECKING: + from pip._vendor.rich.console import RenderableType + +_GIBBERISH = """aihwerij235234ljsdnp34ksodfipwoe234234jlskjdf""" + + +def is_renderable(check_object: Any) -> bool: + """Check if an object may be rendered by Rich.""" + return ( + isinstance(check_object, str) + or hasattr(check_object, "__rich__") + or hasattr(check_object, "__rich_console__") + ) + + +def rich_cast(renderable: object) -> "RenderableType": + """Cast an object to a renderable by calling __rich__ if present. + + Args: + renderable (object): A potentially renderable object + + Returns: + object: The result of recursively calling __rich__. + """ + from pip._vendor.rich.console import RenderableType + + rich_visited_set: Set[type] = set() # Prevent potential infinite loop + while hasattr(renderable, "__rich__") and not isclass(renderable): + # Detect object which claim to have all the attributes + if hasattr(renderable, _GIBBERISH): + return repr(renderable) + cast_method = getattr(renderable, "__rich__") + renderable = cast_method() + renderable_type = type(renderable) + if renderable_type in rich_visited_set: + break + rich_visited_set.add(renderable_type) + + return cast(RenderableType, renderable) diff --git a/news/275aa0e8-ebb1-4eaf-aee0-e5582a8c5d58.trivial.rst b/src/pip/_vendor/rich/py.typed similarity index 100% rename from news/275aa0e8-ebb1-4eaf-aee0-e5582a8c5d58.trivial.rst rename to src/pip/_vendor/rich/py.typed diff --git a/src/pip/_vendor/rich/region.py b/src/pip/_vendor/rich/region.py new file mode 100644 index 00000000000..75b3631c387 --- /dev/null +++ b/src/pip/_vendor/rich/region.py @@ -0,0 +1,10 @@ +from typing import NamedTuple + + +class Region(NamedTuple): + """Defines a rectangular region of the screen.""" + + x: int + y: int + width: int + height: int diff --git a/src/pip/_vendor/rich/repr.py b/src/pip/_vendor/rich/repr.py new file mode 100644 index 00000000000..17147fd4be2 --- /dev/null +++ b/src/pip/_vendor/rich/repr.py @@ -0,0 +1,151 @@ +from functools import partial +import inspect + +from typing import ( + Any, + Callable, + Iterable, + List, + Optional, + overload, + Union, + Tuple, + Type, + TypeVar, +) + + +T = TypeVar("T") + + +Result = Iterable[Union[Any, Tuple[Any], Tuple[str, Any], Tuple[str, Any, Any]]] +RichReprResult = Result + + +class ReprError(Exception): + """An error occurred when attempting to build a repr.""" + + +@overload +def auto(cls: Optional[T]) -> T: + ... + + +@overload +def auto(*, angular: bool = False) -> Callable[[T], T]: + ... + + +def auto( + cls: Optional[T] = None, *, angular: Optional[bool] = None +) -> Union[T, Callable[[T], T]]: + """Class decorator to create __repr__ from __rich_repr__""" + + def do_replace(cls: Type[T], angular: Optional[bool] = None) -> Type[T]: + def auto_repr(self: Type[T]) -> str: + """Create repr string from __rich_repr__""" + repr_str: List[str] = [] + append = repr_str.append + + angular = getattr(self.__rich_repr__, "angular", False) # type: ignore + for arg in self.__rich_repr__(): # type: ignore + if isinstance(arg, tuple): + if len(arg) == 1: + append(repr(arg[0])) + else: + key, value, *default = arg + if key is None: + append(repr(value)) + else: + if len(default) and default[0] == value: + continue + append(f"{key}={value!r}") + else: + append(repr(arg)) + if angular: + return f"<{self.__class__.__name__} {' '.join(repr_str)}>" + else: + return f"{self.__class__.__name__}({', '.join(repr_str)})" + + def auto_rich_repr(self: Type[T]) -> Result: + """Auto generate __rich_rep__ from signature of __init__""" + try: + signature = inspect.signature(self.__init__) ## type: ignore + for name, param in signature.parameters.items(): + if param.kind == param.POSITIONAL_ONLY: + yield getattr(self, name) + elif param.kind in ( + param.POSITIONAL_OR_KEYWORD, + param.KEYWORD_ONLY, + ): + if param.default == param.empty: + yield getattr(self, param.name) + else: + yield param.name, getattr(self, param.name), param.default + except Exception as error: + raise ReprError( + f"Failed to auto generate __rich_repr__; {error}" + ) from None + + if not hasattr(cls, "__rich_repr__"): + auto_rich_repr.__doc__ = "Build a rich repr" + cls.__rich_repr__ = auto_rich_repr # type: ignore + + auto_repr.__doc__ = "Return repr(self)" + cls.__repr__ = auto_repr # type: ignore + if angular is not None: + cls.__rich_repr__.angular = angular # type: ignore + return cls + + if cls is None: + return partial(do_replace, angular=angular) # type: ignore + else: + return do_replace(cls, angular=angular) # type: ignore + + +@overload +def rich_repr(cls: Optional[T]) -> T: + ... + + +@overload +def rich_repr(*, angular: bool = False) -> Callable[[T], T]: + ... + + +def rich_repr( + cls: Optional[T] = None, *, angular: bool = False +) -> Union[T, Callable[[T], T]]: + if cls is None: + return auto(angular=angular) + else: + return auto(cls) + + +if __name__ == "__main__": + + @auto + class Foo: + def __rich_repr__(self) -> Result: + yield "foo" + yield "bar", {"shopping": ["eggs", "ham", "pineapple"]} + yield "buy", "hand sanitizer" + + foo = Foo() + from pip._vendor.rich.console import Console + + console = Console() + + console.rule("Standard repr") + console.print(foo) + + console.print(foo, width=60) + console.print(foo, width=30) + + console.rule("Angular repr") + Foo.__rich_repr__.angular = True # type: ignore + + console.print(foo) + + console.print(foo, width=60) + console.print(foo, width=30) diff --git a/src/pip/_vendor/rich/rule.py b/src/pip/_vendor/rich/rule.py new file mode 100644 index 00000000000..ce4754f6a8c --- /dev/null +++ b/src/pip/_vendor/rich/rule.py @@ -0,0 +1,115 @@ +from typing import Union + +from .align import AlignMethod +from .cells import cell_len, set_cell_size +from .console import Console, ConsoleOptions, RenderResult +from .jupyter import JupyterMixin +from .style import Style +from .text import Text + + +class Rule(JupyterMixin): + """A console renderable to draw a horizontal rule (line). + + Args: + title (Union[str, Text], optional): Text to render in the rule. Defaults to "". + characters (str, optional): Character(s) used to draw the line. Defaults to "─". + style (StyleType, optional): Style of Rule. Defaults to "rule.line". + end (str, optional): Character at end of Rule. defaults to "\\\\n" + align (str, optional): How to align the title, one of "left", "center", or "right". Defaults to "center". + """ + + def __init__( + self, + title: Union[str, Text] = "", + *, + characters: str = "─", + style: Union[str, Style] = "rule.line", + end: str = "\n", + align: AlignMethod = "center", + ) -> None: + if cell_len(characters) < 1: + raise ValueError( + "'characters' argument must have a cell width of at least 1" + ) + if align not in ("left", "center", "right"): + raise ValueError( + f'invalid value for align, expected "left", "center", "right" (not {align!r})' + ) + self.title = title + self.characters = characters + self.style = style + self.end = end + self.align = align + + def __repr__(self) -> str: + return f"Rule({self.title!r}, {self.characters!r})" + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + width = options.max_width + + # Python3.6 doesn't have an isascii method on str + isascii = getattr(str, "isascii", None) or ( + lambda s: all(ord(c) < 128 for c in s) + ) + characters = ( + "-" + if (options.ascii_only and not isascii(self.characters)) + else self.characters + ) + + chars_len = cell_len(characters) + if not self.title: + rule_text = Text(characters * ((width // chars_len) + 1), self.style) + rule_text.truncate(width) + rule_text.plain = set_cell_size(rule_text.plain, width) + yield rule_text + return + + if isinstance(self.title, Text): + title_text = self.title + else: + title_text = console.render_str(self.title, style="rule.text") + + title_text.plain = title_text.plain.replace("\n", " ") + title_text.expand_tabs() + rule_text = Text(end=self.end) + + if self.align == "center": + title_text.truncate(width - 4, overflow="ellipsis") + side_width = (width - cell_len(title_text.plain)) // 2 + left = Text(characters * (side_width // chars_len + 1)) + left.truncate(side_width - 1) + right_length = width - cell_len(left.plain) - cell_len(title_text.plain) + right = Text(characters * (side_width // chars_len + 1)) + right.truncate(right_length) + rule_text.append(left.plain + " ", self.style) + rule_text.append(title_text) + rule_text.append(" " + right.plain, self.style) + elif self.align == "left": + title_text.truncate(width - 2, overflow="ellipsis") + rule_text.append(title_text) + rule_text.append(" ") + rule_text.append(characters * (width - rule_text.cell_len), self.style) + elif self.align == "right": + title_text.truncate(width - 2, overflow="ellipsis") + rule_text.append(characters * (width - title_text.cell_len - 1), self.style) + rule_text.append(" ") + rule_text.append(title_text) + + rule_text.plain = set_cell_size(rule_text.plain, width) + yield rule_text + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console + import sys + + try: + text = sys.argv[1] + except IndexError: + text = "Hello, World" + console = Console() + console.print(Rule(title=text)) diff --git a/src/pip/_vendor/rich/scope.py b/src/pip/_vendor/rich/scope.py new file mode 100644 index 00000000000..6822b8ca542 --- /dev/null +++ b/src/pip/_vendor/rich/scope.py @@ -0,0 +1,86 @@ +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Optional, Tuple + +from .highlighter import ReprHighlighter +from .panel import Panel +from .pretty import Pretty +from .table import Table +from .text import Text, TextType + +if TYPE_CHECKING: + from .console import ConsoleRenderable + + +def render_scope( + scope: "Mapping[str, Any]", + *, + title: Optional[TextType] = None, + sort_keys: bool = True, + indent_guides: bool = False, + max_length: Optional[int] = None, + max_string: Optional[int] = None, +) -> "ConsoleRenderable": + """Render python variables in a given scope. + + Args: + scope (Mapping): A mapping containing variable names and values. + title (str, optional): Optional title. Defaults to None. + sort_keys (bool, optional): Enable sorting of items. Defaults to True. + indent_guides (bool, optional): Enable indentaton guides. Defaults to False. + max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to None. + max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to None. + + Returns: + ConsoleRenderable: A renderable object. + """ + highlighter = ReprHighlighter() + items_table = Table.grid(padding=(0, 1), expand=False) + items_table.add_column(justify="right") + + def sort_items(item: Tuple[str, Any]) -> Tuple[bool, str]: + """Sort special variables first, then alphabetically.""" + key, _ = item + return (not key.startswith("__"), key.lower()) + + items = sorted(scope.items(), key=sort_items) if sort_keys else scope.items() + for key, value in items: + key_text = Text.assemble( + (key, "scope.key.special" if key.startswith("__") else "scope.key"), + (" =", "scope.equals"), + ) + items_table.add_row( + key_text, + Pretty( + value, + highlighter=highlighter, + indent_guides=indent_guides, + max_length=max_length, + max_string=max_string, + ), + ) + return Panel.fit( + items_table, + title=title, + border_style="scope.border", + padding=(0, 1), + ) + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich import print + + print() + + def test(foo: float, bar: float) -> None: + list_of_things = [1, 2, 3, None, 4, True, False, "Hello World"] + dict_of_things = { + "version": "1.1", + "method": "confirmFruitPurchase", + "params": [["apple", "orange", "mangoes", "pomelo"], 1.123], + "id": "194521489", + } + print(render_scope(locals(), title="[i]locals", sort_keys=False)) + + test(20.3423, 3.1427) + print() diff --git a/src/pip/_vendor/rich/screen.py b/src/pip/_vendor/rich/screen.py new file mode 100644 index 00000000000..7f416e1e799 --- /dev/null +++ b/src/pip/_vendor/rich/screen.py @@ -0,0 +1,54 @@ +from typing import Optional, TYPE_CHECKING + +from .segment import Segment +from .style import StyleType +from ._loop import loop_last + + +if TYPE_CHECKING: + from .console import ( + Console, + ConsoleOptions, + RenderResult, + RenderableType, + Group, + ) + + +class Screen: + """A renderable that fills the terminal screen and crops excess. + + Args: + renderable (RenderableType): Child renderable. + style (StyleType, optional): Optional background style. Defaults to None. + """ + + renderable: "RenderableType" + + def __init__( + self, + *renderables: "RenderableType", + style: Optional[StyleType] = None, + application_mode: bool = False, + ) -> None: + from pip._vendor.rich.console import Group + + self.renderable = Group(*renderables) + self.style = style + self.application_mode = application_mode + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + width, height = options.size + style = console.get_style(self.style) if self.style else None + render_options = options.update(width=width, height=height) + lines = console.render_lines( + self.renderable or "", render_options, style=style, pad=True + ) + lines = Segment.set_shape(lines, width, height, style=style) + new_line = Segment("\n\r") if self.application_mode else Segment.line() + for last, line in loop_last(lines): + yield from line + if not last: + yield new_line diff --git a/src/pip/_vendor/rich/segment.py b/src/pip/_vendor/rich/segment.py new file mode 100644 index 00000000000..94ca73076d8 --- /dev/null +++ b/src/pip/_vendor/rich/segment.py @@ -0,0 +1,720 @@ +from enum import IntEnum +from functools import lru_cache +from itertools import filterfalse +from logging import getLogger +from operator import attrgetter +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Sequence, + Tuple, + Type, + Union, +) + +from .cells import ( + _is_single_cell_widths, + cell_len, + get_character_cell_size, + set_cell_size, +) +from .repr import Result, rich_repr +from .style import Style + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderResult + +log = getLogger("rich") + + +class ControlType(IntEnum): + """Non-printable control codes which typically translate to ANSI codes.""" + + BELL = 1 + CARRIAGE_RETURN = 2 + HOME = 3 + CLEAR = 4 + SHOW_CURSOR = 5 + HIDE_CURSOR = 6 + ENABLE_ALT_SCREEN = 7 + DISABLE_ALT_SCREEN = 8 + CURSOR_UP = 9 + CURSOR_DOWN = 10 + CURSOR_FORWARD = 11 + CURSOR_BACKWARD = 12 + CURSOR_MOVE_TO_COLUMN = 13 + CURSOR_MOVE_TO = 14 + ERASE_IN_LINE = 15 + + +ControlCode = Union[ + Tuple[ControlType], Tuple[ControlType, int], Tuple[ControlType, int, int] +] + + +@rich_repr() +class Segment(NamedTuple): + """A piece of text with associated style. Segments are produced by the Console render process and + are ultimately converted in to strings to be written to the terminal. + + Args: + text (str): A piece of text. + style (:class:`~rich.style.Style`, optional): An optional style to apply to the text. + control (Tuple[ControlCode..], optional): Optional sequence of control codes. + """ + + text: str = "" + """Raw text.""" + style: Optional[Style] = None + """An optional style.""" + control: Optional[Sequence[ControlCode]] = None + """Optional sequence of control codes.""" + + def __rich_repr__(self) -> Result: + yield self.text + if self.control is None: + if self.style is not None: + yield self.style + else: + yield self.style + yield self.control + + def __bool__(self) -> bool: + """Check if the segment contains text.""" + return bool(self.text) + + @property + def cell_length(self) -> int: + """Get cell length of segment.""" + return 0 if self.control else cell_len(self.text) + + @property + def is_control(self) -> bool: + """Check if the segment contains control codes.""" + return self.control is not None + + @classmethod + @lru_cache(1024 * 16) + def _split_cells(cls, segment: "Segment", cut: int) -> Tuple["Segment", "Segment"]: # type: ignore + + text, style, control = segment + _Segment = Segment + + cell_length = segment.cell_length + if cut >= cell_length: + return segment, _Segment("", style, control) + + cell_size = get_character_cell_size + + pos = int((cut / cell_length) * len(text)) + + before = text[:pos] + cell_pos = cell_len(before) + if cell_pos == cut: + return ( + _Segment(before, style, control), + _Segment(text[pos:], style, control), + ) + while pos < len(text): + char = text[pos] + pos += 1 + cell_pos += cell_size(char) + before = text[:pos] + if cell_pos == cut: + return ( + _Segment(before, style, control), + _Segment(text[pos:], style, control), + ) + if cell_pos > cut: + return ( + _Segment(before[: pos - 1] + " ", style, control), + _Segment(" " + text[pos:], style, control), + ) + + def split_cells(self, cut: int) -> Tuple["Segment", "Segment"]: + """Split segment in to two segments at the specified column. + + If the cut point falls in the middle of a 2-cell wide character then it is replaced + by two spaces, to preserve the display width of the parent segment. + + Returns: + Tuple[Segment, Segment]: Two segments. + """ + text, style, control = self + + if _is_single_cell_widths(text): + # Fast path with all 1 cell characters + if cut >= len(text): + return self, Segment("", style, control) + return ( + Segment(text[:cut], style, control), + Segment(text[cut:], style, control), + ) + + return self._split_cells(self, cut) + + @classmethod + def line(cls) -> "Segment": + """Make a new line segment.""" + return cls("\n") + + @classmethod + def apply_style( + cls, + segments: Iterable["Segment"], + style: Optional[Style] = None, + post_style: Optional[Style] = None, + ) -> Iterable["Segment"]: + """Apply style(s) to an iterable of segments. + + Returns an iterable of segments where the style is replaced by ``style + segment.style + post_style``. + + Args: + segments (Iterable[Segment]): Segments to process. + style (Style, optional): Base style. Defaults to None. + post_style (Style, optional): Style to apply on top of segment style. Defaults to None. + + Returns: + Iterable[Segments]: A new iterable of segments (possibly the same iterable). + """ + result_segments = segments + if style: + apply = style.__add__ + result_segments = ( + cls(text, None if control else apply(_style), control) + for text, _style, control in result_segments + ) + if post_style: + result_segments = ( + cls( + text, + ( + None + if control + else (_style + post_style if _style else post_style) + ), + control, + ) + for text, _style, control in result_segments + ) + return result_segments + + @classmethod + def filter_control( + cls, segments: Iterable["Segment"], is_control: bool = False + ) -> Iterable["Segment"]: + """Filter segments by ``is_control`` attribute. + + Args: + segments (Iterable[Segment]): An iterable of Segment instances. + is_control (bool, optional): is_control flag to match in search. + + Returns: + Iterable[Segment]: And iterable of Segment instances. + + """ + if is_control: + return filter(attrgetter("control"), segments) + else: + return filterfalse(attrgetter("control"), segments) + + @classmethod + def split_lines(cls, segments: Iterable["Segment"]) -> Iterable[List["Segment"]]: + """Split a sequence of segments in to a list of lines. + + Args: + segments (Iterable[Segment]): Segments potentially containing line feeds. + + Yields: + Iterable[List[Segment]]: Iterable of segment lists, one per line. + """ + line: List[Segment] = [] + append = line.append + + for segment in segments: + if "\n" in segment.text and not segment.control: + text, style, _ = segment + while text: + _text, new_line, text = text.partition("\n") + if _text: + append(cls(_text, style)) + if new_line: + yield line + line = [] + append = line.append + else: + append(segment) + if line: + yield line + + @classmethod + def split_and_crop_lines( + cls, + segments: Iterable["Segment"], + length: int, + style: Optional[Style] = None, + pad: bool = True, + include_new_lines: bool = True, + ) -> Iterable[List["Segment"]]: + """Split segments in to lines, and crop lines greater than a given length. + + Args: + segments (Iterable[Segment]): An iterable of segments, probably + generated from console.render. + length (int): Desired line length. + style (Style, optional): Style to use for any padding. + pad (bool): Enable padding of lines that are less than `length`. + + Returns: + Iterable[List[Segment]]: An iterable of lines of segments. + """ + line: List[Segment] = [] + append = line.append + + adjust_line_length = cls.adjust_line_length + new_line_segment = cls("\n") + + for segment in segments: + if "\n" in segment.text and not segment.control: + text, style, _ = segment + while text: + _text, new_line, text = text.partition("\n") + if _text: + append(cls(_text, style)) + if new_line: + cropped_line = adjust_line_length( + line, length, style=style, pad=pad + ) + if include_new_lines: + cropped_line.append(new_line_segment) + yield cropped_line + del line[:] + else: + append(segment) + if line: + yield adjust_line_length(line, length, style=style, pad=pad) + + @classmethod + def adjust_line_length( + cls, + line: List["Segment"], + length: int, + style: Optional[Style] = None, + pad: bool = True, + ) -> List["Segment"]: + """Adjust a line to a given width (cropping or padding as required). + + Args: + segments (Iterable[Segment]): A list of segments in a single line. + length (int): The desired width of the line. + style (Style, optional): The style of padding if used (space on the end). Defaults to None. + pad (bool, optional): Pad lines with spaces if they are shorter than `length`. Defaults to True. + + Returns: + List[Segment]: A line of segments with the desired length. + """ + line_length = sum(segment.cell_length for segment in line) + new_line: List[Segment] + + if line_length < length: + if pad: + new_line = line + [cls(" " * (length - line_length), style)] + else: + new_line = line[:] + elif line_length > length: + new_line = [] + append = new_line.append + line_length = 0 + for segment in line: + segment_length = segment.cell_length + if line_length + segment_length < length or segment.control: + append(segment) + line_length += segment_length + else: + text, segment_style, _ = segment + text = set_cell_size(text, length - line_length) + append(cls(text, segment_style)) + break + else: + new_line = line[:] + return new_line + + @classmethod + def get_line_length(cls, line: List["Segment"]) -> int: + """Get the length of list of segments. + + Args: + line (List[Segment]): A line encoded as a list of Segments (assumes no '\\\\n' characters), + + Returns: + int: The length of the line. + """ + _cell_len = cell_len + return sum(_cell_len(segment.text) for segment in line) + + @classmethod + def get_shape(cls, lines: List[List["Segment"]]) -> Tuple[int, int]: + """Get the shape (enclosing rectangle) of a list of lines. + + Args: + lines (List[List[Segment]]): A list of lines (no '\\\\n' characters). + + Returns: + Tuple[int, int]: Width and height in characters. + """ + get_line_length = cls.get_line_length + max_width = max(get_line_length(line) for line in lines) if lines else 0 + return (max_width, len(lines)) + + @classmethod + def set_shape( + cls, + lines: List[List["Segment"]], + width: int, + height: Optional[int] = None, + style: Optional[Style] = None, + new_lines: bool = False, + ) -> List[List["Segment"]]: + """Set the shape of a list of lines (enclosing rectangle). + + Args: + lines (List[List[Segment]]): A list of lines. + width (int): Desired width. + height (int, optional): Desired height or None for no change. + style (Style, optional): Style of any padding added. + new_lines (bool, optional): Padded lines should include "\n". Defaults to False. + + Returns: + List[List[Segment]]: New list of lines. + """ + _height = height or len(lines) + + blank = ( + [cls(" " * width + "\n", style)] if new_lines else [cls(" " * width, style)] + ) + + adjust_line_length = cls.adjust_line_length + shaped_lines = lines[:_height] + shaped_lines[:] = [ + adjust_line_length(line, width, style=style) for line in lines + ] + if len(shaped_lines) < _height: + shaped_lines.extend([blank] * (_height - len(shaped_lines))) + return shaped_lines + + @classmethod + def align_top( + cls: Type["Segment"], + lines: List[List["Segment"]], + width: int, + height: int, + style: Style, + new_lines: bool = False, + ) -> List[List["Segment"]]: + """Aligns lines to top (adds extra lines to bottom as required). + + Args: + lines (List[List[Segment]]): A list of lines. + width (int): Desired width. + height (int, optional): Desired height or None for no change. + style (Style): Style of any padding added. + new_lines (bool, optional): Padded lines should include "\n". Defaults to False. + + Returns: + List[List[Segment]]: New list of lines. + """ + extra_lines = height - len(lines) + if not extra_lines: + return lines[:] + lines = lines[:height] + blank = cls(" " * width + "\n", style) if new_lines else cls(" " * width, style) + lines = lines + [[blank]] * extra_lines + return lines + + @classmethod + def align_bottom( + cls: Type["Segment"], + lines: List[List["Segment"]], + width: int, + height: int, + style: Style, + new_lines: bool = False, + ) -> List[List["Segment"]]: + """Aligns render to bottom (adds extra lines above as required). + + Args: + lines (List[List[Segment]]): A list of lines. + width (int): Desired width. + height (int, optional): Desired height or None for no change. + style (Style): Style of any padding added. Defaults to None. + new_lines (bool, optional): Padded lines should include "\n". Defaults to False. + + Returns: + List[List[Segment]]: New list of lines. + """ + extra_lines = height - len(lines) + if not extra_lines: + return lines[:] + lines = lines[:height] + blank = cls(" " * width + "\n", style) if new_lines else cls(" " * width, style) + lines = [[blank]] * extra_lines + lines + return lines + + @classmethod + def align_middle( + cls: Type["Segment"], + lines: List[List["Segment"]], + width: int, + height: int, + style: Style, + new_lines: bool = False, + ) -> List[List["Segment"]]: + """Aligns lines to middle (adds extra lines to above and below as required). + + Args: + lines (List[List[Segment]]): A list of lines. + width (int): Desired width. + height (int, optional): Desired height or None for no change. + style (Style): Style of any padding added. + new_lines (bool, optional): Padded lines should include "\n". Defaults to False. + + Returns: + List[List[Segment]]: New list of lines. + """ + extra_lines = height - len(lines) + if not extra_lines: + return lines[:] + lines = lines[:height] + blank = cls(" " * width + "\n", style) if new_lines else cls(" " * width, style) + top_lines = extra_lines // 2 + bottom_lines = extra_lines - top_lines + lines = [[blank]] * top_lines + lines + [[blank]] * bottom_lines + return lines + + @classmethod + def simplify(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]: + """Simplify an iterable of segments by combining contiguous segments with the same style. + + Args: + segments (Iterable[Segment]): An iterable of segments. + + Returns: + Iterable[Segment]: A possibly smaller iterable of segments that will render the same way. + """ + iter_segments = iter(segments) + try: + last_segment = next(iter_segments) + except StopIteration: + return + + _Segment = Segment + for segment in iter_segments: + if last_segment.style == segment.style and not segment.control: + last_segment = _Segment( + last_segment.text + segment.text, last_segment.style + ) + else: + yield last_segment + last_segment = segment + yield last_segment + + @classmethod + def strip_links(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]: + """Remove all links from an iterable of styles. + + Args: + segments (Iterable[Segment]): An iterable segments. + + Yields: + Segment: Segments with link removed. + """ + for segment in segments: + if segment.control or segment.style is None: + yield segment + else: + text, style, _control = segment + yield cls(text, style.update_link(None) if style else None) + + @classmethod + def strip_styles(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]: + """Remove all styles from an iterable of segments. + + Args: + segments (Iterable[Segment]): An iterable segments. + + Yields: + Segment: Segments with styles replace with None + """ + for text, _style, control in segments: + yield cls(text, None, control) + + @classmethod + def remove_color(cls, segments: Iterable["Segment"]) -> Iterable["Segment"]: + """Remove all color from an iterable of segments. + + Args: + segments (Iterable[Segment]): An iterable segments. + + Yields: + Segment: Segments with colorless style. + """ + + cache: Dict[Style, Style] = {} + for text, style, control in segments: + if style: + colorless_style = cache.get(style) + if colorless_style is None: + colorless_style = style.without_color + cache[style] = colorless_style + yield cls(text, colorless_style, control) + else: + yield cls(text, None, control) + + @classmethod + def divide( + cls, segments: Iterable["Segment"], cuts: Iterable[int] + ) -> Iterable[List["Segment"]]: + """Divides an iterable of segments in to portions. + + Args: + cuts (Iterable[int]): Cell positions where to divide. + + Yields: + [Iterable[List[Segment]]]: An iterable of Segments in List. + """ + split_segments: List["Segment"] = [] + add_segment = split_segments.append + + iter_cuts = iter(cuts) + + while True: + try: + cut = next(iter_cuts) + except StopIteration: + return [] + if cut != 0: + break + yield [] + pos = 0 + + for segment in segments: + while segment.text: + end_pos = pos + segment.cell_length + if end_pos < cut: + add_segment(segment) + pos = end_pos + break + + try: + if end_pos == cut: + add_segment(segment) + yield split_segments[:] + del split_segments[:] + pos = end_pos + break + else: + before, segment = segment.split_cells(cut - pos) + add_segment(before) + yield split_segments[:] + del split_segments[:] + pos = cut + finally: + try: + cut = next(iter_cuts) + except StopIteration: + if split_segments: + yield split_segments[:] + return + yield split_segments[:] + + +class Segments: + """A simple renderable to render an iterable of segments. This class may be useful if + you want to print segments outside of a __rich_console__ method. + + Args: + segments (Iterable[Segment]): An iterable of segments. + new_lines (bool, optional): Add new lines between segments. Defaults to False. + """ + + def __init__(self, segments: Iterable[Segment], new_lines: bool = False) -> None: + self.segments = list(segments) + self.new_lines = new_lines + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + if self.new_lines: + line = Segment.line() + for segment in self.segments: + yield segment + yield line + else: + yield from self.segments + + +class SegmentLines: + def __init__(self, lines: Iterable[List[Segment]], new_lines: bool = False) -> None: + """A simple renderable containing a number of lines of segments. May be used as an intermediate + in rendering process. + + Args: + lines (Iterable[List[Segment]]): Lists of segments forming lines. + new_lines (bool, optional): Insert new lines after each line. Defaults to False. + """ + self.lines = list(lines) + self.new_lines = new_lines + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + if self.new_lines: + new_line = Segment.line() + for line in self.lines: + yield from line + yield new_line + else: + for line in self.lines: + yield from line + + +if __name__ == "__main__": + + if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console + from pip._vendor.rich.syntax import Syntax + from pip._vendor.rich.text import Text + + code = """from rich.console import Console + console = Console() + text = Text.from_markup("Hello, [bold magenta]World[/]!") + console.print(text)""" + + text = Text.from_markup("Hello, [bold magenta]World[/]!") + + console = Console() + + console.rule("rich.Segment") + console.print( + "A Segment is the last step in the Rich render process before generating text with ANSI codes." + ) + console.print("\nConsider the following code:\n") + console.print(Syntax(code, "python", line_numbers=True)) + console.print() + console.print( + "When you call [b]print()[/b], Rich [i]renders[/i] the object in to the the following:\n" + ) + fragments = list(console.render(text)) + console.print(fragments) + console.print() + console.print( + "The Segments are then processed to produce the following output:\n" + ) + console.print(text) + console.print( + "\nYou will only need to know this if you are implementing your own Rich renderables." + ) diff --git a/src/pip/_vendor/rich/spinner.py b/src/pip/_vendor/rich/spinner.py new file mode 100644 index 00000000000..5b13b1e9ba2 --- /dev/null +++ b/src/pip/_vendor/rich/spinner.py @@ -0,0 +1,134 @@ +from typing import cast, List, Optional, TYPE_CHECKING + +from ._spinners import SPINNERS +from .measure import Measurement +from .table import Table +from .text import Text + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderResult, RenderableType + from .style import StyleType + + +class Spinner: + def __init__( + self, + name: str, + text: "RenderableType" = "", + *, + style: Optional["StyleType"] = None, + speed: float = 1.0, + ) -> None: + """A spinner animation. + + Args: + name (str): Name of spinner (run python -m rich.spinner). + text (RenderableType, optional): A renderable to display at the right of the spinner (str or Text typically). Defaults to "". + style (StyleType, optional): Style for spinner animation. Defaults to None. + speed (float, optional): Speed factor for animation. Defaults to 1.0. + + Raises: + KeyError: If name isn't one of the supported spinner animations. + """ + try: + spinner = SPINNERS[name] + except KeyError: + raise KeyError(f"no spinner called {name!r}") + self.text = Text.from_markup(text) if isinstance(text, str) else text + self.frames = cast(List[str], spinner["frames"])[:] + self.interval = cast(float, spinner["interval"]) + self.start_time: Optional[float] = None + self.style = style + self.speed = speed + self.frame_no_offset: float = 0.0 + self._update_speed = 0.0 + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + yield self.render(console.get_time()) + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> Measurement: + text = self.render(0) + return Measurement.get(console, options, text) + + def render(self, time: float) -> "RenderableType": + """Render the spinner for a given time. + + Args: + time (float): Time in seconds. + + Returns: + RenderableType: A renderable containing animation frame. + """ + if self.start_time is None: + self.start_time = time + + frame_no = ((time - self.start_time) * self.speed) / ( + self.interval / 1000.0 + ) + self.frame_no_offset + frame = Text( + self.frames[int(frame_no) % len(self.frames)], style=self.style or "" + ) + + if self._update_speed: + self.frame_no_offset = frame_no + self.start_time = time + self.speed = self._update_speed + self._update_speed = 0.0 + + if not self.text: + return frame + elif isinstance(self.text, (str, Text)): + return Text.assemble(frame, " ", self.text) + else: + table = Table.grid(padding=1) + table.add_row(frame, self.text) + return table + + def update( + self, + *, + text: "RenderableType" = "", + style: Optional["StyleType"] = None, + speed: Optional[float] = None, + ) -> None: + """Updates attributes of a spinner after it has been started. + + Args: + text (RenderableType, optional): A renderable to display at the right of the spinner (str or Text typically). Defaults to "". + style (StyleType, optional): Style for spinner animation. Defaults to None. + speed (float, optional): Speed factor for animation. Defaults to None. + """ + if text: + self.text = Text.from_markup(text) if isinstance(text, str) else text + if style: + self.style = style + if speed: + self._update_speed = speed + + +if __name__ == "__main__": # pragma: no cover + from time import sleep + + from .columns import Columns + from .panel import Panel + from .live import Live + + all_spinners = Columns( + [ + Spinner(spinner_name, text=Text(repr(spinner_name), style="green")) + for spinner_name in sorted(SPINNERS.keys()) + ], + column_first=True, + expand=True, + ) + + with Live( + Panel(all_spinners, title="Spinners", border_style="blue"), + refresh_per_second=20, + ) as live: + while True: + sleep(0.1) diff --git a/src/pip/_vendor/rich/status.py b/src/pip/_vendor/rich/status.py new file mode 100644 index 00000000000..09eff405ec1 --- /dev/null +++ b/src/pip/_vendor/rich/status.py @@ -0,0 +1,132 @@ +from types import TracebackType +from typing import Optional, Type + +from .console import Console, RenderableType +from .jupyter import JupyterMixin +from .live import Live +from .spinner import Spinner +from .style import StyleType + + +class Status(JupyterMixin): + """Displays a status indicator with a 'spinner' animation. + + Args: + status (RenderableType): A status renderable (str or Text typically). + console (Console, optional): Console instance to use, or None for global console. Defaults to None. + spinner (str, optional): Name of spinner animation (see python -m rich.spinner). Defaults to "dots". + spinner_style (StyleType, optional): Style of spinner. Defaults to "status.spinner". + speed (float, optional): Speed factor for spinner animation. Defaults to 1.0. + refresh_per_second (float, optional): Number of refreshes per second. Defaults to 12.5. + """ + + def __init__( + self, + status: RenderableType, + *, + console: Optional[Console] = None, + spinner: str = "dots", + spinner_style: StyleType = "status.spinner", + speed: float = 1.0, + refresh_per_second: float = 12.5, + ): + self.status = status + self.spinner_style = spinner_style + self.speed = speed + self._spinner = Spinner(spinner, text=status, style=spinner_style, speed=speed) + self._live = Live( + self.renderable, + console=console, + refresh_per_second=refresh_per_second, + transient=True, + ) + + @property + def renderable(self) -> Spinner: + return self._spinner + + @property + def console(self) -> "Console": + """Get the Console used by the Status objects.""" + return self._live.console + + def update( + self, + status: Optional[RenderableType] = None, + *, + spinner: Optional[str] = None, + spinner_style: Optional[StyleType] = None, + speed: Optional[float] = None, + ) -> None: + """Update status. + + Args: + status (Optional[RenderableType], optional): New status renderable or None for no change. Defaults to None. + spinner (Optional[str], optional): New spinner or None for no change. Defaults to None. + spinner_style (Optional[StyleType], optional): New spinner style or None for no change. Defaults to None. + speed (Optional[float], optional): Speed factor for spinner animation or None for no change. Defaults to None. + """ + if status is not None: + self.status = status + if spinner_style is not None: + self.spinner_style = spinner_style + if speed is not None: + self.speed = speed + if spinner is not None: + self._spinner = Spinner( + spinner, text=self.status, style=self.spinner_style, speed=self.speed + ) + self._live.update(self.renderable, refresh=True) + else: + self._spinner.update( + text=self.status, style=self.spinner_style, speed=self.speed + ) + + def start(self) -> None: + """Start the status animation.""" + self._live.start() + + def stop(self) -> None: + """Stop the spinner animation.""" + self._live.stop() + + def __rich__(self) -> RenderableType: + return self.renderable + + def __enter__(self) -> "Status": + self.start() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.stop() + + +if __name__ == "__main__": # pragma: no cover + + from time import sleep + + from .console import Console + + console = Console() + with console.status("[magenta]Covid detector booting up") as status: + sleep(3) + console.log("Importing advanced AI") + sleep(3) + console.log("Advanced Covid AI Ready") + sleep(3) + status.update(status="[bold blue] Scanning for Covid", spinner="earth") + sleep(3) + console.log("Found 10,000,000,000 copies of Covid32.exe") + sleep(3) + status.update( + status="[bold red]Moving Covid32.exe to Trash", + spinner="bouncingBall", + spinner_style="yellow", + ) + sleep(5) + console.print("[bold green]Covid deleted successfully") diff --git a/src/pip/_vendor/rich/style.py b/src/pip/_vendor/rich/style.py new file mode 100644 index 00000000000..0787c33147b --- /dev/null +++ b/src/pip/_vendor/rich/style.py @@ -0,0 +1,785 @@ +import sys +from functools import lru_cache +from marshal import loads, dumps +from random import randint +from typing import Any, cast, Dict, Iterable, List, Optional, Type, Union + +from . import errors +from .color import Color, ColorParseError, ColorSystem, blend_rgb +from .repr import rich_repr, Result +from .terminal_theme import DEFAULT_TERMINAL_THEME, TerminalTheme + + +# Style instances and style definitions are often interchangeable +StyleType = Union[str, "Style"] + + +class _Bit: + """A descriptor to get/set a style attribute bit.""" + + __slots__ = ["bit"] + + def __init__(self, bit_no: int) -> None: + self.bit = 1 << bit_no + + def __get__(self, obj: "Style", objtype: Type["Style"]) -> Optional[bool]: + if obj._set_attributes & self.bit: + return obj._attributes & self.bit != 0 + return None + + +@rich_repr +class Style: + """A terminal style. + + A terminal style consists of a color (`color`), a background color (`bgcolor`), and a number of attributes, such + as bold, italic etc. The attributes have 3 states: they can either be on + (``True``), off (``False``), or not set (``None``). + + Args: + color (Union[Color, str], optional): Color of terminal text. Defaults to None. + bgcolor (Union[Color, str], optional): Color of terminal background. Defaults to None. + bold (bool, optional): Enable bold text. Defaults to None. + dim (bool, optional): Enable dim text. Defaults to None. + italic (bool, optional): Enable italic text. Defaults to None. + underline (bool, optional): Enable underlined text. Defaults to None. + blink (bool, optional): Enabled blinking text. Defaults to None. + blink2 (bool, optional): Enable fast blinking text. Defaults to None. + reverse (bool, optional): Enabled reverse text. Defaults to None. + conceal (bool, optional): Enable concealed text. Defaults to None. + strike (bool, optional): Enable strikethrough text. Defaults to None. + underline2 (bool, optional): Enable doubly underlined text. Defaults to None. + frame (bool, optional): Enable framed text. Defaults to None. + encircle (bool, optional): Enable encircled text. Defaults to None. + overline (bool, optional): Enable overlined text. Defaults to None. + link (str, link): Link URL. Defaults to None. + + """ + + _color: Optional[Color] + _bgcolor: Optional[Color] + _attributes: int + _set_attributes: int + _hash: int + _null: bool + _meta: Optional[bytes] + + __slots__ = [ + "_color", + "_bgcolor", + "_attributes", + "_set_attributes", + "_link", + "_link_id", + "_ansi", + "_style_definition", + "_hash", + "_null", + "_meta", + ] + + # maps bits on to SGR parameter + _style_map = { + 0: "1", + 1: "2", + 2: "3", + 3: "4", + 4: "5", + 5: "6", + 6: "7", + 7: "8", + 8: "9", + 9: "21", + 10: "51", + 11: "52", + 12: "53", + } + + STYLE_ATTRIBUTES = { + "dim": "dim", + "d": "dim", + "bold": "bold", + "b": "bold", + "italic": "italic", + "i": "italic", + "underline": "underline", + "u": "underline", + "blink": "blink", + "blink2": "blink2", + "reverse": "reverse", + "r": "reverse", + "conceal": "conceal", + "c": "conceal", + "strike": "strike", + "s": "strike", + "underline2": "underline2", + "uu": "underline2", + "frame": "frame", + "encircle": "encircle", + "overline": "overline", + "o": "overline", + } + + def __init__( + self, + *, + color: Optional[Union[Color, str]] = None, + bgcolor: Optional[Union[Color, str]] = None, + bold: Optional[bool] = None, + dim: Optional[bool] = None, + italic: Optional[bool] = None, + underline: Optional[bool] = None, + blink: Optional[bool] = None, + blink2: Optional[bool] = None, + reverse: Optional[bool] = None, + conceal: Optional[bool] = None, + strike: Optional[bool] = None, + underline2: Optional[bool] = None, + frame: Optional[bool] = None, + encircle: Optional[bool] = None, + overline: Optional[bool] = None, + link: Optional[str] = None, + meta: Optional[Dict[str, Any]] = None, + ): + self._ansi: Optional[str] = None + self._style_definition: Optional[str] = None + + def _make_color(color: Union[Color, str]) -> Color: + return color if isinstance(color, Color) else Color.parse(color) + + self._color = None if color is None else _make_color(color) + self._bgcolor = None if bgcolor is None else _make_color(bgcolor) + self._set_attributes = sum( + ( + bold is not None, + dim is not None and 2, + italic is not None and 4, + underline is not None and 8, + blink is not None and 16, + blink2 is not None and 32, + reverse is not None and 64, + conceal is not None and 128, + strike is not None and 256, + underline2 is not None and 512, + frame is not None and 1024, + encircle is not None and 2048, + overline is not None and 4096, + ) + ) + self._attributes = ( + sum( + ( + bold and 1 or 0, + dim and 2 or 0, + italic and 4 or 0, + underline and 8 or 0, + blink and 16 or 0, + blink2 and 32 or 0, + reverse and 64 or 0, + conceal and 128 or 0, + strike and 256 or 0, + underline2 and 512 or 0, + frame and 1024 or 0, + encircle and 2048 or 0, + overline and 4096 or 0, + ) + ) + if self._set_attributes + else 0 + ) + + self._link = link + self._link_id = f"{randint(0, 999999)}" if link else "" + self._meta = None if meta is None else dumps(meta) + self._hash = hash( + ( + self._color, + self._bgcolor, + self._attributes, + self._set_attributes, + link, + self._meta, + ) + ) + self._null = not (self._set_attributes or color or bgcolor or link or meta) + + @classmethod + def null(cls) -> "Style": + """Create an 'null' style, equivalent to Style(), but more performant.""" + return NULL_STYLE + + @classmethod + def from_color( + cls, color: Optional[Color] = None, bgcolor: Optional[Color] = None + ) -> "Style": + """Create a new style with colors and no attributes. + + Returns: + color (Optional[Color]): A (foreground) color, or None for no color. Defaults to None. + bgcolor (Optional[Color]): A (background) color, or None for no color. Defaults to None. + """ + style: Style = cls.__new__(Style) + style._ansi = None + style._style_definition = None + style._color = color + style._bgcolor = bgcolor + style._set_attributes = 0 + style._attributes = 0 + style._link = None + style._link_id = "" + style._meta = None + style._hash = hash( + ( + color, + bgcolor, + None, + None, + None, + None, + ) + ) + style._null = not (color or bgcolor) + return style + + @classmethod + def from_meta(cls, meta: Optional[Dict[str, Any]]) -> "Style": + """Create a new style with meta data. + + Returns: + meta (Optional[Dict[str, Any]]): A dictionary of meta data. Defaults to None. + """ + style: Style = cls.__new__(Style) + style._ansi = None + style._style_definition = None + style._color = None + style._bgcolor = None + style._set_attributes = 0 + style._attributes = 0 + style._link = None + style._link_id = "" + style._meta = dumps(meta) + style._hash = hash( + ( + None, + None, + None, + None, + None, + style._meta, + ) + ) + style._null = not (meta) + return style + + @classmethod + def on(cls, meta: Optional[Dict[str, Any]] = None, **handlers: Any) -> "Style": + """Create a blank style with meta information. + + Example: + style = Style.on(click=self.on_click) + + Args: + meta (Optiona[Dict[str, Any]], optional): An optional dict of meta information. + **handlers (Any): Keyword arguments are translated in to handlers. + + Returns: + Style: A Style with meta information attached. + """ + meta = {} if meta is None else meta + meta.update({f"@{key}": value for key, value in handlers.items()}) + return cls.from_meta(meta) + + bold = _Bit(0) + dim = _Bit(1) + italic = _Bit(2) + underline = _Bit(3) + blink = _Bit(4) + blink2 = _Bit(5) + reverse = _Bit(6) + conceal = _Bit(7) + strike = _Bit(8) + underline2 = _Bit(9) + frame = _Bit(10) + encircle = _Bit(11) + overline = _Bit(12) + + @property + def link_id(self) -> str: + """Get a link id, used in ansi code for links.""" + return self._link_id + + def __str__(self) -> str: + """Re-generate style definition from attributes.""" + if self._style_definition is None: + attributes: List[str] = [] + append = attributes.append + bits = self._set_attributes + if bits & 0b0000000001111: + if bits & 1: + append("bold" if self.bold else "not bold") + if bits & (1 << 1): + append("dim" if self.dim else "not dim") + if bits & (1 << 2): + append("italic" if self.italic else "not italic") + if bits & (1 << 3): + append("underline" if self.underline else "not underline") + if bits & 0b0000111110000: + if bits & (1 << 4): + append("blink" if self.blink else "not blink") + if bits & (1 << 5): + append("blink2" if self.blink2 else "not blink2") + if bits & (1 << 6): + append("reverse" if self.reverse else "not reverse") + if bits & (1 << 7): + append("conceal" if self.conceal else "not conceal") + if bits & (1 << 8): + append("strike" if self.strike else "not strike") + if bits & 0b1111000000000: + if bits & (1 << 9): + append("underline2" if self.underline2 else "not underline2") + if bits & (1 << 10): + append("frame" if self.frame else "not frame") + if bits & (1 << 11): + append("encircle" if self.encircle else "not encircle") + if bits & (1 << 12): + append("overline" if self.overline else "not overline") + if self._color is not None: + append(self._color.name) + if self._bgcolor is not None: + append("on") + append(self._bgcolor.name) + if self._link: + append("link") + append(self._link) + self._style_definition = " ".join(attributes) or "none" + return self._style_definition + + def __bool__(self) -> bool: + """A Style is false if it has no attributes, colors, or links.""" + return not self._null + + def _make_ansi_codes(self, color_system: ColorSystem) -> str: + """Generate ANSI codes for this style. + + Args: + color_system (ColorSystem): Color system. + + Returns: + str: String containing codes. + """ + if self._ansi is None: + sgr: List[str] = [] + append = sgr.append + _style_map = self._style_map + attributes = self._attributes & self._set_attributes + if attributes: + if attributes & 1: + append(_style_map[0]) + if attributes & 2: + append(_style_map[1]) + if attributes & 4: + append(_style_map[2]) + if attributes & 8: + append(_style_map[3]) + if attributes & 0b0000111110000: + for bit in range(4, 9): + if attributes & (1 << bit): + append(_style_map[bit]) + if attributes & 0b1111000000000: + for bit in range(9, 13): + if attributes & (1 << bit): + append(_style_map[bit]) + if self._color is not None: + sgr.extend(self._color.downgrade(color_system).get_ansi_codes()) + if self._bgcolor is not None: + sgr.extend( + self._bgcolor.downgrade(color_system).get_ansi_codes( + foreground=False + ) + ) + self._ansi = ";".join(sgr) + return self._ansi + + @classmethod + @lru_cache(maxsize=1024) + def normalize(cls, style: str) -> str: + """Normalize a style definition so that styles with the same effect have the same string + representation. + + Args: + style (str): A style definition. + + Returns: + str: Normal form of style definition. + """ + try: + return str(cls.parse(style)) + except errors.StyleSyntaxError: + return style.strip().lower() + + @classmethod + def pick_first(cls, *values: Optional[StyleType]) -> StyleType: + """Pick first non-None style.""" + for value in values: + if value is not None: + return value + raise ValueError("expected at least one non-None style") + + def __rich_repr__(self) -> Result: + yield "color", self.color, None + yield "bgcolor", self.bgcolor, None + yield "bold", self.bold, None, + yield "dim", self.dim, None, + yield "italic", self.italic, None + yield "underline", self.underline, None, + yield "blink", self.blink, None + yield "blink2", self.blink2, None + yield "reverse", self.reverse, None + yield "conceal", self.conceal, None + yield "strike", self.strike, None + yield "underline2", self.underline2, None + yield "frame", self.frame, None + yield "encircle", self.encircle, None + yield "link", self.link, None + if self._meta: + yield "meta", self.meta + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Style): + return NotImplemented + return ( + self._color == other._color + and self._bgcolor == other._bgcolor + and self._set_attributes == other._set_attributes + and self._attributes == other._attributes + and self._link == other._link + and self._meta == other._meta + ) + + def __hash__(self) -> int: + return self._hash + + @property + def color(self) -> Optional[Color]: + """The foreground color or None if it is not set.""" + return self._color + + @property + def bgcolor(self) -> Optional[Color]: + """The background color or None if it is not set.""" + return self._bgcolor + + @property + def link(self) -> Optional[str]: + """Link text, if set.""" + return self._link + + @property + def transparent_background(self) -> bool: + """Check if the style specified a transparent background.""" + return self.bgcolor is None or self.bgcolor.is_default + + @property + def background_style(self) -> "Style": + """A Style with background only.""" + return Style(bgcolor=self.bgcolor) + + @property + def meta(self) -> Dict[str, Any]: + """Get meta information (can not be changed after construction).""" + return {} if self._meta is None else cast(Dict[str, Any], loads(self._meta)) + + @property + def without_color(self) -> "Style": + """Get a copy of the style with color removed.""" + if self._null: + return NULL_STYLE + style: Style = self.__new__(Style) + style._ansi = None + style._style_definition = None + style._color = None + style._bgcolor = None + style._attributes = self._attributes + style._set_attributes = self._set_attributes + style._link = self._link + style._link_id = f"{randint(0, 999999)}" if self._link else "" + style._hash = self._hash + style._null = False + style._meta = None + return style + + @classmethod + @lru_cache(maxsize=4096) + def parse(cls, style_definition: str) -> "Style": + """Parse a style definition. + + Args: + style_definition (str): A string containing a style. + + Raises: + errors.StyleSyntaxError: If the style definition syntax is invalid. + + Returns: + `Style`: A Style instance. + """ + if style_definition.strip() == "none" or not style_definition: + return cls.null() + + STYLE_ATTRIBUTES = cls.STYLE_ATTRIBUTES + color: Optional[str] = None + bgcolor: Optional[str] = None + attributes: Dict[str, Optional[Any]] = {} + link: Optional[str] = None + + words = iter(style_definition.split()) + for original_word in words: + word = original_word.lower() + if word == "on": + word = next(words, "") + if not word: + raise errors.StyleSyntaxError("color expected after 'on'") + try: + Color.parse(word) is None + except ColorParseError as error: + raise errors.StyleSyntaxError( + f"unable to parse {word!r} as background color; {error}" + ) from None + bgcolor = word + + elif word == "not": + word = next(words, "") + attribute = STYLE_ATTRIBUTES.get(word) + if attribute is None: + raise errors.StyleSyntaxError( + f"expected style attribute after 'not', found {word!r}" + ) + attributes[attribute] = False + + elif word == "link": + word = next(words, "") + if not word: + raise errors.StyleSyntaxError("URL expected after 'link'") + link = word + + elif word in STYLE_ATTRIBUTES: + attributes[STYLE_ATTRIBUTES[word]] = True + + else: + try: + Color.parse(word) + except ColorParseError as error: + raise errors.StyleSyntaxError( + f"unable to parse {word!r} as color; {error}" + ) from None + color = word + style = Style(color=color, bgcolor=bgcolor, link=link, **attributes) + return style + + @lru_cache(maxsize=1024) + def get_html_style(self, theme: Optional[TerminalTheme] = None) -> str: + """Get a CSS style rule.""" + theme = theme or DEFAULT_TERMINAL_THEME + css: List[str] = [] + append = css.append + + color = self.color + bgcolor = self.bgcolor + if self.reverse: + color, bgcolor = bgcolor, color + if self.dim: + foreground_color = ( + theme.foreground_color if color is None else color.get_truecolor(theme) + ) + color = Color.from_triplet( + blend_rgb(foreground_color, theme.background_color, 0.5) + ) + if color is not None: + theme_color = color.get_truecolor(theme) + append(f"color: {theme_color.hex}") + append(f"text-decoration-color: {theme_color.hex}") + if bgcolor is not None: + theme_color = bgcolor.get_truecolor(theme, foreground=False) + append(f"background-color: {theme_color.hex}") + if self.bold: + append("font-weight: bold") + if self.italic: + append("font-style: italic") + if self.underline: + append("text-decoration: underline") + if self.strike: + append("text-decoration: line-through") + if self.overline: + append("text-decoration: overline") + return "; ".join(css) + + @classmethod + def combine(cls, styles: Iterable["Style"]) -> "Style": + """Combine styles and get result. + + Args: + styles (Iterable[Style]): Styles to combine. + + Returns: + Style: A new style instance. + """ + iter_styles = iter(styles) + return sum(iter_styles, next(iter_styles)) + + @classmethod + def chain(cls, *styles: "Style") -> "Style": + """Combine styles from positional argument in to a single style. + + Args: + *styles (Iterable[Style]): Styles to combine. + + Returns: + Style: A new style instance. + """ + iter_styles = iter(styles) + return sum(iter_styles, next(iter_styles)) + + def copy(self) -> "Style": + """Get a copy of this style. + + Returns: + Style: A new Style instance with identical attributes. + """ + if self._null: + return NULL_STYLE + style: Style = self.__new__(Style) + style._ansi = self._ansi + style._style_definition = self._style_definition + style._color = self._color + style._bgcolor = self._bgcolor + style._attributes = self._attributes + style._set_attributes = self._set_attributes + style._link = self._link + style._link_id = f"{randint(0, 999999)}" if self._link else "" + style._hash = self._hash + style._null = False + style._meta = self._meta + return style + + def update_link(self, link: Optional[str] = None) -> "Style": + """Get a copy with a different value for link. + + Args: + link (str, optional): New value for link. Defaults to None. + + Returns: + Style: A new Style instance. + """ + style: Style = self.__new__(Style) + style._ansi = self._ansi + style._style_definition = self._style_definition + style._color = self._color + style._bgcolor = self._bgcolor + style._attributes = self._attributes + style._set_attributes = self._set_attributes + style._link = link + style._link_id = f"{randint(0, 999999)}" if link else "" + style._hash = self._hash + style._null = False + style._meta = self._meta + return style + + def render( + self, + text: str = "", + *, + color_system: Optional[ColorSystem] = ColorSystem.TRUECOLOR, + legacy_windows: bool = False, + ) -> str: + """Render the ANSI codes for the style. + + Args: + text (str, optional): A string to style. Defaults to "". + color_system (Optional[ColorSystem], optional): Color system to render to. Defaults to ColorSystem.TRUECOLOR. + + Returns: + str: A string containing ANSI style codes. + """ + if not text or color_system is None: + return text + attrs = self._make_ansi_codes(color_system) + rendered = f"\x1b[{attrs}m{text}\x1b[0m" if attrs else text + if self._link and not legacy_windows: + rendered = ( + f"\x1b]8;id={self._link_id};{self._link}\x1b\\{rendered}\x1b]8;;\x1b\\" + ) + return rendered + + def test(self, text: Optional[str] = None) -> None: + """Write text with style directly to terminal. + + This method is for testing purposes only. + + Args: + text (Optional[str], optional): Text to style or None for style name. + + """ + text = text or str(self) + sys.stdout.write(f"{self.render(text)}\n") + + def __add__(self, style: Optional["Style"]) -> "Style": + if not (isinstance(style, Style) or style is None): + return NotImplemented + if style is None or style._null: + return self + if self._null: + return style + new_style: Style = self.__new__(Style) + new_style._ansi = None + new_style._style_definition = None + new_style._color = style._color or self._color + new_style._bgcolor = style._bgcolor or self._bgcolor + new_style._attributes = (self._attributes & ~style._set_attributes) | ( + style._attributes & style._set_attributes + ) + new_style._set_attributes = self._set_attributes | style._set_attributes + new_style._link = style._link or self._link + new_style._link_id = style._link_id or self._link_id + new_style._hash = style._hash + new_style._null = self._null or style._null + if self._meta and style._meta: + new_style._meta = dumps({**self.meta, **style.meta}) + else: + new_style._meta = self._meta or style._meta + return new_style + + +NULL_STYLE = Style() + + +class StyleStack: + """A stack of styles.""" + + __slots__ = ["_stack"] + + def __init__(self, default_style: "Style") -> None: + self._stack: List[Style] = [default_style] + + def __repr__(self) -> str: + return f"" + + @property + def current(self) -> Style: + """Get the Style at the top of the stack.""" + return self._stack[-1] + + def push(self, style: Style) -> None: + """Push a new style on to the stack. + + Args: + style (Style): New style to combine with current style. + """ + self._stack.append(self._stack[-1] + style) + + def pop(self) -> Style: + """Pop last style and discard. + + Returns: + Style: New current style (also available as stack.current) + """ + self._stack.pop() + return self._stack[-1] diff --git a/src/pip/_vendor/rich/styled.py b/src/pip/_vendor/rich/styled.py new file mode 100644 index 00000000000..91cd0db31c1 --- /dev/null +++ b/src/pip/_vendor/rich/styled.py @@ -0,0 +1,42 @@ +from typing import TYPE_CHECKING + +from .measure import Measurement +from .segment import Segment +from .style import StyleType + +if TYPE_CHECKING: + from .console import Console, ConsoleOptions, RenderResult, RenderableType + + +class Styled: + """Apply a style to a renderable. + + Args: + renderable (RenderableType): Any renderable. + style (StyleType): A style to apply across the entire renderable. + """ + + def __init__(self, renderable: "RenderableType", style: "StyleType") -> None: + self.renderable = renderable + self.style = style + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + style = console.get_style(self.style) + rendered_segments = console.render(self.renderable, options) + segments = Segment.apply_style(rendered_segments, style) + return segments + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> Measurement: + return Measurement.get(console, options, self.renderable) + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich import print + from pip._vendor.rich.panel import Panel + + panel = Styled(Panel("hello"), "on blue") + print(panel) diff --git a/src/pip/_vendor/rich/syntax.py b/src/pip/_vendor/rich/syntax.py new file mode 100644 index 00000000000..58cc1037fae --- /dev/null +++ b/src/pip/_vendor/rich/syntax.py @@ -0,0 +1,735 @@ +import os.path +import platform +from pip._vendor.rich.containers import Lines +import textwrap +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union + +from pip._vendor.pygments.lexer import Lexer +from pip._vendor.pygments.lexers import get_lexer_by_name, guess_lexer_for_filename +from pip._vendor.pygments.style import Style as PygmentsStyle +from pip._vendor.pygments.styles import get_style_by_name +from pip._vendor.pygments.token import ( + Comment, + Error, + Generic, + Keyword, + Name, + Number, + Operator, + String, + Token, + Whitespace, +) +from pip._vendor.pygments.util import ClassNotFound + +from ._loop import loop_first +from .color import Color, blend_rgb +from .console import Console, ConsoleOptions, JustifyMethod, RenderResult +from .jupyter import JupyterMixin +from .measure import Measurement +from .segment import Segment +from .style import Style +from .text import Text + +TokenType = Tuple[str, ...] + +WINDOWS = platform.system() == "Windows" +DEFAULT_THEME = "monokai" + +# The following styles are based on https://github.com/pygments/pygments/blob/master/pygments/formatters/terminal.py +# A few modifications were made + +ANSI_LIGHT: Dict[TokenType, Style] = { + Token: Style(), + Whitespace: Style(color="white"), + Comment: Style(dim=True), + Comment.Preproc: Style(color="cyan"), + Keyword: Style(color="blue"), + Keyword.Type: Style(color="cyan"), + Operator.Word: Style(color="magenta"), + Name.Builtin: Style(color="cyan"), + Name.Function: Style(color="green"), + Name.Namespace: Style(color="cyan", underline=True), + Name.Class: Style(color="green", underline=True), + Name.Exception: Style(color="cyan"), + Name.Decorator: Style(color="magenta", bold=True), + Name.Variable: Style(color="red"), + Name.Constant: Style(color="red"), + Name.Attribute: Style(color="cyan"), + Name.Tag: Style(color="bright_blue"), + String: Style(color="yellow"), + Number: Style(color="blue"), + Generic.Deleted: Style(color="bright_red"), + Generic.Inserted: Style(color="green"), + Generic.Heading: Style(bold=True), + Generic.Subheading: Style(color="magenta", bold=True), + Generic.Prompt: Style(bold=True), + Generic.Error: Style(color="bright_red"), + Error: Style(color="red", underline=True), +} + +ANSI_DARK: Dict[TokenType, Style] = { + Token: Style(), + Whitespace: Style(color="bright_black"), + Comment: Style(dim=True), + Comment.Preproc: Style(color="bright_cyan"), + Keyword: Style(color="bright_blue"), + Keyword.Type: Style(color="bright_cyan"), + Operator.Word: Style(color="bright_magenta"), + Name.Builtin: Style(color="bright_cyan"), + Name.Function: Style(color="bright_green"), + Name.Namespace: Style(color="bright_cyan", underline=True), + Name.Class: Style(color="bright_green", underline=True), + Name.Exception: Style(color="bright_cyan"), + Name.Decorator: Style(color="bright_magenta", bold=True), + Name.Variable: Style(color="bright_red"), + Name.Constant: Style(color="bright_red"), + Name.Attribute: Style(color="bright_cyan"), + Name.Tag: Style(color="bright_blue"), + String: Style(color="yellow"), + Number: Style(color="bright_blue"), + Generic.Deleted: Style(color="bright_red"), + Generic.Inserted: Style(color="bright_green"), + Generic.Heading: Style(bold=True), + Generic.Subheading: Style(color="bright_magenta", bold=True), + Generic.Prompt: Style(bold=True), + Generic.Error: Style(color="bright_red"), + Error: Style(color="red", underline=True), +} + +RICH_SYNTAX_THEMES = {"ansi_light": ANSI_LIGHT, "ansi_dark": ANSI_DARK} + + +class SyntaxTheme(ABC): + """Base class for a syntax theme.""" + + @abstractmethod + def get_style_for_token(self, token_type: TokenType) -> Style: + """Get a style for a given Pygments token.""" + raise NotImplementedError # pragma: no cover + + @abstractmethod + def get_background_style(self) -> Style: + """Get the background color.""" + raise NotImplementedError # pragma: no cover + + +class PygmentsSyntaxTheme(SyntaxTheme): + """Syntax theme that delegates to Pygments theme.""" + + def __init__(self, theme: Union[str, Type[PygmentsStyle]]) -> None: + self._style_cache: Dict[TokenType, Style] = {} + if isinstance(theme, str): + try: + self._pygments_style_class = get_style_by_name(theme) + except ClassNotFound: + self._pygments_style_class = get_style_by_name("default") + else: + self._pygments_style_class = theme + + self._background_color = self._pygments_style_class.background_color + self._background_style = Style(bgcolor=self._background_color) + + def get_style_for_token(self, token_type: TokenType) -> Style: + """Get a style from a Pygments class.""" + try: + return self._style_cache[token_type] + except KeyError: + try: + pygments_style = self._pygments_style_class.style_for_token(token_type) + except KeyError: + style = Style.null() + else: + color = pygments_style["color"] + bgcolor = pygments_style["bgcolor"] + style = Style( + color="#" + color if color else "#000000", + bgcolor="#" + bgcolor if bgcolor else self._background_color, + bold=pygments_style["bold"], + italic=pygments_style["italic"], + underline=pygments_style["underline"], + ) + self._style_cache[token_type] = style + return style + + def get_background_style(self) -> Style: + return self._background_style + + +class ANSISyntaxTheme(SyntaxTheme): + """Syntax theme to use standard colors.""" + + def __init__(self, style_map: Dict[TokenType, Style]) -> None: + self.style_map = style_map + self._missing_style = Style.null() + self._background_style = Style.null() + self._style_cache: Dict[TokenType, Style] = {} + + def get_style_for_token(self, token_type: TokenType) -> Style: + """Look up style in the style map.""" + try: + return self._style_cache[token_type] + except KeyError: + # Styles form a hierarchy + # We need to go from most to least specific + # e.g. ("foo", "bar", "baz") to ("foo", "bar") to ("foo",) + get_style = self.style_map.get + token = tuple(token_type) + style = self._missing_style + while token: + _style = get_style(token) + if _style is not None: + style = _style + break + token = token[:-1] + self._style_cache[token_type] = style + return style + + def get_background_style(self) -> Style: + return self._background_style + + +class Syntax(JupyterMixin): + """Construct a Syntax object to render syntax highlighted code. + + Args: + code (str): Code to highlight. + lexer (Lexer | str): Lexer to use (see https://pygments.org/docs/lexers/) + theme (str, optional): Color theme, aka Pygments style (see https://pygments.org/docs/styles/#getting-a-list-of-available-styles). Defaults to "monokai". + dedent (bool, optional): Enable stripping of initial whitespace. Defaults to False. + line_numbers (bool, optional): Enable rendering of line numbers. Defaults to False. + start_line (int, optional): Starting number for line numbers. Defaults to 1. + line_range (Tuple[int, int], optional): If given should be a tuple of the start and end line to render. + highlight_lines (Set[int]): A set of line numbers to highlight. + code_width: Width of code to render (not including line numbers), or ``None`` to use all available width. + tab_size (int, optional): Size of tabs. Defaults to 4. + word_wrap (bool, optional): Enable word wrapping. + background_color (str, optional): Optional background color, or None to use theme color. Defaults to None. + indent_guides (bool, optional): Show indent guides. Defaults to False. + """ + + _pygments_style_class: Type[PygmentsStyle] + _theme: SyntaxTheme + + @classmethod + def get_theme(cls, name: Union[str, SyntaxTheme]) -> SyntaxTheme: + """Get a syntax theme instance.""" + if isinstance(name, SyntaxTheme): + return name + theme: SyntaxTheme + if name in RICH_SYNTAX_THEMES: + theme = ANSISyntaxTheme(RICH_SYNTAX_THEMES[name]) + else: + theme = PygmentsSyntaxTheme(name) + return theme + + def __init__( + self, + code: str, + lexer: Union[Lexer, str], + *, + theme: Union[str, SyntaxTheme] = DEFAULT_THEME, + dedent: bool = False, + line_numbers: bool = False, + start_line: int = 1, + line_range: Optional[Tuple[int, int]] = None, + highlight_lines: Optional[Set[int]] = None, + code_width: Optional[int] = None, + tab_size: int = 4, + word_wrap: bool = False, + background_color: Optional[str] = None, + indent_guides: bool = False, + ) -> None: + self.code = code + self._lexer = lexer + self.dedent = dedent + self.line_numbers = line_numbers + self.start_line = start_line + self.line_range = line_range + self.highlight_lines = highlight_lines or set() + self.code_width = code_width + self.tab_size = tab_size + self.word_wrap = word_wrap + self.background_color = background_color + self.background_style = ( + Style(bgcolor=background_color) if background_color else Style() + ) + self.indent_guides = indent_guides + + self._theme = self.get_theme(theme) + + @classmethod + def from_path( + cls, + path: str, + encoding: str = "utf-8", + theme: Union[str, SyntaxTheme] = DEFAULT_THEME, + dedent: bool = False, + line_numbers: bool = False, + line_range: Optional[Tuple[int, int]] = None, + start_line: int = 1, + highlight_lines: Optional[Set[int]] = None, + code_width: Optional[int] = None, + tab_size: int = 4, + word_wrap: bool = False, + background_color: Optional[str] = None, + indent_guides: bool = False, + ) -> "Syntax": + """Construct a Syntax object from a file. + + Args: + path (str): Path to file to highlight. + encoding (str): Encoding of file. + theme (str, optional): Color theme, aka Pygments style (see https://pygments.org/docs/styles/#getting-a-list-of-available-styles). Defaults to "emacs". + dedent (bool, optional): Enable stripping of initial whitespace. Defaults to True. + line_numbers (bool, optional): Enable rendering of line numbers. Defaults to False. + start_line (int, optional): Starting number for line numbers. Defaults to 1. + line_range (Tuple[int, int], optional): If given should be a tuple of the start and end line to render. + highlight_lines (Set[int]): A set of line numbers to highlight. + code_width: Width of code to render (not including line numbers), or ``None`` to use all available width. + tab_size (int, optional): Size of tabs. Defaults to 4. + word_wrap (bool, optional): Enable word wrapping of code. + background_color (str, optional): Optional background color, or None to use theme color. Defaults to None. + indent_guides (bool, optional): Show indent guides. Defaults to False. + + Returns: + [Syntax]: A Syntax object that may be printed to the console + """ + with open(path, "rt", encoding=encoding) as code_file: + code = code_file.read() + + lexer = None + lexer_name = "default" + try: + _, ext = os.path.splitext(path) + if ext: + extension = ext.lstrip(".").lower() + lexer = get_lexer_by_name(extension) + lexer_name = lexer.name + except ClassNotFound: + pass + + if lexer is None: + try: + lexer_name = guess_lexer_for_filename(path, code).name + except ClassNotFound: + pass + + return cls( + code, + lexer_name, + theme=theme, + dedent=dedent, + line_numbers=line_numbers, + line_range=line_range, + start_line=start_line, + highlight_lines=highlight_lines, + code_width=code_width, + tab_size=tab_size, + word_wrap=word_wrap, + background_color=background_color, + indent_guides=indent_guides, + ) + + def _get_base_style(self) -> Style: + """Get the base style.""" + default_style = self._theme.get_background_style() + self.background_style + return default_style + + def _get_token_color(self, token_type: TokenType) -> Optional[Color]: + """Get a color (if any) for the given token. + + Args: + token_type (TokenType): A token type tuple from Pygments. + + Returns: + Optional[Color]: Color from theme, or None for no color. + """ + style = self._theme.get_style_for_token(token_type) + return style.color + + @property + def lexer(self) -> Optional[Lexer]: + """The lexer for this syntax, or None if no lexer was found. + + Tries to find the lexer by name if a string was passed to the constructor. + """ + + if isinstance(self._lexer, Lexer): + return self._lexer + try: + return get_lexer_by_name( + self._lexer, + stripnl=False, + ensurenl=True, + tabsize=self.tab_size, + ) + except ClassNotFound: + return None + + def highlight( + self, code: str, line_range: Optional[Tuple[int, int]] = None + ) -> Text: + """Highlight code and return a Text instance. + + Args: + code (str): Code to highlight. + line_range(Tuple[int, int], optional): Optional line range to highlight. + + Returns: + Text: A text instance containing highlighted syntax. + """ + + base_style = self._get_base_style() + justify: JustifyMethod = ( + "default" if base_style.transparent_background else "left" + ) + + text = Text( + justify=justify, + style=base_style, + tab_size=self.tab_size, + no_wrap=not self.word_wrap, + ) + _get_theme_style = self._theme.get_style_for_token + + lexer = self.lexer + + if lexer is None: + text.append(code) + else: + if line_range: + # More complicated path to only stylize a portion of the code + # This speeds up further operations as there are less spans to process + line_start, line_end = line_range + + def line_tokenize() -> Iterable[Tuple[Any, str]]: + """Split tokens to one per line.""" + assert lexer + + for token_type, token in lexer.get_tokens(code): + while token: + line_token, new_line, token = token.partition("\n") + yield token_type, line_token + new_line + + def tokens_to_spans() -> Iterable[Tuple[str, Optional[Style]]]: + """Convert tokens to spans.""" + tokens = iter(line_tokenize()) + line_no = 0 + _line_start = line_start - 1 + + # Skip over tokens until line start + while line_no < _line_start: + _token_type, token = next(tokens) + yield (token, None) + if token.endswith("\n"): + line_no += 1 + # Generate spans until line end + for token_type, token in tokens: + yield (token, _get_theme_style(token_type)) + if token.endswith("\n"): + line_no += 1 + if line_no >= line_end: + break + + text.append_tokens(tokens_to_spans()) + + else: + text.append_tokens( + (token, _get_theme_style(token_type)) + for token_type, token in lexer.get_tokens(code) + ) + if self.background_color is not None: + text.stylize(f"on {self.background_color}") + return text + + def _get_line_numbers_color(self, blend: float = 0.3) -> Color: + background_style = self._theme.get_background_style() + self.background_style + background_color = background_style.bgcolor + if background_color is None or background_color.is_system_defined: + return Color.default() + foreground_color = self._get_token_color(Token.Text) + if foreground_color is None or foreground_color.is_system_defined: + return foreground_color or Color.default() + new_color = blend_rgb( + background_color.get_truecolor(), + foreground_color.get_truecolor(), + cross_fade=blend, + ) + return Color.from_triplet(new_color) + + @property + def _numbers_column_width(self) -> int: + """Get the number of characters used to render the numbers column.""" + column_width = 0 + if self.line_numbers: + column_width = len(str(self.start_line + self.code.count("\n"))) + 2 + return column_width + + def _get_number_styles(self, console: Console) -> Tuple[Style, Style, Style]: + """Get background, number, and highlight styles for line numbers.""" + background_style = self._get_base_style() + if background_style.transparent_background: + return Style.null(), Style(dim=True), Style.null() + if console.color_system in ("256", "truecolor"): + number_style = Style.chain( + background_style, + self._theme.get_style_for_token(Token.Text), + Style(color=self._get_line_numbers_color()), + self.background_style, + ) + highlight_number_style = Style.chain( + background_style, + self._theme.get_style_for_token(Token.Text), + Style(bold=True, color=self._get_line_numbers_color(0.9)), + self.background_style, + ) + else: + number_style = background_style + Style(dim=True) + highlight_number_style = background_style + Style(dim=False) + return background_style, number_style, highlight_number_style + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + if self.code_width is not None: + width = self.code_width + self._numbers_column_width + return Measurement(self._numbers_column_width, width) + return Measurement(self._numbers_column_width, options.max_width) + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + + transparent_background = self._get_base_style().transparent_background + code_width = ( + ( + (options.max_width - self._numbers_column_width - 1) + if self.line_numbers + else options.max_width + ) + if self.code_width is None + else self.code_width + ) + + line_offset = 0 + if self.line_range: + start_line, end_line = self.line_range + line_offset = max(0, start_line - 1) + + ends_on_nl = self.code.endswith("\n") + code = self.code if ends_on_nl else self.code + "\n" + code = textwrap.dedent(code) if self.dedent else code + code = code.expandtabs(self.tab_size) + text = self.highlight(code, self.line_range) + + ( + background_style, + number_style, + highlight_number_style, + ) = self._get_number_styles(console) + + if not self.line_numbers and not self.word_wrap and not self.line_range: + if not ends_on_nl: + text.remove_suffix("\n") + # Simple case of just rendering text + style = ( + self._get_base_style() + + self._theme.get_style_for_token(Comment) + + Style(dim=True) + + self.background_style + ) + if self.indent_guides and not options.ascii_only: + text = text.with_indent_guides(self.tab_size, style=style) + text.overflow = "crop" + if style.transparent_background: + yield from console.render( + text, options=options.update(width=code_width) + ) + else: + syntax_lines = console.render_lines( + text, + options.update(width=code_width, height=None), + style=self.background_style, + pad=True, + new_lines=True, + ) + for syntax_line in syntax_lines: + yield from syntax_line + return + + lines: Union[List[Text], Lines] = text.split("\n", allow_blank=ends_on_nl) + if self.line_range: + lines = lines[line_offset:end_line] + + if self.indent_guides and not options.ascii_only: + style = ( + self._get_base_style() + + self._theme.get_style_for_token(Comment) + + Style(dim=True) + + self.background_style + ) + lines = ( + Text("\n") + .join(lines) + .with_indent_guides(self.tab_size, style=style) + .split("\n", allow_blank=True) + ) + + numbers_column_width = self._numbers_column_width + render_options = options.update(width=code_width) + + highlight_line = self.highlight_lines.__contains__ + _Segment = Segment + padding = _Segment(" " * numbers_column_width + " ", background_style) + new_line = _Segment("\n") + + line_pointer = "> " if options.legacy_windows else "❱ " + + for line_no, line in enumerate(lines, self.start_line + line_offset): + if self.word_wrap: + wrapped_lines = console.render_lines( + line, + render_options.update(height=None), + style=background_style, + pad=not transparent_background, + ) + + else: + segments = list(line.render(console, end="")) + if options.no_wrap: + wrapped_lines = [segments] + else: + wrapped_lines = [ + _Segment.adjust_line_length( + segments, + render_options.max_width, + style=background_style, + pad=not transparent_background, + ) + ] + if self.line_numbers: + for first, wrapped_line in loop_first(wrapped_lines): + if first: + line_column = str(line_no).rjust(numbers_column_width - 2) + " " + if highlight_line(line_no): + yield _Segment(line_pointer, Style(color="red")) + yield _Segment(line_column, highlight_number_style) + else: + yield _Segment(" ", highlight_number_style) + yield _Segment(line_column, number_style) + else: + yield padding + yield from wrapped_line + yield new_line + else: + for wrapped_line in wrapped_lines: + yield from wrapped_line + yield new_line + + +if __name__ == "__main__": # pragma: no cover + + import argparse + import sys + + parser = argparse.ArgumentParser( + description="Render syntax to the console with Rich" + ) + parser.add_argument( + "path", + metavar="PATH", + help="path to file, or - for stdin", + ) + parser.add_argument( + "-c", + "--force-color", + dest="force_color", + action="store_true", + default=None, + help="force color for non-terminals", + ) + parser.add_argument( + "-i", + "--indent-guides", + dest="indent_guides", + action="store_true", + default=False, + help="display indent guides", + ) + parser.add_argument( + "-l", + "--line-numbers", + dest="line_numbers", + action="store_true", + help="render line numbers", + ) + parser.add_argument( + "-w", + "--width", + type=int, + dest="width", + default=None, + help="width of output (default will auto-detect)", + ) + parser.add_argument( + "-r", + "--wrap", + dest="word_wrap", + action="store_true", + default=False, + help="word wrap long lines", + ) + parser.add_argument( + "-s", + "--soft-wrap", + action="store_true", + dest="soft_wrap", + default=False, + help="enable soft wrapping mode", + ) + parser.add_argument( + "-t", "--theme", dest="theme", default="monokai", help="pygments theme" + ) + parser.add_argument( + "-b", + "--background-color", + dest="background_color", + default=None, + help="Override background color", + ) + parser.add_argument( + "-x", + "--lexer", + default="default", + dest="lexer_name", + help="Lexer name", + ) + args = parser.parse_args() + + from pip._vendor.rich.console import Console + + console = Console(force_terminal=args.force_color, width=args.width) + + if args.path == "-": + code = sys.stdin.read() + syntax = Syntax( + code=code, + lexer=args.lexer_name, + line_numbers=args.line_numbers, + word_wrap=args.word_wrap, + theme=args.theme, + background_color=args.background_color, + indent_guides=args.indent_guides, + ) + else: + syntax = Syntax.from_path( + args.path, + line_numbers=args.line_numbers, + word_wrap=args.word_wrap, + theme=args.theme, + background_color=args.background_color, + indent_guides=args.indent_guides, + ) + console.print(syntax, soft_wrap=args.soft_wrap) diff --git a/src/pip/_vendor/rich/table.py b/src/pip/_vendor/rich/table.py new file mode 100644 index 00000000000..da4386085a8 --- /dev/null +++ b/src/pip/_vendor/rich/table.py @@ -0,0 +1,968 @@ +from dataclasses import dataclass, field, replace +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Sequence, + Tuple, + Union, +) + +from . import box, errors +from ._loop import loop_first_last, loop_last +from ._pick import pick_bool +from ._ratio import ratio_distribute, ratio_reduce +from .align import VerticalAlignMethod +from .jupyter import JupyterMixin +from .measure import Measurement +from .padding import Padding, PaddingDimensions +from .protocol import is_renderable +from .segment import Segment +from .style import Style, StyleType +from .text import Text, TextType + +if TYPE_CHECKING: + from .console import ( + Console, + ConsoleOptions, + JustifyMethod, + OverflowMethod, + RenderableType, + RenderResult, + ) + + +@dataclass +class Column: + """Defines a column in a table.""" + + header: "RenderableType" = "" + """RenderableType: Renderable for the header (typically a string)""" + + footer: "RenderableType" = "" + """RenderableType: Renderable for the footer (typically a string)""" + + header_style: StyleType = "" + """StyleType: The style of the header.""" + + footer_style: StyleType = "" + """StyleType: The style of the footer.""" + + style: StyleType = "" + """StyleType: The style of the column.""" + + justify: "JustifyMethod" = "left" + """str: How to justify text within the column ("left", "center", "right", or "full")""" + + vertical: "VerticalAlignMethod" = "top" + """str: How to vertically align content ("top", "middle", or "bottom")""" + + overflow: "OverflowMethod" = "ellipsis" + """str: Overflow method.""" + + width: Optional[int] = None + """Optional[int]: Width of the column, or ``None`` (default) to auto calculate width.""" + + min_width: Optional[int] = None + """Optional[int]: Minimum width of column, or ``None`` for no minimum. Defaults to None.""" + + max_width: Optional[int] = None + """Optional[int]: Maximum width of column, or ``None`` for no maximum. Defaults to None.""" + + ratio: Optional[int] = None + """Optional[int]: Ratio to use when calculating column width, or ``None`` (default) to adapt to column contents.""" + + no_wrap: bool = False + """bool: Prevent wrapping of text within the column. Defaults to ``False``.""" + + _index: int = 0 + """Index of column.""" + + _cells: List["RenderableType"] = field(default_factory=list) + + def copy(self) -> "Column": + """Return a copy of this Column.""" + return replace(self, _cells=[]) + + @property + def cells(self) -> Iterable["RenderableType"]: + """Get all cells in the column, not including header.""" + yield from self._cells + + @property + def flexible(self) -> bool: + """Check if this column is flexible.""" + return self.ratio is not None + + +@dataclass +class Row: + """Information regarding a row.""" + + style: Optional[StyleType] = None + """Style to apply to row.""" + + end_section: bool = False + """Indicated end of section, which will force a line beneath the row.""" + + +class _Cell(NamedTuple): + """A single cell in a table.""" + + style: StyleType + """Style to apply to cell.""" + renderable: "RenderableType" + """Cell renderable.""" + vertical: VerticalAlignMethod + """Cell vertical alignment.""" + + +class Table(JupyterMixin): + """A console renderable to draw a table. + + Args: + *headers (Union[Column, str]): Column headers, either as a string, or :class:`~rich.table.Column` instance. + title (Union[str, Text], optional): The title of the table rendered at the top. Defaults to None. + caption (Union[str, Text], optional): The table caption rendered below. Defaults to None. + width (int, optional): The width in characters of the table, or ``None`` to automatically fit. Defaults to None. + min_width (Optional[int], optional): The minimum width of the table, or ``None`` for no minimum. Defaults to None. + box (box.Box, optional): One of the constants in box.py used to draw the edges (see :ref:`appendix_box`), or ``None`` for no box lines. Defaults to box.HEAVY_HEAD. + safe_box (Optional[bool], optional): Disable box characters that don't display on windows legacy terminal with *raster* fonts. Defaults to True. + padding (PaddingDimensions, optional): Padding for cells (top, right, bottom, left). Defaults to (0, 1). + collapse_padding (bool, optional): Enable collapsing of padding around cells. Defaults to False. + pad_edge (bool, optional): Enable padding of edge cells. Defaults to True. + expand (bool, optional): Expand the table to fit the available space if ``True``, otherwise the table width will be auto-calculated. Defaults to False. + show_header (bool, optional): Show a header row. Defaults to True. + show_footer (bool, optional): Show a footer row. Defaults to False. + show_edge (bool, optional): Draw a box around the outside of the table. Defaults to True. + show_lines (bool, optional): Draw lines between every row. Defaults to False. + leading (bool, optional): Number of blank lines between rows (precludes ``show_lines``). Defaults to 0. + style (Union[str, Style], optional): Default style for the table. Defaults to "none". + row_styles (List[Union, str], optional): Optional list of row styles, if more than one style is given then the styles will alternate. Defaults to None. + header_style (Union[str, Style], optional): Style of the header. Defaults to "table.header". + footer_style (Union[str, Style], optional): Style of the footer. Defaults to "table.footer". + border_style (Union[str, Style], optional): Style of the border. Defaults to None. + title_style (Union[str, Style], optional): Style of the title. Defaults to None. + caption_style (Union[str, Style], optional): Style of the caption. Defaults to None. + title_justify (str, optional): Justify method for title. Defaults to "center". + caption_justify (str, optional): Justify method for caption. Defaults to "center". + highlight (bool, optional): Highlight cell contents (if str). Defaults to False. + """ + + columns: List[Column] + rows: List[Row] + + def __init__( + self, + *headers: Union[Column, str], + title: Optional[TextType] = None, + caption: Optional[TextType] = None, + width: Optional[int] = None, + min_width: Optional[int] = None, + box: Optional[box.Box] = box.HEAVY_HEAD, + safe_box: Optional[bool] = None, + padding: PaddingDimensions = (0, 1), + collapse_padding: bool = False, + pad_edge: bool = True, + expand: bool = False, + show_header: bool = True, + show_footer: bool = False, + show_edge: bool = True, + show_lines: bool = False, + leading: int = 0, + style: StyleType = "none", + row_styles: Optional[Iterable[StyleType]] = None, + header_style: Optional[StyleType] = "table.header", + footer_style: Optional[StyleType] = "table.footer", + border_style: Optional[StyleType] = None, + title_style: Optional[StyleType] = None, + caption_style: Optional[StyleType] = None, + title_justify: "JustifyMethod" = "center", + caption_justify: "JustifyMethod" = "center", + highlight: bool = False, + ) -> None: + + self.columns: List[Column] = [] + self.rows: List[Row] = [] + self.title = title + self.caption = caption + self.width = width + self.min_width = min_width + self.box = box + self.safe_box = safe_box + self._padding = Padding.unpack(padding) + self.pad_edge = pad_edge + self._expand = expand + self.show_header = show_header + self.show_footer = show_footer + self.show_edge = show_edge + self.show_lines = show_lines + self.leading = leading + self.collapse_padding = collapse_padding + self.style = style + self.header_style = header_style or "" + self.footer_style = footer_style or "" + self.border_style = border_style + self.title_style = title_style + self.caption_style = caption_style + self.title_justify: "JustifyMethod" = title_justify + self.caption_justify: "JustifyMethod" = caption_justify + self.highlight = highlight + self.row_styles: Sequence[StyleType] = list(row_styles or []) + append_column = self.columns.append + for header in headers: + if isinstance(header, str): + self.add_column(header=header) + else: + header._index = len(self.columns) + append_column(header) + + @classmethod + def grid( + cls, + *headers: Union[Column, str], + padding: PaddingDimensions = 0, + collapse_padding: bool = True, + pad_edge: bool = False, + expand: bool = False, + ) -> "Table": + """Get a table with no lines, headers, or footer. + + Args: + *headers (Union[Column, str]): Column headers, either as a string, or :class:`~rich.table.Column` instance. + padding (PaddingDimensions, optional): Get padding around cells. Defaults to 0. + collapse_padding (bool, optional): Enable collapsing of padding around cells. Defaults to True. + pad_edge (bool, optional): Enable padding around edges of table. Defaults to False. + expand (bool, optional): Expand the table to fit the available space if ``True``, otherwise the table width will be auto-calculated. Defaults to False. + + Returns: + Table: A table instance. + """ + return cls( + *headers, + box=None, + padding=padding, + collapse_padding=collapse_padding, + show_header=False, + show_footer=False, + show_edge=False, + pad_edge=pad_edge, + expand=expand, + ) + + @property + def expand(self) -> bool: + """Setting a non-None self.width implies expand.""" + return self._expand or self.width is not None + + @expand.setter + def expand(self, expand: bool) -> None: + """Set expand.""" + self._expand = expand + + @property + def _extra_width(self) -> int: + """Get extra width to add to cell content.""" + width = 0 + if self.box and self.show_edge: + width += 2 + if self.box: + width += len(self.columns) - 1 + return width + + @property + def row_count(self) -> int: + """Get the current number of rows.""" + return len(self.rows) + + def get_row_style(self, console: "Console", index: int) -> StyleType: + """Get the current row style.""" + style = Style.null() + if self.row_styles: + style += console.get_style(self.row_styles[index % len(self.row_styles)]) + row_style = self.rows[index].style + if row_style is not None: + style += console.get_style(row_style) + return style + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> Measurement: + max_width = options.max_width + if self.width is not None: + max_width = self.width + if max_width < 0: + return Measurement(0, 0) + + extra_width = self._extra_width + max_width = sum( + self._calculate_column_widths( + console, options.update_width(max_width - extra_width) + ) + ) + _measure_column = self._measure_column + + measurements = [ + _measure_column(console, options.update_width(max_width), column) + for column in self.columns + ] + minimum_width = ( + sum(measurement.minimum for measurement in measurements) + extra_width + ) + maximum_width = ( + sum(measurement.maximum for measurement in measurements) + extra_width + if (self.width is None) + else self.width + ) + measurement = Measurement(minimum_width, maximum_width) + measurement = measurement.clamp(self.min_width) + return measurement + + @property + def padding(self) -> Tuple[int, int, int, int]: + """Get cell padding.""" + return self._padding + + @padding.setter + def padding(self, padding: PaddingDimensions) -> "Table": + """Set cell padding.""" + self._padding = Padding.unpack(padding) + return self + + def add_column( + self, + header: "RenderableType" = "", + footer: "RenderableType" = "", + *, + header_style: Optional[StyleType] = None, + footer_style: Optional[StyleType] = None, + style: Optional[StyleType] = None, + justify: "JustifyMethod" = "left", + vertical: "VerticalAlignMethod" = "top", + overflow: "OverflowMethod" = "ellipsis", + width: Optional[int] = None, + min_width: Optional[int] = None, + max_width: Optional[int] = None, + ratio: Optional[int] = None, + no_wrap: bool = False, + ) -> None: + """Add a column to the table. + + Args: + header (RenderableType, optional): Text or renderable for the header. + Defaults to "". + footer (RenderableType, optional): Text or renderable for the footer. + Defaults to "". + header_style (Union[str, Style], optional): Style for the header, or None for default. Defaults to None. + footer_style (Union[str, Style], optional): Style for the footer, or None for default. Defaults to None. + style (Union[str, Style], optional): Style for the column cells, or None for default. Defaults to None. + justify (JustifyMethod, optional): Alignment for cells. Defaults to "left". + vertical (VerticalAlignMethod, optional): Vertical alignment, one of "top", "middle", or "bottom". Defaults to "top". + overflow (OverflowMethod): Overflow method: "crop", "fold", "ellipsis". Defaults to "ellipsis". + width (int, optional): Desired width of column in characters, or None to fit to contents. Defaults to None. + min_width (Optional[int], optional): Minimum width of column, or ``None`` for no minimum. Defaults to None. + max_width (Optional[int], optional): Maximum width of column, or ``None`` for no maximum. Defaults to None. + ratio (int, optional): Flexible ratio for the column (requires ``Table.expand`` or ``Table.width``). Defaults to None. + no_wrap (bool, optional): Set to ``True`` to disable wrapping of this column. + """ + + column = Column( + _index=len(self.columns), + header=header, + footer=footer, + header_style=header_style or "", + footer_style=footer_style or "", + style=style or "", + justify=justify, + vertical=vertical, + overflow=overflow, + width=width, + min_width=min_width, + max_width=max_width, + ratio=ratio, + no_wrap=no_wrap, + ) + self.columns.append(column) + + def add_row( + self, + *renderables: Optional["RenderableType"], + style: Optional[StyleType] = None, + end_section: bool = False, + ) -> None: + """Add a row of renderables. + + Args: + *renderables (None or renderable): Each cell in a row must be a renderable object (including str), + or ``None`` for a blank cell. + style (StyleType, optional): An optional style to apply to the entire row. Defaults to None. + end_section (bool, optional): End a section and draw a line. Defaults to False. + + Raises: + errors.NotRenderableError: If you add something that can't be rendered. + """ + + def add_cell(column: Column, renderable: "RenderableType") -> None: + column._cells.append(renderable) + + cell_renderables: List[Optional["RenderableType"]] = list(renderables) + + columns = self.columns + if len(cell_renderables) < len(columns): + cell_renderables = [ + *cell_renderables, + *[None] * (len(columns) - len(cell_renderables)), + ] + for index, renderable in enumerate(cell_renderables): + if index == len(columns): + column = Column(_index=index) + for _ in self.rows: + add_cell(column, Text("")) + self.columns.append(column) + else: + column = columns[index] + if renderable is None: + add_cell(column, "") + elif is_renderable(renderable): + add_cell(column, renderable) + else: + raise errors.NotRenderableError( + f"unable to render {type(renderable).__name__}; a string or other renderable object is required" + ) + self.rows.append(Row(style=style, end_section=end_section)) + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + + if not self.columns: + yield Segment("\n") + return + + max_width = options.max_width + if self.width is not None: + max_width = self.width + + extra_width = self._extra_width + widths = self._calculate_column_widths( + console, options.update_width(max_width - extra_width) + ) + table_width = sum(widths) + extra_width + + render_options = options.update( + width=table_width, highlight=self.highlight, height=None + ) + + def render_annotation( + text: TextType, style: StyleType, justify: "JustifyMethod" = "center" + ) -> "RenderResult": + render_text = ( + console.render_str(text, style=style, highlight=False) + if isinstance(text, str) + else text + ) + return console.render( + render_text, options=render_options.update(justify=justify) + ) + + if self.title: + yield from render_annotation( + self.title, + style=Style.pick_first(self.title_style, "table.title"), + justify=self.title_justify, + ) + yield from self._render(console, render_options, widths) + if self.caption: + yield from render_annotation( + self.caption, + style=Style.pick_first(self.caption_style, "table.caption"), + justify=self.caption_justify, + ) + + def _calculate_column_widths( + self, console: "Console", options: "ConsoleOptions" + ) -> List[int]: + """Calculate the widths of each column, including padding, not including borders.""" + max_width = options.max_width + columns = self.columns + width_ranges = [ + self._measure_column(console, options, column) for column in columns + ] + widths = [_range.maximum or 1 for _range in width_ranges] + get_padding_width = self._get_padding_width + extra_width = self._extra_width + if self.expand: + ratios = [col.ratio or 0 for col in columns if col.flexible] + if any(ratios): + fixed_widths = [ + 0 if column.flexible else _range.maximum + for _range, column in zip(width_ranges, columns) + ] + flex_minimum = [ + (column.width or 1) + get_padding_width(column._index) + for column in columns + if column.flexible + ] + flexible_width = max_width - sum(fixed_widths) + flex_widths = ratio_distribute(flexible_width, ratios, flex_minimum) + iter_flex_widths = iter(flex_widths) + for index, column in enumerate(columns): + if column.flexible: + widths[index] = fixed_widths[index] + next(iter_flex_widths) + table_width = sum(widths) + + if table_width > max_width: + widths = self._collapse_widths( + widths, + [(column.width is None and not column.no_wrap) for column in columns], + max_width, + ) + table_width = sum(widths) + # last resort, reduce columns evenly + if table_width > max_width: + excess_width = table_width - max_width + widths = ratio_reduce(excess_width, [1] * len(widths), widths, widths) + table_width = sum(widths) + + width_ranges = [ + self._measure_column(console, options.update_width(width), column) + for width, column in zip(widths, columns) + ] + widths = [_range.maximum or 0 for _range in width_ranges] + + if (table_width < max_width and self.expand) or ( + self.min_width is not None and table_width < (self.min_width - extra_width) + ): + _max_width = ( + max_width + if self.min_width is None + else min(self.min_width - extra_width, max_width) + ) + pad_widths = ratio_distribute(_max_width - table_width, widths) + widths = [_width + pad for _width, pad in zip(widths, pad_widths)] + + return widths + + @classmethod + def _collapse_widths( + cls, widths: List[int], wrapable: List[bool], max_width: int + ) -> List[int]: + """Reduce widths so that the total is under max_width. + + Args: + widths (List[int]): List of widths. + wrapable (List[bool]): List of booleans that indicate if a column may shrink. + max_width (int): Maximum width to reduce to. + + Returns: + List[int]: A new list of widths. + """ + total_width = sum(widths) + excess_width = total_width - max_width + if any(wrapable): + while total_width and excess_width > 0: + max_column = max( + width for width, allow_wrap in zip(widths, wrapable) if allow_wrap + ) + second_max_column = max( + width if allow_wrap and width != max_column else 0 + for width, allow_wrap in zip(widths, wrapable) + ) + column_difference = max_column - second_max_column + ratios = [ + (1 if (width == max_column and allow_wrap) else 0) + for width, allow_wrap in zip(widths, wrapable) + ] + if not any(ratios) or not column_difference: + break + max_reduce = [min(excess_width, column_difference)] * len(widths) + widths = ratio_reduce(excess_width, ratios, max_reduce, widths) + + total_width = sum(widths) + excess_width = total_width - max_width + return widths + + def _get_cells( + self, console: "Console", column_index: int, column: Column + ) -> Iterable[_Cell]: + """Get all the cells with padding and optional header.""" + + collapse_padding = self.collapse_padding + pad_edge = self.pad_edge + padding = self.padding + any_padding = any(padding) + + first_column = column_index == 0 + last_column = column_index == len(self.columns) - 1 + + _padding_cache: Dict[Tuple[bool, bool], Tuple[int, int, int, int]] = {} + + def get_padding(first_row: bool, last_row: bool) -> Tuple[int, int, int, int]: + cached = _padding_cache.get((first_row, last_row)) + if cached: + return cached + top, right, bottom, left = padding + + if collapse_padding: + if not first_column: + left = max(0, left - right) + if not last_row: + bottom = max(0, top - bottom) + + if not pad_edge: + if first_column: + left = 0 + if last_column: + right = 0 + if first_row: + top = 0 + if last_row: + bottom = 0 + _padding = (top, right, bottom, left) + _padding_cache[(first_row, last_row)] = _padding + return _padding + + raw_cells: List[Tuple[StyleType, "RenderableType"]] = [] + _append = raw_cells.append + get_style = console.get_style + if self.show_header: + header_style = get_style(self.header_style or "") + get_style( + column.header_style + ) + _append((header_style, column.header)) + cell_style = get_style(column.style or "") + for cell in column.cells: + _append((cell_style, cell)) + if self.show_footer: + footer_style = get_style(self.footer_style or "") + get_style( + column.footer_style + ) + _append((footer_style, column.footer)) + + if any_padding: + _Padding = Padding + for first, last, (style, renderable) in loop_first_last(raw_cells): + yield _Cell( + style, + _Padding(renderable, get_padding(first, last)), + getattr(renderable, "vertical", None) or column.vertical, + ) + else: + for (style, renderable) in raw_cells: + yield _Cell( + style, + renderable, + getattr(renderable, "vertical", None) or column.vertical, + ) + + def _get_padding_width(self, column_index: int) -> int: + """Get extra width from padding.""" + _, pad_right, _, pad_left = self.padding + if self.collapse_padding: + if column_index > 0: + pad_left = max(0, pad_left - pad_right) + return pad_left + pad_right + + def _measure_column( + self, + console: "Console", + options: "ConsoleOptions", + column: Column, + ) -> Measurement: + """Get the minimum and maximum width of the column.""" + + max_width = options.max_width + if max_width < 1: + return Measurement(0, 0) + + padding_width = self._get_padding_width(column._index) + + if column.width is not None: + # Fixed width column + return Measurement( + column.width + padding_width, column.width + padding_width + ).with_maximum(max_width) + # Flexible column, we need to measure contents + min_widths: List[int] = [] + max_widths: List[int] = [] + append_min = min_widths.append + append_max = max_widths.append + get_render_width = Measurement.get + for cell in self._get_cells(console, column._index, column): + _min, _max = get_render_width(console, options, cell.renderable) + append_min(_min) + append_max(_max) + + measurement = Measurement( + max(min_widths) if min_widths else 1, + max(max_widths) if max_widths else max_width, + ).with_maximum(max_width) + measurement = measurement.clamp( + None if column.min_width is None else column.min_width + padding_width, + None if column.max_width is None else column.max_width + padding_width, + ) + return measurement + + def _render( + self, console: "Console", options: "ConsoleOptions", widths: List[int] + ) -> "RenderResult": + table_style = console.get_style(self.style or "") + + border_style = table_style + console.get_style(self.border_style or "") + _column_cells = ( + self._get_cells(console, column_index, column) + for column_index, column in enumerate(self.columns) + ) + row_cells: List[Tuple[_Cell, ...]] = list(zip(*_column_cells)) + _box = ( + self.box.substitute( + options, safe=pick_bool(self.safe_box, console.safe_box) + ) + if self.box + else None + ) + + # _box = self.box + new_line = Segment.line() + + columns = self.columns + show_header = self.show_header + show_footer = self.show_footer + show_edge = self.show_edge + show_lines = self.show_lines + leading = self.leading + + _Segment = Segment + if _box: + box_segments = [ + ( + _Segment(_box.head_left, border_style), + _Segment(_box.head_right, border_style), + _Segment(_box.head_vertical, border_style), + ), + ( + _Segment(_box.foot_left, border_style), + _Segment(_box.foot_right, border_style), + _Segment(_box.foot_vertical, border_style), + ), + ( + _Segment(_box.mid_left, border_style), + _Segment(_box.mid_right, border_style), + _Segment(_box.mid_vertical, border_style), + ), + ] + if show_edge: + yield _Segment(_box.get_top(widths), border_style) + yield new_line + else: + box_segments = [] + + get_row_style = self.get_row_style + get_style = console.get_style + + for index, (first, last, row_cell) in enumerate(loop_first_last(row_cells)): + header_row = first and show_header + footer_row = last and show_footer + row = ( + self.rows[index - show_header] + if (not header_row and not footer_row) + else None + ) + max_height = 1 + cells: List[List[List[Segment]]] = [] + if header_row or footer_row: + row_style = Style.null() + else: + row_style = get_style( + get_row_style(console, index - 1 if show_header else index) + ) + for width, cell, column in zip(widths, row_cell, columns): + render_options = options.update( + width=width, + justify=column.justify, + no_wrap=column.no_wrap, + overflow=column.overflow, + height=None, + ) + lines = console.render_lines( + cell.renderable, + render_options, + style=get_style(cell.style) + row_style, + ) + max_height = max(max_height, len(lines)) + cells.append(lines) + + row_height = max(len(cell) for cell in cells) + + def align_cell( + cell: List[List[Segment]], + vertical: "VerticalAlignMethod", + width: int, + style: Style, + ) -> List[List[Segment]]: + if header_row: + vertical = "bottom" + elif footer_row: + vertical = "top" + + if vertical == "top": + return _Segment.align_top(cell, width, row_height, style) + elif vertical == "middle": + return _Segment.align_middle(cell, width, row_height, style) + return _Segment.align_bottom(cell, width, row_height, style) + + cells[:] = [ + _Segment.set_shape( + align_cell( + cell, + _cell.vertical, + width, + get_style(_cell.style) + row_style, + ), + width, + max_height, + ) + for width, _cell, cell, column in zip(widths, row_cell, cells, columns) + ] + + if _box: + if last and show_footer: + yield _Segment( + _box.get_row(widths, "foot", edge=show_edge), border_style + ) + yield new_line + left, right, _divider = box_segments[0 if first else (2 if last else 1)] + + # If the column divider is whitespace also style it with the row background + divider = ( + _divider + if _divider.text.strip() + else _Segment( + _divider.text, row_style.background_style + _divider.style + ) + ) + for line_no in range(max_height): + if show_edge: + yield left + for last_cell, rendered_cell in loop_last(cells): + yield from rendered_cell[line_no] + if not last_cell: + yield divider + if show_edge: + yield right + yield new_line + else: + for line_no in range(max_height): + for rendered_cell in cells: + yield from rendered_cell[line_no] + yield new_line + if _box and first and show_header: + yield _Segment( + _box.get_row(widths, "head", edge=show_edge), border_style + ) + yield new_line + end_section = row and row.end_section + if _box and (show_lines or leading or end_section): + if ( + not last + and not (show_footer and index >= len(row_cells) - 2) + and not (show_header and header_row) + ): + if leading: + yield _Segment( + _box.get_row(widths, "mid", edge=show_edge) * leading, + border_style, + ) + else: + yield _Segment( + _box.get_row(widths, "row", edge=show_edge), border_style + ) + yield new_line + + if _box and show_edge: + yield _Segment(_box.get_bottom(widths), border_style) + yield new_line + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console + from pip._vendor.rich.highlighter import ReprHighlighter + from pip._vendor.rich.table import Table as Table + + from ._timer import timer + + with timer("Table render"): + table = Table( + title="Star Wars Movies", + caption="Rich example table", + caption_justify="right", + ) + + table.add_column( + "Released", header_style="bright_cyan", style="cyan", no_wrap=True + ) + table.add_column("Title", style="magenta") + table.add_column("Box Office", justify="right", style="green") + + table.add_row( + "Dec 20, 2019", + "Star Wars: The Rise of Skywalker", + "$952,110,690", + ) + table.add_row("May 25, 2018", "Solo: A Star Wars Story", "$393,151,347") + table.add_row( + "Dec 15, 2017", + "Star Wars Ep. V111: The Last Jedi", + "$1,332,539,889", + style="on black", + end_section=True, + ) + table.add_row( + "Dec 16, 2016", + "Rogue One: A Star Wars Story", + "$1,332,439,889", + ) + + def header(text: str) -> None: + console.print() + console.rule(highlight(text)) + console.print() + + console = Console() + highlight = ReprHighlighter() + header("Example Table") + console.print(table, justify="center") + + table.expand = True + header("expand=True") + console.print(table) + + table.width = 50 + header("width=50") + + console.print(table, justify="center") + + table.width = None + table.expand = False + table.row_styles = ["dim", "none"] + header("row_styles=['dim', 'none']") + + console.print(table, justify="center") + + table.width = None + table.expand = False + table.row_styles = ["dim", "none"] + table.leading = 1 + header("leading=1, row_styles=['dim', 'none']") + console.print(table, justify="center") + + table.width = None + table.expand = False + table.row_styles = ["dim", "none"] + table.show_lines = True + table.leading = 0 + header("show_lines=True, row_styles=['dim', 'none']") + console.print(table, justify="center") diff --git a/src/pip/_vendor/rich/tabulate.py b/src/pip/_vendor/rich/tabulate.py new file mode 100644 index 00000000000..6889f2d331b --- /dev/null +++ b/src/pip/_vendor/rich/tabulate.py @@ -0,0 +1,51 @@ +from collections.abc import Mapping +from typing import Any, Optional +import warnings + +from pip._vendor.rich.console import JustifyMethod + +from . import box +from .highlighter import ReprHighlighter +from .pretty import Pretty +from .table import Table + + +def tabulate_mapping( + mapping: "Mapping[Any, Any]", + title: Optional[str] = None, + caption: Optional[str] = None, + title_justify: Optional[JustifyMethod] = None, + caption_justify: Optional[JustifyMethod] = None, +) -> Table: + """Generate a simple table from a mapping. + + Args: + mapping (Mapping): A mapping object (e.g. a dict); + title (str, optional): Optional title to be displayed over the table. + caption (str, optional): Optional caption to be displayed below the table. + title_justify (str, optional): Justify method for title. Defaults to None. + caption_justify (str, optional): Justify method for caption. Defaults to None. + + Returns: + Table: A table instance which may be rendered by the Console. + """ + warnings.warn("tabulate_mapping will be deprecated in Rich v11", DeprecationWarning) + table = Table( + show_header=False, + title=title, + caption=caption, + box=box.ROUNDED, + border_style="blue", + ) + table.title = title + table.caption = caption + if title_justify is not None: + table.title_justify = title_justify + if caption_justify is not None: + table.caption_justify = caption_justify + highlighter = ReprHighlighter() + for key, value in mapping.items(): + table.add_row( + Pretty(key, highlighter=highlighter), Pretty(value, highlighter=highlighter) + ) + return table diff --git a/src/pip/_vendor/rich/terminal_theme.py b/src/pip/_vendor/rich/terminal_theme.py new file mode 100644 index 00000000000..801ac0b7b85 --- /dev/null +++ b/src/pip/_vendor/rich/terminal_theme.py @@ -0,0 +1,55 @@ +from typing import List, Optional, Tuple + +from .color_triplet import ColorTriplet +from .palette import Palette + +_ColorTuple = Tuple[int, int, int] + + +class TerminalTheme: + """A color theme used when exporting console content. + + Args: + background (Tuple[int, int, int]): The background color. + foreground (Tuple[int, int, int]): The foreground (text) color. + normal (List[Tuple[int, int, int]]): A list of 8 normal intensity colors. + bright (List[Tuple[int, int, int]], optional): A list of 8 bright colors, or None + to repeat normal intensity. Defaults to None. + """ + + def __init__( + self, + background: _ColorTuple, + foreground: _ColorTuple, + normal: List[_ColorTuple], + bright: Optional[List[_ColorTuple]] = None, + ) -> None: + self.background_color = ColorTriplet(*background) + self.foreground_color = ColorTriplet(*foreground) + self.ansi_colors = Palette(normal + (bright or normal)) + + +DEFAULT_TERMINAL_THEME = TerminalTheme( + (255, 255, 255), + (0, 0, 0), + [ + (0, 0, 0), + (128, 0, 0), + (0, 128, 0), + (128, 128, 0), + (0, 0, 128), + (128, 0, 128), + (0, 128, 128), + (192, 192, 192), + ], + [ + (128, 128, 128), + (255, 0, 0), + (0, 255, 0), + (255, 255, 0), + (0, 0, 255), + (255, 0, 255), + (0, 255, 255), + (255, 255, 255), + ], +) diff --git a/src/pip/_vendor/rich/text.py b/src/pip/_vendor/rich/text.py new file mode 100644 index 00000000000..ea12c09d729 --- /dev/null +++ b/src/pip/_vendor/rich/text.py @@ -0,0 +1,1282 @@ +import re +from functools import partial, reduce +from math import gcd +from operator import itemgetter +from pip._vendor.rich.emoji import EmojiVariant +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + NamedTuple, + Optional, + Tuple, + Union, +) + +from ._loop import loop_last +from ._pick import pick_bool +from ._wrap import divide_line +from .align import AlignMethod +from .cells import cell_len, set_cell_size +from .containers import Lines +from .control import strip_control_codes +from .emoji import EmojiVariant +from .jupyter import JupyterMixin +from .measure import Measurement +from .segment import Segment +from .style import Style, StyleType + +if TYPE_CHECKING: # pragma: no cover + from .console import Console, ConsoleOptions, JustifyMethod, OverflowMethod + +DEFAULT_JUSTIFY: "JustifyMethod" = "default" +DEFAULT_OVERFLOW: "OverflowMethod" = "fold" + + +_re_whitespace = re.compile(r"\s+$") + +TextType = Union[str, "Text"] + +GetStyleCallable = Callable[[str], Optional[StyleType]] + + +class Span(NamedTuple): + """A marked up region in some text.""" + + start: int + """Span start index.""" + end: int + """Span end index.""" + style: Union[str, Style] + """Style associated with the span.""" + + def __repr__(self) -> str: + return ( + f"Span({self.start}, {self.end}, {self.style!r})" + if (isinstance(self.style, Style) and self.style._meta) + else f"Span({self.start}, {self.end}, {repr(self.style)})" + ) + + def __bool__(self) -> bool: + return self.end > self.start + + def split(self, offset: int) -> Tuple["Span", Optional["Span"]]: + """Split a span in to 2 from a given offset.""" + + if offset < self.start: + return self, None + if offset >= self.end: + return self, None + + start, end, style = self + span1 = Span(start, min(end, offset), style) + span2 = Span(span1.end, end, style) + return span1, span2 + + def move(self, offset: int) -> "Span": + """Move start and end by a given offset. + + Args: + offset (int): Number of characters to add to start and end. + + Returns: + TextSpan: A new TextSpan with adjusted position. + """ + start, end, style = self + return Span(start + offset, end + offset, style) + + def right_crop(self, offset: int) -> "Span": + """Crop the span at the given offset. + + Args: + offset (int): A value between start and end. + + Returns: + Span: A new (possibly smaller) span. + """ + start, end, style = self + if offset >= end: + return self + return Span(start, min(offset, end), style) + + +class Text(JupyterMixin): + """Text with color / style. + + Args: + text (str, optional): Default unstyled text. Defaults to "". + style (Union[str, Style], optional): Base style for text. Defaults to "". + justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None. + overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None. + no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None. + end (str, optional): Character to end text with. Defaults to "\\\\n". + tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to 8. + spans (List[Span], optional). A list of predefined style spans. Defaults to None. + """ + + __slots__ = [ + "_text", + "style", + "justify", + "overflow", + "no_wrap", + "end", + "tab_size", + "_spans", + "_length", + ] + + def __init__( + self, + text: str = "", + style: Union[str, Style] = "", + *, + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + no_wrap: Optional[bool] = None, + end: str = "\n", + tab_size: Optional[int] = 8, + spans: Optional[List[Span]] = None, + ) -> None: + self._text = [strip_control_codes(text)] + self.style = style + self.justify: Optional["JustifyMethod"] = justify + self.overflow: Optional["OverflowMethod"] = overflow + self.no_wrap = no_wrap + self.end = end + self.tab_size = tab_size + self._spans: List[Span] = spans or [] + self._length: int = len(text) + + def __len__(self) -> int: + return self._length + + def __bool__(self) -> bool: + return bool(self._length) + + def __str__(self) -> str: + return self.plain + + def __repr__(self) -> str: + return f"" + + def __add__(self, other: Any) -> "Text": + if isinstance(other, (str, Text)): + result = self.copy() + result.append(other) + return result + return NotImplemented + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Text): + return NotImplemented + return self.plain == other.plain and self._spans == other._spans + + def __contains__(self, other: object) -> bool: + if isinstance(other, str): + return other in self.plain + elif isinstance(other, Text): + return other.plain in self.plain + return False + + def __getitem__(self, slice: Union[int, slice]) -> "Text": + def get_text_at(offset: int) -> "Text": + _Span = Span + text = Text( + self.plain[offset], + spans=[ + _Span(0, 1, style) + for start, end, style in self._spans + if end > offset >= start + ], + end="", + ) + return text + + if isinstance(slice, int): + return get_text_at(slice) + else: + start, stop, step = slice.indices(len(self.plain)) + if step == 1: + lines = self.divide([start, stop]) + return lines[1] + else: + # This would be a bit of work to implement efficiently + # For now, its not required + raise TypeError("slices with step!=1 are not supported") + + @property + def cell_len(self) -> int: + """Get the number of cells required to render this text.""" + return cell_len(self.plain) + + @property + def markup(self) -> str: + """Get console markup to render this Text. + + Returns: + str: A string potentially creating markup tags. + """ + from .markup import escape + + output: List[str] = [] + + plain = self.plain + markup_spans = [ + (0, False, self.style), + *((span.start, False, span.style) for span in self._spans), + *((span.end, True, span.style) for span in self._spans), + (len(plain), True, self.style), + ] + markup_spans.sort(key=itemgetter(0, 1)) + position = 0 + append = output.append + for offset, closing, style in markup_spans: + if offset > position: + append(escape(plain[position:offset])) + position = offset + if style: + append(f"[/{style}]" if closing else f"[{style}]") + markup = "".join(output) + return markup + + @classmethod + def from_markup( + cls, + text: str, + *, + style: Union[str, Style] = "", + emoji: bool = True, + emoji_variant: Optional[EmojiVariant] = None, + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + ) -> "Text": + """Create Text instance from markup. + + Args: + text (str): A string containing console markup. + emoji (bool, optional): Also render emoji code. Defaults to True. + justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None. + overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None. + + Returns: + Text: A Text instance with markup rendered. + """ + from .markup import render + + rendered_text = render(text, style, emoji=emoji, emoji_variant=emoji_variant) + rendered_text.justify = justify + rendered_text.overflow = overflow + return rendered_text + + @classmethod + def from_ansi( + cls, + text: str, + *, + style: Union[str, Style] = "", + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + no_wrap: Optional[bool] = None, + end: str = "\n", + tab_size: Optional[int] = 8, + ) -> "Text": + """Create a Text object from a string containing ANSI escape codes. + + Args: + text (str): A string containing escape codes. + style (Union[str, Style], optional): Base style for text. Defaults to "". + justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None. + overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None. + no_wrap (bool, optional): Disable text wrapping, or None for default. Defaults to None. + end (str, optional): Character to end text with. Defaults to "\\\\n". + tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to 8. + """ + from .ansi import AnsiDecoder + + joiner = Text( + "\n", + justify=justify, + overflow=overflow, + no_wrap=no_wrap, + end=end, + tab_size=tab_size, + style=style, + ) + decoder = AnsiDecoder() + result = joiner.join(line for line in decoder.decode(text)) + return result + + @classmethod + def styled( + cls, + text: str, + style: StyleType = "", + *, + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + ) -> "Text": + """Construct a Text instance with a pre-applied styled. A style applied in this way won't be used + to pad the text when it is justified. + + Args: + text (str): A string containing console markup. + style (Union[str, Style]): Style to apply to the text. Defaults to "". + justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None. + overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None. + + Returns: + Text: A text instance with a style applied to the entire string. + """ + styled_text = cls(text, justify=justify, overflow=overflow) + styled_text.stylize(style) + return styled_text + + @classmethod + def assemble( + cls, + *parts: Union[str, "Text", Tuple[str, StyleType]], + style: Union[str, Style] = "", + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + no_wrap: Optional[bool] = None, + end: str = "\n", + tab_size: int = 8, + meta: Optional[Dict[str, Any]] = None, + ) -> "Text": + """Construct a text instance by combining a sequence of strings with optional styles. + The positional arguments should be either strings, or a tuple of string + style. + + Args: + style (Union[str, Style], optional): Base style for text. Defaults to "". + justify (str, optional): Justify method: "left", "center", "full", "right". Defaults to None. + overflow (str, optional): Overflow method: "crop", "fold", "ellipsis". Defaults to None. + end (str, optional): Character to end text with. Defaults to "\\\\n". + tab_size (int): Number of spaces per tab, or ``None`` to use ``console.tab_size``. Defaults to 8. + meta (Dict[str, Any], optional). Meta data to apply to text, or None for no meta data. Default to None + + Returns: + Text: A new text instance. + """ + text = cls( + style=style, + justify=justify, + overflow=overflow, + no_wrap=no_wrap, + end=end, + tab_size=tab_size, + ) + append = text.append + _Text = Text + for part in parts: + if isinstance(part, (_Text, str)): + append(part) + else: + append(*part) + if meta: + text.apply_meta(meta) + return text + + @property + def plain(self) -> str: + """Get the text as a single string.""" + if len(self._text) != 1: + self._text[:] = ["".join(self._text)] + return self._text[0] + + @plain.setter + def plain(self, new_text: str) -> None: + """Set the text to a new value.""" + if new_text != self.plain: + self._text[:] = [new_text] + old_length = self._length + self._length = len(new_text) + if old_length > self._length: + self._trim_spans() + + @property + def spans(self) -> List[Span]: + """Get a reference to the internal list of spans.""" + return self._spans + + @spans.setter + def spans(self, spans: List[Span]) -> None: + """Set spans.""" + self._spans = spans[:] + + def blank_copy(self, plain: str = "") -> "Text": + """Return a new Text instance with copied meta data (but not the string or spans).""" + copy_self = Text( + plain, + style=self.style, + justify=self.justify, + overflow=self.overflow, + no_wrap=self.no_wrap, + end=self.end, + tab_size=self.tab_size, + ) + return copy_self + + def copy(self) -> "Text": + """Return a copy of this instance.""" + copy_self = Text( + self.plain, + style=self.style, + justify=self.justify, + overflow=self.overflow, + no_wrap=self.no_wrap, + end=self.end, + tab_size=self.tab_size, + ) + copy_self._spans[:] = self._spans + return copy_self + + def stylize( + self, + style: Union[str, Style], + start: int = 0, + end: Optional[int] = None, + ) -> None: + """Apply a style to the text, or a portion of the text. + + Args: + style (Union[str, Style]): Style instance or style definition to apply. + start (int): Start offset (negative indexing is supported). Defaults to 0. + end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None. + + """ + if style: + length = len(self) + if start < 0: + start = length + start + if end is None: + end = length + if end < 0: + end = length + end + if start >= length or end <= start: + # Span not in text or not valid + return + self._spans.append(Span(start, min(length, end), style)) + + def apply_meta( + self, meta: Dict[str, Any], start: int = 0, end: Optional[int] = None + ) -> None: + """Apply meta data to the text, or a portion of the text. + + Args: + meta (Dict[str, Any]): A dict of meta information. + start (int): Start offset (negative indexing is supported). Defaults to 0. + end (Optional[int], optional): End offset (negative indexing is supported), or None for end of text. Defaults to None. + + """ + style = Style.from_meta(meta) + self.stylize(style, start=start, end=end) + + def on(self, meta: Optional[Dict[str, Any]] = None, **handlers: Any) -> "Text": + """Apply event handlers (used by Textual project). + + Example: + >>> from rich.text import Text + >>> text = Text("hello world") + >>> text.on(click="view.toggle('world')") + + Args: + meta (Dict[str, Any]): Mapping of meta information. + **handlers: Keyword args are prefixed with "@" to defined handlers. + + Returns: + Text: Self is returned to method may be chained. + """ + meta = {} if meta is None else meta + meta.update({f"@{key}": value for key, value in handlers.items()}) + self.stylize(Style.from_meta(meta)) + return self + + def remove_suffix(self, suffix: str) -> None: + """Remove a suffix if it exists. + + Args: + suffix (str): Suffix to remove. + """ + if self.plain.endswith(suffix): + self.right_crop(len(suffix)) + + def get_style_at_offset(self, console: "Console", offset: int) -> Style: + """Get the style of a character at give offset. + + Args: + console (~Console): Console where text will be rendered. + offset (int): Offset in to text (negative indexing supported) + + Returns: + Style: A Style instance. + """ + # TODO: This is a little inefficient, it is only used by full justify + if offset < 0: + offset = len(self) + offset + get_style = console.get_style + style = get_style(self.style).copy() + for start, end, span_style in self._spans: + if end > offset >= start: + style += get_style(span_style, default="") + return style + + def highlight_regex( + self, + re_highlight: str, + style: Optional[Union[GetStyleCallable, StyleType]] = None, + *, + style_prefix: str = "", + ) -> int: + """Highlight text with a regular expression, where group names are + translated to styles. + + Args: + re_highlight (str): A regular expression. + style (Union[GetStyleCallable, StyleType]): Optional style to apply to whole match, or a callable + which accepts the matched text and returns a style. Defaults to None. + style_prefix (str, optional): Optional prefix to add to style group names. + + Returns: + int: Number of regex matches + """ + count = 0 + append_span = self._spans.append + _Span = Span + plain = self.plain + for match in re.finditer(re_highlight, plain): + get_span = match.span + if style: + start, end = get_span() + match_style = style(plain[start:end]) if callable(style) else style + if match_style is not None and end > start: + append_span(_Span(start, end, match_style)) + + count += 1 + for name in match.groupdict().keys(): + start, end = get_span(name) + if start != -1 and end > start: + append_span(_Span(start, end, f"{style_prefix}{name}")) + return count + + def highlight_words( + self, + words: Iterable[str], + style: Union[str, Style], + *, + case_sensitive: bool = True, + ) -> int: + """Highlight words with a style. + + Args: + words (Iterable[str]): Worlds to highlight. + style (Union[str, Style]): Style to apply. + case_sensitive (bool, optional): Enable case sensitive matchings. Defaults to True. + + Returns: + int: Number of words highlighted. + """ + re_words = "|".join(re.escape(word) for word in words) + add_span = self._spans.append + count = 0 + _Span = Span + for match in re.finditer( + re_words, self.plain, flags=0 if case_sensitive else re.IGNORECASE + ): + start, end = match.span(0) + add_span(_Span(start, end, style)) + count += 1 + return count + + def rstrip(self) -> None: + """Strip whitespace from end of text.""" + self.plain = self.plain.rstrip() + + def rstrip_end(self, size: int) -> None: + """Remove whitespace beyond a certain width at the end of the text. + + Args: + size (int): The desired size of the text. + """ + text_length = len(self) + if text_length > size: + excess = text_length - size + whitespace_match = _re_whitespace.search(self.plain) + if whitespace_match is not None: + whitespace_count = len(whitespace_match.group(0)) + self.right_crop(min(whitespace_count, excess)) + + def set_length(self, new_length: int) -> None: + """Set new length of the text, clipping or padding is required.""" + length = len(self) + if length != new_length: + if length < new_length: + self.pad_right(new_length - length) + else: + self.right_crop(length - new_length) + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> Iterable[Segment]: + tab_size: int = console.tab_size or self.tab_size or 8 + justify = self.justify or options.justify or DEFAULT_JUSTIFY + + overflow = self.overflow or options.overflow or DEFAULT_OVERFLOW + + lines = self.wrap( + console, + options.max_width, + justify=justify, + overflow=overflow, + tab_size=tab_size or 8, + no_wrap=pick_bool(self.no_wrap, options.no_wrap, False), + ) + all_lines = Text("\n").join(lines) + yield from all_lines.render(console, end=self.end) + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> Measurement: + text = self.plain + lines = text.splitlines() + max_text_width = max(cell_len(line) for line in lines) if lines else 0 + words = text.split() + min_text_width = ( + max(cell_len(word) for word in words) if words else max_text_width + ) + return Measurement(min_text_width, max_text_width) + + def render(self, console: "Console", end: str = "") -> Iterable["Segment"]: + """Render the text as Segments. + + Args: + console (Console): Console instance. + end (Optional[str], optional): Optional end character. + + Returns: + Iterable[Segment]: Result of render that may be written to the console. + """ + _Segment = Segment + text = self.plain + if not self._spans: + yield Segment(text) + if end: + yield _Segment(end) + return + get_style = partial(console.get_style, default=Style.null()) + + enumerated_spans = list(enumerate(self._spans, 1)) + style_map = {index: get_style(span.style) for index, span in enumerated_spans} + style_map[0] = get_style(self.style) + + spans = [ + (0, False, 0), + *((span.start, False, index) for index, span in enumerated_spans), + *((span.end, True, index) for index, span in enumerated_spans), + (len(text), True, 0), + ] + spans.sort(key=itemgetter(0, 1)) + + stack: List[int] = [] + stack_append = stack.append + stack_pop = stack.remove + + style_cache: Dict[Tuple[Style, ...], Style] = {} + style_cache_get = style_cache.get + combine = Style.combine + + def get_current_style() -> Style: + """Construct current style from stack.""" + styles = tuple(style_map[_style_id] for _style_id in sorted(stack)) + cached_style = style_cache_get(styles) + if cached_style is not None: + return cached_style + current_style = combine(styles) + style_cache[styles] = current_style + return current_style + + for (offset, leaving, style_id), (next_offset, _, _) in zip(spans, spans[1:]): + if leaving: + stack_pop(style_id) + else: + stack_append(style_id) + if next_offset > offset: + yield _Segment(text[offset:next_offset], get_current_style()) + if end: + yield _Segment(end) + + def join(self, lines: Iterable["Text"]) -> "Text": + """Join text together with this instance as the separator. + + Args: + lines (Iterable[Text]): An iterable of Text instances to join. + + Returns: + Text: A new text instance containing join text. + """ + + new_text = self.blank_copy() + + def iter_text() -> Iterable["Text"]: + if self.plain: + for last, line in loop_last(lines): + yield line + if not last: + yield self + else: + yield from lines + + extend_text = new_text._text.extend + append_span = new_text._spans.append + extend_spans = new_text._spans.extend + offset = 0 + _Span = Span + + for text in iter_text(): + extend_text(text._text) + if text.style: + append_span(_Span(offset, offset + len(text), text.style)) + extend_spans( + _Span(offset + start, offset + end, style) + for start, end, style in text._spans + ) + offset += len(text) + new_text._length = offset + return new_text + + def expand_tabs(self, tab_size: Optional[int] = None) -> None: + """Converts tabs to spaces. + + Args: + tab_size (int, optional): Size of tabs. Defaults to 8. + + """ + if "\t" not in self.plain: + return + pos = 0 + if tab_size is None: + tab_size = self.tab_size + assert tab_size is not None + result = self.blank_copy() + append = result.append + + _style = self.style + for line in self.split("\n", include_separator=True): + parts = line.split("\t", include_separator=True) + for part in parts: + if part.plain.endswith("\t"): + part._text = [part.plain[:-1] + " "] + append(part) + pos += len(part) + spaces = tab_size - ((pos - 1) % tab_size) - 1 + if spaces: + append(" " * spaces, _style) + pos += spaces + else: + append(part) + self._text = [result.plain] + self._length = len(self.plain) + self._spans[:] = result._spans + + def truncate( + self, + max_width: int, + *, + overflow: Optional["OverflowMethod"] = None, + pad: bool = False, + ) -> None: + """Truncate text if it is longer that a given width. + + Args: + max_width (int): Maximum number of characters in text. + overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None, to use self.overflow. + pad (bool, optional): Pad with spaces if the length is less than max_width. Defaults to False. + """ + _overflow = overflow or self.overflow or DEFAULT_OVERFLOW + if _overflow != "ignore": + length = cell_len(self.plain) + if length > max_width: + if _overflow == "ellipsis": + self.plain = set_cell_size(self.plain, max_width - 1) + "…" + else: + self.plain = set_cell_size(self.plain, max_width) + if pad and length < max_width: + spaces = max_width - length + self._text = [f"{self.plain}{' ' * spaces}"] + self._length = len(self.plain) + + def _trim_spans(self) -> None: + """Remove or modify any spans that are over the end of the text.""" + max_offset = len(self.plain) + _Span = Span + self._spans[:] = [ + ( + span + if span.end < max_offset + else _Span(span.start, min(max_offset, span.end), span.style) + ) + for span in self._spans + if span.start < max_offset + ] + + def pad(self, count: int, character: str = " ") -> None: + """Pad left and right with a given number of characters. + + Args: + count (int): Width of padding. + """ + assert len(character) == 1, "Character must be a string of length 1" + if count: + pad_characters = character * count + self.plain = f"{pad_characters}{self.plain}{pad_characters}" + _Span = Span + self._spans[:] = [ + _Span(start + count, end + count, style) + for start, end, style in self._spans + ] + + def pad_left(self, count: int, character: str = " ") -> None: + """Pad the left with a given character. + + Args: + count (int): Number of characters to pad. + character (str, optional): Character to pad with. Defaults to " ". + """ + assert len(character) == 1, "Character must be a string of length 1" + if count: + self.plain = f"{character * count}{self.plain}" + _Span = Span + self._spans[:] = [ + _Span(start + count, end + count, style) + for start, end, style in self._spans + ] + + def pad_right(self, count: int, character: str = " ") -> None: + """Pad the right with a given character. + + Args: + count (int): Number of characters to pad. + character (str, optional): Character to pad with. Defaults to " ". + """ + assert len(character) == 1, "Character must be a string of length 1" + if count: + self.plain = f"{self.plain}{character * count}" + + def align(self, align: AlignMethod, width: int, character: str = " ") -> None: + """Align text to a given width. + + Args: + align (AlignMethod): One of "left", "center", or "right". + width (int): Desired width. + character (str, optional): Character to pad with. Defaults to " ". + """ + self.truncate(width) + excess_space = width - cell_len(self.plain) + if excess_space: + if align == "left": + self.pad_right(excess_space, character) + elif align == "center": + left = excess_space // 2 + self.pad_left(left, character) + self.pad_right(excess_space - left, character) + else: + self.pad_left(excess_space, character) + + def append( + self, text: Union["Text", str], style: Optional[Union[str, "Style"]] = None + ) -> "Text": + """Add text with an optional style. + + Args: + text (Union[Text, str]): A str or Text to append. + style (str, optional): A style name. Defaults to None. + + Returns: + Text: Returns self for chaining. + """ + + if not isinstance(text, (str, Text)): + raise TypeError("Only str or Text can be appended to Text") + + if len(text): + if isinstance(text, str): + text = strip_control_codes(text) + self._text.append(text) + offset = len(self) + text_length = len(text) + if style is not None: + self._spans.append(Span(offset, offset + text_length, style)) + self._length += text_length + elif isinstance(text, Text): + _Span = Span + if style is not None: + raise ValueError( + "style must not be set when appending Text instance" + ) + text_length = self._length + if text.style is not None: + self._spans.append( + _Span(text_length, text_length + len(text), text.style) + ) + self._text.append(text.plain) + self._spans.extend( + _Span(start + text_length, end + text_length, style) + for start, end, style in text._spans + ) + self._length += len(text) + return self + + def append_text(self, text: "Text") -> "Text": + """Append another Text instance. This method is more performant that Text.append, but + only works for Text. + + Returns: + Text: Returns self for chaining. + """ + _Span = Span + text_length = self._length + if text.style is not None: + self._spans.append(_Span(text_length, text_length + len(text), text.style)) + self._text.append(text.plain) + self._spans.extend( + _Span(start + text_length, end + text_length, style) + for start, end, style in text._spans + ) + self._length += len(text) + return self + + def append_tokens( + self, tokens: Iterable[Tuple[str, Optional[StyleType]]] + ) -> "Text": + """Append iterable of str and style. Style may be a Style instance or a str style definition. + + Args: + pairs (Iterable[Tuple[str, Optional[StyleType]]]): An iterable of tuples containing str content and style. + + Returns: + Text: Returns self for chaining. + """ + append_text = self._text.append + append_span = self._spans.append + _Span = Span + offset = len(self) + for content, style in tokens: + append_text(content) + if style is not None: + append_span(_Span(offset, offset + len(content), style)) + offset += len(content) + self._length = offset + return self + + def copy_styles(self, text: "Text") -> None: + """Copy styles from another Text instance. + + Args: + text (Text): A Text instance to copy styles from, must be the same length. + """ + self._spans.extend(text._spans) + + def split( + self, + separator: str = "\n", + *, + include_separator: bool = False, + allow_blank: bool = False, + ) -> Lines: + """Split rich text in to lines, preserving styles. + + Args: + separator (str, optional): String to split on. Defaults to "\\\\n". + include_separator (bool, optional): Include the separator in the lines. Defaults to False. + allow_blank (bool, optional): Return a blank line if the text ends with a separator. Defaults to False. + + Returns: + List[RichText]: A list of rich text, one per line of the original. + """ + assert separator, "separator must not be empty" + + text = self.plain + if separator not in text: + return Lines([self.copy()]) + + if include_separator: + lines = self.divide( + match.end() for match in re.finditer(re.escape(separator), text) + ) + else: + + def flatten_spans() -> Iterable[int]: + for match in re.finditer(re.escape(separator), text): + start, end = match.span() + yield start + yield end + + lines = Lines( + line for line in self.divide(flatten_spans()) if line.plain != separator + ) + + if not allow_blank and text.endswith(separator): + lines.pop() + + return lines + + def divide(self, offsets: Iterable[int]) -> Lines: + """Divide text in to a number of lines at given offsets. + + Args: + offsets (Iterable[int]): Offsets used to divide text. + + Returns: + Lines: New RichText instances between offsets. + """ + _offsets = list(offsets) + + if not _offsets: + return Lines([self.copy()]) + + text = self.plain + text_length = len(text) + divide_offsets = [0, *_offsets, text_length] + line_ranges = list(zip(divide_offsets, divide_offsets[1:])) + + style = self.style + justify = self.justify + overflow = self.overflow + _Text = Text + new_lines = Lines( + _Text( + text[start:end], + style=style, + justify=justify, + overflow=overflow, + ) + for start, end in line_ranges + ) + if not self._spans: + return new_lines + + _line_appends = [line._spans.append for line in new_lines._lines] + line_count = len(line_ranges) + _Span = Span + + for span_start, span_end, style in self._spans: + + lower_bound = 0 + upper_bound = line_count + start_line_no = (lower_bound + upper_bound) // 2 + + while True: + line_start, line_end = line_ranges[start_line_no] + if span_start < line_start: + upper_bound = start_line_no - 1 + elif span_start > line_end: + lower_bound = start_line_no + 1 + else: + break + start_line_no = (lower_bound + upper_bound) // 2 + + if span_end < line_end: + end_line_no = start_line_no + else: + end_line_no = lower_bound = start_line_no + upper_bound = line_count + + while True: + line_start, line_end = line_ranges[end_line_no] + if span_end < line_start: + upper_bound = end_line_no - 1 + elif span_end > line_end: + lower_bound = end_line_no + 1 + else: + break + end_line_no = (lower_bound + upper_bound) // 2 + + for line_no in range(start_line_no, end_line_no + 1): + line_start, line_end = line_ranges[line_no] + new_start = max(0, span_start - line_start) + new_end = min(span_end - line_start, line_end - line_start) + if new_end > new_start: + _line_appends[line_no](_Span(new_start, new_end, style)) + + return new_lines + + def right_crop(self, amount: int = 1) -> None: + """Remove a number of characters from the end of the text.""" + max_offset = len(self.plain) - amount + _Span = Span + self._spans[:] = [ + ( + span + if span.end < max_offset + else _Span(span.start, min(max_offset, span.end), span.style) + ) + for span in self._spans + if span.start < max_offset + ] + self._text = [self.plain[:-amount]] + self._length -= amount + + def wrap( + self, + console: "Console", + width: int, + *, + justify: Optional["JustifyMethod"] = None, + overflow: Optional["OverflowMethod"] = None, + tab_size: int = 8, + no_wrap: Optional[bool] = None, + ) -> Lines: + """Word wrap the text. + + Args: + console (Console): Console instance. + width (int): Number of characters per line. + emoji (bool, optional): Also render emoji code. Defaults to True. + justify (str, optional): Justify method: "default", "left", "center", "full", "right". Defaults to "default". + overflow (str, optional): Overflow method: "crop", "fold", or "ellipsis". Defaults to None. + tab_size (int, optional): Default tab size. Defaults to 8. + no_wrap (bool, optional): Disable wrapping, Defaults to False. + + Returns: + Lines: Number of lines. + """ + wrap_justify = justify or self.justify or DEFAULT_JUSTIFY + wrap_overflow = overflow or self.overflow or DEFAULT_OVERFLOW + + no_wrap = pick_bool(no_wrap, self.no_wrap, False) or overflow == "ignore" + + lines = Lines() + for line in self.split(allow_blank=True): + if "\t" in line: + line.expand_tabs(tab_size) + if no_wrap: + new_lines = Lines([line]) + else: + offsets = divide_line(str(line), width, fold=wrap_overflow == "fold") + new_lines = line.divide(offsets) + for line in new_lines: + line.rstrip_end(width) + if wrap_justify: + new_lines.justify( + console, width, justify=wrap_justify, overflow=wrap_overflow + ) + for line in new_lines: + line.truncate(width, overflow=wrap_overflow) + lines.extend(new_lines) + return lines + + def fit(self, width: int) -> Lines: + """Fit the text in to given width by chopping in to lines. + + Args: + width (int): Maximum characters in a line. + + Returns: + Lines: List of lines. + """ + lines: Lines = Lines() + append = lines.append + for line in self.split(): + line.set_length(width) + append(line) + return lines + + def detect_indentation(self) -> int: + """Auto-detect indentation of code. + + Returns: + int: Number of spaces used to indent code. + """ + + _indentations = { + len(match.group(1)) + for match in re.finditer(r"^( *)(.*)$", self.plain, flags=re.MULTILINE) + } + + try: + indentation = ( + reduce(gcd, [indent for indent in _indentations if not indent % 2]) or 1 + ) + except TypeError: + indentation = 1 + + return indentation + + def with_indent_guides( + self, + indent_size: Optional[int] = None, + *, + character: str = "│", + style: StyleType = "dim green", + ) -> "Text": + """Adds indent guide lines to text. + + Args: + indent_size (Optional[int]): Size of indentation, or None to auto detect. Defaults to None. + character (str, optional): Character to use for indentation. Defaults to "│". + style (Union[Style, str], optional): Style of indent guides. + + Returns: + Text: New text with indentation guides. + """ + + _indent_size = self.detect_indentation() if indent_size is None else indent_size + + text = self.copy() + text.expand_tabs() + indent_line = f"{character}{' ' * (_indent_size - 1)}" + + re_indent = re.compile(r"^( *)(.*)$") + new_lines: List[Text] = [] + add_line = new_lines.append + blank_lines = 0 + for line in text.split(allow_blank=True): + match = re_indent.match(line.plain) + if not match or not match.group(2): + blank_lines += 1 + continue + indent = match.group(1) + full_indents, remaining_space = divmod(len(indent), _indent_size) + new_indent = f"{indent_line * full_indents}{' ' * remaining_space}" + line.plain = new_indent + line.plain[len(new_indent) :] + line.stylize(style, 0, len(new_indent)) + if blank_lines: + new_lines.extend([Text(new_indent, style=style)] * blank_lines) + blank_lines = 0 + add_line(line) + if blank_lines: + new_lines.extend([Text("", style=style)] * blank_lines) + + new_text = text.blank_copy("\n").join(new_lines) + return new_text + + +if __name__ == "__main__": # pragma: no cover + from pip._vendor.rich.console import Console + + text = Text( + """\nLorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n""" + ) + text.highlight_words(["Lorem"], "bold") + text.highlight_words(["ipsum"], "italic") + + console = Console() + + console.rule("justify='left'") + console.print(text, style="red") + console.print() + + console.rule("justify='center'") + console.print(text, style="green", justify="center") + console.print() + + console.rule("justify='right'") + console.print(text, style="blue", justify="right") + console.print() + + console.rule("justify='full'") + console.print(text, style="magenta", justify="full") + console.print() diff --git a/src/pip/_vendor/rich/theme.py b/src/pip/_vendor/rich/theme.py new file mode 100644 index 00000000000..bfb3c7f8215 --- /dev/null +++ b/src/pip/_vendor/rich/theme.py @@ -0,0 +1,112 @@ +import configparser +from typing import Dict, List, IO, Mapping, Optional + +from .default_styles import DEFAULT_STYLES +from .style import Style, StyleType + + +class Theme: + """A container for style information, used by :class:`~rich.console.Console`. + + Args: + styles (Dict[str, Style], optional): A mapping of style names on to styles. Defaults to None for a theme with no styles. + inherit (bool, optional): Inherit default styles. Defaults to True. + """ + + styles: Dict[str, Style] + + def __init__( + self, styles: Optional[Mapping[str, StyleType]] = None, inherit: bool = True + ): + self.styles = DEFAULT_STYLES.copy() if inherit else {} + if styles is not None: + self.styles.update( + { + name: style if isinstance(style, Style) else Style.parse(style) + for name, style in styles.items() + } + ) + + @property + def config(self) -> str: + """Get contents of a config file for this theme.""" + config = "[styles]\n" + "\n".join( + f"{name} = {style}" for name, style in sorted(self.styles.items()) + ) + return config + + @classmethod + def from_file( + cls, config_file: IO[str], source: Optional[str] = None, inherit: bool = True + ) -> "Theme": + """Load a theme from a text mode file. + + Args: + config_file (IO[str]): An open conf file. + source (str, optional): The filename of the open file. Defaults to None. + inherit (bool, optional): Inherit default styles. Defaults to True. + + Returns: + Theme: A New theme instance. + """ + config = configparser.ConfigParser() + config.read_file(config_file, source=source) + styles = {name: Style.parse(value) for name, value in config.items("styles")} + theme = Theme(styles, inherit=inherit) + return theme + + @classmethod + def read(cls, path: str, inherit: bool = True) -> "Theme": + """Read a theme from a path. + + Args: + path (str): Path to a config file readable by Python configparser module. + inherit (bool, optional): Inherit default styles. Defaults to True. + + Returns: + Theme: A new theme instance. + """ + with open(path, "rt") as config_file: + return cls.from_file(config_file, source=path, inherit=inherit) + + +class ThemeStackError(Exception): + """Base exception for errors related to the theme stack.""" + + +class ThemeStack: + """A stack of themes. + + Args: + theme (Theme): A theme instance + """ + + def __init__(self, theme: Theme) -> None: + self._entries: List[Dict[str, Style]] = [theme.styles] + self.get = self._entries[-1].get + + def push_theme(self, theme: Theme, inherit: bool = True) -> None: + """Push a theme on the top of the stack. + + Args: + theme (Theme): A Theme instance. + inherit (boolean, optional): Inherit styles from current top of stack. + """ + styles: Dict[str, Style] + styles = ( + {**self._entries[-1], **theme.styles} if inherit else theme.styles.copy() + ) + self._entries.append(styles) + self.get = self._entries[-1].get + + def pop_theme(self) -> None: + """Pop (and discard) the top-most theme.""" + if len(self._entries) == 1: + raise ThemeStackError("Unable to pop base theme") + self._entries.pop() + self.get = self._entries[-1].get + + +if __name__ == "__main__": # pragma: no cover + theme = Theme() + print(theme.config) diff --git a/src/pip/_vendor/rich/themes.py b/src/pip/_vendor/rich/themes.py new file mode 100644 index 00000000000..bf6db104a2c --- /dev/null +++ b/src/pip/_vendor/rich/themes.py @@ -0,0 +1,5 @@ +from .default_styles import DEFAULT_STYLES +from .theme import Theme + + +DEFAULT = Theme(DEFAULT_STYLES) diff --git a/src/pip/_vendor/rich/traceback.py b/src/pip/_vendor/rich/traceback.py new file mode 100644 index 00000000000..66a39ebab3c --- /dev/null +++ b/src/pip/_vendor/rich/traceback.py @@ -0,0 +1,678 @@ +from __future__ import absolute_import + +import os +import platform +import sys +from dataclasses import dataclass, field +from traceback import walk_tb +from types import ModuleType, TracebackType +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Type, Union + +from pip._vendor.pygments.lexers import guess_lexer_for_filename +from pip._vendor.pygments.token import Comment, Keyword, Name, Number, Operator, String +from pip._vendor.pygments.token import Text as TextToken +from pip._vendor.pygments.token import Token + +from . import pretty +from ._loop import loop_first, loop_last +from .columns import Columns +from .console import Console, ConsoleOptions, ConsoleRenderable, RenderResult, group +from .constrain import Constrain +from .highlighter import RegexHighlighter, ReprHighlighter +from .panel import Panel +from .scope import render_scope +from .style import Style +from .syntax import Syntax +from .text import Text +from .theme import Theme + +WINDOWS = platform.system() == "Windows" + +LOCALS_MAX_LENGTH = 10 +LOCALS_MAX_STRING = 80 + + +def install( + *, + console: Optional[Console] = None, + width: Optional[int] = 100, + extra_lines: int = 3, + theme: Optional[str] = None, + word_wrap: bool = False, + show_locals: bool = False, + indent_guides: bool = True, + suppress: Iterable[Union[str, ModuleType]] = (), + max_frames: int = 100, +) -> Callable[[Type[BaseException], BaseException, Optional[TracebackType]], Any]: + """Install a rich traceback handler. + + Once installed, any tracebacks will be printed with syntax highlighting and rich formatting. + + + Args: + console (Optional[Console], optional): Console to write exception to. Default uses internal Console instance. + width (Optional[int], optional): Width (in characters) of traceback. Defaults to 100. + extra_lines (int, optional): Extra lines of code. Defaults to 3. + theme (Optional[str], optional): Pygments theme to use in traceback. Defaults to ``None`` which will pick + a theme appropriate for the platform. + word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False. + show_locals (bool, optional): Enable display of local variables. Defaults to False. + indent_guides (bool, optional): Enable indent guides in code and locals. Defaults to True. + suppress (Sequence[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback. + + Returns: + Callable: The previous exception handler that was replaced. + + """ + traceback_console = Console(file=sys.stderr) if console is None else console + + def excepthook( + type_: Type[BaseException], + value: BaseException, + traceback: Optional[TracebackType], + ) -> None: + traceback_console.print( + Traceback.from_exception( + type_, + value, + traceback, + width=width, + extra_lines=extra_lines, + theme=theme, + word_wrap=word_wrap, + show_locals=show_locals, + indent_guides=indent_guides, + suppress=suppress, + max_frames=max_frames, + ) + ) + + def ipy_excepthook_closure(ip: Any) -> None: # pragma: no cover + tb_data = {} # store information about showtraceback call + default_showtraceback = ip.showtraceback # keep reference of default traceback + + def ipy_show_traceback(*args: Any, **kwargs: Any) -> None: + """wrap the default ip.showtraceback to store info for ip._showtraceback""" + nonlocal tb_data + tb_data = kwargs + default_showtraceback(*args, **kwargs) + + def ipy_display_traceback( + *args: Any, is_syntax: bool = False, **kwargs: Any + ) -> None: + """Internally called traceback from ip._showtraceback""" + nonlocal tb_data + exc_tuple = ip._get_exc_info() + + # do not display trace on syntax error + tb: Optional[TracebackType] = None if is_syntax else exc_tuple[2] + + # determine correct tb_offset + compiled = tb_data.get("running_compiled_code", False) + tb_offset = tb_data.get("tb_offset", 1 if compiled else 0) + # remove ipython internal frames from trace with tb_offset + for _ in range(tb_offset): + if tb is None: + break + tb = tb.tb_next + + excepthook(exc_tuple[0], exc_tuple[1], tb) + tb_data = {} # clear data upon usage + + # replace _showtraceback instead of showtraceback to allow ipython features such as debugging to work + # this is also what the ipython docs recommends to modify when subclassing InteractiveShell + ip._showtraceback = ipy_display_traceback + # add wrapper to capture tb_data + ip.showtraceback = ipy_show_traceback + ip.showsyntaxerror = lambda *args, **kwargs: ipy_display_traceback( + *args, is_syntax=True, **kwargs + ) + + try: # pragma: no cover + # if within ipython, use customized traceback + ip = get_ipython() # type: ignore + ipy_excepthook_closure(ip) + return sys.excepthook + except Exception: + # otherwise use default system hook + old_excepthook = sys.excepthook + sys.excepthook = excepthook + return old_excepthook + + +@dataclass +class Frame: + filename: str + lineno: int + name: str + line: str = "" + locals: Optional[Dict[str, pretty.Node]] = None + + +@dataclass +class _SyntaxError: + offset: int + filename: str + line: str + lineno: int + msg: str + + +@dataclass +class Stack: + exc_type: str + exc_value: str + syntax_error: Optional[_SyntaxError] = None + is_cause: bool = False + frames: List[Frame] = field(default_factory=list) + + +@dataclass +class Trace: + stacks: List[Stack] + + +class PathHighlighter(RegexHighlighter): + highlights = [r"(?P.*/)(?P.+)"] + + +class Traceback: + """A Console renderable that renders a traceback. + + Args: + trace (Trace, optional): A `Trace` object produced from `extract`. Defaults to None, which uses + the last exception. + width (Optional[int], optional): Number of characters used to traceback. Defaults to 100. + extra_lines (int, optional): Additional lines of code to render. Defaults to 3. + theme (str, optional): Override pygments theme used in traceback. + word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False. + show_locals (bool, optional): Enable display of local variables. Defaults to False. + indent_guides (bool, optional): Enable indent guides in code and locals. Defaults to True. + locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to 10. + locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80. + suppress (Sequence[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback. + max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100. + + """ + + LEXERS = { + "": "text", + ".py": "python", + ".pxd": "cython", + ".pyx": "cython", + ".pxi": "pyrex", + } + + def __init__( + self, + trace: Optional[Trace] = None, + width: Optional[int] = 100, + extra_lines: int = 3, + theme: Optional[str] = None, + word_wrap: bool = False, + show_locals: bool = False, + indent_guides: bool = True, + locals_max_length: int = LOCALS_MAX_LENGTH, + locals_max_string: int = LOCALS_MAX_STRING, + suppress: Iterable[Union[str, ModuleType]] = (), + max_frames: int = 100, + ): + if trace is None: + exc_type, exc_value, traceback = sys.exc_info() + if exc_type is None or exc_value is None or traceback is None: + raise ValueError( + "Value for 'trace' required if not called in except: block" + ) + trace = self.extract( + exc_type, exc_value, traceback, show_locals=show_locals + ) + self.trace = trace + self.width = width + self.extra_lines = extra_lines + self.theme = Syntax.get_theme(theme or "ansi_dark") + self.word_wrap = word_wrap + self.show_locals = show_locals + self.indent_guides = indent_guides + self.locals_max_length = locals_max_length + self.locals_max_string = locals_max_string + + self.suppress: Sequence[str] = [] + for suppress_entity in suppress: + if not isinstance(suppress_entity, str): + assert ( + suppress_entity.__file__ is not None + ), f"{suppress_entity!r} must be a module with '__file__' attribute" + path = os.path.dirname(suppress_entity.__file__) + else: + path = suppress_entity + path = os.path.normpath(os.path.abspath(path)) + self.suppress.append(path) + self.max_frames = max(4, max_frames) if max_frames > 0 else 0 + + @classmethod + def from_exception( + cls, + exc_type: Type[Any], + exc_value: BaseException, + traceback: Optional[TracebackType], + width: Optional[int] = 100, + extra_lines: int = 3, + theme: Optional[str] = None, + word_wrap: bool = False, + show_locals: bool = False, + indent_guides: bool = True, + locals_max_length: int = LOCALS_MAX_LENGTH, + locals_max_string: int = LOCALS_MAX_STRING, + suppress: Iterable[Union[str, ModuleType]] = (), + max_frames: int = 100, + ) -> "Traceback": + """Create a traceback from exception info + + Args: + exc_type (Type[BaseException]): Exception type. + exc_value (BaseException): Exception value. + traceback (TracebackType): Python Traceback object. + width (Optional[int], optional): Number of characters used to traceback. Defaults to 100. + extra_lines (int, optional): Additional lines of code to render. Defaults to 3. + theme (str, optional): Override pygments theme used in traceback. + word_wrap (bool, optional): Enable word wrapping of long lines. Defaults to False. + show_locals (bool, optional): Enable display of local variables. Defaults to False. + indent_guides (bool, optional): Enable indent guides in code and locals. Defaults to True. + locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to 10. + locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80. + suppress (Iterable[Union[str, ModuleType]]): Optional sequence of modules or paths to exclude from traceback. + max_frames (int): Maximum number of frames to show in a traceback, 0 for no maximum. Defaults to 100. + + Returns: + Traceback: A Traceback instance that may be printed. + """ + rich_traceback = cls.extract( + exc_type, exc_value, traceback, show_locals=show_locals + ) + return cls( + rich_traceback, + width=width, + extra_lines=extra_lines, + theme=theme, + word_wrap=word_wrap, + show_locals=show_locals, + indent_guides=indent_guides, + locals_max_length=locals_max_length, + locals_max_string=locals_max_string, + suppress=suppress, + max_frames=max_frames, + ) + + @classmethod + def extract( + cls, + exc_type: Type[BaseException], + exc_value: BaseException, + traceback: Optional[TracebackType], + show_locals: bool = False, + locals_max_length: int = LOCALS_MAX_LENGTH, + locals_max_string: int = LOCALS_MAX_STRING, + ) -> Trace: + """Extract traceback information. + + Args: + exc_type (Type[BaseException]): Exception type. + exc_value (BaseException): Exception value. + traceback (TracebackType): Python Traceback object. + show_locals (bool, optional): Enable display of local variables. Defaults to False. + locals_max_length (int, optional): Maximum length of containers before abbreviating, or None for no abbreviation. + Defaults to 10. + locals_max_string (int, optional): Maximum length of string before truncating, or None to disable. Defaults to 80. + + Returns: + Trace: A Trace instance which you can use to construct a `Traceback`. + """ + + stacks: List[Stack] = [] + is_cause = False + + from pip._vendor.rich import _IMPORT_CWD + + def safe_str(_object: Any) -> str: + """Don't allow exceptions from __str__ to propegate.""" + try: + return str(_object) + except Exception: + return "" + + while True: + stack = Stack( + exc_type=safe_str(exc_type.__name__), + exc_value=safe_str(exc_value), + is_cause=is_cause, + ) + + if isinstance(exc_value, SyntaxError): + stack.syntax_error = _SyntaxError( + offset=exc_value.offset or 0, + filename=exc_value.filename or "?", + lineno=exc_value.lineno or 0, + line=exc_value.text or "", + msg=exc_value.msg, + ) + + stacks.append(stack) + append = stack.frames.append + + for frame_summary, line_no in walk_tb(traceback): + filename = frame_summary.f_code.co_filename + if filename and not filename.startswith("<"): + if not os.path.isabs(filename): + filename = os.path.join(_IMPORT_CWD, filename) + frame = Frame( + filename=filename or "?", + lineno=line_no, + name=frame_summary.f_code.co_name, + locals={ + key: pretty.traverse( + value, + max_length=locals_max_length, + max_string=locals_max_string, + ) + for key, value in frame_summary.f_locals.items() + } + if show_locals + else None, + ) + append(frame) + if "_rich_traceback_guard" in frame_summary.f_locals: + del stack.frames[:] + + cause = getattr(exc_value, "__cause__", None) + if cause and cause.__traceback__: + exc_type = cause.__class__ + exc_value = cause + traceback = cause.__traceback__ + if traceback: + is_cause = True + continue + + cause = exc_value.__context__ + if ( + cause + and cause.__traceback__ + and not getattr(exc_value, "__suppress_context__", False) + ): + exc_type = cause.__class__ + exc_value = cause + traceback = cause.__traceback__ + if traceback: + is_cause = False + continue + # No cover, code is reached but coverage doesn't recognize it. + break # pragma: no cover + + trace = Trace(stacks=stacks) + return trace + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + theme = self.theme + background_style = theme.get_background_style() + token_style = theme.get_style_for_token + + traceback_theme = Theme( + { + "pretty": token_style(TextToken), + "pygments.text": token_style(Token), + "pygments.string": token_style(String), + "pygments.function": token_style(Name.Function), + "pygments.number": token_style(Number), + "repr.indent": token_style(Comment) + Style(dim=True), + "repr.str": token_style(String), + "repr.brace": token_style(TextToken) + Style(bold=True), + "repr.number": token_style(Number), + "repr.bool_true": token_style(Keyword.Constant), + "repr.bool_false": token_style(Keyword.Constant), + "repr.none": token_style(Keyword.Constant), + "scope.border": token_style(String.Delimiter), + "scope.equals": token_style(Operator), + "scope.key": token_style(Name), + "scope.key.special": token_style(Name.Constant) + Style(dim=True), + }, + inherit=False, + ) + + highlighter = ReprHighlighter() + for last, stack in loop_last(reversed(self.trace.stacks)): + if stack.frames: + stack_renderable: ConsoleRenderable = Panel( + self._render_stack(stack), + title="[traceback.title]Traceback [dim](most recent call last)", + style=background_style, + border_style="traceback.border", + expand=True, + padding=(0, 1), + ) + stack_renderable = Constrain(stack_renderable, self.width) + with console.use_theme(traceback_theme): + yield stack_renderable + if stack.syntax_error is not None: + with console.use_theme(traceback_theme): + yield Constrain( + Panel( + self._render_syntax_error(stack.syntax_error), + style=background_style, + border_style="traceback.border.syntax_error", + expand=True, + padding=(0, 1), + width=self.width, + ), + self.width, + ) + yield Text.assemble( + (f"{stack.exc_type}: ", "traceback.exc_type"), + highlighter(stack.syntax_error.msg), + ) + elif stack.exc_value: + yield Text.assemble( + (f"{stack.exc_type}: ", "traceback.exc_type"), + highlighter(stack.exc_value), + ) + else: + yield Text.assemble((f"{stack.exc_type}", "traceback.exc_type")) + + if not last: + if stack.is_cause: + yield Text.from_markup( + "\n[i]The above exception was the direct cause of the following exception:\n", + ) + else: + yield Text.from_markup( + "\n[i]During handling of the above exception, another exception occurred:\n", + ) + + @group() + def _render_syntax_error(self, syntax_error: _SyntaxError) -> RenderResult: + highlighter = ReprHighlighter() + path_highlighter = PathHighlighter() + if syntax_error.filename != "": + text = Text.assemble( + (f" {syntax_error.filename}", "pygments.string"), + (":", "pygments.text"), + (str(syntax_error.lineno), "pygments.number"), + style="pygments.text", + ) + yield path_highlighter(text) + syntax_error_text = highlighter(syntax_error.line.rstrip()) + syntax_error_text.no_wrap = True + offset = min(syntax_error.offset - 1, len(syntax_error_text)) + syntax_error_text.stylize("bold underline", offset, offset) + syntax_error_text += Text.from_markup( + "\n" + " " * offset + "[traceback.offset]▲[/]", + style="pygments.text", + ) + yield syntax_error_text + + @classmethod + def _guess_lexer(cls, filename: str, code: str) -> str: + ext = os.path.splitext(filename)[-1] + if not ext: + # No extension, look at first line to see if it is a hashbang + # Note, this is an educated guess and not a guarantee + # If it fails, the only downside is that the code is highlighted strangely + new_line_index = code.index("\n") + first_line = code[:new_line_index] if new_line_index != -1 else code + if first_line.startswith("#!") and "python" in first_line.lower(): + return "python" + lexer_name = ( + cls.LEXERS.get(ext) or guess_lexer_for_filename(filename, code).name + ) + return lexer_name + + @group() + def _render_stack(self, stack: Stack) -> RenderResult: + path_highlighter = PathHighlighter() + theme = self.theme + code_cache: Dict[str, str] = {} + + def read_code(filename: str) -> str: + """Read files, and cache results on filename. + + Args: + filename (str): Filename to read + + Returns: + str: Contents of file + """ + code = code_cache.get(filename) + if code is None: + with open( + filename, "rt", encoding="utf-8", errors="replace" + ) as code_file: + code = code_file.read() + code_cache[filename] = code + return code + + def render_locals(frame: Frame) -> Iterable[ConsoleRenderable]: + if frame.locals: + yield render_scope( + frame.locals, + title="locals", + indent_guides=self.indent_guides, + max_length=self.locals_max_length, + max_string=self.locals_max_string, + ) + + exclude_frames: Optional[range] = None + if self.max_frames != 0: + exclude_frames = range( + self.max_frames // 2, + len(stack.frames) - self.max_frames // 2, + ) + + excluded = False + for frame_index, frame in enumerate(stack.frames): + + if exclude_frames and frame_index in exclude_frames: + excluded = True + continue + + if excluded: + assert exclude_frames is not None + yield Text( + f"\n... {len(exclude_frames)} frames hidden ...", + justify="center", + style="traceback.error", + ) + excluded = False + + first = frame_index == 1 + frame_filename = frame.filename + suppressed = any(frame_filename.startswith(path) for path in self.suppress) + + text = Text.assemble( + path_highlighter(Text(frame.filename, style="pygments.string")), + (":", "pygments.text"), + (str(frame.lineno), "pygments.number"), + " in ", + (frame.name, "pygments.function"), + style="pygments.text", + ) + if not frame.filename.startswith("<") and not first: + yield "" + yield text + if frame.filename.startswith("<"): + yield from render_locals(frame) + continue + if not suppressed: + try: + code = read_code(frame.filename) + lexer_name = self._guess_lexer(frame.filename, code) + syntax = Syntax( + code, + lexer_name, + theme=theme, + line_numbers=True, + line_range=( + frame.lineno - self.extra_lines, + frame.lineno + self.extra_lines, + ), + highlight_lines={frame.lineno}, + word_wrap=self.word_wrap, + code_width=88, + indent_guides=self.indent_guides, + dedent=False, + ) + yield "" + except Exception as error: + yield Text.assemble( + (f"\n{error}", "traceback.error"), + ) + else: + yield ( + Columns( + [ + syntax, + *render_locals(frame), + ], + padding=1, + ) + if frame.locals + else syntax + ) + + +if __name__ == "__main__": # pragma: no cover + + from .console import Console + + console = Console() + import sys + + def bar(a: Any) -> None: # 这是对亚洲语言支持的测试。面对模棱两可的想法,拒绝猜测的诱惑 + one = 1 + print(one / a) + + def foo(a: Any) -> None: + _rich_traceback_guard = True + zed = { + "characters": { + "Paul Atreides", + "Vladimir Harkonnen", + "Thufir Hawat", + "Duncan Idaho", + }, + "atomic_types": (None, False, True), + } + bar(a) + + def error() -> None: + + try: + try: + foo(0) + except: + slfkjsldkfj # type: ignore + except: + console.print_exception(show_locals=True) + + error() diff --git a/src/pip/_vendor/rich/tree.py b/src/pip/_vendor/rich/tree.py new file mode 100644 index 00000000000..c5ec27da932 --- /dev/null +++ b/src/pip/_vendor/rich/tree.py @@ -0,0 +1,249 @@ +from typing import Iterator, List, Optional, Tuple + +from ._loop import loop_first, loop_last +from .console import Console, ConsoleOptions, RenderableType, RenderResult +from .jupyter import JupyterMixin +from .measure import Measurement +from .segment import Segment +from .style import Style, StyleStack, StyleType +from .styled import Styled + + +class Tree(JupyterMixin): + """A renderable for a tree structure. + + Args: + label (RenderableType): The renderable or str for the tree label. + style (StyleType, optional): Style of this tree. Defaults to "tree". + guide_style (StyleType, optional): Style of the guide lines. Defaults to "tree.line". + expanded (bool, optional): Also display children. Defaults to True. + highlight (bool, optional): Highlight renderable (if str). Defaults to False. + """ + + def __init__( + self, + label: RenderableType, + *, + style: StyleType = "tree", + guide_style: StyleType = "tree.line", + expanded: bool = True, + highlight: bool = False, + hide_root: bool = False, + ) -> None: + self.label = label + self.style = style + self.guide_style = guide_style + self.children: List[Tree] = [] + self.expanded = expanded + self.highlight = highlight + self.hide_root = hide_root + + def add( + self, + label: RenderableType, + *, + style: Optional[StyleType] = None, + guide_style: Optional[StyleType] = None, + expanded: bool = True, + highlight: bool = False, + ) -> "Tree": + """Add a child tree. + + Args: + label (RenderableType): The renderable or str for the tree label. + style (StyleType, optional): Style of this tree. Defaults to "tree". + guide_style (StyleType, optional): Style of the guide lines. Defaults to "tree.line". + expanded (bool, optional): Also display children. Defaults to True. + highlight (Optional[bool], optional): Highlight renderable (if str). Defaults to False. + + Returns: + Tree: A new child Tree, which may be further modified. + """ + node = Tree( + label, + style=self.style if style is None else style, + guide_style=self.guide_style if guide_style is None else guide_style, + expanded=expanded, + highlight=self.highlight if highlight is None else highlight, + ) + self.children.append(node) + return node + + def __rich_console__( + self, console: "Console", options: "ConsoleOptions" + ) -> "RenderResult": + + stack: List[Iterator[Tuple[bool, Tree]]] = [] + pop = stack.pop + push = stack.append + new_line = Segment.line() + + get_style = console.get_style + null_style = Style.null() + guide_style = get_style(self.guide_style, default="") or null_style + SPACE, CONTINUE, FORK, END = range(4) + + ASCII_GUIDES = (" ", "| ", "+-- ", "`-- ") + TREE_GUIDES = [ + (" ", "│ ", "├── ", "└── "), + (" ", "┃ ", "┣━━ ", "┗━━ "), + (" ", "║ ", "╠══ ", "╚══ "), + ] + _Segment = Segment + + def make_guide(index: int, style: Style) -> Segment: + """Make a Segment for a level of the guide lines.""" + if options.ascii_only: + line = ASCII_GUIDES[index] + else: + guide = 1 if style.bold else (2 if style.underline2 else 0) + line = TREE_GUIDES[0 if options.legacy_windows else guide][index] + return _Segment(line, style) + + levels: List[Segment] = [make_guide(CONTINUE, guide_style)] + push(iter(loop_last([self]))) + + guide_style_stack = StyleStack(get_style(self.guide_style)) + style_stack = StyleStack(get_style(self.style)) + remove_guide_styles = Style(bold=False, underline2=False) + + depth = 0 + + while stack: + stack_node = pop() + try: + last, node = next(stack_node) + except StopIteration: + levels.pop() + if levels: + guide_style = levels[-1].style or null_style + levels[-1] = make_guide(FORK, guide_style) + guide_style_stack.pop() + style_stack.pop() + continue + push(stack_node) + if last: + levels[-1] = make_guide(END, levels[-1].style or null_style) + + guide_style = guide_style_stack.current + get_style(node.guide_style) + style = style_stack.current + get_style(node.style) + prefix = levels[(2 if self.hide_root else 1) :] + renderable_lines = console.render_lines( + Styled(node.label, style), + options.update( + width=options.max_width + - sum(level.cell_length for level in prefix), + highlight=self.highlight, + height=None, + ), + ) + + if not (depth == 0 and self.hide_root): + for first, line in loop_first(renderable_lines): + if prefix: + yield from _Segment.apply_style( + prefix, + style.background_style, + post_style=remove_guide_styles, + ) + yield from line + yield new_line + if first and prefix: + prefix[-1] = make_guide( + SPACE if last else CONTINUE, prefix[-1].style or null_style + ) + + if node.expanded and node.children: + levels[-1] = make_guide( + SPACE if last else CONTINUE, levels[-1].style or null_style + ) + levels.append( + make_guide(END if len(node.children) == 1 else FORK, guide_style) + ) + style_stack.push(get_style(node.style)) + guide_style_stack.push(get_style(node.guide_style)) + push(iter(loop_last(node.children))) + depth += 1 + + def __rich_measure__( + self, console: "Console", options: "ConsoleOptions" + ) -> "Measurement": + stack: List[Iterator[Tree]] = [iter([self])] + pop = stack.pop + push = stack.append + minimum = 0 + maximum = 0 + measure = Measurement.get + level = 0 + while stack: + iter_tree = pop() + try: + tree = next(iter_tree) + except StopIteration: + level -= 1 + continue + push(iter_tree) + min_measure, max_measure = measure(console, options, tree.label) + indent = level * 4 + minimum = max(min_measure + indent, minimum) + maximum = max(max_measure + indent, maximum) + if tree.expanded and tree.children: + push(iter(tree.children)) + level += 1 + return Measurement(minimum, maximum) + + +if __name__ == "__main__": # pragma: no cover + + from pip._vendor.rich.console import Group + from pip._vendor.rich.markdown import Markdown + from pip._vendor.rich.panel import Panel + from pip._vendor.rich.syntax import Syntax + from pip._vendor.rich.table import Table + + table = Table(row_styles=["", "dim"]) + + table.add_column("Released", style="cyan", no_wrap=True) + table.add_column("Title", style="magenta") + table.add_column("Box Office", justify="right", style="green") + + table.add_row("Dec 20, 2019", "Star Wars: The Rise of Skywalker", "$952,110,690") + table.add_row("May 25, 2018", "Solo: A Star Wars Story", "$393,151,347") + table.add_row("Dec 15, 2017", "Star Wars Ep. V111: The Last Jedi", "$1,332,539,889") + table.add_row("Dec 16, 2016", "Rogue One: A Star Wars Story", "$1,332,439,889") + + code = """\ +class Segment(NamedTuple): + text: str = "" + style: Optional[Style] = None + is_control: bool = False +""" + syntax = Syntax(code, "python", theme="monokai", line_numbers=True) + + markdown = Markdown( + """\ +### example.md +> Hello, World! +> +> Markdown _all_ the things +""" + ) + + root = Tree("🌲 [b green]Rich Tree", highlight=True, hide_root=True) + + node = root.add(":file_folder: Renderables", guide_style="red") + simple_node = node.add(":file_folder: [bold yellow]Atomic", guide_style="uu green") + simple_node.add(Group("📄 Syntax", syntax)) + simple_node.add(Group("📄 Markdown", Panel(markdown, border_style="green"))) + + containers_node = node.add( + ":file_folder: [bold magenta]Containers", guide_style="bold magenta" + ) + containers_node.expanded = True + panel = Panel.fit("Just a panel", border_style="red") + containers_node.add(Group("📄 Panels", panel)) + + containers_node.add(Group("📄 [b magenta]Table", table)) + + console = Console() + console.print(root) diff --git a/src/pip/_vendor/six.py b/src/pip/_vendor/six.py index 83f69783d1a..4e15675d8b5 100644 --- a/src/pip/_vendor/six.py +++ b/src/pip/_vendor/six.py @@ -29,7 +29,7 @@ import types __author__ = "Benjamin Peterson " -__version__ = "1.15.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ def __len__(self): MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -186,6 +191,11 @@ def find_module(self, fullname, path=None): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -223,6 +233,12 @@ def get_code(self, fullname): return None get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) diff --git a/src/pip/_vendor/retrying.LICENSE b/src/pip/_vendor/tenacity/LICENSE similarity index 100% rename from src/pip/_vendor/retrying.LICENSE rename to src/pip/_vendor/tenacity/LICENSE diff --git a/src/pip/_vendor/tenacity/__init__.py b/src/pip/_vendor/tenacity/__init__.py new file mode 100644 index 00000000000..086ad46e1d6 --- /dev/null +++ b/src/pip/_vendor/tenacity/__init__.py @@ -0,0 +1,517 @@ +# Copyright 2016-2018 Julien Danjou +# Copyright 2017 Elisey Zanko +# Copyright 2016 Étienne Bersac +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import sys +import threading +import time +import typing as t +import warnings +from abc import ABC, abstractmethod +from concurrent import futures +from inspect import iscoroutinefunction + +# Import all built-in retry strategies for easier usage. +from .retry import retry_base # noqa +from .retry import retry_all # noqa +from .retry import retry_always # noqa +from .retry import retry_any # noqa +from .retry import retry_if_exception # noqa +from .retry import retry_if_exception_type # noqa +from .retry import retry_if_not_exception_type # noqa +from .retry import retry_if_not_result # noqa +from .retry import retry_if_result # noqa +from .retry import retry_never # noqa +from .retry import retry_unless_exception_type # noqa +from .retry import retry_if_exception_message # noqa +from .retry import retry_if_not_exception_message # noqa + +# Import all nap strategies for easier usage. +from .nap import sleep # noqa +from .nap import sleep_using_event # noqa + +# Import all built-in stop strategies for easier usage. +from .stop import stop_after_attempt # noqa +from .stop import stop_after_delay # noqa +from .stop import stop_all # noqa +from .stop import stop_any # noqa +from .stop import stop_never # noqa +from .stop import stop_when_event_set # noqa + +# Import all built-in wait strategies for easier usage. +from .wait import wait_chain # noqa +from .wait import wait_combine # noqa +from .wait import wait_exponential # noqa +from .wait import wait_fixed # noqa +from .wait import wait_incrementing # noqa +from .wait import wait_none # noqa +from .wait import wait_random # noqa +from .wait import wait_random_exponential # noqa +from .wait import wait_random_exponential as wait_full_jitter # noqa + +# Import all built-in before strategies for easier usage. +from .before import before_log # noqa +from .before import before_nothing # noqa + +# Import all built-in after strategies for easier usage. +from .after import after_log # noqa +from .after import after_nothing # noqa + +# Import all built-in after strategies for easier usage. +from .before_sleep import before_sleep_log # noqa +from .before_sleep import before_sleep_nothing # noqa + +# Replace a conditional import with a hard-coded None so that pip does +# not attempt to use tornado even if it is present in the environment. +# If tornado is non-None, tenacity will attempt to execute some code +# that is sensitive to the version of tornado, which could break pip +# if an old version is found. +tornado = None # type: ignore + +if t.TYPE_CHECKING: + import types + + from .wait import wait_base + from .stop import stop_base + + +WrappedFn = t.TypeVar("WrappedFn", bound=t.Callable) +_RetValT = t.TypeVar("_RetValT") + + +@t.overload +def retry(fn: WrappedFn) -> WrappedFn: + pass + + +@t.overload +def retry(*dargs: t.Any, **dkw: t.Any) -> t.Callable[[WrappedFn], WrappedFn]: # noqa + pass + + +def retry(*dargs: t.Any, **dkw: t.Any) -> t.Union[WrappedFn, t.Callable[[WrappedFn], WrappedFn]]: # noqa + """Wrap a function with a new `Retrying` object. + + :param dargs: positional arguments passed to Retrying object + :param dkw: keyword arguments passed to the Retrying object + """ + # support both @retry and @retry() as valid syntax + if len(dargs) == 1 and callable(dargs[0]): + return retry()(dargs[0]) + else: + + def wrap(f: WrappedFn) -> WrappedFn: + if isinstance(f, retry_base): + warnings.warn( + f"Got retry_base instance ({f.__class__.__name__}) as callable argument, " + f"this will probably hang indefinitely (did you mean retry={f.__class__.__name__}(...)?)" + ) + if iscoroutinefunction(f): + r: "BaseRetrying" = AsyncRetrying(*dargs, **dkw) + elif tornado and hasattr(tornado.gen, "is_coroutine_function") and tornado.gen.is_coroutine_function(f): + r = TornadoRetrying(*dargs, **dkw) + else: + r = Retrying(*dargs, **dkw) + + return r.wraps(f) + + return wrap + + +class TryAgain(Exception): + """Always retry the executed function when raised.""" + + +NO_RESULT = object() + + +class DoAttempt: + pass + + +class DoSleep(float): + pass + + +class BaseAction: + """Base class for representing actions to take by retry object. + + Concrete implementations must define: + - __init__: to initialize all necessary fields + - REPR_FIELDS: class variable specifying attributes to include in repr(self) + - NAME: for identification in retry object methods and callbacks + """ + + REPR_FIELDS: t.Sequence[str] = () + NAME: t.Optional[str] = None + + def __repr__(self) -> str: + state_str = ", ".join(f"{field}={getattr(self, field)!r}" for field in self.REPR_FIELDS) + return f"{self.__class__.__name__}({state_str})" + + def __str__(self) -> str: + return repr(self) + + +class RetryAction(BaseAction): + REPR_FIELDS = ("sleep",) + NAME = "retry" + + def __init__(self, sleep: t.SupportsFloat) -> None: + self.sleep = float(sleep) + + +_unset = object() + + +def _first_set(first: t.Union[t.Any, object], second: t.Any) -> t.Any: + return second if first is _unset else first + + +class RetryError(Exception): + """Encapsulates the last attempt instance right before giving up.""" + + def __init__(self, last_attempt: "Future") -> None: + self.last_attempt = last_attempt + super().__init__(last_attempt) + + def reraise(self) -> "t.NoReturn": + if self.last_attempt.failed: + raise self.last_attempt.result() + raise self + + def __str__(self) -> str: + return f"{self.__class__.__name__}[{self.last_attempt}]" + + +class AttemptManager: + """Manage attempt context.""" + + def __init__(self, retry_state: "RetryCallState"): + self.retry_state = retry_state + + def __enter__(self) -> None: + pass + + def __exit__( + self, + exc_type: t.Optional[t.Type[BaseException]], + exc_value: t.Optional[BaseException], + traceback: t.Optional["types.TracebackType"], + ) -> t.Optional[bool]: + if isinstance(exc_value, BaseException): + self.retry_state.set_exception((exc_type, exc_value, traceback)) + return True # Swallow exception. + else: + # We don't have the result, actually. + self.retry_state.set_result(None) + return None + + +class BaseRetrying(ABC): + def __init__( + self, + sleep: t.Callable[[t.Union[int, float]], None] = sleep, + stop: "stop_base" = stop_never, + wait: "wait_base" = wait_none(), + retry: retry_base = retry_if_exception_type(), + before: t.Callable[["RetryCallState"], None] = before_nothing, + after: t.Callable[["RetryCallState"], None] = after_nothing, + before_sleep: t.Optional[t.Callable[["RetryCallState"], None]] = None, + reraise: bool = False, + retry_error_cls: t.Type[RetryError] = RetryError, + retry_error_callback: t.Optional[t.Callable[["RetryCallState"], t.Any]] = None, + ): + self.sleep = sleep + self.stop = stop + self.wait = wait + self.retry = retry + self.before = before + self.after = after + self.before_sleep = before_sleep + self.reraise = reraise + self._local = threading.local() + self.retry_error_cls = retry_error_cls + self.retry_error_callback = retry_error_callback + + def copy( + self, + sleep: t.Union[t.Callable[[t.Union[int, float]], None], object] = _unset, + stop: t.Union["stop_base", object] = _unset, + wait: t.Union["wait_base", object] = _unset, + retry: t.Union[retry_base, object] = _unset, + before: t.Union[t.Callable[["RetryCallState"], None], object] = _unset, + after: t.Union[t.Callable[["RetryCallState"], None], object] = _unset, + before_sleep: t.Union[t.Optional[t.Callable[["RetryCallState"], None]], object] = _unset, + reraise: t.Union[bool, object] = _unset, + retry_error_cls: t.Union[t.Type[RetryError], object] = _unset, + retry_error_callback: t.Union[t.Optional[t.Callable[["RetryCallState"], t.Any]], object] = _unset, + ) -> "BaseRetrying": + """Copy this object with some parameters changed if needed.""" + return self.__class__( + sleep=_first_set(sleep, self.sleep), + stop=_first_set(stop, self.stop), + wait=_first_set(wait, self.wait), + retry=_first_set(retry, self.retry), + before=_first_set(before, self.before), + after=_first_set(after, self.after), + before_sleep=_first_set(before_sleep, self.before_sleep), + reraise=_first_set(reraise, self.reraise), + retry_error_cls=_first_set(retry_error_cls, self.retry_error_cls), + retry_error_callback=_first_set(retry_error_callback, self.retry_error_callback), + ) + + def __repr__(self) -> str: + return ( + f"<{self.__class__.__name__} object at 0x{id(self):x} (" + f"stop={self.stop}, " + f"wait={self.wait}, " + f"sleep={self.sleep}, " + f"retry={self.retry}, " + f"before={self.before}, " + f"after={self.after})>" + ) + + @property + def statistics(self) -> t.Dict[str, t.Any]: + """Return a dictionary of runtime statistics. + + This dictionary will be empty when the controller has never been + ran. When it is running or has ran previously it should have (but + may not) have useful and/or informational keys and values when + running is underway and/or completed. + + .. warning:: The keys in this dictionary **should** be some what + stable (not changing), but there existence **may** + change between major releases as new statistics are + gathered or removed so before accessing keys ensure that + they actually exist and handle when they do not. + + .. note:: The values in this dictionary are local to the thread + running call (so if multiple threads share the same retrying + object - either directly or indirectly) they will each have + there own view of statistics they have collected (in the + future we may provide a way to aggregate the various + statistics from each thread). + """ + try: + return self._local.statistics + except AttributeError: + self._local.statistics = {} + return self._local.statistics + + def wraps(self, f: WrappedFn) -> WrappedFn: + """Wrap a function for retrying. + + :param f: A function to wraps for retrying. + """ + + @functools.wraps(f) + def wrapped_f(*args: t.Any, **kw: t.Any) -> t.Any: + return self(f, *args, **kw) + + def retry_with(*args: t.Any, **kwargs: t.Any) -> WrappedFn: + return self.copy(*args, **kwargs).wraps(f) + + wrapped_f.retry = self + wrapped_f.retry_with = retry_with + + return wrapped_f + + def begin(self) -> None: + self.statistics.clear() + self.statistics["start_time"] = time.monotonic() + self.statistics["attempt_number"] = 1 + self.statistics["idle_for"] = 0 + + def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.Any]: # noqa + fut = retry_state.outcome + if fut is None: + if self.before is not None: + self.before(retry_state) + return DoAttempt() + + is_explicit_retry = retry_state.outcome.failed and isinstance(retry_state.outcome.exception(), TryAgain) + if not (is_explicit_retry or self.retry(retry_state=retry_state)): + return fut.result() + + if self.after is not None: + self.after(retry_state) + + self.statistics["delay_since_first_attempt"] = retry_state.seconds_since_start + if self.stop(retry_state=retry_state): + if self.retry_error_callback: + return self.retry_error_callback(retry_state) + retry_exc = self.retry_error_cls(fut) + if self.reraise: + raise retry_exc.reraise() + raise retry_exc from fut.exception() + + if self.wait: + sleep = self.wait(retry_state=retry_state) + else: + sleep = 0.0 + retry_state.next_action = RetryAction(sleep) + retry_state.idle_for += sleep + self.statistics["idle_for"] += sleep + self.statistics["attempt_number"] += 1 + + if self.before_sleep is not None: + self.before_sleep(retry_state) + + return DoSleep(sleep) + + def __iter__(self) -> t.Generator[AttemptManager, None, None]: + self.begin() + + retry_state = RetryCallState(self, fn=None, args=(), kwargs={}) + while True: + do = self.iter(retry_state=retry_state) + if isinstance(do, DoAttempt): + yield AttemptManager(retry_state=retry_state) + elif isinstance(do, DoSleep): + retry_state.prepare_for_next_attempt() + self.sleep(do) + else: + break + + @abstractmethod + def __call__(self, fn: t.Callable[..., _RetValT], *args: t.Any, **kwargs: t.Any) -> _RetValT: + pass + + +class Retrying(BaseRetrying): + """Retrying controller.""" + + def __call__(self, fn: t.Callable[..., _RetValT], *args: t.Any, **kwargs: t.Any) -> _RetValT: + self.begin() + + retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs) + while True: + do = self.iter(retry_state=retry_state) + if isinstance(do, DoAttempt): + try: + result = fn(*args, **kwargs) + except BaseException: # noqa: B902 + retry_state.set_exception(sys.exc_info()) + else: + retry_state.set_result(result) + elif isinstance(do, DoSleep): + retry_state.prepare_for_next_attempt() + self.sleep(do) + else: + return do + + +class Future(futures.Future): + """Encapsulates a (future or past) attempted call to a target function.""" + + def __init__(self, attempt_number: int) -> None: + super().__init__() + self.attempt_number = attempt_number + + @property + def failed(self) -> bool: + """Return whether a exception is being held in this future.""" + return self.exception() is not None + + @classmethod + def construct(cls, attempt_number: int, value: t.Any, has_exception: bool) -> "Future": + """Construct a new Future object.""" + fut = cls(attempt_number) + if has_exception: + fut.set_exception(value) + else: + fut.set_result(value) + return fut + + +class RetryCallState: + """State related to a single call wrapped with Retrying.""" + + def __init__( + self, + retry_object: BaseRetrying, + fn: t.Optional[WrappedFn], + args: t.Any, + kwargs: t.Any, + ) -> None: + #: Retry call start timestamp + self.start_time = time.monotonic() + #: Retry manager object + self.retry_object = retry_object + #: Function wrapped by this retry call + self.fn = fn + #: Arguments of the function wrapped by this retry call + self.args = args + #: Keyword arguments of the function wrapped by this retry call + self.kwargs = kwargs + + #: The number of the current attempt + self.attempt_number: int = 1 + #: Last outcome (result or exception) produced by the function + self.outcome: t.Optional[Future] = None + #: Timestamp of the last outcome + self.outcome_timestamp: t.Optional[float] = None + #: Time spent sleeping in retries + self.idle_for: float = 0.0 + #: Next action as decided by the retry manager + self.next_action: t.Optional[RetryAction] = None + + @property + def seconds_since_start(self) -> t.Optional[float]: + if self.outcome_timestamp is None: + return None + return self.outcome_timestamp - self.start_time + + def prepare_for_next_attempt(self) -> None: + self.outcome = None + self.outcome_timestamp = None + self.attempt_number += 1 + self.next_action = None + + def set_result(self, val: t.Any) -> None: + ts = time.monotonic() + fut = Future(self.attempt_number) + fut.set_result(val) + self.outcome, self.outcome_timestamp = fut, ts + + def set_exception(self, exc_info: t.Tuple[t.Type[BaseException], BaseException, "types.TracebackType"]) -> None: + ts = time.monotonic() + fut = Future(self.attempt_number) + fut.set_exception(exc_info[1]) + self.outcome, self.outcome_timestamp = fut, ts + + def __repr__(self): + if self.outcome is None: + result = "none yet" + elif self.outcome.failed: + exception = self.outcome.exception() + result = f"failed ({exception.__class__.__name__} {exception})" + else: + result = f"returned {self.outcome.result()}" + + slept = float(round(self.idle_for, 2)) + clsname = self.__class__.__name__ + return f"<{clsname} {id(self)}: attempt #{self.attempt_number}; slept for {slept}; last result: {result}>" + + +from pip._vendor.tenacity._asyncio import AsyncRetrying # noqa:E402,I100 + +if tornado: + from pip._vendor.tenacity.tornadoweb import TornadoRetrying diff --git a/src/pip/_vendor/tenacity/_asyncio.py b/src/pip/_vendor/tenacity/_asyncio.py new file mode 100644 index 00000000000..0f32b5f6207 --- /dev/null +++ b/src/pip/_vendor/tenacity/_asyncio.py @@ -0,0 +1,92 @@ +# Copyright 2016 Étienne Bersac +# Copyright 2016 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import sys +import typing +from asyncio import sleep + +from pip._vendor.tenacity import AttemptManager +from pip._vendor.tenacity import BaseRetrying +from pip._vendor.tenacity import DoAttempt +from pip._vendor.tenacity import DoSleep +from pip._vendor.tenacity import RetryCallState + +WrappedFn = typing.TypeVar("WrappedFn", bound=typing.Callable) +_RetValT = typing.TypeVar("_RetValT") + + +class AsyncRetrying(BaseRetrying): + def __init__(self, sleep: typing.Callable[[float], typing.Awaitable] = sleep, **kwargs: typing.Any) -> None: + super().__init__(**kwargs) + self.sleep = sleep + + async def __call__( # type: ignore # Change signature from supertype + self, + fn: typing.Callable[..., typing.Awaitable[_RetValT]], + *args: typing.Any, + **kwargs: typing.Any, + ) -> _RetValT: + self.begin() + + retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs) + while True: + do = self.iter(retry_state=retry_state) + if isinstance(do, DoAttempt): + try: + result = await fn(*args, **kwargs) + except BaseException: # noqa: B902 + retry_state.set_exception(sys.exc_info()) + else: + retry_state.set_result(result) + elif isinstance(do, DoSleep): + retry_state.prepare_for_next_attempt() + await self.sleep(do) + else: + return do + + def __aiter__(self) -> "AsyncRetrying": + self.begin() + self._retry_state = RetryCallState(self, fn=None, args=(), kwargs={}) + return self + + async def __anext__(self) -> typing.Union[AttemptManager, typing.Any]: + while True: + do = self.iter(retry_state=self._retry_state) + if do is None: + raise StopAsyncIteration + elif isinstance(do, DoAttempt): + return AttemptManager(retry_state=self._retry_state) + elif isinstance(do, DoSleep): + self._retry_state.prepare_for_next_attempt() + await self.sleep(do) + else: + return do + + def wraps(self, fn: WrappedFn) -> WrappedFn: + fn = super().wraps(fn) + # Ensure wrapper is recognized as a coroutine function. + + @functools.wraps(fn) + async def async_wrapped(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + return await fn(*args, **kwargs) + + # Preserve attributes + async_wrapped.retry = fn.retry + async_wrapped.retry_with = fn.retry_with + + return async_wrapped diff --git a/src/pip/_vendor/tenacity/_utils.py b/src/pip/_vendor/tenacity/_utils.py new file mode 100644 index 00000000000..d5c4c9de591 --- /dev/null +++ b/src/pip/_vendor/tenacity/_utils.py @@ -0,0 +1,68 @@ +# Copyright 2016 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import typing + + +# sys.maxsize: +# An integer giving the maximum value a variable of type Py_ssize_t can take. +MAX_WAIT = sys.maxsize / 2 + + +def find_ordinal(pos_num: int) -> str: + # See: https://en.wikipedia.org/wiki/English_numerals#Ordinal_numbers + if pos_num == 0: + return "th" + elif pos_num == 1: + return "st" + elif pos_num == 2: + return "nd" + elif pos_num == 3: + return "rd" + elif 4 <= pos_num <= 20: + return "th" + else: + return find_ordinal(pos_num % 10) + + +def to_ordinal(pos_num: int) -> str: + return f"{pos_num}{find_ordinal(pos_num)}" + + +def get_callback_name(cb: typing.Callable[..., typing.Any]) -> str: + """Get a callback fully-qualified name. + + If no name can be produced ``repr(cb)`` is called and returned. + """ + segments = [] + try: + segments.append(cb.__qualname__) + except AttributeError: + try: + segments.append(cb.__name__) + except AttributeError: + pass + if not segments: + return repr(cb) + else: + try: + # When running under sphinx it appears this can be none? + if cb.__module__: + segments.insert(0, cb.__module__) + except AttributeError: + pass + return ".".join(segments) diff --git a/src/pip/_vendor/tenacity/after.py b/src/pip/_vendor/tenacity/after.py new file mode 100644 index 00000000000..c056700f9fa --- /dev/null +++ b/src/pip/_vendor/tenacity/after.py @@ -0,0 +1,46 @@ +# Copyright 2016 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing + +from pip._vendor.tenacity import _utils + +if typing.TYPE_CHECKING: + import logging + + from pip._vendor.tenacity import RetryCallState + + +def after_nothing(retry_state: "RetryCallState") -> None: + """After call strategy that does nothing.""" + + +def after_log( + logger: "logging.Logger", + log_level: int, + sec_format: str = "%0.3f", +) -> typing.Callable[["RetryCallState"], None]: + """After call strategy that logs to some logger the finished attempt.""" + + def log_it(retry_state: "RetryCallState") -> None: + logger.log( + log_level, + f"Finished call to '{_utils.get_callback_name(retry_state.fn)}' " + f"after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it.", + ) + + return log_it diff --git a/src/pip/_vendor/tenacity/before.py b/src/pip/_vendor/tenacity/before.py new file mode 100644 index 00000000000..a72c2c5f70e --- /dev/null +++ b/src/pip/_vendor/tenacity/before.py @@ -0,0 +1,41 @@ +# Copyright 2016 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing + +from pip._vendor.tenacity import _utils + +if typing.TYPE_CHECKING: + import logging + + from pip._vendor.tenacity import RetryCallState + + +def before_nothing(retry_state: "RetryCallState") -> None: + """Before call strategy that does nothing.""" + + +def before_log(logger: "logging.Logger", log_level: int) -> typing.Callable[["RetryCallState"], None]: + """Before call strategy that logs to some logger the attempt.""" + + def log_it(retry_state: "RetryCallState") -> None: + logger.log( + log_level, + f"Starting call to '{_utils.get_callback_name(retry_state.fn)}', " + f"this is the {_utils.to_ordinal(retry_state.attempt_number)} time calling it.", + ) + + return log_it diff --git a/src/pip/_vendor/tenacity/before_sleep.py b/src/pip/_vendor/tenacity/before_sleep.py new file mode 100644 index 00000000000..b35564fbad8 --- /dev/null +++ b/src/pip/_vendor/tenacity/before_sleep.py @@ -0,0 +1,58 @@ +# Copyright 2016 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing + +from pip._vendor.tenacity import _utils + +if typing.TYPE_CHECKING: + import logging + + from pip._vendor.tenacity import RetryCallState + + +def before_sleep_nothing(retry_state: "RetryCallState") -> None: + """Before call strategy that does nothing.""" + + +def before_sleep_log( + logger: "logging.Logger", + log_level: int, + exc_info: bool = False, +) -> typing.Callable[["RetryCallState"], None]: + """Before call strategy that logs to some logger the attempt.""" + + def log_it(retry_state: "RetryCallState") -> None: + if retry_state.outcome.failed: + ex = retry_state.outcome.exception() + verb, value = "raised", f"{ex.__class__.__name__}: {ex}" + + if exc_info: + local_exc_info = retry_state.outcome.exception() + else: + local_exc_info = False + else: + verb, value = "returned", retry_state.outcome.result() + local_exc_info = False # exc_info does not apply when no exception + + logger.log( + log_level, + f"Retrying {_utils.get_callback_name(retry_state.fn)} " + f"in {retry_state.next_action.sleep} seconds as it {verb} {value}.", + exc_info=local_exc_info, + ) + + return log_it diff --git a/src/pip/_vendor/tenacity/nap.py b/src/pip/_vendor/tenacity/nap.py new file mode 100644 index 00000000000..72aa5bfd4b6 --- /dev/null +++ b/src/pip/_vendor/tenacity/nap.py @@ -0,0 +1,43 @@ +# Copyright 2016 Étienne Bersac +# Copyright 2016 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import typing + +if typing.TYPE_CHECKING: + import threading + + +def sleep(seconds: float) -> None: + """ + Sleep strategy that delays execution for a given number of seconds. + + This is the default strategy, and may be mocked out for unit testing. + """ + time.sleep(seconds) + + +class sleep_using_event: + """Sleep strategy that waits on an event to be set.""" + + def __init__(self, event: "threading.Event") -> None: + self.event = event + + def __call__(self, timeout: typing.Optional[float]) -> None: + # NOTE(harlowja): this may *not* actually wait for timeout + # seconds if the event is set (ie this may eject out early). + self.event.wait(timeout=timeout) diff --git a/news/2905cccb-2fe8-4b0d-8734-303510a7e4ce.trivial.rst b/src/pip/_vendor/tenacity/py.typed similarity index 100% rename from news/2905cccb-2fe8-4b0d-8734-303510a7e4ce.trivial.rst rename to src/pip/_vendor/tenacity/py.typed diff --git a/src/pip/_vendor/tenacity/retry.py b/src/pip/_vendor/tenacity/retry.py new file mode 100644 index 00000000000..1d727e9b346 --- /dev/null +++ b/src/pip/_vendor/tenacity/retry.py @@ -0,0 +1,213 @@ +# Copyright 2016–2021 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import re +import typing + +if typing.TYPE_CHECKING: + from pip._vendor.tenacity import RetryCallState + + +class retry_base(abc.ABC): + """Abstract base class for retry strategies.""" + + @abc.abstractmethod + def __call__(self, retry_state: "RetryCallState") -> bool: + pass + + def __and__(self, other: "retry_base") -> "retry_all": + return retry_all(self, other) + + def __or__(self, other: "retry_base") -> "retry_any": + return retry_any(self, other) + + +class _retry_never(retry_base): + """Retry strategy that never rejects any result.""" + + def __call__(self, retry_state: "RetryCallState") -> bool: + return False + + +retry_never = _retry_never() + + +class _retry_always(retry_base): + """Retry strategy that always rejects any result.""" + + def __call__(self, retry_state: "RetryCallState") -> bool: + return True + + +retry_always = _retry_always() + + +class retry_if_exception(retry_base): + """Retry strategy that retries if an exception verifies a predicate.""" + + def __init__(self, predicate: typing.Callable[[BaseException], bool]) -> None: + self.predicate = predicate + + def __call__(self, retry_state: "RetryCallState") -> bool: + if retry_state.outcome.failed: + return self.predicate(retry_state.outcome.exception()) + else: + return False + + +class retry_if_exception_type(retry_if_exception): + """Retries if an exception has been raised of one or more types.""" + + def __init__( + self, + exception_types: typing.Union[ + typing.Type[BaseException], + typing.Tuple[typing.Type[BaseException], ...], + ] = Exception, + ) -> None: + self.exception_types = exception_types + super().__init__(lambda e: isinstance(e, exception_types)) + + +class retry_if_not_exception_type(retry_if_exception): + """Retries except an exception has been raised of one or more types.""" + + def __init__( + self, + exception_types: typing.Union[ + typing.Type[BaseException], + typing.Tuple[typing.Type[BaseException], ...], + ] = Exception, + ) -> None: + self.exception_types = exception_types + super().__init__(lambda e: not isinstance(e, exception_types)) + + +class retry_unless_exception_type(retry_if_exception): + """Retries until an exception is raised of one or more types.""" + + def __init__( + self, + exception_types: typing.Union[ + typing.Type[BaseException], + typing.Tuple[typing.Type[BaseException], ...], + ] = Exception, + ) -> None: + self.exception_types = exception_types + super().__init__(lambda e: not isinstance(e, exception_types)) + + def __call__(self, retry_state: "RetryCallState") -> bool: + # always retry if no exception was raised + if not retry_state.outcome.failed: + return True + return self.predicate(retry_state.outcome.exception()) + + +class retry_if_result(retry_base): + """Retries if the result verifies a predicate.""" + + def __init__(self, predicate: typing.Callable[[typing.Any], bool]) -> None: + self.predicate = predicate + + def __call__(self, retry_state: "RetryCallState") -> bool: + if not retry_state.outcome.failed: + return self.predicate(retry_state.outcome.result()) + else: + return False + + +class retry_if_not_result(retry_base): + """Retries if the result refutes a predicate.""" + + def __init__(self, predicate: typing.Callable[[typing.Any], bool]) -> None: + self.predicate = predicate + + def __call__(self, retry_state: "RetryCallState") -> bool: + if not retry_state.outcome.failed: + return not self.predicate(retry_state.outcome.result()) + else: + return False + + +class retry_if_exception_message(retry_if_exception): + """Retries if an exception message equals or matches.""" + + def __init__( + self, + message: typing.Optional[str] = None, + match: typing.Optional[str] = None, + ) -> None: + if message and match: + raise TypeError(f"{self.__class__.__name__}() takes either 'message' or 'match', not both") + + # set predicate + if message: + + def message_fnc(exception: BaseException) -> bool: + return message == str(exception) + + predicate = message_fnc + elif match: + prog = re.compile(match) + + def match_fnc(exception: BaseException) -> bool: + return bool(prog.match(str(exception))) + + predicate = match_fnc + else: + raise TypeError(f"{self.__class__.__name__}() missing 1 required argument 'message' or 'match'") + + super().__init__(predicate) + + +class retry_if_not_exception_message(retry_if_exception_message): + """Retries until an exception message equals or matches.""" + + def __init__( + self, + message: typing.Optional[str] = None, + match: typing.Optional[str] = None, + ) -> None: + super().__init__(message, match) + # invert predicate + if_predicate = self.predicate + self.predicate = lambda *args_, **kwargs_: not if_predicate(*args_, **kwargs_) + + def __call__(self, retry_state: "RetryCallState") -> bool: + if not retry_state.outcome.failed: + return True + return self.predicate(retry_state.outcome.exception()) + + +class retry_any(retry_base): + """Retries if any of the retries condition is valid.""" + + def __init__(self, *retries: retry_base) -> None: + self.retries = retries + + def __call__(self, retry_state: "RetryCallState") -> bool: + return any(r(retry_state) for r in self.retries) + + +class retry_all(retry_base): + """Retries if all the retries condition are valid.""" + + def __init__(self, *retries: retry_base) -> None: + self.retries = retries + + def __call__(self, retry_state: "RetryCallState") -> bool: + return all(r(retry_state) for r in self.retries) diff --git a/src/pip/_vendor/tenacity/stop.py b/src/pip/_vendor/tenacity/stop.py new file mode 100644 index 00000000000..faaae9a8ddb --- /dev/null +++ b/src/pip/_vendor/tenacity/stop.py @@ -0,0 +1,96 @@ +# Copyright 2016–2021 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +import typing + +if typing.TYPE_CHECKING: + import threading + + from pip._vendor.tenacity import RetryCallState + + +class stop_base(abc.ABC): + """Abstract base class for stop strategies.""" + + @abc.abstractmethod + def __call__(self, retry_state: "RetryCallState") -> bool: + pass + + def __and__(self, other: "stop_base") -> "stop_all": + return stop_all(self, other) + + def __or__(self, other: "stop_base") -> "stop_any": + return stop_any(self, other) + + +class stop_any(stop_base): + """Stop if any of the stop condition is valid.""" + + def __init__(self, *stops: stop_base) -> None: + self.stops = stops + + def __call__(self, retry_state: "RetryCallState") -> bool: + return any(x(retry_state) for x in self.stops) + + +class stop_all(stop_base): + """Stop if all the stop conditions are valid.""" + + def __init__(self, *stops: stop_base) -> None: + self.stops = stops + + def __call__(self, retry_state: "RetryCallState") -> bool: + return all(x(retry_state) for x in self.stops) + + +class _stop_never(stop_base): + """Never stop.""" + + def __call__(self, retry_state: "RetryCallState") -> bool: + return False + + +stop_never = _stop_never() + + +class stop_when_event_set(stop_base): + """Stop when the given event is set.""" + + def __init__(self, event: "threading.Event") -> None: + self.event = event + + def __call__(self, retry_state: "RetryCallState") -> bool: + return self.event.is_set() + + +class stop_after_attempt(stop_base): + """Stop when the previous attempt >= max_attempt.""" + + def __init__(self, max_attempt_number: int) -> None: + self.max_attempt_number = max_attempt_number + + def __call__(self, retry_state: "RetryCallState") -> bool: + return retry_state.attempt_number >= self.max_attempt_number + + +class stop_after_delay(stop_base): + """Stop when the time from the first attempt >= limit.""" + + def __init__(self, max_delay: float) -> None: + self.max_delay = max_delay + + def __call__(self, retry_state: "RetryCallState") -> bool: + return retry_state.seconds_since_start >= self.max_delay diff --git a/src/pip/_vendor/tenacity/tornadoweb.py b/src/pip/_vendor/tenacity/tornadoweb.py new file mode 100644 index 00000000000..8f7731af0e6 --- /dev/null +++ b/src/pip/_vendor/tenacity/tornadoweb.py @@ -0,0 +1,59 @@ +# Copyright 2017 Elisey Zanko +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import typing + +from pip._vendor.tenacity import BaseRetrying +from pip._vendor.tenacity import DoAttempt +from pip._vendor.tenacity import DoSleep +from pip._vendor.tenacity import RetryCallState + +from tornado import gen + +if typing.TYPE_CHECKING: + from tornado.concurrent import Future + +_RetValT = typing.TypeVar("_RetValT") + + +class TornadoRetrying(BaseRetrying): + def __init__(self, sleep: "typing.Callable[[float], Future[None]]" = gen.sleep, **kwargs: typing.Any) -> None: + super().__init__(**kwargs) + self.sleep = sleep + + @gen.coroutine + def __call__( # type: ignore # Change signature from supertype + self, + fn: "typing.Callable[..., typing.Union[typing.Generator[typing.Any, typing.Any, _RetValT], Future[_RetValT]]]", + *args: typing.Any, + **kwargs: typing.Any, + ) -> "typing.Generator[typing.Any, typing.Any, _RetValT]": + self.begin() + + retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs) + while True: + do = self.iter(retry_state=retry_state) + if isinstance(do, DoAttempt): + try: + result = yield fn(*args, **kwargs) + except BaseException: # noqa: B902 + retry_state.set_exception(sys.exc_info()) + else: + retry_state.set_result(result) + elif isinstance(do, DoSleep): + retry_state.prepare_for_next_attempt() + yield self.sleep(do) + else: + raise gen.Return(do) diff --git a/src/pip/_vendor/tenacity/wait.py b/src/pip/_vendor/tenacity/wait.py new file mode 100644 index 00000000000..6ed97a7bcdc --- /dev/null +++ b/src/pip/_vendor/tenacity/wait.py @@ -0,0 +1,191 @@ +# Copyright 2016–2021 Julien Danjou +# Copyright 2016 Joshua Harlow +# Copyright 2013-2014 Ray Holder +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import random +import typing + +from pip._vendor.tenacity import _utils + +if typing.TYPE_CHECKING: + from pip._vendor.tenacity import RetryCallState + + +class wait_base(abc.ABC): + """Abstract base class for wait strategies.""" + + @abc.abstractmethod + def __call__(self, retry_state: "RetryCallState") -> float: + pass + + def __add__(self, other: "wait_base") -> "wait_combine": + return wait_combine(self, other) + + def __radd__(self, other: "wait_base") -> typing.Union["wait_combine", "wait_base"]: + # make it possible to use multiple waits with the built-in sum function + if other == 0: + return self + return self.__add__(other) + + +class wait_fixed(wait_base): + """Wait strategy that waits a fixed amount of time between each retry.""" + + def __init__(self, wait: float) -> None: + self.wait_fixed = wait + + def __call__(self, retry_state: "RetryCallState") -> float: + return self.wait_fixed + + +class wait_none(wait_fixed): + """Wait strategy that doesn't wait at all before retrying.""" + + def __init__(self) -> None: + super().__init__(0) + + +class wait_random(wait_base): + """Wait strategy that waits a random amount of time between min/max.""" + + def __init__(self, min: typing.Union[int, float] = 0, max: typing.Union[int, float] = 1) -> None: # noqa + self.wait_random_min = min + self.wait_random_max = max + + def __call__(self, retry_state: "RetryCallState") -> float: + return self.wait_random_min + (random.random() * (self.wait_random_max - self.wait_random_min)) + + +class wait_combine(wait_base): + """Combine several waiting strategies.""" + + def __init__(self, *strategies: wait_base) -> None: + self.wait_funcs = strategies + + def __call__(self, retry_state: "RetryCallState") -> float: + return sum(x(retry_state=retry_state) for x in self.wait_funcs) + + +class wait_chain(wait_base): + """Chain two or more waiting strategies. + + If all strategies are exhausted, the very last strategy is used + thereafter. + + For example:: + + @retry(wait=wait_chain(*[wait_fixed(1) for i in range(3)] + + [wait_fixed(2) for j in range(5)] + + [wait_fixed(5) for k in range(4))) + def wait_chained(): + print("Wait 1s for 3 attempts, 2s for 5 attempts and 5s + thereafter.") + """ + + def __init__(self, *strategies: wait_base) -> None: + self.strategies = strategies + + def __call__(self, retry_state: "RetryCallState") -> float: + wait_func_no = min(max(retry_state.attempt_number, 1), len(self.strategies)) + wait_func = self.strategies[wait_func_no - 1] + return wait_func(retry_state=retry_state) + + +class wait_incrementing(wait_base): + """Wait an incremental amount of time after each attempt. + + Starting at a starting value and incrementing by a value for each attempt + (and restricting the upper limit to some maximum value). + """ + + def __init__( + self, + start: typing.Union[int, float] = 0, + increment: typing.Union[int, float] = 100, + max: typing.Union[int, float] = _utils.MAX_WAIT, # noqa + ) -> None: + self.start = start + self.increment = increment + self.max = max + + def __call__(self, retry_state: "RetryCallState") -> float: + result = self.start + (self.increment * (retry_state.attempt_number - 1)) + return max(0, min(result, self.max)) + + +class wait_exponential(wait_base): + """Wait strategy that applies exponential backoff. + + It allows for a customized multiplier and an ability to restrict the + upper and lower limits to some maximum and minimum value. + + The intervals are fixed (i.e. there is no jitter), so this strategy is + suitable for balancing retries against latency when a required resource is + unavailable for an unknown duration, but *not* suitable for resolving + contention between multiple processes for a shared resource. Use + wait_random_exponential for the latter case. + """ + + def __init__( + self, + multiplier: typing.Union[int, float] = 1, + max: typing.Union[int, float] = _utils.MAX_WAIT, # noqa + exp_base: typing.Union[int, float] = 2, + min: typing.Union[int, float] = 0, # noqa + ) -> None: + self.multiplier = multiplier + self.min = min + self.max = max + self.exp_base = exp_base + + def __call__(self, retry_state: "RetryCallState") -> float: + try: + exp = self.exp_base ** (retry_state.attempt_number - 1) + result = self.multiplier * exp + except OverflowError: + return self.max + return max(max(0, self.min), min(result, self.max)) + + +class wait_random_exponential(wait_exponential): + """Random wait with exponentially widening window. + + An exponential backoff strategy used to mediate contention between multiple + uncoordinated processes for a shared resource in distributed systems. This + is the sense in which "exponential backoff" is meant in e.g. Ethernet + networking, and corresponds to the "Full Jitter" algorithm described in + this blog post: + + https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + + Each retry occurs at a random time in a geometrically expanding interval. + It allows for a custom multiplier and an ability to restrict the upper + limit of the random interval to some maximum value. + + Example:: + + wait_random_exponential(multiplier=0.5, # initial window 0.5s + max=60) # max 60s timeout + + When waiting for an unavailable resource to become available again, as + opposed to trying to resolve contention for a shared resource, the + wait_exponential strategy (which uses a fixed interval) may be preferable. + + """ + + def __call__(self, retry_state: "RetryCallState") -> float: + high = super().__call__(retry_state=retry_state) + return random.uniform(0, high) diff --git a/src/pip/_vendor/toml.pyi b/src/pip/_vendor/toml.pyi deleted file mode 100644 index 018a1ad1061..00000000000 --- a/src/pip/_vendor/toml.pyi +++ /dev/null @@ -1 +0,0 @@ -from toml import * \ No newline at end of file diff --git a/src/pip/_vendor/toml/__init__.py b/src/pip/_vendor/toml/__init__.py deleted file mode 100644 index 34a5eabb6ea..00000000000 --- a/src/pip/_vendor/toml/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Python module which parses and emits TOML. - -Released under the MIT license. -""" - -from pip._vendor.toml import encoder -from pip._vendor.toml import decoder - -__version__ = "0.10.2" -_spec_ = "0.5.0" - -load = decoder.load -loads = decoder.loads -TomlDecoder = decoder.TomlDecoder -TomlDecodeError = decoder.TomlDecodeError -TomlPreserveCommentDecoder = decoder.TomlPreserveCommentDecoder - -dump = encoder.dump -dumps = encoder.dumps -TomlEncoder = encoder.TomlEncoder -TomlArraySeparatorEncoder = encoder.TomlArraySeparatorEncoder -TomlPreserveInlineDictEncoder = encoder.TomlPreserveInlineDictEncoder -TomlNumpyEncoder = encoder.TomlNumpyEncoder -TomlPreserveCommentEncoder = encoder.TomlPreserveCommentEncoder -TomlPathlibEncoder = encoder.TomlPathlibEncoder diff --git a/src/pip/_vendor/toml/decoder.py b/src/pip/_vendor/toml/decoder.py deleted file mode 100644 index e071100de0f..00000000000 --- a/src/pip/_vendor/toml/decoder.py +++ /dev/null @@ -1,1057 +0,0 @@ -import datetime -import io -from os import linesep -import re -import sys - -from pip._vendor.toml.tz import TomlTz - -if sys.version_info < (3,): - _range = xrange # noqa: F821 -else: - unicode = str - _range = range - basestring = str - unichr = chr - - -def _detect_pathlib_path(p): - if (3, 4) <= sys.version_info: - import pathlib - if isinstance(p, pathlib.PurePath): - return True - return False - - -def _ispath(p): - if isinstance(p, (bytes, basestring)): - return True - return _detect_pathlib_path(p) - - -def _getpath(p): - if (3, 6) <= sys.version_info: - import os - return os.fspath(p) - if _detect_pathlib_path(p): - return str(p) - return p - - -try: - FNFError = FileNotFoundError -except NameError: - FNFError = IOError - - -TIME_RE = re.compile(r"([0-9]{2}):([0-9]{2}):([0-9]{2})(\.([0-9]{3,6}))?") - - -class TomlDecodeError(ValueError): - """Base toml Exception / Error.""" - - def __init__(self, msg, doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - colno = pos - doc.rfind('\n', 0, pos) - emsg = '{} (line {} column {} char {})'.format(msg, lineno, colno, pos) - ValueError.__init__(self, emsg) - self.msg = msg - self.doc = doc - self.pos = pos - self.lineno = lineno - self.colno = colno - - -# Matches a TOML number, which allows underscores for readability -_number_with_underscores = re.compile('([0-9])(_([0-9]))*') - - -class CommentValue(object): - def __init__(self, val, comment, beginline, _dict): - self.val = val - separator = "\n" if beginline else " " - self.comment = separator + comment - self._dict = _dict - - def __getitem__(self, key): - return self.val[key] - - def __setitem__(self, key, value): - self.val[key] = value - - def dump(self, dump_value_func): - retstr = dump_value_func(self.val) - if isinstance(self.val, self._dict): - return self.comment + "\n" + unicode(retstr) - else: - return unicode(retstr) + self.comment - - -def _strictly_valid_num(n): - n = n.strip() - if not n: - return False - if n[0] == '_': - return False - if n[-1] == '_': - return False - if "_." in n or "._" in n: - return False - if len(n) == 1: - return True - if n[0] == '0' and n[1] not in ['.', 'o', 'b', 'x']: - return False - if n[0] == '+' or n[0] == '-': - n = n[1:] - if len(n) > 1 and n[0] == '0' and n[1] != '.': - return False - if '__' in n: - return False - return True - - -def load(f, _dict=dict, decoder=None): - """Parses named file or files as toml and returns a dictionary - - Args: - f: Path to the file to open, array of files to read into single dict - or a file descriptor - _dict: (optional) Specifies the class of the returned toml dictionary - decoder: The decoder to use - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError -- When f is invalid type - TomlDecodeError: Error while decoding toml - IOError / FileNotFoundError -- When an array with no valid (existing) - (Python 2 / Python 3) file paths is passed - """ - - if _ispath(f): - with io.open(_getpath(f), encoding='utf-8') as ffile: - return loads(ffile.read(), _dict, decoder) - elif isinstance(f, list): - from os import path as op - from warnings import warn - if not [path for path in f if op.exists(path)]: - error_msg = "Load expects a list to contain filenames only." - error_msg += linesep - error_msg += ("The list needs to contain the path of at least one " - "existing file.") - raise FNFError(error_msg) - if decoder is None: - decoder = TomlDecoder(_dict) - d = decoder.get_empty_table() - for l in f: # noqa: E741 - if op.exists(l): - d.update(load(l, _dict, decoder)) - else: - warn("Non-existent filename in list with at least one valid " - "filename") - return d - else: - try: - return loads(f.read(), _dict, decoder) - except AttributeError: - raise TypeError("You can only load a file descriptor, filename or " - "list") - - -_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') - - -def loads(s, _dict=dict, decoder=None): - """Parses string as toml - - Args: - s: String to be parsed - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError: When a non-string is passed - TomlDecodeError: Error while decoding toml - """ - - implicitgroups = [] - if decoder is None: - decoder = TomlDecoder(_dict) - retval = decoder.get_empty_table() - currentlevel = retval - if not isinstance(s, basestring): - raise TypeError("Expecting something like a string") - - if not isinstance(s, unicode): - s = s.decode('utf8') - - original = s - sl = list(s) - openarr = 0 - openstring = False - openstrchar = "" - multilinestr = False - arrayoftables = False - beginline = True - keygroup = False - dottedkey = False - keyname = 0 - key = '' - prev_key = '' - line_no = 1 - - for i, item in enumerate(sl): - if item == '\r' and sl[i + 1] == '\n': - sl[i] = ' ' - continue - if keyname: - key += item - if item == '\n': - raise TomlDecodeError("Key name found without value." - " Reached end of line.", original, i) - if openstring: - if item == openstrchar: - oddbackslash = False - k = 1 - while i >= k and sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - if not oddbackslash: - keyname = 2 - openstring = False - openstrchar = "" - continue - elif keyname == 1: - if item.isspace(): - keyname = 2 - continue - elif item == '.': - dottedkey = True - continue - elif item.isalnum() or item == '_' or item == '-': - continue - elif (dottedkey and sl[i - 1] == '.' and - (item == '"' or item == "'")): - openstring = True - openstrchar = item - continue - elif keyname == 2: - if item.isspace(): - if dottedkey: - nextitem = sl[i + 1] - if not nextitem.isspace() and nextitem != '.': - keyname = 1 - continue - if item == '.': - dottedkey = True - nextitem = sl[i + 1] - if not nextitem.isspace() and nextitem != '.': - keyname = 1 - continue - if item == '=': - keyname = 0 - prev_key = key[:-1].rstrip() - key = '' - dottedkey = False - else: - raise TomlDecodeError("Found invalid character in key name: '" + - item + "'. Try quoting the key name.", - original, i) - if item == "'" and openstrchar != '"': - k = 1 - try: - while sl[i - k] == "'": - k += 1 - if k == 3: - break - except IndexError: - pass - if k == 3: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = "'" - else: - openstrchar = "" - if item == '"' and openstrchar != "'": - oddbackslash = False - k = 1 - tripquote = False - try: - while sl[i - k] == '"': - k += 1 - if k == 3: - tripquote = True - break - if k == 1 or (k == 3 and tripquote): - while sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - except IndexError: - pass - if not oddbackslash: - if tripquote: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = '"' - else: - openstrchar = "" - if item == '#' and (not openstring and not keygroup and - not arrayoftables): - j = i - comment = "" - try: - while sl[j] != '\n': - comment += s[j] - sl[j] = ' ' - j += 1 - except IndexError: - break - if not openarr: - decoder.preserve_comment(line_no, prev_key, comment, beginline) - if item == '[' and (not openstring and not keygroup and - not arrayoftables): - if beginline: - if len(sl) > i + 1 and sl[i + 1] == '[': - arrayoftables = True - else: - keygroup = True - else: - openarr += 1 - if item == ']' and not openstring: - if keygroup: - keygroup = False - elif arrayoftables: - if sl[i - 1] == ']': - arrayoftables = False - else: - openarr -= 1 - if item == '\n': - if openstring or multilinestr: - if not multilinestr: - raise TomlDecodeError("Unbalanced quotes", original, i) - if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( - sl[i - 2] == sl[i - 1])): - sl[i] = sl[i - 1] - if sl[i - 3] == sl[i - 1]: - sl[i - 3] = ' ' - elif openarr: - sl[i] = ' ' - else: - beginline = True - line_no += 1 - elif beginline and sl[i] != ' ' and sl[i] != '\t': - beginline = False - if not keygroup and not arrayoftables: - if sl[i] == '=': - raise TomlDecodeError("Found empty keyname. ", original, i) - keyname = 1 - key += item - if keyname: - raise TomlDecodeError("Key name found without value." - " Reached end of file.", original, len(s)) - if openstring: # reached EOF and have an unterminated string - raise TomlDecodeError("Unterminated string found." - " Reached end of file.", original, len(s)) - s = ''.join(sl) - s = s.split('\n') - multikey = None - multilinestr = "" - multibackslash = False - pos = 0 - for idx, line in enumerate(s): - if idx > 0: - pos += len(s[idx - 1]) + 1 - - decoder.embed_comments(idx, currentlevel) - - if not multilinestr or multibackslash or '\n' not in multilinestr: - line = line.strip() - if line == "" and (not multikey or multibackslash): - continue - if multikey: - if multibackslash: - multilinestr += line - else: - multilinestr += line - multibackslash = False - closed = False - if multilinestr[0] == '[': - closed = line[-1] == ']' - elif len(line) > 2: - closed = (line[-1] == multilinestr[0] and - line[-2] == multilinestr[0] and - line[-3] == multilinestr[0]) - if closed: - try: - value, vtype = decoder.load_value(multilinestr) - except ValueError as err: - raise TomlDecodeError(str(err), original, pos) - currentlevel[multikey] = value - multikey = None - multilinestr = "" - else: - k = len(multilinestr) - 1 - while k > -1 and multilinestr[k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = multilinestr[:-1] - else: - multilinestr += "\n" - continue - if line[0] == '[': - arrayoftables = False - if len(line) == 1: - raise TomlDecodeError("Opening key group bracket on line by " - "itself.", original, pos) - if line[1] == '[': - arrayoftables = True - line = line[2:] - splitstr = ']]' - else: - line = line[1:] - splitstr = ']' - i = 1 - quotesplits = decoder._get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and splitstr in quotesplit: - break - i += quotesplit.count(splitstr) - quoted = not quoted - line = line.split(splitstr, i) - if len(line) < i + 1 or line[-1].strip() != "": - raise TomlDecodeError("Key group not on a line by itself.", - original, pos) - groups = splitstr.join(line[:-1]).split('.') - i = 0 - while i < len(groups): - groups[i] = groups[i].strip() - if len(groups[i]) > 0 and (groups[i][0] == '"' or - groups[i][0] == "'"): - groupstr = groups[i] - j = i + 1 - while ((not groupstr[0] == groupstr[-1]) or - len(groupstr) == 1): - j += 1 - if j > len(groups) + 2: - raise TomlDecodeError("Invalid group name '" + - groupstr + "' Something " + - "went wrong.", original, pos) - groupstr = '.'.join(groups[i:j]).strip() - groups[i] = groupstr[1:-1] - groups[i + 1:j] = [] - else: - if not _groupname_re.match(groups[i]): - raise TomlDecodeError("Invalid group name '" + - groups[i] + "'. Try quoting it.", - original, pos) - i += 1 - currentlevel = retval - for i in _range(len(groups)): - group = groups[i] - if group == "": - raise TomlDecodeError("Can't have a keygroup with an empty " - "name", original, pos) - try: - currentlevel[group] - if i == len(groups) - 1: - if group in implicitgroups: - implicitgroups.remove(group) - if arrayoftables: - raise TomlDecodeError("An implicitly defined " - "table can't be an array", - original, pos) - elif arrayoftables: - currentlevel[group].append(decoder.get_empty_table() - ) - else: - raise TomlDecodeError("What? " + group + - " already exists?" + - str(currentlevel), - original, pos) - except TypeError: - currentlevel = currentlevel[-1] - if group not in currentlevel: - currentlevel[group] = decoder.get_empty_table() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [decoder.get_empty_table()] - except KeyError: - if i != len(groups) - 1: - implicitgroups.append(group) - currentlevel[group] = decoder.get_empty_table() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [decoder.get_empty_table()] - currentlevel = currentlevel[group] - if arrayoftables: - try: - currentlevel = currentlevel[-1] - except KeyError: - pass - elif line[0] == "{": - if line[-1] != "}": - raise TomlDecodeError("Line breaks are not allowed in inline" - "objects", original, pos) - try: - decoder.load_inline_object(line, currentlevel, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err), original, pos) - elif "=" in line: - try: - ret = decoder.load_line(line, currentlevel, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err), original, pos) - if ret is not None: - multikey, multilinestr, multibackslash = ret - return retval - - -def _load_date(val): - microsecond = 0 - tz = None - try: - if len(val) > 19: - if val[19] == '.': - if val[-1].upper() == 'Z': - subsecondval = val[20:-1] - tzval = "Z" - else: - subsecondvalandtz = val[20:] - if '+' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('+') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - elif '-' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('-') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - else: - tzval = None - subsecondval = subsecondvalandtz - if tzval is not None: - tz = TomlTz(tzval) - microsecond = int(int(subsecondval) * - (10 ** (6 - len(subsecondval)))) - else: - tz = TomlTz(val[19:]) - except ValueError: - tz = None - if "-" not in val[1:]: - return None - try: - if len(val) == 10: - d = datetime.date( - int(val[:4]), int(val[5:7]), - int(val[8:10])) - else: - d = datetime.datetime( - int(val[:4]), int(val[5:7]), - int(val[8:10]), int(val[11:13]), - int(val[14:16]), int(val[17:19]), microsecond, tz) - except ValueError: - return None - return d - - -def _load_unicode_escapes(v, hexbytes, prefix): - skip = False - i = len(v) - 1 - while i > -1 and v[i] == '\\': - skip = not skip - i -= 1 - for hx in hexbytes: - if skip: - skip = False - i = len(hx) - 1 - while i > -1 and hx[i] == '\\': - skip = not skip - i -= 1 - v += prefix - v += hx - continue - hxb = "" - i = 0 - hxblen = 4 - if prefix == "\\U": - hxblen = 8 - hxb = ''.join(hx[i:i + hxblen]).lower() - if hxb.strip('0123456789abcdef'): - raise ValueError("Invalid escape sequence: " + hxb) - if hxb[0] == "d" and hxb[1].strip('01234567'): - raise ValueError("Invalid escape sequence: " + hxb + - ". Only scalar unicode points are allowed.") - v += unichr(int(hxb, 16)) - v += unicode(hx[len(hxb):]) - return v - - -# Unescape TOML string values. - -# content after the \ -_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] -# What it should be replaced by -_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] -# Used for substitution -_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) - - -def _unescape(v): - """Unescape characters in a TOML string.""" - i = 0 - backslash = False - while i < len(v): - if backslash: - backslash = False - if v[i] in _escapes: - v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] - elif v[i] == '\\': - v = v[:i - 1] + v[i:] - elif v[i] == 'u' or v[i] == 'U': - i += 1 - else: - raise ValueError("Reserved escape sequence used") - continue - elif v[i] == '\\': - backslash = True - i += 1 - return v - - -class InlineTableDict(object): - """Sentinel subclass of dict for inline tables.""" - - -class TomlDecoder(object): - - def __init__(self, _dict=dict): - self._dict = _dict - - def get_empty_table(self): - return self._dict() - - def get_empty_inline_table(self): - class DynamicInlineTableDict(self._dict, InlineTableDict): - """Concrete sentinel subclass for inline tables. - It is a subclass of _dict which is passed in dynamically at load - time - - It is also a subclass of InlineTableDict - """ - - return DynamicInlineTableDict() - - def load_inline_object(self, line, currentlevel, multikey=False, - multibackslash=False): - candidate_groups = line[1:-1].split(",") - groups = [] - if len(candidate_groups) == 1 and not candidate_groups[0].strip(): - candidate_groups.pop() - while len(candidate_groups) > 0: - candidate_group = candidate_groups.pop(0) - try: - _, value = candidate_group.split('=', 1) - except ValueError: - raise ValueError("Invalid inline table encountered") - value = value.strip() - if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( - value[0] in '-0123456789' or - value in ('true', 'false') or - (value[0] == "[" and value[-1] == "]") or - (value[0] == '{' and value[-1] == '}'))): - groups.append(candidate_group) - elif len(candidate_groups) > 0: - candidate_groups[0] = (candidate_group + "," + - candidate_groups[0]) - else: - raise ValueError("Invalid inline table value encountered") - for group in groups: - status = self.load_line(group, currentlevel, multikey, - multibackslash) - if status is not None: - break - - def _get_split_on_quotes(self, line): - doublequotesplits = line.split('"') - quoted = False - quotesplits = [] - if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: - singlequotesplits = doublequotesplits[0].split("'") - doublequotesplits = doublequotesplits[1:] - while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): - singlequotesplits[-1] += '"' + doublequotesplits[0] - doublequotesplits = doublequotesplits[1:] - if "'" in singlequotesplits[-1]: - singlequotesplits = (singlequotesplits[:-1] + - singlequotesplits[-1].split("'")) - quotesplits += singlequotesplits - for doublequotesplit in doublequotesplits: - if quoted: - quotesplits.append(doublequotesplit) - else: - quotesplits += doublequotesplit.split("'") - quoted = not quoted - return quotesplits - - def load_line(self, line, currentlevel, multikey, multibackslash): - i = 1 - quotesplits = self._get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and '=' in quotesplit: - break - i += quotesplit.count('=') - quoted = not quoted - pair = line.split('=', i) - strictly_valid = _strictly_valid_num(pair[-1]) - if _number_with_underscores.match(pair[-1]): - pair[-1] = pair[-1].replace('_', '') - while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and - pair[-1][0] != "'" and pair[-1][0] != '"' and - pair[-1][0] != '[' and pair[-1][0] != '{' and - pair[-1].strip() != 'true' and - pair[-1].strip() != 'false'): - try: - float(pair[-1]) - break - except ValueError: - pass - if _load_date(pair[-1]) is not None: - break - if TIME_RE.match(pair[-1]): - break - i += 1 - prev_val = pair[-1] - pair = line.split('=', i) - if prev_val == pair[-1]: - raise ValueError("Invalid date or number") - if strictly_valid: - strictly_valid = _strictly_valid_num(pair[-1]) - pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] - if '.' in pair[0]: - if '"' in pair[0] or "'" in pair[0]: - quotesplits = self._get_split_on_quotes(pair[0]) - quoted = False - levels = [] - for quotesplit in quotesplits: - if quoted: - levels.append(quotesplit) - else: - levels += [level.strip() for level in - quotesplit.split('.')] - quoted = not quoted - else: - levels = pair[0].split('.') - while levels[-1] == "": - levels = levels[:-1] - for level in levels[:-1]: - if level == "": - continue - if level not in currentlevel: - currentlevel[level] = self.get_empty_table() - currentlevel = currentlevel[level] - pair[0] = levels[-1].strip() - elif (pair[0][0] == '"' or pair[0][0] == "'") and \ - (pair[0][-1] == pair[0][0]): - pair[0] = _unescape(pair[0][1:-1]) - k, koffset = self._load_line_multiline_str(pair[1]) - if k > -1: - while k > -1 and pair[1][k + koffset] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = pair[1][:-1] - else: - multilinestr = pair[1] + "\n" - multikey = pair[0] - else: - value, vtype = self.load_value(pair[1], strictly_valid) - try: - currentlevel[pair[0]] - raise ValueError("Duplicate keys!") - except TypeError: - raise ValueError("Duplicate keys!") - except KeyError: - if multikey: - return multikey, multilinestr, multibackslash - else: - currentlevel[pair[0]] = value - - def _load_line_multiline_str(self, p): - poffset = 0 - if len(p) < 3: - return -1, poffset - if p[0] == '[' and (p.strip()[-1] != ']' and - self._load_array_isstrarray(p)): - newp = p[1:].strip().split(',') - while len(newp) > 1 and newp[-1][0] != '"' and newp[-1][0] != "'": - newp = newp[:-2] + [newp[-2] + ',' + newp[-1]] - newp = newp[-1] - poffset = len(p) - len(newp) - p = newp - if p[0] != '"' and p[0] != "'": - return -1, poffset - if p[1] != p[0] or p[2] != p[0]: - return -1, poffset - if len(p) > 5 and p[-1] == p[0] and p[-2] == p[0] and p[-3] == p[0]: - return -1, poffset - return len(p) - 1, poffset - - def load_value(self, v, strictly_valid=True): - if not v: - raise ValueError("Empty value is invalid") - if v == 'true': - return (True, "bool") - elif v.lower() == 'true': - raise ValueError("Only all lowercase booleans allowed") - elif v == 'false': - return (False, "bool") - elif v.lower() == 'false': - raise ValueError("Only all lowercase booleans allowed") - elif v[0] == '"' or v[0] == "'": - quotechar = v[0] - testv = v[1:].split(quotechar) - triplequote = False - triplequotecount = 0 - if len(testv) > 1 and testv[0] == '' and testv[1] == '': - testv = testv[2:] - triplequote = True - closed = False - for tv in testv: - if tv == '': - if triplequote: - triplequotecount += 1 - else: - closed = True - else: - oddbackslash = False - try: - i = -1 - j = tv[i] - while j == '\\': - oddbackslash = not oddbackslash - i -= 1 - j = tv[i] - except IndexError: - pass - if not oddbackslash: - if closed: - raise ValueError("Found tokens after a closed " + - "string. Invalid TOML.") - else: - if not triplequote or triplequotecount > 1: - closed = True - else: - triplequotecount = 0 - if quotechar == '"': - escapeseqs = v.split('\\')[1:] - backslash = False - for i in escapeseqs: - if i == '': - backslash = not backslash - else: - if i[0] not in _escapes and (i[0] != 'u' and - i[0] != 'U' and - not backslash): - raise ValueError("Reserved escape sequence used") - if backslash: - backslash = False - for prefix in ["\\u", "\\U"]: - if prefix in v: - hexbytes = v.split(prefix) - v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], - prefix) - v = _unescape(v) - if len(v) > 1 and v[1] == quotechar and (len(v) < 3 or - v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == '[': - return (self.load_array(v), "array") - elif v[0] == '{': - inline_object = self.get_empty_inline_table() - self.load_inline_object(v, inline_object) - return (inline_object, "inline_object") - elif TIME_RE.match(v): - h, m, s, _, ms = TIME_RE.match(v).groups() - time = datetime.time(int(h), int(m), int(s), int(ms) if ms else 0) - return (time, "time") - else: - parsed_date = _load_date(v) - if parsed_date is not None: - return (parsed_date, "date") - if not strictly_valid: - raise ValueError("Weirdness with leading zeroes or " - "underscores in your number.") - itype = "int" - neg = False - if v[0] == '-': - neg = True - v = v[1:] - elif v[0] == '+': - v = v[1:] - v = v.replace('_', '') - lowerv = v.lower() - if '.' in v or ('x' not in v and ('e' in v or 'E' in v)): - if '.' in v and v.split('.', 1)[1] == '': - raise ValueError("This float is missing digits after " - "the point") - if v[0] not in '0123456789': - raise ValueError("This float doesn't have a leading " - "digit") - v = float(v) - itype = "float" - elif len(lowerv) == 3 and (lowerv == 'inf' or lowerv == 'nan'): - v = float(v) - itype = "float" - if itype == "int": - v = int(v, 0) - if neg: - return (0 - v, itype) - return (v, itype) - - def bounded_string(self, s): - if len(s) == 0: - return True - if s[-1] != s[0]: - return False - i = -2 - backslash = False - while len(s) + i > 0: - if s[i] == "\\": - backslash = not backslash - i -= 1 - else: - break - return not backslash - - def _load_array_isstrarray(self, a): - a = a[1:-1].strip() - if a != '' and (a[0] == '"' or a[0] == "'"): - return True - return False - - def load_array(self, a): - atype = None - retval = [] - a = a.strip() - if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): - strarray = self._load_array_isstrarray(a) - if not a[1:-1].strip().startswith('{'): - a = a[1:-1].split(',') - else: - # a is an inline object, we must find the matching parenthesis - # to define groups - new_a = [] - start_group_index = 1 - end_group_index = 2 - open_bracket_count = 1 if a[start_group_index] == '{' else 0 - in_str = False - while end_group_index < len(a[1:]): - if a[end_group_index] == '"' or a[end_group_index] == "'": - if in_str: - backslash_index = end_group_index - 1 - while (backslash_index > -1 and - a[backslash_index] == '\\'): - in_str = not in_str - backslash_index -= 1 - in_str = not in_str - if not in_str and a[end_group_index] == '{': - open_bracket_count += 1 - if in_str or a[end_group_index] != '}': - end_group_index += 1 - continue - elif a[end_group_index] == '}' and open_bracket_count > 1: - open_bracket_count -= 1 - end_group_index += 1 - continue - - # Increase end_group_index by 1 to get the closing bracket - end_group_index += 1 - - new_a.append(a[start_group_index:end_group_index]) - - # The next start index is at least after the closing - # bracket, a closing bracket can be followed by a comma - # since we are in an array. - start_group_index = end_group_index + 1 - while (start_group_index < len(a[1:]) and - a[start_group_index] != '{'): - start_group_index += 1 - end_group_index = start_group_index + 1 - a = new_a - b = 0 - if strarray: - while b < len(a) - 1: - ab = a[b].strip() - while (not self.bounded_string(ab) or - (len(ab) > 2 and - ab[0] == ab[1] == ab[2] and - ab[-2] != ab[0] and - ab[-3] != ab[0])): - a[b] = a[b] + ',' + a[b + 1] - ab = a[b].strip() - if b < len(a) - 2: - a = a[:b + 1] + a[b + 2:] - else: - a = a[:b + 1] - b += 1 - else: - al = list(a[1:-1]) - a = [] - openarr = 0 - j = 0 - for i in _range(len(al)): - if al[i] == '[': - openarr += 1 - elif al[i] == ']': - openarr -= 1 - elif al[i] == ',' and not openarr: - a.append(''.join(al[j:i])) - j = i + 1 - a.append(''.join(al[j:])) - for i in _range(len(a)): - a[i] = a[i].strip() - if a[i] != '': - nval, ntype = self.load_value(a[i]) - if atype: - if ntype != atype: - raise ValueError("Not a homogeneous array") - else: - atype = ntype - retval.append(nval) - return retval - - def preserve_comment(self, line_no, key, comment, beginline): - pass - - def embed_comments(self, idx, currentlevel): - pass - - -class TomlPreserveCommentDecoder(TomlDecoder): - - def __init__(self, _dict=dict): - self.saved_comments = {} - super(TomlPreserveCommentDecoder, self).__init__(_dict) - - def preserve_comment(self, line_no, key, comment, beginline): - self.saved_comments[line_no] = (key, comment, beginline) - - def embed_comments(self, idx, currentlevel): - if idx not in self.saved_comments: - return - - key, comment, beginline = self.saved_comments[idx] - currentlevel[key] = CommentValue(currentlevel[key], comment, beginline, - self._dict) diff --git a/src/pip/_vendor/toml/encoder.py b/src/pip/_vendor/toml/encoder.py deleted file mode 100644 index 7fb94da98ac..00000000000 --- a/src/pip/_vendor/toml/encoder.py +++ /dev/null @@ -1,304 +0,0 @@ -import datetime -import re -import sys -from decimal import Decimal - -from pip._vendor.toml.decoder import InlineTableDict - -if sys.version_info >= (3,): - unicode = str - - -def dump(o, f, encoder=None): - """Writes out dict as toml to a file - - Args: - o: Object to dump into toml - f: File descriptor where the toml should be stored - encoder: The ``TomlEncoder`` to use for constructing the output string - - Returns: - String containing the toml corresponding to dictionary - - Raises: - TypeError: When anything other than file descriptor is passed - """ - - if not f.write: - raise TypeError("You can only dump an object to a file descriptor") - d = dumps(o, encoder=encoder) - f.write(d) - return d - - -def dumps(o, encoder=None): - """Stringifies input dict as toml - - Args: - o: Object to dump into toml - encoder: The ``TomlEncoder`` to use for constructing the output string - - Returns: - String containing the toml corresponding to dict - - Examples: - ```python - >>> import toml - >>> output = { - ... 'a': "I'm a string", - ... 'b': ["I'm", "a", "list"], - ... 'c': 2400 - ... } - >>> toml.dumps(output) - 'a = "I\'m a string"\nb = [ "I\'m", "a", "list",]\nc = 2400\n' - ``` - """ - - retval = "" - if encoder is None: - encoder = TomlEncoder(o.__class__) - addtoretval, sections = encoder.dump_sections(o, "") - retval += addtoretval - outer_objs = [id(o)] - while sections: - section_ids = [id(section) for section in sections.values()] - for outer_obj in outer_objs: - if outer_obj in section_ids: - raise ValueError("Circular reference detected") - outer_objs += section_ids - newsections = encoder.get_empty_table() - for section in sections: - addtoretval, addtosections = encoder.dump_sections( - sections[section], section) - - if addtoretval or (not addtoretval and not addtosections): - if retval and retval[-2:] != "\n\n": - retval += "\n" - retval += "[" + section + "]\n" - if addtoretval: - retval += addtoretval - for s in addtosections: - newsections[section + "." + s] = addtosections[s] - sections = newsections - return retval - - -def _dump_str(v): - if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): - v = v.decode('utf-8') - v = "%r" % v - if v[0] == 'u': - v = v[1:] - singlequote = v.startswith("'") - if singlequote or v.startswith('"'): - v = v[1:-1] - if singlequote: - v = v.replace("\\'", "'") - v = v.replace('"', '\\"') - v = v.split("\\x") - while len(v) > 1: - i = -1 - if not v[0]: - v = v[1:] - v[0] = v[0].replace("\\\\", "\\") - # No, I don't know why != works and == breaks - joinx = v[0][i] != "\\" - while v[0][:i] and v[0][i] == "\\": - joinx = not joinx - i -= 1 - if joinx: - joiner = "x" - else: - joiner = "u00" - v = [v[0] + joiner + v[1]] + v[2:] - return unicode('"' + v[0] + '"') - - -def _dump_float(v): - return "{}".format(v).replace("e+0", "e+").replace("e-0", "e-") - - -def _dump_time(v): - utcoffset = v.utcoffset() - if utcoffset is None: - return v.isoformat() - # The TOML norm specifies that it's local time thus we drop the offset - return v.isoformat()[:-6] - - -class TomlEncoder(object): - - def __init__(self, _dict=dict, preserve=False): - self._dict = _dict - self.preserve = preserve - self.dump_funcs = { - str: _dump_str, - unicode: _dump_str, - list: self.dump_list, - bool: lambda v: unicode(v).lower(), - int: lambda v: v, - float: _dump_float, - Decimal: _dump_float, - datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), - datetime.time: _dump_time, - datetime.date: lambda v: v.isoformat() - } - - def get_empty_table(self): - return self._dict() - - def dump_list(self, v): - retval = "[" - for u in v: - retval += " " + unicode(self.dump_value(u)) + "," - retval += "]" - return retval - - def dump_inline_table(self, section): - """Preserve inline table in its compact syntax instead of expanding - into subsection. - - https://github.com/toml-lang/toml#user-content-inline-table - """ - retval = "" - if isinstance(section, dict): - val_list = [] - for k, v in section.items(): - val = self.dump_inline_table(v) - val_list.append(k + " = " + val) - retval += "{ " + ", ".join(val_list) + " }\n" - return retval - else: - return unicode(self.dump_value(section)) - - def dump_value(self, v): - # Lookup function corresponding to v's type - dump_fn = self.dump_funcs.get(type(v)) - if dump_fn is None and hasattr(v, '__iter__'): - dump_fn = self.dump_funcs[list] - # Evaluate function (if it exists) else return v - return dump_fn(v) if dump_fn is not None else self.dump_funcs[str](v) - - def dump_sections(self, o, sup): - retstr = "" - if sup != "" and sup[-1] != ".": - sup += '.' - retdict = self._dict() - arraystr = "" - for section in o: - section = unicode(section) - qsection = section - if not re.match(r'^[A-Za-z0-9_-]+$', section): - qsection = _dump_str(section) - if not isinstance(o[section], dict): - arrayoftables = False - if isinstance(o[section], list): - for a in o[section]: - if isinstance(a, dict): - arrayoftables = True - if arrayoftables: - for a in o[section]: - arraytabstr = "\n" - arraystr += "[[" + sup + qsection + "]]\n" - s, d = self.dump_sections(a, sup + qsection) - if s: - if s[0] == "[": - arraytabstr += s - else: - arraystr += s - while d: - newd = self._dict() - for dsec in d: - s1, d1 = self.dump_sections(d[dsec], sup + - qsection + "." + - dsec) - if s1: - arraytabstr += ("[" + sup + qsection + - "." + dsec + "]\n") - arraytabstr += s1 - for s1 in d1: - newd[dsec + "." + s1] = d1[s1] - d = newd - arraystr += arraytabstr - else: - if o[section] is not None: - retstr += (qsection + " = " + - unicode(self.dump_value(o[section])) + '\n') - elif self.preserve and isinstance(o[section], InlineTableDict): - retstr += (qsection + " = " + - self.dump_inline_table(o[section])) - else: - retdict[qsection] = o[section] - retstr += arraystr - return (retstr, retdict) - - -class TomlPreserveInlineDictEncoder(TomlEncoder): - - def __init__(self, _dict=dict): - super(TomlPreserveInlineDictEncoder, self).__init__(_dict, True) - - -class TomlArraySeparatorEncoder(TomlEncoder): - - def __init__(self, _dict=dict, preserve=False, separator=","): - super(TomlArraySeparatorEncoder, self).__init__(_dict, preserve) - if separator.strip() == "": - separator = "," + separator - elif separator.strip(' \t\n\r,'): - raise ValueError("Invalid separator for arrays") - self.separator = separator - - def dump_list(self, v): - t = [] - retval = "[" - for u in v: - t.append(self.dump_value(u)) - while t != []: - s = [] - for u in t: - if isinstance(u, list): - for r in u: - s.append(r) - else: - retval += " " + unicode(u) + self.separator - t = s - retval += "]" - return retval - - -class TomlNumpyEncoder(TomlEncoder): - - def __init__(self, _dict=dict, preserve=False): - import numpy as np - super(TomlNumpyEncoder, self).__init__(_dict, preserve) - self.dump_funcs[np.float16] = _dump_float - self.dump_funcs[np.float32] = _dump_float - self.dump_funcs[np.float64] = _dump_float - self.dump_funcs[np.int16] = self._dump_int - self.dump_funcs[np.int32] = self._dump_int - self.dump_funcs[np.int64] = self._dump_int - - def _dump_int(self, v): - return "{}".format(int(v)) - - -class TomlPreserveCommentEncoder(TomlEncoder): - - def __init__(self, _dict=dict, preserve=False): - from pip._vendor.toml.decoder import CommentValue - super(TomlPreserveCommentEncoder, self).__init__(_dict, preserve) - self.dump_funcs[CommentValue] = lambda v: v.dump(self.dump_value) - - -class TomlPathlibEncoder(TomlEncoder): - - def _dump_pathlib_path(self, v): - return _dump_str(str(v)) - - def dump_value(self, v): - if (3, 4) <= sys.version_info: - import pathlib - if isinstance(v, pathlib.PurePath): - v = str(v) - return super(TomlPathlibEncoder, self).dump_value(v) diff --git a/src/pip/_vendor/toml/ordered.py b/src/pip/_vendor/toml/ordered.py deleted file mode 100644 index 6052016e8e6..00000000000 --- a/src/pip/_vendor/toml/ordered.py +++ /dev/null @@ -1,15 +0,0 @@ -from collections import OrderedDict -from pip._vendor.toml import TomlEncoder -from pip._vendor.toml import TomlDecoder - - -class TomlOrderedDecoder(TomlDecoder): - - def __init__(self): - super(self.__class__, self).__init__(_dict=OrderedDict) - - -class TomlOrderedEncoder(TomlEncoder): - - def __init__(self): - super(self.__class__, self).__init__(_dict=OrderedDict) diff --git a/src/pip/_vendor/toml/tz.py b/src/pip/_vendor/toml/tz.py deleted file mode 100644 index bf20593a264..00000000000 --- a/src/pip/_vendor/toml/tz.py +++ /dev/null @@ -1,24 +0,0 @@ -from datetime import tzinfo, timedelta - - -class TomlTz(tzinfo): - def __init__(self, toml_offset): - if toml_offset == "Z": - self._raw_offset = "+00:00" - else: - self._raw_offset = toml_offset - self._sign = -1 if self._raw_offset[0] == '-' else 1 - self._hours = int(self._raw_offset[1:3]) - self._minutes = int(self._raw_offset[4:6]) - - def __deepcopy__(self, memo): - return self.__class__(self._raw_offset) - - def tzname(self, dt): - return "UTC" + self._raw_offset - - def utcoffset(self, dt): - return self._sign * timedelta(hours=self._hours, minutes=self._minutes) - - def dst(self, dt): - return timedelta(0) diff --git a/src/pip/_vendor/toml/LICENSE b/src/pip/_vendor/tomli/LICENSE similarity index 74% rename from src/pip/_vendor/toml/LICENSE rename to src/pip/_vendor/tomli/LICENSE index 5010e3075e6..e859590f886 100644 --- a/src/pip/_vendor/toml/LICENSE +++ b/src/pip/_vendor/tomli/LICENSE @@ -1,12 +1,6 @@ -The MIT License +MIT License -Copyright 2013-2019 William Pearson -Copyright 2015-2016 Julien Enselme -Copyright 2016 Google Inc. -Copyright 2017 Samuel Vasko -Copyright 2017 Nate Prewitt -Copyright 2017 Jack Evans -Copyright 2019 Filippo Broggini +Copyright (c) 2021 Taneli Hukkinen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -15,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/pip/_vendor/tomli/__init__.py b/src/pip/_vendor/tomli/__init__.py new file mode 100644 index 00000000000..1cd8e07279a --- /dev/null +++ b/src/pip/_vendor/tomli/__init__.py @@ -0,0 +1,6 @@ +"""A lil' TOML parser.""" + +__all__ = ("loads", "load", "TOMLDecodeError") +__version__ = "1.0.3" # DO NOT EDIT THIS LINE MANUALLY. LET bump2version UTILITY DO IT + +from pip._vendor.tomli._parser import TOMLDecodeError, load, loads diff --git a/src/pip/_vendor/tomli/_parser.py b/src/pip/_vendor/tomli/_parser.py new file mode 100644 index 00000000000..730a746843b --- /dev/null +++ b/src/pip/_vendor/tomli/_parser.py @@ -0,0 +1,703 @@ +import string +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + FrozenSet, + Iterable, + Optional, + TextIO, + Tuple, +) + +from pip._vendor.tomli._re import ( + RE_BIN, + RE_DATETIME, + RE_HEX, + RE_LOCALTIME, + RE_NUMBER, + RE_OCT, + match_to_datetime, + match_to_localtime, + match_to_number, +) + +if TYPE_CHECKING: + from re import Pattern + + +ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) + +# Neither of these sets include quotation mark or backslash. They are +# currently handled as separate cases in the parser functions. +ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t") +ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n\r") + +ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS +ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ASCII_CTRL - frozenset("\t\n") + +ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS + +TOML_WS = frozenset(" \t") +TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") +BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") + +BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( + { + "\\b": "\u0008", # backspace + "\\t": "\u0009", # tab + "\\n": "\u000A", # linefeed + "\\f": "\u000C", # form feed + "\\r": "\u000D", # carriage return + '\\"': "\u0022", # quote + "\\\\": "\u005C", # backslash + } +) + +# Type annotations +ParseFloat = Callable[[str], Any] +Key = Tuple[str, ...] +Pos = int + + +class TOMLDecodeError(ValueError): + """An error raised if a document is not valid TOML.""" + + +def load(fp: TextIO, *, parse_float: ParseFloat = float) -> Dict[str, Any]: + """Parse TOML from a file object.""" + s = fp.read() + return loads(s, parse_float=parse_float) + + +def loads(s: str, *, parse_float: ParseFloat = float) -> Dict[str, Any]: # noqa: C901 + """Parse TOML from a string.""" + + # The spec allows converting "\r\n" to "\n", even in string + # literals. Let's do so to simplify parsing. + src = s.replace("\r\n", "\n") + pos = 0 + state = State() + + # Parse one statement at a time + # (typically means one line in TOML source) + while True: + # 1. Skip line leading whitespace + pos = skip_chars(src, pos, TOML_WS) + + # 2. Parse rules. Expect one of the following: + # - end of file + # - end of line + # - comment + # - key/value pair + # - append dict to list (and move to its namespace) + # - create dict (and move to its namespace) + # Skip trailing whitespace when applicable. + try: + char = src[pos] + except IndexError: + break + if char == "\n": + pos += 1 + continue + if char in KEY_INITIAL_CHARS: + pos = key_value_rule(src, pos, state, parse_float) + pos = skip_chars(src, pos, TOML_WS) + elif char == "[": + try: + second_char: Optional[str] = src[pos + 1] + except IndexError: + second_char = None + if second_char == "[": + pos = create_list_rule(src, pos, state) + else: + pos = create_dict_rule(src, pos, state) + pos = skip_chars(src, pos, TOML_WS) + elif char != "#": + raise suffixed_err(src, pos, "Invalid statement") + + # 3. Skip comment + pos = skip_comment(src, pos) + + # 4. Expect end of line or end of file + try: + char = src[pos] + except IndexError: + break + if char != "\n": + raise suffixed_err( + src, pos, "Expected newline or end of document after a statement" + ) + pos += 1 + + return state.out.dict + + +class State: + def __init__(self) -> None: + # Mutable, read-only + self.out = NestedDict() + self.flags = Flags() + + # Immutable, read and write + self.header_namespace: Key = () + + +class Flags: + """Flags that map to parsed keys/namespaces.""" + + # Marks an immutable namespace (inline array or inline table). + FROZEN = 0 + # Marks a nest that has been explicitly created and can no longer + # be opened using the "[table]" syntax. + EXPLICIT_NEST = 1 + + def __init__(self) -> None: + self._flags: Dict[str, dict] = {} + + def unset_all(self, key: Key) -> None: + cont = self._flags + for k in key[:-1]: + if k not in cont: + return + cont = cont[k]["nested"] + cont.pop(key[-1], None) + + def set_for_relative_key(self, head_key: Key, rel_key: Key, flag: int) -> None: + cont = self._flags + for k in head_key: + if k not in cont: + cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont = cont[k]["nested"] + for k in rel_key: + if k in cont: + cont[k]["flags"].add(flag) + else: + cont[k] = {"flags": {flag}, "recursive_flags": set(), "nested": {}} + cont = cont[k]["nested"] + + def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003 + cont = self._flags + key_parent, key_stem = key[:-1], key[-1] + for k in key_parent: + if k not in cont: + cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont = cont[k]["nested"] + if key_stem not in cont: + cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag) + + def is_(self, key: Key, flag: int) -> bool: + if not key: + return False # document root has no flags + cont = self._flags + for k in key[:-1]: + if k not in cont: + return False + inner_cont = cont[k] + if flag in inner_cont["recursive_flags"]: + return True + cont = inner_cont["nested"] + key_stem = key[-1] + if key_stem in cont: + cont = cont[key_stem] + return flag in cont["flags"] or flag in cont["recursive_flags"] + return False + + +class NestedDict: + def __init__(self) -> None: + # The parsed content of the TOML document + self.dict: Dict[str, Any] = {} + + def get_or_create_nest( + self, + key: Key, + *, + access_lists: bool = True, + ) -> dict: + cont: Any = self.dict + for k in key: + if k not in cont: + cont[k] = {} + cont = cont[k] + if access_lists and isinstance(cont, list): + cont = cont[-1] + if not isinstance(cont, dict): + raise KeyError("There is no nest behind this key") + return cont + + def append_nest_to_list(self, key: Key) -> None: + cont = self.get_or_create_nest(key[:-1]) + last_key = key[-1] + if last_key in cont: + list_ = cont[last_key] + if not isinstance(list_, list): + raise KeyError("An object other than list found behind this key") + list_.append({}) + else: + cont[last_key] = [{}] + + +def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: + try: + while src[pos] in chars: + pos += 1 + except IndexError: + pass + return pos + + +def skip_until( + src: str, + pos: Pos, + expect: str, + *, + error_on: FrozenSet[str], + error_on_eof: bool, +) -> Pos: + try: + new_pos = src.index(expect, pos) + except ValueError: + new_pos = len(src) + if error_on_eof: + raise suffixed_err(src, new_pos, f'Expected "{expect!r}"') + + bad_chars = error_on.intersection(src[pos:new_pos]) + if bad_chars: + bad_char = next(iter(bad_chars)) + bad_pos = src.index(bad_char, pos) + raise suffixed_err(src, bad_pos, f'Found invalid character "{bad_char!r}"') + return new_pos + + +def skip_comment(src: str, pos: Pos) -> Pos: + try: + char: Optional[str] = src[pos] + except IndexError: + char = None + if char == "#": + return skip_until( + src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False + ) + return pos + + +def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos: + while True: + pos_before_skip = pos + pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) + pos = skip_comment(src, pos) + if pos == pos_before_skip: + return pos + + +def create_dict_rule(src: str, pos: Pos, state: State) -> Pos: + pos += 1 # Skip "[" + pos = skip_chars(src, pos, TOML_WS) + pos, key = parse_key(src, pos) + + if state.flags.is_(key, Flags.EXPLICIT_NEST) or state.flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Can not declare {key} twice") + state.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) + try: + state.out.get_or_create_nest(key) + except KeyError: + raise suffixed_err(src, pos, "Can not overwrite a value") + state.header_namespace = key + + if src[pos : pos + 1] != "]": + raise suffixed_err(src, pos, 'Expected "]" at the end of a table declaration') + return pos + 1 + + +def create_list_rule(src: str, pos: Pos, state: State) -> Pos: + pos += 2 # Skip "[[" + pos = skip_chars(src, pos, TOML_WS) + pos, key = parse_key(src, pos) + + if state.flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Can not mutate immutable namespace {key}") + # Free the namespace now that it points to another empty list item... + state.flags.unset_all(key) + # ...but this key precisely is still prohibited from table declaration + state.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) + try: + state.out.append_nest_to_list(key) + except KeyError: + raise suffixed_err(src, pos, "Can not overwrite a value") + state.header_namespace = key + + end_marker = src[pos : pos + 2] + if end_marker != "]]": + raise suffixed_err( + src, + pos, + f'Found "{end_marker!r}" at the end of an array declaration.' + ' Expected "]]"', + ) + return pos + 2 + + +def key_value_rule(src: str, pos: Pos, state: State, parse_float: ParseFloat) -> Pos: + pos, key, value = parse_key_value_pair(src, pos, parse_float) + key_parent, key_stem = key[:-1], key[-1] + abs_key_parent = state.header_namespace + key_parent + + if state.flags.is_(abs_key_parent, Flags.FROZEN): + raise suffixed_err( + src, pos, f"Can not mutate immutable namespace {abs_key_parent}" + ) + # Containers in the relative path can't be opened with the table syntax after this + state.flags.set_for_relative_key(state.header_namespace, key, Flags.EXPLICIT_NEST) + try: + nest = state.out.get_or_create_nest(abs_key_parent) + except KeyError: + raise suffixed_err(src, pos, "Can not overwrite a value") + if key_stem in nest: + raise suffixed_err(src, pos, "Can not overwrite a value") + # Mark inline table and array namespaces recursively immutable + if isinstance(value, (dict, list)): + abs_key = state.header_namespace + key + state.flags.set(abs_key, Flags.FROZEN, recursive=True) + nest[key_stem] = value + return pos + + +def parse_key_value_pair( + src: str, pos: Pos, parse_float: ParseFloat +) -> Tuple[Pos, Key, Any]: + pos, key = parse_key(src, pos) + try: + char: Optional[str] = src[pos] + except IndexError: + char = None + if char != "=": + raise suffixed_err(src, pos, 'Expected "=" after a key in a key/value pair') + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + pos, value = parse_value(src, pos, parse_float) + return pos, key, value + + +def parse_key(src: str, pos: Pos) -> Tuple[Pos, Key]: + pos, key_part = parse_key_part(src, pos) + key = [key_part] + pos = skip_chars(src, pos, TOML_WS) + while True: + try: + char: Optional[str] = src[pos] + except IndexError: + char = None + if char != ".": + return pos, tuple(key) + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + pos, key_part = parse_key_part(src, pos) + key.append(key_part) + pos = skip_chars(src, pos, TOML_WS) + + +def parse_key_part(src: str, pos: Pos) -> Tuple[Pos, str]: + try: + char: Optional[str] = src[pos] + except IndexError: + char = None + if char in BARE_KEY_CHARS: + start_pos = pos + pos = skip_chars(src, pos, BARE_KEY_CHARS) + return pos, src[start_pos:pos] + if char == "'": + return parse_literal_str(src, pos) + if char == '"': + return parse_one_line_basic_str(src, pos) + raise suffixed_err(src, pos, "Invalid initial character for a key part") + + +def parse_one_line_basic_str(src: str, pos: Pos) -> Tuple[Pos, str]: + pos += 1 + return parse_basic_str(src, pos, multiline=False) + + +def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> Tuple[Pos, list]: + pos += 1 + array: list = [] + + pos = skip_comments_and_array_ws(src, pos) + if src[pos : pos + 1] == "]": + return pos + 1, array + while True: + pos, val = parse_value(src, pos, parse_float) + array.append(val) + pos = skip_comments_and_array_ws(src, pos) + + c = src[pos : pos + 1] + if c == "]": + return pos + 1, array + if c != ",": + raise suffixed_err(src, pos, "Unclosed array") + pos += 1 + + pos = skip_comments_and_array_ws(src, pos) + if src[pos : pos + 1] == "]": + return pos + 1, array + + +def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> Tuple[Pos, dict]: + pos += 1 + nested_dict = NestedDict() + flags = Flags() + + pos = skip_chars(src, pos, TOML_WS) + if src[pos : pos + 1] == "}": + return pos + 1, nested_dict.dict + while True: + pos, key, value = parse_key_value_pair(src, pos, parse_float) + key_parent, key_stem = key[:-1], key[-1] + if flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Can not mutate immutable namespace {key}") + try: + nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) + except KeyError: + raise suffixed_err(src, pos, "Can not overwrite a value") + if key_stem in nest: + raise suffixed_err(src, pos, f'Duplicate inline table key "{key_stem}"') + nest[key_stem] = value + pos = skip_chars(src, pos, TOML_WS) + c = src[pos : pos + 1] + if c == "}": + return pos + 1, nested_dict.dict + if c != ",": + raise suffixed_err(src, pos, "Unclosed inline table") + if isinstance(value, (dict, list)): + flags.set(key, Flags.FROZEN, recursive=True) + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + + +def parse_basic_str_escape( + src: str, pos: Pos, *, multiline: bool = False +) -> Tuple[Pos, str]: + escape_id = src[pos : pos + 2] + pos += 2 + if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}: + # Skip whitespace until next non-whitespace character or end of + # the doc. Error if non-whitespace is found before newline. + if escape_id != "\\\n": + pos = skip_chars(src, pos, TOML_WS) + char = src[pos : pos + 1] + if not char: + return pos, "" + if char != "\n": + raise suffixed_err(src, pos, 'Unescaped "\\" in a string') + pos += 1 + pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) + return pos, "" + if escape_id == "\\u": + return parse_hex_char(src, pos, 4) + if escape_id == "\\U": + return parse_hex_char(src, pos, 8) + try: + return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] + except KeyError: + if len(escape_id) != 2: + raise suffixed_err(src, pos, "Unterminated string") + raise suffixed_err(src, pos, 'Unescaped "\\" in a string') + + +def parse_basic_str_escape_multiline(src: str, pos: Pos) -> Tuple[Pos, str]: + return parse_basic_str_escape(src, pos, multiline=True) + + +def parse_hex_char(src: str, pos: Pos, hex_len: int) -> Tuple[Pos, str]: + hex_str = src[pos : pos + hex_len] + if len(hex_str) != hex_len or any(c not in string.hexdigits for c in hex_str): + raise suffixed_err(src, pos, "Invalid hex value") + pos += hex_len + hex_int = int(hex_str, 16) + if not is_unicode_scalar_value(hex_int): + raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + return pos, chr(hex_int) + + +def parse_literal_str(src: str, pos: Pos) -> Tuple[Pos, str]: + pos += 1 # Skip starting apostrophe + start_pos = pos + pos = skip_until( + src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True + ) + return pos + 1, src[start_pos:pos] # Skip ending apostrophe + + +def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> Tuple[Pos, str]: + pos += 3 + if src[pos : pos + 1] == "\n": + pos += 1 + + if literal: + delim = "'" + end_pos = skip_until( + src, + pos, + "'''", + error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS, + error_on_eof=True, + ) + result = src[pos:end_pos] + pos = end_pos + 3 + else: + delim = '"' + pos, result = parse_basic_str(src, pos, multiline=True) + + # Add at maximum two extra apostrophes/quotes if the end sequence + # is 4 or 5 chars long instead of just 3. + if src[pos : pos + 1] != delim: + return pos, result + pos += 1 + if src[pos : pos + 1] != delim: + return pos, result + delim + pos += 1 + return pos, result + (delim * 2) + + +def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> Tuple[Pos, str]: + if multiline: + error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS + parse_escapes = parse_basic_str_escape_multiline + else: + error_on = ILLEGAL_BASIC_STR_CHARS + parse_escapes = parse_basic_str_escape + result = "" + start_pos = pos + while True: + try: + char = src[pos] + except IndexError: + raise suffixed_err(src, pos, "Unterminated string") + if char == '"': + if not multiline: + return pos + 1, result + src[start_pos:pos] + if src[pos + 1 : pos + 3] == '""': + return pos + 3, result + src[start_pos:pos] + pos += 1 + continue + if char == "\\": + result += src[start_pos:pos] + pos, parsed_escape = parse_escapes(src, pos) + result += parsed_escape + start_pos = pos + continue + if char in error_on: + raise suffixed_err(src, pos, f'Illegal character "{char!r}"') + pos += 1 + + +def parse_regex(src: str, pos: Pos, regex: "Pattern") -> Tuple[Pos, str]: + match = regex.match(src, pos) + if not match: + raise suffixed_err(src, pos, "Unexpected sequence") + return match.end(), match.group() + + +def parse_value( # noqa: C901 + src: str, pos: Pos, parse_float: ParseFloat +) -> Tuple[Pos, Any]: + try: + char: Optional[str] = src[pos] + except IndexError: + char = None + + # Basic strings + if char == '"': + if src[pos + 1 : pos + 3] == '""': + return parse_multiline_str(src, pos, literal=False) + return parse_one_line_basic_str(src, pos) + + # Literal strings + if char == "'": + if src[pos + 1 : pos + 3] == "''": + return parse_multiline_str(src, pos, literal=True) + return parse_literal_str(src, pos) + + # Booleans + if char == "t": + if src[pos + 1 : pos + 4] == "rue": + return pos + 4, True + if char == "f": + if src[pos + 1 : pos + 5] == "alse": + return pos + 5, False + + # Dates and times + datetime_match = RE_DATETIME.match(src, pos) + if datetime_match: + try: + datetime_obj = match_to_datetime(datetime_match) + except ValueError: + raise suffixed_err(src, pos, "Invalid date or datetime") + return datetime_match.end(), datetime_obj + localtime_match = RE_LOCALTIME.match(src, pos) + if localtime_match: + return localtime_match.end(), match_to_localtime(localtime_match) + + # Non-decimal integers + if char == "0": + second_char = src[pos + 1 : pos + 2] + if second_char == "x": + pos, hex_str = parse_regex(src, pos + 2, RE_HEX) + return pos, int(hex_str, 16) + if second_char == "o": + pos, oct_str = parse_regex(src, pos + 2, RE_OCT) + return pos, int(oct_str, 8) + if second_char == "b": + pos, bin_str = parse_regex(src, pos + 2, RE_BIN) + return pos, int(bin_str, 2) + + # Decimal integers and "normal" floats. + # The regex will greedily match any type starting with a decimal + # char, so needs to be located after handling of non-decimal ints, + # and dates and times. + number_match = RE_NUMBER.match(src, pos) + if number_match: + return number_match.end(), match_to_number(number_match, parse_float) + + # Arrays + if char == "[": + return parse_array(src, pos, parse_float) + + # Inline tables + if char == "{": + return parse_inline_table(src, pos, parse_float) + + # Special floats + first_three = src[pos : pos + 3] + if first_three in {"inf", "nan"}: + return pos + 3, parse_float(first_three) + first_four = src[pos : pos + 4] + if first_four in {"-inf", "+inf", "-nan", "+nan"}: + return pos + 4, parse_float(first_four) + + raise suffixed_err(src, pos, "Invalid value") + + +def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: + """Return a `TOMLDecodeError` where error message is suffixed with + coordinates in source.""" + + def coord_repr(src: str, pos: Pos) -> str: + if pos >= len(src): + return "end of document" + line = src.count("\n", 0, pos) + 1 + if line == 1: + column = pos + 1 + else: + column = pos - src.rindex("\n", 0, pos) + return f"line {line}, column {column}" + + return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + + +def is_unicode_scalar_value(codepoint: int) -> bool: + return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111) diff --git a/src/pip/_vendor/tomli/_re.py b/src/pip/_vendor/tomli/_re.py new file mode 100644 index 00000000000..3883fdd9c90 --- /dev/null +++ b/src/pip/_vendor/tomli/_re.py @@ -0,0 +1,83 @@ +from datetime import date, datetime, time, timedelta, timezone, tzinfo +import re +from typing import TYPE_CHECKING, Any, Optional, Union + +if TYPE_CHECKING: + from re import Match + + from pip._vendor.tomli._parser import ParseFloat + +# E.g. +# - 00:32:00.999999 +# - 00:32:00 +_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?" + +RE_HEX = re.compile(r"[0-9A-Fa-f](?:_?[0-9A-Fa-f])*") +RE_BIN = re.compile(r"[01](?:_?[01])*") +RE_OCT = re.compile(r"[0-7](?:_?[0-7])*") +RE_NUMBER = re.compile( + r"[+-]?(?:0|[1-9](?:_?[0-9])*)" # integer + + r"(?:\.[0-9](?:_?[0-9])*)?" # optional fractional part + + r"(?:[eE][+-]?[0-9](?:_?[0-9])*)?" # optional exponent part +) +RE_LOCALTIME = re.compile(_TIME_RE_STR) +RE_DATETIME = re.compile( + r"([0-9]{4})-(0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-9]|3[01])" # date, e.g. 1988-10-27 + + r"(?:" + + r"[T ]" + + _TIME_RE_STR + + r"(?:(Z)|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))?" # time offset + + r")?" +) + + +def match_to_datetime(match: "Match") -> Union[datetime, date]: + """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`. + + Raises ValueError if the match does not correspond to a valid date + or datetime. + """ + ( + year_str, + month_str, + day_str, + hour_str, + minute_str, + sec_str, + micros_str, + zulu_time, + offset_dir_str, + offset_hour_str, + offset_minute_str, + ) = match.groups() + year, month, day = int(year_str), int(month_str), int(day_str) + if hour_str is None: + return date(year, month, day) + hour, minute, sec = int(hour_str), int(minute_str), int(sec_str) + micros = int(micros_str[1:].ljust(6, "0")[:6]) if micros_str else 0 + if offset_dir_str: + offset_dir = 1 if offset_dir_str == "+" else -1 + tz: Optional[tzinfo] = timezone( + timedelta( + hours=offset_dir * int(offset_hour_str), + minutes=offset_dir * int(offset_minute_str), + ) + ) + elif zulu_time: + tz = timezone.utc + else: # local date-time + tz = None + return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) + + +def match_to_localtime(match: "Match") -> time: + hour_str, minute_str, sec_str, micros_str = match.groups() + micros = int(micros_str[1:].ljust(6, "0")[:6]) if micros_str else 0 + return time(int(hour_str), int(minute_str), int(sec_str), micros) + + +def match_to_number(match: "Match", parse_float: "ParseFloat") -> Any: + match_str = match.group() + if "." in match_str or "e" in match_str or "E" in match_str: + return parse_float(match_str) + return int(match_str) diff --git a/src/pip/_vendor/tomli/py.typed b/src/pip/_vendor/tomli/py.typed new file mode 100644 index 00000000000..7632ecf7754 --- /dev/null +++ b/src/pip/_vendor/tomli/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/src/pip/_vendor/typing_extensions.LICENSE b/src/pip/_vendor/typing_extensions.LICENSE new file mode 100644 index 00000000000..583f9f6e617 --- /dev/null +++ b/src/pip/_vendor/typing_extensions.LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are +retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/src/pip/_vendor/typing_extensions.py b/src/pip/_vendor/typing_extensions.py new file mode 100644 index 00000000000..9f1c7aa31e2 --- /dev/null +++ b/src/pip/_vendor/typing_extensions.py @@ -0,0 +1,2296 @@ +import abc +import collections +import collections.abc +import operator +import sys +import typing + +# After PEP 560, internal typing API was substantially reworked. +# This is especially important for Protocol class which uses internal APIs +# quite extensively. +PEP_560 = sys.version_info[:3] >= (3, 7, 0) + +if PEP_560: + GenericMeta = type +else: + # 3.6 + from typing import GenericMeta, _type_vars # noqa + +# The two functions below are copies of typing internal helpers. +# They are needed by _ProtocolMeta + + +def _no_slots_copy(dct): + dict_copy = dict(dct) + if '__slots__' in dict_copy: + for slot in dict_copy['__slots__']: + dict_copy.pop(slot, None) + return dict_copy + + +def _check_generic(cls, parameters): + if not cls.__parameters__: + raise TypeError(f"{cls} is not a generic class") + alen = len(parameters) + elen = len(cls.__parameters__) + if alen != elen: + raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" + f" actual {alen}, expected {elen}") + + +# Please keep __all__ alphabetized within each category. +__all__ = [ + # Super-special typing primitives. + 'ClassVar', + 'Concatenate', + 'Final', + 'ParamSpec', + 'Self', + 'Type', + + # ABCs (from collections.abc). + 'Awaitable', + 'AsyncIterator', + 'AsyncIterable', + 'Coroutine', + 'AsyncGenerator', + 'AsyncContextManager', + 'ChainMap', + + # Concrete collection types. + 'ContextManager', + 'Counter', + 'Deque', + 'DefaultDict', + 'OrderedDict', + 'TypedDict', + + # Structural checks, a.k.a. protocols. + 'SupportsIndex', + + # One-off things. + 'Annotated', + 'final', + 'IntVar', + 'Literal', + 'NewType', + 'overload', + 'Protocol', + 'runtime', + 'runtime_checkable', + 'Text', + 'TypeAlias', + 'TypeGuard', + 'TYPE_CHECKING', +] + +if PEP_560: + __all__.extend(["get_args", "get_origin", "get_type_hints"]) + +# 3.6.2+ +if hasattr(typing, 'NoReturn'): + NoReturn = typing.NoReturn +# 3.6.0-3.6.1 +else: + class _NoReturn(typing._FinalTypingBase, _root=True): + """Special type indicating functions that never return. + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + This type is invalid in other positions, e.g., ``List[NoReturn]`` + will fail in static type checkers. + """ + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("NoReturn cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("NoReturn cannot be used with issubclass().") + + NoReturn = _NoReturn(_root=True) + +# Some unconstrained type variables. These are used by the container types. +# (These are not for export.) +T = typing.TypeVar('T') # Any type. +KT = typing.TypeVar('KT') # Key type. +VT = typing.TypeVar('VT') # Value type. +T_co = typing.TypeVar('T_co', covariant=True) # Any type covariant containers. +T_contra = typing.TypeVar('T_contra', contravariant=True) # Ditto contravariant. + +ClassVar = typing.ClassVar + +# On older versions of typing there is an internal class named "Final". +# 3.8+ +if hasattr(typing, 'Final') and sys.version_info[:2] >= (3, 7): + Final = typing.Final +# 3.7 +elif sys.version_info[:2] >= (3, 7): + class _FinalForm(typing._SpecialForm, _root=True): + + def __repr__(self): + return 'typing_extensions.' + self._name + + def __getitem__(self, parameters): + item = typing._type_check(parameters, + f'{self._name} accepts only single type') + return typing._GenericAlias(self, (item,)) + + Final = _FinalForm('Final', + doc="""A special typing construct to indicate that a name + cannot be re-assigned or overridden in a subclass. + For example: + + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker + + class Connection: + TIMEOUT: Final[int] = 10 + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker + + There is no runtime checking of these properties.""") +# 3.6 +else: + class _Final(typing._FinalTypingBase, _root=True): + """A special typing construct to indicate that a name + cannot be re-assigned or overridden in a subclass. + For example: + + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker + + class Connection: + TIMEOUT: Final[int] = 10 + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker + + There is no runtime checking of these properties. + """ + + __slots__ = ('__type__',) + + def __init__(self, tp=None, **kwds): + self.__type__ = tp + + def __getitem__(self, item): + cls = type(self) + if self.__type__ is None: + return cls(typing._type_check(item, + f'{cls.__name__[1:]} accepts only single type.'), + _root=True) + raise TypeError(f'{cls.__name__[1:]} cannot be further subscripted') + + def _eval_type(self, globalns, localns): + new_tp = typing._eval_type(self.__type__, globalns, localns) + if new_tp == self.__type__: + return self + return type(self)(new_tp, _root=True) + + def __repr__(self): + r = super().__repr__() + if self.__type__ is not None: + r += f'[{typing._type_repr(self.__type__)}]' + return r + + def __hash__(self): + return hash((type(self).__name__, self.__type__)) + + def __eq__(self, other): + if not isinstance(other, _Final): + return NotImplemented + if self.__type__ is not None: + return self.__type__ == other.__type__ + return self is other + + Final = _Final(_root=True) + + +# 3.8+ +if hasattr(typing, 'final'): + final = typing.final +# 3.6-3.7 +else: + def final(f): + """This decorator can be used to indicate to type checkers that + the decorated method cannot be overridden, and decorated class + cannot be subclassed. For example: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... + + There is no runtime checking of these properties. + """ + return f + + +def IntVar(name): + return typing.TypeVar(name) + + +# 3.8+: +if hasattr(typing, 'Literal'): + Literal = typing.Literal +# 3.7: +elif sys.version_info[:2] >= (3, 7): + class _LiteralForm(typing._SpecialForm, _root=True): + + def __repr__(self): + return 'typing_extensions.' + self._name + + def __getitem__(self, parameters): + return typing._GenericAlias(self, parameters) + + Literal = _LiteralForm('Literal', + doc="""A type that can be used to indicate to type checkers + that the corresponding value has a value literally equivalent + to the provided parameter. For example: + + var: Literal[4] = 4 + + The type checker understands that 'var' is literally equal to + the value 4 and no other value. + + Literal[...] cannot be subclassed. There is no runtime + checking verifying that the parameter is actually a value + instead of a type.""") +# 3.6: +else: + class _Literal(typing._FinalTypingBase, _root=True): + """A type that can be used to indicate to type checkers that the + corresponding value has a value literally equivalent to the + provided parameter. For example: + + var: Literal[4] = 4 + + The type checker understands that 'var' is literally equal to the + value 4 and no other value. + + Literal[...] cannot be subclassed. There is no runtime checking + verifying that the parameter is actually a value instead of a type. + """ + + __slots__ = ('__values__',) + + def __init__(self, values=None, **kwds): + self.__values__ = values + + def __getitem__(self, values): + cls = type(self) + if self.__values__ is None: + if not isinstance(values, tuple): + values = (values,) + return cls(values, _root=True) + raise TypeError(f'{cls.__name__[1:]} cannot be further subscripted') + + def _eval_type(self, globalns, localns): + return self + + def __repr__(self): + r = super().__repr__() + if self.__values__ is not None: + r += f'[{", ".join(map(typing._type_repr, self.__values__))}]' + return r + + def __hash__(self): + return hash((type(self).__name__, self.__values__)) + + def __eq__(self, other): + if not isinstance(other, _Literal): + return NotImplemented + if self.__values__ is not None: + return self.__values__ == other.__values__ + return self is other + + Literal = _Literal(_root=True) + + +_overload_dummy = typing._overload_dummy # noqa +overload = typing.overload + + +# This is not a real generic class. Don't use outside annotations. +Type = typing.Type + +# Various ABCs mimicking those in collections.abc. +# A few are simply re-exported for completeness. + + +class _ExtensionsGenericMeta(GenericMeta): + def __subclasscheck__(self, subclass): + """This mimics a more modern GenericMeta.__subclasscheck__() logic + (that does not have problems with recursion) to work around interactions + between collections, typing, and typing_extensions on older + versions of Python, see https://github.com/python/typing/issues/501. + """ + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if not self.__extra__: + return super().__subclasscheck__(subclass) + res = self.__extra__.__subclasshook__(subclass) + if res is not NotImplemented: + return res + if self.__extra__ in subclass.__mro__: + return True + for scls in self.__extra__.__subclasses__(): + if isinstance(scls, GenericMeta): + continue + if issubclass(subclass, scls): + return True + return False + + +Awaitable = typing.Awaitable +Coroutine = typing.Coroutine +AsyncIterable = typing.AsyncIterable +AsyncIterator = typing.AsyncIterator + +# 3.6.1+ +if hasattr(typing, 'Deque'): + Deque = typing.Deque +# 3.6.0 +else: + class Deque(collections.deque, typing.MutableSequence[T], + metaclass=_ExtensionsGenericMeta, + extra=collections.deque): + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls._gorg is Deque: + return collections.deque(*args, **kwds) + return typing._generic_new(collections.deque, cls, *args, **kwds) + +ContextManager = typing.ContextManager +# 3.6.2+ +if hasattr(typing, 'AsyncContextManager'): + AsyncContextManager = typing.AsyncContextManager +# 3.6.0-3.6.1 +else: + from _collections_abc import _check_methods as _check_methods_in_mro # noqa + + class AsyncContextManager(typing.Generic[T_co]): + __slots__ = () + + async def __aenter__(self): + return self + + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncContextManager: + return _check_methods_in_mro(C, "__aenter__", "__aexit__") + return NotImplemented + +DefaultDict = typing.DefaultDict + +# 3.7.2+ +if hasattr(typing, 'OrderedDict'): + OrderedDict = typing.OrderedDict +# 3.7.0-3.7.2 +elif (3, 7, 0) <= sys.version_info[:3] < (3, 7, 2): + OrderedDict = typing._alias(collections.OrderedDict, (KT, VT)) +# 3.6 +else: + class OrderedDict(collections.OrderedDict, typing.MutableMapping[KT, VT], + metaclass=_ExtensionsGenericMeta, + extra=collections.OrderedDict): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls._gorg is OrderedDict: + return collections.OrderedDict(*args, **kwds) + return typing._generic_new(collections.OrderedDict, cls, *args, **kwds) + +# 3.6.2+ +if hasattr(typing, 'Counter'): + Counter = typing.Counter +# 3.6.0-3.6.1 +else: + class Counter(collections.Counter, + typing.Dict[T, int], + metaclass=_ExtensionsGenericMeta, extra=collections.Counter): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls._gorg is Counter: + return collections.Counter(*args, **kwds) + return typing._generic_new(collections.Counter, cls, *args, **kwds) + +# 3.6.1+ +if hasattr(typing, 'ChainMap'): + ChainMap = typing.ChainMap +elif hasattr(collections, 'ChainMap'): + class ChainMap(collections.ChainMap, typing.MutableMapping[KT, VT], + metaclass=_ExtensionsGenericMeta, + extra=collections.ChainMap): + + __slots__ = () + + def __new__(cls, *args, **kwds): + if cls._gorg is ChainMap: + return collections.ChainMap(*args, **kwds) + return typing._generic_new(collections.ChainMap, cls, *args, **kwds) + +# 3.6.1+ +if hasattr(typing, 'AsyncGenerator'): + AsyncGenerator = typing.AsyncGenerator +# 3.6.0 +else: + class AsyncGenerator(AsyncIterator[T_co], typing.Generic[T_co, T_contra], + metaclass=_ExtensionsGenericMeta, + extra=collections.abc.AsyncGenerator): + __slots__ = () + +NewType = typing.NewType +Text = typing.Text +TYPE_CHECKING = typing.TYPE_CHECKING + + +def _gorg(cls): + """This function exists for compatibility with old typing versions.""" + assert isinstance(cls, GenericMeta) + if hasattr(cls, '_gorg'): + return cls._gorg + while cls.__origin__ is not None: + cls = cls.__origin__ + return cls + + +_PROTO_WHITELIST = ['Callable', 'Awaitable', + 'Iterable', 'Iterator', 'AsyncIterable', 'AsyncIterator', + 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', + 'ContextManager', 'AsyncContextManager'] + + +def _get_protocol_attrs(cls): + attrs = set() + for base in cls.__mro__[:-1]: # without object + if base.__name__ in ('Protocol', 'Generic'): + continue + annotations = getattr(base, '__annotations__', {}) + for attr in list(base.__dict__.keys()) + list(annotations.keys()): + if (not attr.startswith('_abc_') and attr not in ( + '__abstractmethods__', '__annotations__', '__weakref__', + '_is_protocol', '_is_runtime_protocol', '__dict__', + '__args__', '__slots__', + '__next_in_mro__', '__parameters__', '__origin__', + '__orig_bases__', '__extra__', '__tree_hash__', + '__doc__', '__subclasshook__', '__init__', '__new__', + '__module__', '_MutableMapping__marker', '_gorg')): + attrs.add(attr) + return attrs + + +def _is_callable_members_only(cls): + return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls)) + + +# 3.8+ +if hasattr(typing, 'Protocol'): + Protocol = typing.Protocol +# 3.7 +elif PEP_560: + from typing import _collect_type_vars # noqa + + def _no_init(self, *args, **kwargs): + if type(self)._is_protocol: + raise TypeError('Protocols cannot be instantiated') + + class _ProtocolMeta(abc.ABCMeta): + # This metaclass is a bit unfortunate and exists only because of the lack + # of __instancehook__. + def __instancecheck__(cls, instance): + # We need this method for situations where attributes are + # assigned in __init__. + if ((not getattr(cls, '_is_protocol', False) or + _is_callable_members_only(cls)) and + issubclass(instance.__class__, cls)): + return True + if cls._is_protocol: + if all(hasattr(instance, attr) and + (not callable(getattr(cls, attr, None)) or + getattr(instance, attr) is not None) + for attr in _get_protocol_attrs(cls)): + return True + return super().__instancecheck__(instance) + + class Protocol(metaclass=_ProtocolMeta): + # There is quite a lot of overlapping code with typing.Generic. + # Unfortunately it is hard to avoid this while these live in two different + # modules. The duplicated code will be removed when Protocol is moved to typing. + """Base class for protocol classes. Protocol classes are defined as:: + + class Proto(Protocol): + def meth(self) -> int: + ... + + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: + + class C: + def meth(self) -> int: + return 0 + + def func(x: Proto) -> int: + return x.meth() + + func(C()) # Passes static type check + + See PEP 544 for details. Protocol classes decorated with + @typing_extensions.runtime act as simple-minded runtime protocol that checks + only the presence of given attributes, ignoring their type signatures. + + Protocol classes can be generic, they are defined as:: + + class GenProto(Protocol[T]): + def meth(self) -> T: + ... + """ + __slots__ = () + _is_protocol = True + + def __new__(cls, *args, **kwds): + if cls is Protocol: + raise TypeError("Type Protocol cannot be instantiated; " + "it can only be used as a base class") + return super().__new__(cls) + + @typing._tp_cache + def __class_getitem__(cls, params): + if not isinstance(params, tuple): + params = (params,) + if not params and cls is not typing.Tuple: + raise TypeError( + f"Parameter list to {cls.__qualname__}[...] cannot be empty") + msg = "Parameters to generic types must be types." + params = tuple(typing._type_check(p, msg) for p in params) # noqa + if cls is Protocol: + # Generic can only be subscripted with unique type variables. + if not all(isinstance(p, typing.TypeVar) for p in params): + i = 0 + while isinstance(params[i], typing.TypeVar): + i += 1 + raise TypeError( + "Parameters to Protocol[...] must all be type variables." + f" Parameter {i + 1} is {params[i]}") + if len(set(params)) != len(params): + raise TypeError( + "Parameters to Protocol[...] must all be unique") + else: + # Subscripting a regular Generic subclass. + _check_generic(cls, params) + return typing._GenericAlias(cls, params) + + def __init_subclass__(cls, *args, **kwargs): + tvars = [] + if '__orig_bases__' in cls.__dict__: + error = typing.Generic in cls.__orig_bases__ + else: + error = typing.Generic in cls.__bases__ + if error: + raise TypeError("Cannot inherit from plain Generic") + if '__orig_bases__' in cls.__dict__: + tvars = _collect_type_vars(cls.__orig_bases__) + # Look for Generic[T1, ..., Tn] or Protocol[T1, ..., Tn]. + # If found, tvars must be a subset of it. + # If not found, tvars is it. + # Also check for and reject plain Generic, + # and reject multiple Generic[...] and/or Protocol[...]. + gvars = None + for base in cls.__orig_bases__: + if (isinstance(base, typing._GenericAlias) and + base.__origin__ in (typing.Generic, Protocol)): + # for error messages + the_base = base.__origin__.__name__ + if gvars is not None: + raise TypeError( + "Cannot inherit from Generic[...]" + " and/or Protocol[...] multiple types.") + gvars = base.__parameters__ + if gvars is None: + gvars = tvars + else: + tvarset = set(tvars) + gvarset = set(gvars) + if not tvarset <= gvarset: + s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) + s_args = ', '.join(str(g) for g in gvars) + raise TypeError(f"Some type variables ({s_vars}) are" + f" not listed in {the_base}[{s_args}]") + tvars = gvars + cls.__parameters__ = tuple(tvars) + + # Determine if this is a protocol or a concrete subclass. + if not cls.__dict__.get('_is_protocol', None): + cls._is_protocol = any(b is Protocol for b in cls.__bases__) + + # Set (or override) the protocol subclass hook. + def _proto_hook(other): + if not cls.__dict__.get('_is_protocol', None): + return NotImplemented + if not getattr(cls, '_is_runtime_protocol', False): + if sys._getframe(2).f_globals['__name__'] in ['abc', 'functools']: + return NotImplemented + raise TypeError("Instance and class checks can only be used with" + " @runtime protocols") + if not _is_callable_members_only(cls): + if sys._getframe(2).f_globals['__name__'] in ['abc', 'functools']: + return NotImplemented + raise TypeError("Protocols with non-method members" + " don't support issubclass()") + if not isinstance(other, type): + # Same error as for issubclass(1, int) + raise TypeError('issubclass() arg 1 must be a class') + for attr in _get_protocol_attrs(cls): + for base in other.__mro__: + if attr in base.__dict__: + if base.__dict__[attr] is None: + return NotImplemented + break + annotations = getattr(base, '__annotations__', {}) + if (isinstance(annotations, typing.Mapping) and + attr in annotations and + isinstance(other, _ProtocolMeta) and + other._is_protocol): + break + else: + return NotImplemented + return True + if '__subclasshook__' not in cls.__dict__: + cls.__subclasshook__ = _proto_hook + + # We have nothing more to do for non-protocols. + if not cls._is_protocol: + return + + # Check consistency of bases. + for base in cls.__bases__: + if not (base in (object, typing.Generic) or + base.__module__ == 'collections.abc' and + base.__name__ in _PROTO_WHITELIST or + isinstance(base, _ProtocolMeta) and base._is_protocol): + raise TypeError('Protocols can only inherit from other' + f' protocols, got {repr(base)}') + cls.__init__ = _no_init +# 3.6 +else: + from typing import _next_in_mro, _type_check # noqa + + def _no_init(self, *args, **kwargs): + if type(self)._is_protocol: + raise TypeError('Protocols cannot be instantiated') + + class _ProtocolMeta(GenericMeta): + """Internal metaclass for Protocol. + + This exists so Protocol classes can be generic without deriving + from Generic. + """ + def __new__(cls, name, bases, namespace, + tvars=None, args=None, origin=None, extra=None, orig_bases=None): + # This is just a version copied from GenericMeta.__new__ that + # includes "Protocol" special treatment. (Comments removed for brevity.) + assert extra is None # Protocols should not have extra + if tvars is not None: + assert origin is not None + assert all(isinstance(t, typing.TypeVar) for t in tvars), tvars + else: + tvars = _type_vars(bases) + gvars = None + for base in bases: + if base is typing.Generic: + raise TypeError("Cannot inherit from plain Generic") + if (isinstance(base, GenericMeta) and + base.__origin__ in (typing.Generic, Protocol)): + if gvars is not None: + raise TypeError( + "Cannot inherit from Generic[...] or" + " Protocol[...] multiple times.") + gvars = base.__parameters__ + if gvars is None: + gvars = tvars + else: + tvarset = set(tvars) + gvarset = set(gvars) + if not tvarset <= gvarset: + s_vars = ", ".join(str(t) for t in tvars if t not in gvarset) + s_args = ", ".join(str(g) for g in gvars) + cls_name = "Generic" if any(b.__origin__ is typing.Generic + for b in bases) else "Protocol" + raise TypeError(f"Some type variables ({s_vars}) are" + f" not listed in {cls_name}[{s_args}]") + tvars = gvars + + initial_bases = bases + if (extra is not None and type(extra) is abc.ABCMeta and + extra not in bases): + bases = (extra,) + bases + bases = tuple(_gorg(b) if isinstance(b, GenericMeta) else b + for b in bases) + if any(isinstance(b, GenericMeta) and b is not typing.Generic for b in bases): + bases = tuple(b for b in bases if b is not typing.Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) + self = super(GenericMeta, cls).__new__(cls, name, bases, namespace, + _root=True) + super(GenericMeta, self).__setattr__('_gorg', + self if not origin else + _gorg(origin)) + self.__parameters__ = tvars + self.__args__ = tuple(... if a is typing._TypingEllipsis else + () if a is typing._TypingEmpty else + a for a in args) if args else None + self.__next_in_mro__ = _next_in_mro(self) + if orig_bases is None: + self.__orig_bases__ = initial_bases + elif origin is not None: + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache + if hasattr(self, '_subs_tree'): + self.__tree_hash__ = (hash(self._subs_tree()) if origin else + super(GenericMeta, self).__hash__()) + return self + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + if not cls.__dict__.get('_is_protocol', None): + cls._is_protocol = any(b is Protocol or + isinstance(b, _ProtocolMeta) and + b.__origin__ is Protocol + for b in cls.__bases__) + if cls._is_protocol: + for base in cls.__mro__[1:]: + if not (base in (object, typing.Generic) or + base.__module__ == 'collections.abc' and + base.__name__ in _PROTO_WHITELIST or + isinstance(base, typing.TypingMeta) and base._is_protocol or + isinstance(base, GenericMeta) and + base.__origin__ is typing.Generic): + raise TypeError(f'Protocols can only inherit from other' + f' protocols, got {repr(base)}') + + cls.__init__ = _no_init + + def _proto_hook(other): + if not cls.__dict__.get('_is_protocol', None): + return NotImplemented + if not isinstance(other, type): + # Same error as for issubclass(1, int) + raise TypeError('issubclass() arg 1 must be a class') + for attr in _get_protocol_attrs(cls): + for base in other.__mro__: + if attr in base.__dict__: + if base.__dict__[attr] is None: + return NotImplemented + break + annotations = getattr(base, '__annotations__', {}) + if (isinstance(annotations, typing.Mapping) and + attr in annotations and + isinstance(other, _ProtocolMeta) and + other._is_protocol): + break + else: + return NotImplemented + return True + if '__subclasshook__' not in cls.__dict__: + cls.__subclasshook__ = _proto_hook + + def __instancecheck__(self, instance): + # We need this method for situations where attributes are + # assigned in __init__. + if ((not getattr(self, '_is_protocol', False) or + _is_callable_members_only(self)) and + issubclass(instance.__class__, self)): + return True + if self._is_protocol: + if all(hasattr(instance, attr) and + (not callable(getattr(self, attr, None)) or + getattr(instance, attr) is not None) + for attr in _get_protocol_attrs(self)): + return True + return super(GenericMeta, self).__instancecheck__(instance) + + def __subclasscheck__(self, cls): + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if (self.__dict__.get('_is_protocol', None) and + not self.__dict__.get('_is_runtime_protocol', None)): + if sys._getframe(1).f_globals['__name__'] in ['abc', + 'functools', + 'typing']: + return False + raise TypeError("Instance and class checks can only be used with" + " @runtime protocols") + if (self.__dict__.get('_is_runtime_protocol', None) and + not _is_callable_members_only(self)): + if sys._getframe(1).f_globals['__name__'] in ['abc', + 'functools', + 'typing']: + return super(GenericMeta, self).__subclasscheck__(cls) + raise TypeError("Protocols with non-method members" + " don't support issubclass()") + return super(GenericMeta, self).__subclasscheck__(cls) + + @typing._tp_cache + def __getitem__(self, params): + # We also need to copy this from GenericMeta.__getitem__ to get + # special treatment of "Protocol". (Comments removed for brevity.) + if not isinstance(params, tuple): + params = (params,) + if not params and _gorg(self) is not typing.Tuple: + raise TypeError( + f"Parameter list to {self.__qualname__}[...] cannot be empty") + msg = "Parameters to generic types must be types." + params = tuple(_type_check(p, msg) for p in params) + if self in (typing.Generic, Protocol): + if not all(isinstance(p, typing.TypeVar) for p in params): + raise TypeError( + f"Parameters to {repr(self)}[...] must all be type variables") + if len(set(params)) != len(params): + raise TypeError( + f"Parameters to {repr(self)}[...] must all be unique") + tvars = params + args = params + elif self in (typing.Tuple, typing.Callable): + tvars = _type_vars(params) + args = params + elif self.__origin__ in (typing.Generic, Protocol): + raise TypeError(f"Cannot subscript already-subscripted {repr(self)}") + else: + _check_generic(self, params) + tvars = _type_vars(params) + args = params + + prepend = (self,) if self.__origin__ is None else () + return self.__class__(self.__name__, + prepend + self.__bases__, + _no_slots_copy(self.__dict__), + tvars=tvars, + args=args, + origin=self, + extra=self.__extra__, + orig_bases=self.__orig_bases__) + + class Protocol(metaclass=_ProtocolMeta): + """Base class for protocol classes. Protocol classes are defined as:: + + class Proto(Protocol): + def meth(self) -> int: + ... + + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: + + class C: + def meth(self) -> int: + return 0 + + def func(x: Proto) -> int: + return x.meth() + + func(C()) # Passes static type check + + See PEP 544 for details. Protocol classes decorated with + @typing_extensions.runtime act as simple-minded runtime protocol that checks + only the presence of given attributes, ignoring their type signatures. + + Protocol classes can be generic, they are defined as:: + + class GenProto(Protocol[T]): + def meth(self) -> T: + ... + """ + __slots__ = () + _is_protocol = True + + def __new__(cls, *args, **kwds): + if _gorg(cls) is Protocol: + raise TypeError("Type Protocol cannot be instantiated; " + "it can be used only as a base class") + return typing._generic_new(cls.__next_in_mro__, cls, *args, **kwds) + + +# 3.8+ +if hasattr(typing, 'runtime_checkable'): + runtime_checkable = typing.runtime_checkable +# 3.6-3.7 +else: + def runtime_checkable(cls): + """Mark a protocol class as a runtime protocol, so that it + can be used with isinstance() and issubclass(). Raise TypeError + if applied to a non-protocol class. + + This allows a simple-minded structural check very similar to the + one-offs in collections.abc such as Hashable. + """ + if not isinstance(cls, _ProtocolMeta) or not cls._is_protocol: + raise TypeError('@runtime_checkable can be only applied to protocol classes,' + f' got {cls!r}') + cls._is_runtime_protocol = True + return cls + + +# Exists for backwards compatibility. +runtime = runtime_checkable + + +# 3.8+ +if hasattr(typing, 'SupportsIndex'): + SupportsIndex = typing.SupportsIndex +# 3.6-3.7 +else: + @runtime_checkable + class SupportsIndex(Protocol): + __slots__ = () + + @abc.abstractmethod + def __index__(self) -> int: + pass + + +if sys.version_info >= (3, 9, 2): + # The standard library TypedDict in Python 3.8 does not store runtime information + # about which (if any) keys are optional. See https://bugs.python.org/issue38834 + # The standard library TypedDict in Python 3.9.0/1 does not honour the "total" + # keyword with old-style TypedDict(). See https://bugs.python.org/issue42059 + TypedDict = typing.TypedDict +else: + def _check_fails(cls, other): + try: + if sys._getframe(1).f_globals['__name__'] not in ['abc', + 'functools', + 'typing']: + # Typed dicts are only for static structural subtyping. + raise TypeError('TypedDict does not support instance and class checks') + except (AttributeError, ValueError): + pass + return False + + def _dict_new(*args, **kwargs): + if not args: + raise TypeError('TypedDict.__new__(): not enough arguments') + _, args = args[0], args[1:] # allow the "cls" keyword be passed + return dict(*args, **kwargs) + + _dict_new.__text_signature__ = '($cls, _typename, _fields=None, /, **kwargs)' + + def _typeddict_new(*args, total=True, **kwargs): + if not args: + raise TypeError('TypedDict.__new__(): not enough arguments') + _, args = args[0], args[1:] # allow the "cls" keyword be passed + if args: + typename, args = args[0], args[1:] # allow the "_typename" keyword be passed + elif '_typename' in kwargs: + typename = kwargs.pop('_typename') + import warnings + warnings.warn("Passing '_typename' as keyword argument is deprecated", + DeprecationWarning, stacklevel=2) + else: + raise TypeError("TypedDict.__new__() missing 1 required positional " + "argument: '_typename'") + if args: + try: + fields, = args # allow the "_fields" keyword be passed + except ValueError: + raise TypeError('TypedDict.__new__() takes from 2 to 3 ' + f'positional arguments but {len(args) + 2} ' + 'were given') + elif '_fields' in kwargs and len(kwargs) == 1: + fields = kwargs.pop('_fields') + import warnings + warnings.warn("Passing '_fields' as keyword argument is deprecated", + DeprecationWarning, stacklevel=2) + else: + fields = None + + if fields is None: + fields = kwargs + elif kwargs: + raise TypeError("TypedDict takes either a dict or keyword arguments," + " but not both") + + ns = {'__annotations__': dict(fields)} + try: + # Setting correct module is necessary to make typed dict classes pickleable. + ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + + return _TypedDictMeta(typename, (), ns, total=total) + + _typeddict_new.__text_signature__ = ('($cls, _typename, _fields=None,' + ' /, *, total=True, **kwargs)') + + class _TypedDictMeta(type): + def __init__(cls, name, bases, ns, total=True): + super().__init__(name, bases, ns) + + def __new__(cls, name, bases, ns, total=True): + # Create new typed dict class object. + # This method is called directly when TypedDict is subclassed, + # or via _typeddict_new when TypedDict is instantiated. This way + # TypedDict supports all three syntaxes described in its docstring. + # Subclasses and instances of TypedDict return actual dictionaries + # via _dict_new. + ns['__new__'] = _typeddict_new if name == 'TypedDict' else _dict_new + tp_dict = super().__new__(cls, name, (dict,), ns) + + annotations = {} + own_annotations = ns.get('__annotations__', {}) + own_annotation_keys = set(own_annotations.keys()) + msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" + own_annotations = { + n: typing._type_check(tp, msg) for n, tp in own_annotations.items() + } + required_keys = set() + optional_keys = set() + + for base in bases: + annotations.update(base.__dict__.get('__annotations__', {})) + required_keys.update(base.__dict__.get('__required_keys__', ())) + optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + annotations.update(own_annotations) + if total: + required_keys.update(own_annotation_keys) + else: + optional_keys.update(own_annotation_keys) + + tp_dict.__annotations__ = annotations + tp_dict.__required_keys__ = frozenset(required_keys) + tp_dict.__optional_keys__ = frozenset(optional_keys) + if not hasattr(tp_dict, '__total__'): + tp_dict.__total__ = total + return tp_dict + + __instancecheck__ = __subclasscheck__ = _check_fails + + TypedDict = _TypedDictMeta('TypedDict', (dict,), {}) + TypedDict.__module__ = __name__ + TypedDict.__doc__ = \ + """A simple typed name space. At runtime it is equivalent to a plain dict. + + TypedDict creates a dictionary type that expects all of its + instances to have a certain set of keys, with each key + associated with a value of a consistent type. This expectation + is not checked at runtime but is only enforced by type checkers. + Usage:: + + class Point2D(TypedDict): + x: int + y: int + label: str + + a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + + assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + + The type info can be accessed via the Point2D.__annotations__ dict, and + the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + TypedDict supports two additional equivalent forms:: + + Point2D = TypedDict('Point2D', x=int, y=int, label=str) + Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) + + The class syntax is only supported in Python 3.6+, while two other + syntax forms work for Python 2.7 and 3.2+ + """ + + +# Python 3.9+ has PEP 593 (Annotated and modified get_type_hints) +if hasattr(typing, 'Annotated'): + Annotated = typing.Annotated + get_type_hints = typing.get_type_hints + # Not exported and not a public API, but needed for get_origin() and get_args() + # to work. + _AnnotatedAlias = typing._AnnotatedAlias +# 3.7-3.8 +elif PEP_560: + class _AnnotatedAlias(typing._GenericAlias, _root=True): + """Runtime representation of an annotated type. + + At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' + with extra annotations. The alias behaves like a normal typing alias, + instantiating is the same as instantiating the underlying type, binding + it to types is also the same. + """ + def __init__(self, origin, metadata): + if isinstance(origin, _AnnotatedAlias): + metadata = origin.__metadata__ + metadata + origin = origin.__origin__ + super().__init__(origin, origin) + self.__metadata__ = metadata + + def copy_with(self, params): + assert len(params) == 1 + new_type = params[0] + return _AnnotatedAlias(new_type, self.__metadata__) + + def __repr__(self): + return (f"typing_extensions.Annotated[{typing._type_repr(self.__origin__)}, " + f"{', '.join(repr(a) for a in self.__metadata__)}]") + + def __reduce__(self): + return operator.getitem, ( + Annotated, (self.__origin__,) + self.__metadata__ + ) + + def __eq__(self, other): + if not isinstance(other, _AnnotatedAlias): + return NotImplemented + if self.__origin__ != other.__origin__: + return False + return self.__metadata__ == other.__metadata__ + + def __hash__(self): + return hash((self.__origin__, self.__metadata__)) + + class Annotated: + """Add context specific metadata to a type. + + Example: Annotated[int, runtime_check.Unsigned] indicates to the + hypothetical runtime_check module that this type is an unsigned int. + Every other consumer of this type can ignore this metadata and treat + this type as int. + + The first argument to Annotated must be a valid type (and will be in + the __origin__ field), the remaining arguments are kept as a tuple in + the __extra__ field. + + Details: + + - It's an error to call `Annotated` with less than two arguments. + - Nested Annotated are flattened:: + + Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] + + - Instantiating an annotated type is equivalent to instantiating the + underlying type:: + + Annotated[C, Ann1](5) == C(5) + + - Annotated can be used as a generic type alias:: + + Optimized = Annotated[T, runtime.Optimize()] + Optimized[int] == Annotated[int, runtime.Optimize()] + + OptimizedList = Annotated[List[T], runtime.Optimize()] + OptimizedList[int] == Annotated[List[int], runtime.Optimize()] + """ + + __slots__ = () + + def __new__(cls, *args, **kwargs): + raise TypeError("Type Annotated cannot be instantiated.") + + @typing._tp_cache + def __class_getitem__(cls, params): + if not isinstance(params, tuple) or len(params) < 2: + raise TypeError("Annotated[...] should be used " + "with at least two arguments (a type and an " + "annotation).") + msg = "Annotated[t, ...]: t must be a type." + origin = typing._type_check(params[0], msg) + metadata = tuple(params[1:]) + return _AnnotatedAlias(origin, metadata) + + def __init_subclass__(cls, *args, **kwargs): + raise TypeError( + f"Cannot subclass {cls.__module__}.Annotated" + ) + + def _strip_annotations(t): + """Strips the annotations from a given type. + """ + if isinstance(t, _AnnotatedAlias): + return _strip_annotations(t.__origin__) + if isinstance(t, typing._GenericAlias): + stripped_args = tuple(_strip_annotations(a) for a in t.__args__) + if stripped_args == t.__args__: + return t + res = t.copy_with(stripped_args) + res._special = t._special + return res + return t + + def get_type_hints(obj, globalns=None, localns=None, include_extras=False): + """Return type hints for an object. + + This is often the same as obj.__annotations__, but it handles + forward references encoded as string literals, adds Optional[t] if a + default value equal to None is set and recursively replaces all + 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). + + The argument may be a module, class, method, or function. The annotations + are returned as a dictionary. For classes, annotations include also + inherited members. + + TypeError is raised if the argument is not of a type that can contain + annotations, and an empty dictionary is returned if no annotations are + present. + + BEWARE -- the behavior of globalns and localns is counterintuitive + (unless you are familiar with how eval() and exec() work). The + search order is locals first, then globals. + + - If no dict arguments are passed, an attempt is made to use the + globals from obj (or the respective module's globals for classes), + and these are also used as the locals. If the object does not appear + to have globals, an empty dictionary is used. + + - If one dict argument is passed, it is used for both globals and + locals. + + - If two dict arguments are passed, they specify globals and + locals, respectively. + """ + hint = typing.get_type_hints(obj, globalns=globalns, localns=localns) + if include_extras: + return hint + return {k: _strip_annotations(t) for k, t in hint.items()} +# 3.6 +else: + + def _is_dunder(name): + """Returns True if name is a __dunder_variable_name__.""" + return len(name) > 4 and name.startswith('__') and name.endswith('__') + + # Prior to Python 3.7 types did not have `copy_with`. A lot of the equality + # checks, argument expansion etc. are done on the _subs_tre. As a result we + # can't provide a get_type_hints function that strips out annotations. + + class AnnotatedMeta(typing.GenericMeta): + """Metaclass for Annotated""" + + def __new__(cls, name, bases, namespace, **kwargs): + if any(b is not object for b in bases): + raise TypeError("Cannot subclass " + str(Annotated)) + return super().__new__(cls, name, bases, namespace, **kwargs) + + @property + def __metadata__(self): + return self._subs_tree()[2] + + def _tree_repr(self, tree): + cls, origin, metadata = tree + if not isinstance(origin, tuple): + tp_repr = typing._type_repr(origin) + else: + tp_repr = origin[0]._tree_repr(origin) + metadata_reprs = ", ".join(repr(arg) for arg in metadata) + return f'{cls}[{tp_repr}, {metadata_reprs}]' + + def _subs_tree(self, tvars=None, args=None): # noqa + if self is Annotated: + return Annotated + res = super()._subs_tree(tvars=tvars, args=args) + # Flatten nested Annotated + if isinstance(res[1], tuple) and res[1][0] is Annotated: + sub_tp = res[1][1] + sub_annot = res[1][2] + return (Annotated, sub_tp, sub_annot + res[2]) + return res + + def _get_cons(self): + """Return the class used to create instance of this type.""" + if self.__origin__ is None: + raise TypeError("Cannot get the underlying type of a " + "non-specialized Annotated type.") + tree = self._subs_tree() + while isinstance(tree, tuple) and tree[0] is Annotated: + tree = tree[1] + if isinstance(tree, tuple): + return tree[0] + else: + return tree + + @typing._tp_cache + def __getitem__(self, params): + if not isinstance(params, tuple): + params = (params,) + if self.__origin__ is not None: # specializing an instantiated type + return super().__getitem__(params) + elif not isinstance(params, tuple) or len(params) < 2: + raise TypeError("Annotated[...] should be instantiated " + "with at least two arguments (a type and an " + "annotation).") + else: + msg = "Annotated[t, ...]: t must be a type." + tp = typing._type_check(params[0], msg) + metadata = tuple(params[1:]) + return self.__class__( + self.__name__, + self.__bases__, + _no_slots_copy(self.__dict__), + tvars=_type_vars((tp,)), + # Metadata is a tuple so it won't be touched by _replace_args et al. + args=(tp, metadata), + origin=self, + ) + + def __call__(self, *args, **kwargs): + cons = self._get_cons() + result = cons(*args, **kwargs) + try: + result.__orig_class__ = self + except AttributeError: + pass + return result + + def __getattr__(self, attr): + # For simplicity we just don't relay all dunder names + if self.__origin__ is not None and not _is_dunder(attr): + return getattr(self._get_cons(), attr) + raise AttributeError(attr) + + def __setattr__(self, attr, value): + if _is_dunder(attr) or attr.startswith('_abc_'): + super().__setattr__(attr, value) + elif self.__origin__ is None: + raise AttributeError(attr) + else: + setattr(self._get_cons(), attr, value) + + def __instancecheck__(self, obj): + raise TypeError("Annotated cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("Annotated cannot be used with issubclass().") + + class Annotated(metaclass=AnnotatedMeta): + """Add context specific metadata to a type. + + Example: Annotated[int, runtime_check.Unsigned] indicates to the + hypothetical runtime_check module that this type is an unsigned int. + Every other consumer of this type can ignore this metadata and treat + this type as int. + + The first argument to Annotated must be a valid type, the remaining + arguments are kept as a tuple in the __metadata__ field. + + Details: + + - It's an error to call `Annotated` with less than two arguments. + - Nested Annotated are flattened:: + + Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] + + - Instantiating an annotated type is equivalent to instantiating the + underlying type:: + + Annotated[C, Ann1](5) == C(5) + + - Annotated can be used as a generic type alias:: + + Optimized = Annotated[T, runtime.Optimize()] + Optimized[int] == Annotated[int, runtime.Optimize()] + + OptimizedList = Annotated[List[T], runtime.Optimize()] + OptimizedList[int] == Annotated[List[int], runtime.Optimize()] + """ + +# Python 3.8 has get_origin() and get_args() but those implementations aren't +# Annotated-aware, so we can't use those. Python 3.9's versions don't support +# ParamSpecArgs and ParamSpecKwargs, so only Python 3.10's versions will do. +if sys.version_info[:2] >= (3, 10): + get_origin = typing.get_origin + get_args = typing.get_args +# 3.7-3.9 +elif PEP_560: + try: + # 3.9+ + from typing import _BaseGenericAlias + except ImportError: + _BaseGenericAlias = typing._GenericAlias + try: + # 3.9+ + from typing import GenericAlias + except ImportError: + GenericAlias = typing._GenericAlias + + def get_origin(tp): + """Get the unsubscripted version of a type. + + This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar + and Annotated. Return None for unsupported types. Examples:: + + get_origin(Literal[42]) is Literal + get_origin(int) is None + get_origin(ClassVar[int]) is ClassVar + get_origin(Generic) is Generic + get_origin(Generic[T]) is Generic + get_origin(Union[T, int]) is Union + get_origin(List[Tuple[T, T]][int]) == list + get_origin(P.args) is P + """ + if isinstance(tp, _AnnotatedAlias): + return Annotated + if isinstance(tp, (typing._GenericAlias, GenericAlias, _BaseGenericAlias, + ParamSpecArgs, ParamSpecKwargs)): + return tp.__origin__ + if tp is typing.Generic: + return typing.Generic + return None + + def get_args(tp): + """Get type arguments with all substitutions performed. + + For unions, basic simplifications used by Union constructor are performed. + Examples:: + get_args(Dict[str, int]) == (str, int) + get_args(int) == () + get_args(Union[int, Union[T, int], str][int]) == (int, str) + get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) + get_args(Callable[[], T][int]) == ([], int) + """ + if isinstance(tp, _AnnotatedAlias): + return (tp.__origin__,) + tp.__metadata__ + if isinstance(tp, (typing._GenericAlias, GenericAlias)): + if getattr(tp, "_special", False): + return () + res = tp.__args__ + if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis: + res = (list(res[:-1]), res[-1]) + return res + return () + + +# 3.10+ +if hasattr(typing, 'TypeAlias'): + TypeAlias = typing.TypeAlias +# 3.9 +elif sys.version_info[:2] >= (3, 9): + class _TypeAliasForm(typing._SpecialForm, _root=True): + def __repr__(self): + return 'typing_extensions.' + self._name + + @_TypeAliasForm + def TypeAlias(self, parameters): + """Special marker indicating that an assignment should + be recognized as a proper type alias definition by type + checkers. + + For example:: + + Predicate: TypeAlias = Callable[..., bool] + + It's invalid when used anywhere except as in the example above. + """ + raise TypeError(f"{self} is not subscriptable") +# 3.7-3.8 +elif sys.version_info[:2] >= (3, 7): + class _TypeAliasForm(typing._SpecialForm, _root=True): + def __repr__(self): + return 'typing_extensions.' + self._name + + TypeAlias = _TypeAliasForm('TypeAlias', + doc="""Special marker indicating that an assignment should + be recognized as a proper type alias definition by type + checkers. + + For example:: + + Predicate: TypeAlias = Callable[..., bool] + + It's invalid when used anywhere except as in the example + above.""") +# 3.6 +else: + class _TypeAliasMeta(typing.TypingMeta): + """Metaclass for TypeAlias""" + + def __repr__(self): + return 'typing_extensions.TypeAlias' + + class _TypeAliasBase(typing._FinalTypingBase, metaclass=_TypeAliasMeta, _root=True): + """Special marker indicating that an assignment should + be recognized as a proper type alias definition by type + checkers. + + For example:: + + Predicate: TypeAlias = Callable[..., bool] + + It's invalid when used anywhere except as in the example above. + """ + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("TypeAlias cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("TypeAlias cannot be used with issubclass().") + + def __repr__(self): + return 'typing_extensions.TypeAlias' + + TypeAlias = _TypeAliasBase(_root=True) + + +# Python 3.10+ has PEP 612 +if hasattr(typing, 'ParamSpecArgs'): + ParamSpecArgs = typing.ParamSpecArgs + ParamSpecKwargs = typing.ParamSpecKwargs +# 3.6-3.9 +else: + class _Immutable: + """Mixin to indicate that object should not be copied.""" + __slots__ = () + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + class ParamSpecArgs(_Immutable): + """The args for a ParamSpec object. + + Given a ParamSpec object P, P.args is an instance of ParamSpecArgs. + + ParamSpecArgs objects have a reference back to their ParamSpec: + + P.args.__origin__ is P + + This type is meant for runtime introspection and has no special meaning to + static type checkers. + """ + def __init__(self, origin): + self.__origin__ = origin + + def __repr__(self): + return f"{self.__origin__.__name__}.args" + + class ParamSpecKwargs(_Immutable): + """The kwargs for a ParamSpec object. + + Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs. + + ParamSpecKwargs objects have a reference back to their ParamSpec: + + P.kwargs.__origin__ is P + + This type is meant for runtime introspection and has no special meaning to + static type checkers. + """ + def __init__(self, origin): + self.__origin__ = origin + + def __repr__(self): + return f"{self.__origin__.__name__}.kwargs" + +# 3.10+ +if hasattr(typing, 'ParamSpec'): + ParamSpec = typing.ParamSpec +# 3.6-3.9 +else: + + # Inherits from list as a workaround for Callable checks in Python < 3.9.2. + class ParamSpec(list): + """Parameter specification variable. + + Usage:: + + P = ParamSpec('P') + + Parameter specification variables exist primarily for the benefit of static + type checkers. They are used to forward the parameter types of one + callable to another callable, a pattern commonly found in higher order + functions and decorators. They are only valid when used in ``Concatenate``, + or s the first argument to ``Callable``. In Python 3.10 and higher, + they are also supported in user-defined Generics at runtime. + See class Generic for more information on generic types. An + example for annotating a decorator:: + + T = TypeVar('T') + P = ParamSpec('P') + + def add_logging(f: Callable[P, T]) -> Callable[P, T]: + '''A type-safe decorator to add logging to a function.''' + def inner(*args: P.args, **kwargs: P.kwargs) -> T: + logging.info(f'{f.__name__} was called') + return f(*args, **kwargs) + return inner + + @add_logging + def add_two(x: float, y: float) -> float: + '''Add two numbers together.''' + return x + y + + Parameter specification variables defined with covariant=True or + contravariant=True can be used to declare covariant or contravariant + generic types. These keyword arguments are valid, but their actual semantics + are yet to be decided. See PEP 612 for details. + + Parameter specification variables can be introspected. e.g.: + + P.__name__ == 'T' + P.__bound__ == None + P.__covariant__ == False + P.__contravariant__ == False + + Note that only parameter specification variables defined in global scope can + be pickled. + """ + + # Trick Generic __parameters__. + __class__ = typing.TypeVar + + @property + def args(self): + return ParamSpecArgs(self) + + @property + def kwargs(self): + return ParamSpecKwargs(self) + + def __init__(self, name, *, bound=None, covariant=False, contravariant=False): + super().__init__([self]) + self.__name__ = name + self.__covariant__ = bool(covariant) + self.__contravariant__ = bool(contravariant) + if bound: + self.__bound__ = typing._type_check(bound, 'Bound must be a type.') + else: + self.__bound__ = None + + # for pickling: + try: + def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + def_mod = None + if def_mod != 'typing_extensions': + self.__module__ = def_mod + + def __repr__(self): + if self.__covariant__: + prefix = '+' + elif self.__contravariant__: + prefix = '-' + else: + prefix = '~' + return prefix + self.__name__ + + def __hash__(self): + return object.__hash__(self) + + def __eq__(self, other): + return self is other + + def __reduce__(self): + return self.__name__ + + # Hack to get typing._type_check to pass. + def __call__(self, *args, **kwargs): + pass + + if not PEP_560: + # Only needed in 3.6. + def _get_type_vars(self, tvars): + if self not in tvars: + tvars.append(self) + + +# 3.6-3.9 +if not hasattr(typing, 'Concatenate'): + # Inherits from list as a workaround for Callable checks in Python < 3.9.2. + class _ConcatenateGenericAlias(list): + + # Trick Generic into looking into this for __parameters__. + if PEP_560: + __class__ = typing._GenericAlias + else: + __class__ = typing._TypingBase + + # Flag in 3.8. + _special = False + # Attribute in 3.6 and earlier. + _gorg = typing.Generic + + def __init__(self, origin, args): + super().__init__(args) + self.__origin__ = origin + self.__args__ = args + + def __repr__(self): + _type_repr = typing._type_repr + return (f'{_type_repr(self.__origin__)}' + f'[{", ".join(_type_repr(arg) for arg in self.__args__)}]') + + def __hash__(self): + return hash((self.__origin__, self.__args__)) + + # Hack to get typing._type_check to pass in Generic. + def __call__(self, *args, **kwargs): + pass + + @property + def __parameters__(self): + return tuple( + tp for tp in self.__args__ if isinstance(tp, (typing.TypeVar, ParamSpec)) + ) + + if not PEP_560: + # Only required in 3.6. + def _get_type_vars(self, tvars): + if self.__origin__ and self.__parameters__: + typing._get_type_vars(self.__parameters__, tvars) + + +# 3.6-3.9 +@typing._tp_cache +def _concatenate_getitem(self, parameters): + if parameters == (): + raise TypeError("Cannot take a Concatenate of no types.") + if not isinstance(parameters, tuple): + parameters = (parameters,) + if not isinstance(parameters[-1], ParamSpec): + raise TypeError("The last parameter to Concatenate should be a " + "ParamSpec variable.") + msg = "Concatenate[arg, ...]: each arg must be a type." + parameters = tuple(typing._type_check(p, msg) for p in parameters) + return _ConcatenateGenericAlias(self, parameters) + + +# 3.10+ +if hasattr(typing, 'Concatenate'): + Concatenate = typing.Concatenate + _ConcatenateGenericAlias = typing._ConcatenateGenericAlias # noqa +# 3.9 +elif sys.version_info[:2] >= (3, 9): + @_TypeAliasForm + def Concatenate(self, parameters): + """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a + higher order function which adds, removes or transforms parameters of a + callable. + + For example:: + + Callable[Concatenate[int, P], int] + + See PEP 612 for detailed information. + """ + return _concatenate_getitem(self, parameters) +# 3.7-8 +elif sys.version_info[:2] >= (3, 7): + class _ConcatenateForm(typing._SpecialForm, _root=True): + def __repr__(self): + return 'typing_extensions.' + self._name + + def __getitem__(self, parameters): + return _concatenate_getitem(self, parameters) + + Concatenate = _ConcatenateForm( + 'Concatenate', + doc="""Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a + higher order function which adds, removes or transforms parameters of a + callable. + + For example:: + + Callable[Concatenate[int, P], int] + + See PEP 612 for detailed information. + """) +# 3.6 +else: + class _ConcatenateAliasMeta(typing.TypingMeta): + """Metaclass for Concatenate.""" + + def __repr__(self): + return 'typing_extensions.Concatenate' + + class _ConcatenateAliasBase(typing._FinalTypingBase, + metaclass=_ConcatenateAliasMeta, + _root=True): + """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a + higher order function which adds, removes or transforms parameters of a + callable. + + For example:: + + Callable[Concatenate[int, P], int] + + See PEP 612 for detailed information. + """ + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError("Concatenate cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError("Concatenate cannot be used with issubclass().") + + def __repr__(self): + return 'typing_extensions.Concatenate' + + def __getitem__(self, parameters): + return _concatenate_getitem(self, parameters) + + Concatenate = _ConcatenateAliasBase(_root=True) + +# 3.10+ +if hasattr(typing, 'TypeGuard'): + TypeGuard = typing.TypeGuard +# 3.9 +elif sys.version_info[:2] >= (3, 9): + class _TypeGuardForm(typing._SpecialForm, _root=True): + def __repr__(self): + return 'typing_extensions.' + self._name + + @_TypeGuardForm + def TypeGuard(self, parameters): + """Special typing form used to annotate the return type of a user-defined + type guard function. ``TypeGuard`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean. + + ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type guard". + + Sometimes it would be convenient to use a user-defined boolean function + as a type guard. Such a function should use ``TypeGuard[...]`` as its + return type to alert static type checkers to this intention. + + Using ``-> TypeGuard`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the type inside ``TypeGuard``. + + For example:: + + def is_str(val: Union[str, float]): + # "isinstance" type guard + if isinstance(val, str): + # Type of ``val`` is narrowed to ``str`` + ... + else: + # Else, type of ``val`` is narrowed to ``float``. + ... + + Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower + form of ``TypeA`` (it can even be a wider form) and this may lead to + type-unsafe results. The main reason is to allow for things like + narrowing ``List[object]`` to ``List[str]`` even though the latter is not + a subtype of the former, since ``List`` is invariant. The responsibility of + writing type-safe type guards is left to the user. + + ``TypeGuard`` also works with type variables. For more information, see + PEP 647 (User-Defined Type Guards). + """ + item = typing._type_check(parameters, f'{self} accepts only single type.') + return typing._GenericAlias(self, (item,)) +# 3.7-3.8 +elif sys.version_info[:2] >= (3, 7): + class _TypeGuardForm(typing._SpecialForm, _root=True): + + def __repr__(self): + return 'typing_extensions.' + self._name + + def __getitem__(self, parameters): + item = typing._type_check(parameters, + f'{self._name} accepts only a single type') + return typing._GenericAlias(self, (item,)) + + TypeGuard = _TypeGuardForm( + 'TypeGuard', + doc="""Special typing form used to annotate the return type of a user-defined + type guard function. ``TypeGuard`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean. + + ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type guard". + + Sometimes it would be convenient to use a user-defined boolean function + as a type guard. Such a function should use ``TypeGuard[...]`` as its + return type to alert static type checkers to this intention. + + Using ``-> TypeGuard`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the type inside ``TypeGuard``. + + For example:: + + def is_str(val: Union[str, float]): + # "isinstance" type guard + if isinstance(val, str): + # Type of ``val`` is narrowed to ``str`` + ... + else: + # Else, type of ``val`` is narrowed to ``float``. + ... + + Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower + form of ``TypeA`` (it can even be a wider form) and this may lead to + type-unsafe results. The main reason is to allow for things like + narrowing ``List[object]`` to ``List[str]`` even though the latter is not + a subtype of the former, since ``List`` is invariant. The responsibility of + writing type-safe type guards is left to the user. + + ``TypeGuard`` also works with type variables. For more information, see + PEP 647 (User-Defined Type Guards). + """) +# 3.6 +else: + class _TypeGuard(typing._FinalTypingBase, _root=True): + """Special typing form used to annotate the return type of a user-defined + type guard function. ``TypeGuard`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean. + + ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type guard". + + Sometimes it would be convenient to use a user-defined boolean function + as a type guard. Such a function should use ``TypeGuard[...]`` as its + return type to alert static type checkers to this intention. + + Using ``-> TypeGuard`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the type inside ``TypeGuard``. + + For example:: + + def is_str(val: Union[str, float]): + # "isinstance" type guard + if isinstance(val, str): + # Type of ``val`` is narrowed to ``str`` + ... + else: + # Else, type of ``val`` is narrowed to ``float``. + ... + + Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower + form of ``TypeA`` (it can even be a wider form) and this may lead to + type-unsafe results. The main reason is to allow for things like + narrowing ``List[object]`` to ``List[str]`` even though the latter is not + a subtype of the former, since ``List`` is invariant. The responsibility of + writing type-safe type guards is left to the user. + + ``TypeGuard`` also works with type variables. For more information, see + PEP 647 (User-Defined Type Guards). + """ + + __slots__ = ('__type__',) + + def __init__(self, tp=None, **kwds): + self.__type__ = tp + + def __getitem__(self, item): + cls = type(self) + if self.__type__ is None: + return cls(typing._type_check(item, + f'{cls.__name__[1:]} accepts only a single type.'), + _root=True) + raise TypeError(f'{cls.__name__[1:]} cannot be further subscripted') + + def _eval_type(self, globalns, localns): + new_tp = typing._eval_type(self.__type__, globalns, localns) + if new_tp == self.__type__: + return self + return type(self)(new_tp, _root=True) + + def __repr__(self): + r = super().__repr__() + if self.__type__ is not None: + r += f'[{typing._type_repr(self.__type__)}]' + return r + + def __hash__(self): + return hash((type(self).__name__, self.__type__)) + + def __eq__(self, other): + if not isinstance(other, _TypeGuard): + return NotImplemented + if self.__type__ is not None: + return self.__type__ == other.__type__ + return self is other + + TypeGuard = _TypeGuard(_root=True) + +if hasattr(typing, "Self"): + Self = typing.Self +elif sys.version_info[:2] >= (3, 7): + # Vendored from cpython typing._SpecialFrom + class _SpecialForm(typing._Final, _root=True): + __slots__ = ('_name', '__doc__', '_getitem') + + def __init__(self, getitem): + self._getitem = getitem + self._name = getitem.__name__ + self.__doc__ = getitem.__doc__ + + def __getattr__(self, item): + if item in {'__name__', '__qualname__'}: + return self._name + + raise AttributeError(item) + + def __mro_entries__(self, bases): + raise TypeError(f"Cannot subclass {self!r}") + + def __repr__(self): + return f'typing_extensions.{self._name}' + + def __reduce__(self): + return self._name + + def __call__(self, *args, **kwds): + raise TypeError(f"Cannot instantiate {self!r}") + + def __or__(self, other): + return typing.Union[self, other] + + def __ror__(self, other): + return typing.Union[other, self] + + def __instancecheck__(self, obj): + raise TypeError(f"{self} cannot be used with isinstance()") + + def __subclasscheck__(self, cls): + raise TypeError(f"{self} cannot be used with issubclass()") + + @typing._tp_cache + def __getitem__(self, parameters): + return self._getitem(self, parameters) + + @_SpecialForm + def Self(self, params): + """Used to spell the type of "self" in classes. + + Example:: + + from typing import Self + + class ReturnsSelf: + def parse(self, data: bytes) -> Self: + ... + return self + + """ + + raise TypeError(f"{self} is not subscriptable") +else: + class _Self(typing._FinalTypingBase, _root=True): + """Used to spell the type of "self" in classes. + + Example:: + + from typing import Self + + class ReturnsSelf: + def parse(self, data: bytes) -> Self: + ... + return self + + """ + + __slots__ = () + + def __instancecheck__(self, obj): + raise TypeError(f"{self} cannot be used with isinstance().") + + def __subclasscheck__(self, cls): + raise TypeError(f"{self} cannot be used with issubclass().") + + Self = _Self(_root=True) + + +if hasattr(typing, 'Required'): + Required = typing.Required + NotRequired = typing.NotRequired +elif sys.version_info[:2] >= (3, 9): + class _ExtensionsSpecialForm(typing._SpecialForm, _root=True): + def __repr__(self): + return 'typing_extensions.' + self._name + + @_ExtensionsSpecialForm + def Required(self, parameters): + """A special typing construct to mark a key of a total=False TypedDict + as required. For example: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """ + item = typing._type_check(parameters, f'{self._name} accepts only single type') + return typing._GenericAlias(self, (item,)) + + @_ExtensionsSpecialForm + def NotRequired(self, parameters): + """A special typing construct to mark a key of a TypedDict as + potentially missing. For example: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """ + item = typing._type_check(parameters, f'{self._name} accepts only single type') + return typing._GenericAlias(self, (item,)) + +elif sys.version_info[:2] >= (3, 7): + class _RequiredForm(typing._SpecialForm, _root=True): + def __repr__(self): + return 'typing_extensions.' + self._name + + def __getitem__(self, parameters): + item = typing._type_check(parameters, + '{} accepts only single type'.format(self._name)) + return typing._GenericAlias(self, (item,)) + + Required = _RequiredForm( + 'Required', + doc="""A special typing construct to mark a key of a total=False TypedDict + as required. For example: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """) + NotRequired = _RequiredForm( + 'NotRequired', + doc="""A special typing construct to mark a key of a TypedDict as + potentially missing. For example: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """) +else: + # NOTE: Modeled after _Final's implementation when _FinalTypingBase available + class _MaybeRequired(typing._FinalTypingBase, _root=True): + __slots__ = ('__type__',) + + def __init__(self, tp=None, **kwds): + self.__type__ = tp + + def __getitem__(self, item): + cls = type(self) + if self.__type__ is None: + return cls(typing._type_check(item, + '{} accepts only single type.'.format(cls.__name__[1:])), + _root=True) + raise TypeError('{} cannot be further subscripted' + .format(cls.__name__[1:])) + + def _eval_type(self, globalns, localns): + new_tp = typing._eval_type(self.__type__, globalns, localns) + if new_tp == self.__type__: + return self + return type(self)(new_tp, _root=True) + + def __repr__(self): + r = super().__repr__() + if self.__type__ is not None: + r += '[{}]'.format(typing._type_repr(self.__type__)) + return r + + def __hash__(self): + return hash((type(self).__name__, self.__type__)) + + def __eq__(self, other): + if not isinstance(other, type(self)): + return NotImplemented + if self.__type__ is not None: + return self.__type__ == other.__type__ + return self is other + + class _Required(_MaybeRequired, _root=True): + """A special typing construct to mark a key of a total=False TypedDict + as required. For example: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """ + + class _NotRequired(_MaybeRequired, _root=True): + """A special typing construct to mark a key of a TypedDict as + potentially missing. For example: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """ + + Required = _Required(_root=True) + NotRequired = _NotRequired(_root=True) diff --git a/src/pip/_vendor/typing_extensions.pyi b/src/pip/_vendor/typing_extensions.pyi new file mode 100644 index 00000000000..547f7a1b019 --- /dev/null +++ b/src/pip/_vendor/typing_extensions.pyi @@ -0,0 +1 @@ +from typing_extensions import * \ No newline at end of file diff --git a/src/pip/_vendor/urllib3/_version.py b/src/pip/_vendor/urllib3/_version.py index 2dba29e3fbe..fa8979d73e3 100644 --- a/src/pip/_vendor/urllib3/_version.py +++ b/src/pip/_vendor/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.2" +__version__ = "1.26.8" diff --git a/src/pip/_vendor/urllib3/connection.py b/src/pip/_vendor/urllib3/connection.py index 660d679c361..4d92ac6d2c3 100644 --- a/src/pip/_vendor/urllib3/connection.py +++ b/src/pip/_vendor/urllib3/connection.py @@ -51,15 +51,16 @@ class BrokenPipeError(Exception): SubjectAltNameWarning, SystemTimeWarning, ) -from .packages.ssl_match_hostname import CertificateError, match_hostname from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection from .util.ssl_ import ( assert_fingerprint, create_urllib3_context, + is_ipaddress, resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, ) +from .util.ssl_match_hostname import CertificateError, match_hostname log = logging.getLogger(__name__) @@ -67,7 +68,7 @@ class BrokenPipeError(Exception): # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2019, 1, 1) +RECENT_DATE = datetime.date(2020, 7, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") @@ -107,6 +108,10 @@ class HTTPConnection(_HTTPConnection, object): #: Whether this connection verifies the host's certificate. is_verified = False + #: Whether this proxy connection (if used) verifies the proxy host's + #: certificate. + proxy_is_verified = None + def __init__(self, *args, **kw): if not six.PY2: kw.pop("strict", None) @@ -201,7 +206,7 @@ def connect(self): self._prepare_conn(conn) def putrequest(self, method, url, *args, **kwargs): - """""" + """ """ # Empty docstring because the indentation of CPython's implementation # is broken but we don't want this method in our documentation. match = _CONTAINS_CONTROL_CHAR_RE.search(method) @@ -214,8 +219,8 @@ def putrequest(self, method, url, *args, **kwargs): return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) def putheader(self, header, *values): - """""" - if SKIP_HEADER not in values: + """ """ + if not any(isinstance(v, str) and v == SKIP_HEADER for v in values): _HTTPConnection.putheader(self, header, *values) elif six.ensure_str(header.lower()) not in SKIPPABLE_HEADERS: raise ValueError( @@ -249,7 +254,7 @@ def request_chunked(self, method, url, body=None, headers=None): self.putheader("User-Agent", _get_default_user_agent()) for header, value in headers.items(): self.putheader(header, value) - if "transfer-encoding" not in headers: + if "transfer-encoding" not in header_keys: self.putheader("Transfer-Encoding", "chunked") self.endheaders() @@ -493,7 +498,7 @@ def _connect_tls_proxy(self, hostname, conn): # If no cert was provided, use only the default options for server # certificate validation - return ssl_wrap_socket( + socket = ssl_wrap_socket( sock=conn, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, @@ -502,8 +507,37 @@ def _connect_tls_proxy(self, hostname, conn): ssl_context=ssl_context, ) + if ssl_context.verify_mode != ssl.CERT_NONE and not getattr( + ssl_context, "check_hostname", False + ): + # While urllib3 attempts to always turn off hostname matching from + # the TLS library, this cannot always be done. So we check whether + # the TLS Library still thinks it's matching hostnames. + cert = socket.getpeercert() + if not cert.get("subjectAltName", ()): + warnings.warn( + ( + "Certificate for {0} has no `subjectAltName`, falling back to check for a " + "`commonName` for now. This feature is being removed by major browsers and " + "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 " + "for details.)".format(hostname) + ), + SubjectAltNameWarning, + ) + _match_hostname(cert, hostname) + + self.proxy_is_verified = ssl_context.verify_mode == ssl.CERT_REQUIRED + return socket + def _match_hostname(cert, asserted_hostname): + # Our upstream implementation of ssl.match_hostname() + # only applies this normalization to IP addresses so it doesn't + # match DNS SANs so we do the same thing! + stripped_hostname = asserted_hostname.strip("u[]") + if is_ipaddress(stripped_hostname): + asserted_hostname = stripped_hostname + try: match_hostname(cert, asserted_hostname) except CertificateError as e: diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py index 4708c5bfc78..15bffcb23a9 100644 --- a/src/pip/_vendor/urllib3/connectionpool.py +++ b/src/pip/_vendor/urllib3/connectionpool.py @@ -2,6 +2,7 @@ import errno import logging +import re import socket import sys import warnings @@ -35,7 +36,6 @@ ) from .packages import six from .packages.six.moves import queue -from .packages.ssl_match_hostname import CertificateError from .request import RequestMethods from .response import HTTPResponse from .util.connection import is_connection_dropped @@ -44,6 +44,7 @@ from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry +from .util.ssl_match_hostname import CertificateError from .util.timeout import Timeout from .util.url import Url, _encode_target from .util.url import _normalize_host as normalize_host @@ -301,8 +302,11 @@ def _put_conn(self, conn): pass except queue.Full: # This should never happen if self.block == True - log.warning("Connection pool is full, discarding connection: %s", self.host) - + log.warning( + "Connection pool is full, discarding connection: %s. Connection pool size: %s", + self.host, + self.pool.qsize(), + ) # Connection never got put back into the pool, close it. if conn: conn.close() @@ -318,7 +322,7 @@ def _prepare_proxy(self, conn): pass def _get_timeout(self, timeout): - """ Helper that always returns a :class:`urllib3.util.Timeout` """ + """Helper that always returns a :class:`urllib3.util.Timeout`""" if timeout is _Default: return self.timeout.clone() @@ -745,7 +749,33 @@ def urlopen( # Discard the connection for these exceptions. It will be # replaced during the next _get_conn() call. clean_exit = False - if isinstance(e, (BaseSSLError, CertificateError)): + + def _is_ssl_error_message_from_http_proxy(ssl_error): + # We're trying to detect the message 'WRONG_VERSION_NUMBER' but + # SSLErrors are kinda all over the place when it comes to the message, + # so we try to cover our bases here! + message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) + return ( + "wrong version number" in message or "unknown protocol" in message + ) + + # Try to detect a common user error with proxies which is to + # set an HTTP proxy to be HTTPS when it should be 'http://' + # (ie {'http': 'http://proxy', 'https': 'https://proxy'}) + # Instead we add a nice error message and point to a URL. + if ( + isinstance(e, BaseSSLError) + and self.proxy + and _is_ssl_error_message_from_http_proxy(e) + ): + e = ProxyError( + "Your proxy appears to only use HTTP and not HTTPS, " + "try changing your proxy URL to be HTTP. See: " + "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" + "#https-proxy-error-http-proxy", + SSLError(e), + ) + elif isinstance(e, (BaseSSLError, CertificateError)): e = SSLError(e) elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: e = ProxyError("Cannot connect to proxy.", e) @@ -1014,12 +1044,23 @@ def _validate_conn(self, conn): ( "Unverified HTTPS request is being made to host '%s'. " "Adding certificate verification is strongly advised. See: " - "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" "#ssl-warnings" % conn.host ), InsecureRequestWarning, ) + if getattr(conn, "proxy_is_verified", None) is False: + warnings.warn( + ( + "Unverified HTTPS connection done to an HTTPS proxy. " + "Adding certificate verification is strongly advised. See: " + "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" + "#ssl-warnings" + ), + InsecureRequestWarning, + ) + def connection_from_url(url, **kw): """ diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py b/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py index 42526be7f55..264d564dbda 100644 --- a/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py +++ b/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py @@ -48,7 +48,7 @@ ) from ctypes.util import find_library -from pip._vendor.urllib3.packages.six import raise_from +from ...packages.six import raise_from if platform.system() != "Darwin": raise ImportError("Only macOS is supported") diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py b/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py index ed8120190c0..fa0b245d279 100644 --- a/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py +++ b/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py @@ -188,6 +188,7 @@ def _cert_array_from_pem(pem_bundle): # We only want to do that if an error occurs: otherwise, the caller # should free. CoreFoundation.CFRelease(cert_array) + raise return cert_array diff --git a/src/pip/_vendor/urllib3/contrib/appengine.py b/src/pip/_vendor/urllib3/contrib/appengine.py index b9d2a6907c4..668538695f9 100644 --- a/src/pip/_vendor/urllib3/contrib/appengine.py +++ b/src/pip/_vendor/urllib3/contrib/appengine.py @@ -111,7 +111,7 @@ def __init__( warnings.warn( "urllib3 is using URLFetch on Google App Engine sandbox instead " "of sockets. To use sockets directly instead of URLFetch see " - "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", + "https://urllib3.readthedocs.io/en/1.26.x/reference/urllib3.contrib.html.", AppEnginePlatformWarning, ) diff --git a/src/pip/_vendor/urllib3/contrib/ntlmpool.py b/src/pip/_vendor/urllib3/contrib/ntlmpool.py index b2df45dcf60..41a8fd174cb 100644 --- a/src/pip/_vendor/urllib3/contrib/ntlmpool.py +++ b/src/pip/_vendor/urllib3/contrib/ntlmpool.py @@ -5,6 +5,7 @@ """ from __future__ import absolute_import +import warnings from logging import getLogger from ntlm import ntlm @@ -12,6 +13,14 @@ from .. import HTTPSConnectionPool from ..packages.six.moves.http_client import HTTPSConnection +warnings.warn( + "The 'urllib3.contrib.ntlmpool' module is deprecated and will be removed " + "in urllib3 v2.0 release, urllib3 is not able to support it properly due " + "to reasons listed in issue: https://github.com/urllib3/urllib3/issues/2282. " + "If you are a user of this module please comment in the mentioned issue.", + DeprecationWarning, +) + log = getLogger(__name__) diff --git a/src/pip/_vendor/urllib3/contrib/pyopenssl.py b/src/pip/_vendor/urllib3/contrib/pyopenssl.py index bc5c114fa7e..3130f51ac06 100644 --- a/src/pip/_vendor/urllib3/contrib/pyopenssl.py +++ b/src/pip/_vendor/urllib3/contrib/pyopenssl.py @@ -28,8 +28,8 @@ .. code-block:: python try: - import urllib3.contrib.pyopenssl - urllib3.contrib.pyopenssl.inject_into_urllib3() + import pip._vendor.urllib3.contrib.pyopenssl as pyopenssl + pyopenssl.inject_into_urllib3() except ImportError: pass @@ -76,6 +76,7 @@ class UnsupportedExtension(Exception): from .. import util from ..packages import six +from ..util.ssl_ import PROTOCOL_TLS_CLIENT __all__ = ["inject_into_urllib3", "extract_from_urllib3"] @@ -85,6 +86,7 @@ class UnsupportedExtension(Exception): # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, + PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py index 8f058f5070b..b4ca80b88be 100644 --- a/src/pip/_vendor/urllib3/contrib/securetransport.py +++ b/src/pip/_vendor/urllib3/contrib/securetransport.py @@ -19,8 +19,8 @@ To use this module, simply import and inject it:: - import urllib3.contrib.securetransport - urllib3.contrib.securetransport.inject_into_urllib3() + import pip._vendor.urllib3.contrib.securetransport as securetransport + securetransport.inject_into_urllib3() Happy TLSing! @@ -67,6 +67,7 @@ from pip._vendor import six from .. import util +from ..util.ssl_ import PROTOCOL_TLS_CLIENT from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( _assert_no_error, @@ -154,7 +155,8 @@ # TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. # TLSv1 to 1.2 are supported on macOS 10.8+ _protocol_to_min_max = { - util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12) + util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), + PROTOCOL_TLS_CLIENT: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), } if hasattr(ssl, "PROTOCOL_SSLv2"): diff --git a/src/pip/_vendor/urllib3/contrib/socks.py b/src/pip/_vendor/urllib3/contrib/socks.py index 93df8325d59..c326e80dd11 100644 --- a/src/pip/_vendor/urllib3/contrib/socks.py +++ b/src/pip/_vendor/urllib3/contrib/socks.py @@ -51,7 +51,7 @@ ( "SOCKS support in urllib3 requires the installation of optional " "dependencies: specifically, PySocks. For more information, see " - "https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies" + "https://urllib3.readthedocs.io/en/1.26.x/contrib.html#socks-proxies" ), DependencyWarning, ) diff --git a/src/pip/_vendor/urllib3/exceptions.py b/src/pip/_vendor/urllib3/exceptions.py index d69958d5dfc..cba6f3f560f 100644 --- a/src/pip/_vendor/urllib3/exceptions.py +++ b/src/pip/_vendor/urllib3/exceptions.py @@ -289,7 +289,17 @@ class ProxySchemeUnknown(AssertionError, URLSchemeUnknown): # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. def __init__(self, scheme): - message = "Not supported proxy scheme %s" % scheme + # 'localhost' is here because our URL parser parses + # localhost:8080 -> scheme=localhost, remove if we fix this. + if scheme == "localhost": + scheme = None + if scheme is None: + message = "Proxy URL had no scheme, should start with http:// or https://" + else: + message = ( + "Proxy URL had unsupported scheme %s, should use http:// or https://" + % scheme + ) super(ProxySchemeUnknown, self).__init__(message) diff --git a/src/pip/_vendor/urllib3/packages/__init__.py b/src/pip/_vendor/urllib3/packages/__init__.py index fce4caa65d2..e69de29bb2d 100644 --- a/src/pip/_vendor/urllib3/packages/__init__.py +++ b/src/pip/_vendor/urllib3/packages/__init__.py @@ -1,5 +0,0 @@ -from __future__ import absolute_import - -from . import ssl_match_hostname - -__all__ = ("ssl_match_hostname",) diff --git a/src/pip/_vendor/urllib3/packages/six.py b/src/pip/_vendor/urllib3/packages/six.py index 314424099f6..ba50acb0624 100644 --- a/src/pip/_vendor/urllib3/packages/six.py +++ b/src/pip/_vendor/urllib3/packages/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2019 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import types __author__ = "Benjamin Peterson " -__version__ = "1.12.0" +__version__ = "1.16.0" # Useful for very coarse version differentiation. @@ -71,6 +71,11 @@ def __len__(self): MAXSIZE = int((1 << 63) - 1) del X +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + def _add_doc(func, doc): """Add documentation to a function.""" @@ -182,6 +187,11 @@ def find_module(self, fullname, path=None): return self return None + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + def __get_module(self, fullname): try: return self.known_modules[fullname] @@ -220,6 +230,12 @@ def get_code(self, fullname): get_source = get_code # same as get_code + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + _importer = _SixMetaPathImporter(__name__) @@ -260,9 +276,19 @@ class _MovedItems(_LazyModule): ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule( + "collections_abc", + "collections", + "collections.abc" if sys.version_info >= (3, 3) else "collections", + ), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule( + "_dummy_thread", + "dummy_thread", + "_dummy_thread" if sys.version_info < (3, 9) else "_thread", + ), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), @@ -307,7 +333,9 @@ class _MovedItems(_LazyModule): ] # Add windows specific modules. if sys.platform == "win32": - _moved_attributes += [MovedModule("winreg", "_winreg")] + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) @@ -476,7 +504,7 @@ class Module_six_moves_urllib_robotparser(_LazyModule): _urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), ] for attr in _urllib_robotparser_moved_attributes: setattr(Module_six_moves_urllib_robotparser, attr.name, attr) @@ -678,9 +706,11 @@ def u(s): if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): @@ -707,6 +737,7 @@ def indexbytes(buf, i): _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -723,6 +754,10 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") @@ -750,7 +785,7 @@ def exec_(_code_, _globs_=None, _locs_=None): del frame elif _locs_ is None: _locs_ = _globs_ - exec("""exec _code_ in _globs_, _locs_""") + exec ("""exec _code_ in _globs_, _locs_""") exec_( """def reraise(tp, value, tb=None): @@ -762,18 +797,7 @@ def exec_(_code_, _globs_=None, _locs_=None): ) -if sys.version_info[:2] == (3, 2): - exec_( - """def raise_from(value, from_value): - try: - if from_value is None: - raise value - raise value from from_value - finally: - value = None -""" - ) -elif sys.version_info[:2] > (3, 2): +if sys.version_info[:2] > (3,): exec_( """def raise_from(value, from_value): try: @@ -863,19 +887,41 @@ def print_(*args, **kwargs): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper( + wrapper, + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ def wraps( wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES, ): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - - return wrapper + return functools.partial( + _update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated + ) + wraps.__doc__ = functools.wraps.__doc__ else: wraps = functools.wraps @@ -888,7 +934,15 @@ def with_metaclass(meta, *bases): # the actual metaclass. class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d["__orig_bases__"] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) @classmethod def __prepare__(cls, name, this_bases): @@ -928,12 +982,11 @@ def ensure_binary(s, encoding="utf-8", errors="strict"): - `str` -> encoded to `bytes` - `bytes` -> `bytes` """ + if isinstance(s, binary_type): + return s if isinstance(s, text_type): return s.encode(encoding, errors) - elif isinstance(s, binary_type): - return s - else: - raise TypeError("not expecting type '%s'" % type(s)) + raise TypeError("not expecting type '%s'" % type(s)) def ensure_str(s, encoding="utf-8", errors="strict"): @@ -947,12 +1000,15 @@ def ensure_str(s, encoding="utf-8", errors="strict"): - `str` -> `str` - `bytes` -> decoded to `str` """ - if not isinstance(s, (text_type, binary_type)): - raise TypeError("not expecting type '%s'" % type(s)) + # Optimization: Fast return for the common case. + if type(s) is str: + return s if PY2 and isinstance(s, text_type): - s = s.encode(encoding, errors) + return s.encode(encoding, errors) elif PY3 and isinstance(s, binary_type): - s = s.decode(encoding, errors) + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) return s @@ -977,7 +1033,7 @@ def ensure_text(s, encoding="utf-8", errors="strict"): def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py deleted file mode 100644 index 6b12fd90aad..00000000000 --- a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -import sys - -try: - # Our match_hostname function is the same as 3.5's, so we only want to - # import the match_hostname function if it's at least that good. - if sys.version_info < (3, 5): - raise ImportError("Fallback to vendored code") - - from ssl import CertificateError, match_hostname -except ImportError: - try: - # Backport of the function from a pypi module - from backports.ssl_match_hostname import ( # type: ignore - CertificateError, - match_hostname, - ) - except ImportError: - # Our vendored copy - from ._implementation import CertificateError, match_hostname # type: ignore - -# Not needed, but documenting what we provide. -__all__ = ("CertificateError", "match_hostname") diff --git a/src/pip/_vendor/urllib3/response.py b/src/pip/_vendor/urllib3/response.py index 38693f4fc6e..776e49dd2b2 100644 --- a/src/pip/_vendor/urllib3/response.py +++ b/src/pip/_vendor/urllib3/response.py @@ -7,10 +7,7 @@ from socket import error as SocketError from socket import timeout as SocketTimeout -try: - import brotli -except ImportError: - brotli = None +brotli = None from ._collections import HTTPHeaderDict from .connection import BaseSSLError, HTTPException diff --git a/src/pip/_vendor/urllib3/util/connection.py b/src/pip/_vendor/urllib3/util/connection.py index f1e5d37f88f..6af1138f260 100644 --- a/src/pip/_vendor/urllib3/util/connection.py +++ b/src/pip/_vendor/urllib3/util/connection.py @@ -2,9 +2,8 @@ import socket -from pip._vendor.urllib3.exceptions import LocationParseError - from ..contrib import _appengine_environ +from ..exceptions import LocationParseError from ..packages import six from .wait import NoWayToWaitForSocketError, wait_for_read @@ -118,7 +117,7 @@ def allowed_gai_family(): def _has_ipv6(host): - """ Returns True if the system can bind an IPv6 address. """ + """Returns True if the system can bind an IPv6 address.""" sock = None has_ipv6 = False diff --git a/src/pip/_vendor/urllib3/util/proxy.py b/src/pip/_vendor/urllib3/util/proxy.py index 34f884d5b31..2199cc7b7f0 100644 --- a/src/pip/_vendor/urllib3/util/proxy.py +++ b/src/pip/_vendor/urllib3/util/proxy.py @@ -45,6 +45,7 @@ def create_proxy_ssl_context( ssl_version=resolve_ssl_version(ssl_version), cert_reqs=resolve_cert_reqs(cert_reqs), ) + if ( not ca_certs and not ca_cert_dir diff --git a/src/pip/_vendor/urllib3/util/request.py b/src/pip/_vendor/urllib3/util/request.py index 25103383ec7..330766ef4f3 100644 --- a/src/pip/_vendor/urllib3/util/request.py +++ b/src/pip/_vendor/urllib3/util/request.py @@ -13,12 +13,6 @@ SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"]) ACCEPT_ENCODING = "gzip,deflate" -try: - import brotli as _unused_module_brotli # noqa: F401 -except ImportError: - pass -else: - ACCEPT_ENCODING += ",br" _FAILEDTELL = object() diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py index ee51f922f84..3398323fd7c 100644 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -37,7 +37,7 @@ class _RetryMeta(type): def DEFAULT_METHOD_WHITELIST(cls): warnings.warn( "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " - "will be removed in v2.0. Use 'Retry.DEFAULT_METHODS_ALLOWED' instead", + "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", DeprecationWarning, ) return cls.DEFAULT_ALLOWED_METHODS @@ -69,6 +69,24 @@ def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value): ) cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value + @property + def BACKOFF_MAX(cls): + warnings.warn( + "Using 'Retry.BACKOFF_MAX' is deprecated and " + "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead", + DeprecationWarning, + ) + return cls.DEFAULT_BACKOFF_MAX + + @BACKOFF_MAX.setter + def BACKOFF_MAX(cls, value): + warnings.warn( + "Using 'Retry.BACKOFF_MAX' is deprecated and " + "will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead", + DeprecationWarning, + ) + cls.DEFAULT_BACKOFF_MAX = value + @six.add_metaclass(_RetryMeta) class Retry(object): @@ -181,7 +199,7 @@ class Retry(object): seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer - than :attr:`Retry.BACKOFF_MAX`. + than :attr:`Retry.DEFAULT_BACKOFF_MAX`. By default, backoff is disabled (set to 0). @@ -220,7 +238,7 @@ class Retry(object): DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) #: Maximum backoff time. - BACKOFF_MAX = 120 + DEFAULT_BACKOFF_MAX = 120 def __init__( self, @@ -253,6 +271,7 @@ def __init__( "Using 'method_whitelist' with Retry is deprecated and " "will be removed in v2.0. Use 'allowed_methods' instead", DeprecationWarning, + stacklevel=2, ) allowed_methods = method_whitelist if allowed_methods is _Default: @@ -320,7 +339,7 @@ def new(self, **kw): @classmethod def from_int(cls, retries, redirect=True, default=None): - """ Backwards-compatibility for the old retries format.""" + """Backwards-compatibility for the old retries format.""" if retries is None: retries = default if default is not None else cls.DEFAULT @@ -347,7 +366,7 @@ def get_backoff_time(self): return 0 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1)) - return min(self.BACKOFF_MAX, backoff_value) + return min(self.DEFAULT_BACKOFF_MAX, backoff_value) def parse_retry_after(self, retry_after): # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 @@ -373,7 +392,7 @@ def parse_retry_after(self, retry_after): return seconds def get_retry_after(self, response): - """ Get the value of Retry-After in seconds. """ + """Get the value of Retry-After in seconds.""" retry_after = response.getheader("Retry-After") @@ -467,7 +486,7 @@ def is_retry(self, method, status_code, has_retry_after=False): ) def is_exhausted(self): - """ Are we out of retries? """ + """Are we out of retries?""" retry_counts = ( self.total, self.connect, diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py index 763da82bb66..2b45d391d4d 100644 --- a/src/pip/_vendor/urllib3/util/ssl_.py +++ b/src/pip/_vendor/urllib3/util/ssl_.py @@ -71,6 +71,11 @@ def _const_compare_digest_backport(a, b): except ImportError: PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 +try: + from ssl import PROTOCOL_TLS_CLIENT +except ImportError: + PROTOCOL_TLS_CLIENT = PROTOCOL_TLS + try: from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 @@ -159,7 +164,7 @@ def wrap_socket(self, socket, server_hostname=None, server_side=False): "urllib3 from configuring SSL appropriately and may cause " "certain SSL connections to fail. You can upgrade to a newer " "version of Python to solve this. For more information, see " - "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" "#ssl-warnings", InsecurePlatformWarning, ) @@ -278,7 +283,11 @@ def create_urllib3_context( Constructed SSLContext object with specified options :rtype: SSLContext """ - context = SSLContext(ssl_version or PROTOCOL_TLS) + # PROTOCOL_TLS is deprecated in Python 3.10 + if not ssl_version or ssl_version == PROTOCOL_TLS: + ssl_version = PROTOCOL_TLS_CLIENT + + context = SSLContext(ssl_version) context.set_ciphers(ciphers or DEFAULT_CIPHERS) @@ -313,13 +322,25 @@ def create_urllib3_context( ) is not None: context.post_handshake_auth = True - context.verify_mode = cert_reqs - if ( - getattr(context, "check_hostname", None) is not None - ): # Platform-specific: Python 3.2 - # We do our own verification, including fingerprints and alternative - # hostnames. So disable it here - context.check_hostname = False + def disable_check_hostname(): + if ( + getattr(context, "check_hostname", None) is not None + ): # Platform-specific: Python 3.2 + # We do our own verification, including fingerprints and alternative + # hostnames. So disable it here + context.check_hostname = False + + # The order of the below lines setting verify_mode and check_hostname + # matter due to safe-guards SSLContext has to prevent an SSLContext with + # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more + # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used + # or not so we don't know the initial state of the freshly created SSLContext. + if cert_reqs == ssl.CERT_REQUIRED: + context.verify_mode = cert_reqs + disable_check_hostname() + else: + disable_check_hostname() + context.verify_mode = cert_reqs # Enable logging of TLS session keys via defacto standard environment variable # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. @@ -401,7 +422,7 @@ def ssl_wrap_socket( try: if hasattr(context, "set_alpn_protocols"): context.set_alpn_protocols(ALPN_PROTOCOLS) - except NotImplementedError: + except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols pass # If we detect server_hostname is an IP address then the SNI @@ -419,7 +440,7 @@ def ssl_wrap_socket( "This may cause the server to present an incorrect TLS " "certificate, which can cause validation failures. You can upgrade to " "a newer version of Python to solve this. For more information, see " - "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" "#ssl-warnings", SNIMissingWarning, ) diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/src/pip/_vendor/urllib3/util/ssl_match_hostname.py similarity index 96% rename from src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py rename to src/pip/_vendor/urllib3/util/ssl_match_hostname.py index 689208d3c63..a4b4a569cb1 100644 --- a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/src/pip/_vendor/urllib3/util/ssl_match_hostname.py @@ -9,7 +9,7 @@ # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used in Python 2.7. +# util.ssl_match_hostname to continue to be used in Python 2.7. try: import ipaddress except ImportError: @@ -78,7 +78,8 @@ def _dnsname_match(dn, hostname, max_wildcards=1): def _to_unicode(obj): if isinstance(obj, str) and sys.version_info < (3,): - obj = unicode(obj, encoding="ascii", errors="strict") + # ignored flake8 # F821 to support python 2.7 function + obj = unicode(obj, encoding="ascii", errors="strict") # noqa: F821 return obj diff --git a/src/pip/_vendor/urllib3/util/ssltransport.py b/src/pip/_vendor/urllib3/util/ssltransport.py index ca00233c931..4a7105d1791 100644 --- a/src/pip/_vendor/urllib3/util/ssltransport.py +++ b/src/pip/_vendor/urllib3/util/ssltransport.py @@ -2,8 +2,8 @@ import socket import ssl -from pip._vendor.urllib3.exceptions import ProxySchemeUnsupported -from pip._vendor.urllib3.packages import six +from ..exceptions import ProxySchemeUnsupported +from ..packages import six SSL_BLOCKSIZE = 16384 @@ -193,7 +193,7 @@ def _wrap_ssl_read(self, len, buffer=None): raise def _ssl_io_loop(self, func, *args): - """ Performs an I/O loop between incoming/outgoing and the socket.""" + """Performs an I/O loop between incoming/outgoing and the socket.""" should_loop = True ret = None diff --git a/src/pip/_vendor/urllib3/util/url.py b/src/pip/_vendor/urllib3/util/url.py index 66c8795b11e..3651c43182a 100644 --- a/src/pip/_vendor/urllib3/util/url.py +++ b/src/pip/_vendor/urllib3/util/url.py @@ -63,12 +63,12 @@ BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") -SUBAUTHORITY_PAT = (u"^(?:(.*)@)?(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( +_HOST_PORT_PAT = ("^(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( REG_NAME_PAT, IPV4_PAT, IPV6_ADDRZ_PAT, ) -SUBAUTHORITY_RE = re.compile(SUBAUTHORITY_PAT, re.UNICODE | re.DOTALL) +_HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL) UNRESERVED_CHARS = set( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" @@ -365,7 +365,9 @@ def parse_url(url): scheme = scheme.lower() if authority: - auth, host, port = SUBAUTHORITY_RE.match(authority).groups() + auth, _, host_port = authority.rpartition("@") + auth = auth or None + host, port = _HOST_PORT_RE.match(host_port).groups() if auth and normalize_uri: auth = _encode_invalid_chars(auth, USERINFO_CHARS) if port == "": diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 1db32f396ae..2c93c0f8f2b 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,23 +1,25 @@ -appdirs==1.4.4 -CacheControl==0.12.6 +CacheControl==0.12.10 # Make sure to update the license in pyproject.toml for this. colorama==0.4.4 -contextlib2==0.6.0.post1 -distlib==0.3.1 -distro==1.5.0 +distlib==0.3.3 +distro==1.6.0 html5lib==1.1 -msgpack==1.0.0 -packaging==20.8 -pep517==0.9.1 -progress==1.5 -pyparsing==2.4.7 -requests==2.25.0 - certifi==2020.11.08 - chardet==3.0.4 - idna==2.10 - urllib3==1.26.2 -resolvelib==0.5.4 -retrying==1.3.3 +msgpack==1.0.3 +packaging==21.3 +pep517==0.12.0 +platformdirs==2.4.1 +progress==1.6 +pyparsing==3.0.7 +requests==2.27.1 + certifi==2021.10.08 + chardet==4.0.0 + idna==3.3 + urllib3==1.26.8 +rich==11.0.0 + pygments==2.11.2 + typing_extensions==4.0.1 +resolvelib==0.8.1 setuptools==44.0.0 -six==1.15.0 -toml==0.10.2 +six==1.16.0 +tenacity==8.0.1 +tomli==1.0.3 webencodings==0.5.1 diff --git a/src/pip/py.typed b/src/pip/py.typed new file mode 100644 index 00000000000..493b53e4e7a --- /dev/null +++ b/src/pip/py.typed @@ -0,0 +1,4 @@ +pip is a command line program. While it is implemented in Python, and so is +available for import, you must not use pip's internal APIs in this way. Typing +information is provided as a convenience only and is not a guarantee. Expect +unannounced changes to the API and types in releases. diff --git a/tests/conftest.py b/tests/conftest.py index 0bb69dae6d7..c137f844129 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,30 +7,44 @@ import subprocess import sys import time -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager +from typing import TYPE_CHECKING, Callable, Dict, Iterable, Iterator, List, Optional +from unittest.mock import patch +import py.path import pytest -from mock import patch -from pip._vendor.contextlib2 import ExitStack, nullcontext + +# Config will be available from the public API in pytest >= 7.0.0: +# https://github.com/pytest-dev/pytest/commit/88d84a57916b592b070f4201dc84f0286d1f9fef +from _pytest.config import Config + +# Parser will be available from the public API in pytest >= 7.0.0: +# https://github.com/pytest-dev/pytest/commit/538b5c24999e9ebb4fab43faabc8bcc28737bcdf +from _pytest.config.argparsing import Parser from setuptools.wheel import Wheel from pip._internal.cli.main import main as pip_entry_point +from pip._internal.locations import _USE_SYSCONFIG from pip._internal.utils.temp_dir import global_tempdir_manager -from pip._internal.utils.typing import MYPY_CHECK_RUNNING from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData -from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key from tests.lib.path import Path +from tests.lib.server import MockServer as _MockServer from tests.lib.server import make_mock_server, server_running -from tests.lib.venv import VirtualEnvironment +from tests.lib.venv import VirtualEnvironment, VirtualEnvironmentType + +from .lib.compat import nullcontext -if MYPY_CHECK_RUNNING: - from typing import Dict, Iterable +if TYPE_CHECKING: + from typing import Protocol - from tests.lib.server import MockServer as _MockServer - from tests.lib.server import Responder + from wsgi import WSGIApplication +else: + # TODO: Protocol was introduced in Python 3.8. Remove this branch when + # dropping support for Python 3.7. + Protocol = object -def pytest_addoption(parser): +def pytest_addoption(parser: Parser) -> None: parser.addoption( "--keep-tmpdir", action="store_true", @@ -56,53 +70,62 @@ def pytest_addoption(parser): default=False, help="run 'pip search' tests", ) + parser.addoption( + "--proxy", + action="store", + default=None, + help="use given proxy in session network tests", + ) -def pytest_collection_modifyitems(config, items): +def pytest_collection_modifyitems(config: Config, items: List[pytest.Item]) -> None: for item in items: - if not hasattr(item, 'module'): # e.g.: DoctestTextfile + if not hasattr(item, "module"): # e.g.: DoctestTextfile continue - if (item.get_closest_marker('search') and - not config.getoption('--run-search')): - item.add_marker(pytest.mark.skip('pip search test skipped')) + if item.get_closest_marker("search") and not config.getoption("--run-search"): + item.add_marker(pytest.mark.skip("pip search test skipped")) if "CI" in os.environ: # Mark network tests as flaky - if item.get_closest_marker('network') is not None: + if item.get_closest_marker("network") is not None: item.add_marker(pytest.mark.flaky(reruns=3, reruns_delay=2)) - if (item.get_closest_marker('incompatible_with_test_venv') and - config.getoption("--use-venv")): - item.add_marker(pytest.mark.skip( - 'Incompatible with test venv')) - if (item.get_closest_marker('incompatible_with_venv') and - sys.prefix != sys.base_prefix): - item.add_marker(pytest.mark.skip( - 'Incompatible with venv')) + if item.get_closest_marker("incompatible_with_test_venv") and config.getoption( + "--use-venv" + ): + item.add_marker(pytest.mark.skip("Incompatible with test venv")) + if ( + item.get_closest_marker("incompatible_with_venv") + and sys.prefix != sys.base_prefix + ): + item.add_marker(pytest.mark.skip("Incompatible with venv")) + if item.get_closest_marker("incompatible_with_sysconfig") and _USE_SYSCONFIG: + item.add_marker(pytest.mark.skip("Incompatible with sysconfig")) + + # "Item" has no attribute "module" + module_file = item.module.__file__ # type: ignore[attr-defined] module_path = os.path.relpath( - item.module.__file__, - os.path.commonprefix([__file__, item.module.__file__]), + module_file, os.path.commonprefix([__file__, module_file]) ) module_root_dir = module_path.split(os.pathsep)[0] - if (module_root_dir.startswith("functional") or - module_root_dir.startswith("integration") or - module_root_dir.startswith("lib")): + if ( + module_root_dir.startswith("functional") + or module_root_dir.startswith("integration") + or module_root_dir.startswith("lib") + ): item.add_marker(pytest.mark.integration) elif module_root_dir.startswith("unit"): item.add_marker(pytest.mark.unit) else: - raise RuntimeError( - f"Unknown test type (filename = {module_path})" - ) + raise RuntimeError(f"Unknown test type (filename = {module_path})") @pytest.fixture(scope="session", autouse=True) -def resolver_variant(request): - """Set environment variable to make pip default to the correct resolver. - """ +def resolver_variant(request: pytest.FixtureRequest) -> Iterator[str]: + """Set environment variable to make pip default to the correct resolver.""" resolver = request.config.getoption("--resolver") # Handle the environment variables for this test. @@ -122,24 +145,23 @@ def resolver_variant(request): yield resolver -@pytest.fixture(scope='session') -def tmpdir_factory(request, tmpdir_factory): - """ Modified `tmpdir_factory` session fixture +@pytest.fixture(scope="session") +def tmpdir_factory( + request: pytest.FixtureRequest, tmpdir_factory: pytest.TempdirFactory +) -> Iterator[pytest.TempdirFactory]: + """Modified `tmpdir_factory` session fixture that will automatically cleanup after itself. """ yield tmpdir_factory if not request.config.getoption("--keep-tmpdir"): - # py.path.remove() uses str paths on Python 2 and cannot - # handle non-ASCII file names. This works around the problem by - # passing a unicode object to rmtree(). shutil.rmtree( - str(tmpdir_factory.getbasetemp()), + tmpdir_factory.getbasetemp(), ignore_errors=True, ) @pytest.fixture -def tmpdir(request, tmpdir): +def tmpdir(request: pytest.FixtureRequest, tmpdir: py.path.local) -> Iterator[Path]: """ Return a temporary directory path object which is unique to each test function invocation, created as a sub directory of the base temporary @@ -155,14 +177,11 @@ def tmpdir(request, tmpdir): # This should prevent us from needing a multiple gigabyte temporary # directory while running the tests. if not request.config.getoption("--keep-tmpdir"): - # py.path.remove() uses str paths on Python 2 and cannot - # handle non-ASCII file names. This works around the problem by - # passing a unicode object to rmtree(). - shutil.rmtree(str(tmpdir), ignore_errors=True) + tmpdir.remove(ignore_errors=True) @pytest.fixture(autouse=True) -def isolate(tmpdir, monkeypatch): +def isolate(tmpdir: Path, monkeypatch: pytest.MonkeyPatch) -> None: """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -182,17 +201,17 @@ def isolate(tmpdir, monkeypatch): fake_root = os.path.join(str(tmpdir), "fake-root") os.makedirs(fake_root) - if sys.platform == 'win32': + if sys.platform == "win32": # Note: this will only take effect in subprocesses... home_drive, home_path = os.path.splitdrive(home_dir) - monkeypatch.setenv('USERPROFILE', home_dir) - monkeypatch.setenv('HOMEDRIVE', home_drive) - monkeypatch.setenv('HOMEPATH', home_path) + monkeypatch.setenv("USERPROFILE", home_dir) + monkeypatch.setenv("HOMEDRIVE", home_drive) + monkeypatch.setenv("HOMEPATH", home_path) for env_var, sub_path in ( - ('APPDATA', 'AppData/Roaming'), - ('LOCALAPPDATA', 'AppData/Local'), + ("APPDATA", "AppData/Roaming"), + ("LOCALAPPDATA", "AppData/Local"), ): - path = os.path.join(home_dir, *sub_path.split('/')) + path = os.path.join(home_dir, *sub_path.split("/")) monkeypatch.setenv(env_var, path) os.makedirs(path) else: @@ -201,23 +220,46 @@ def isolate(tmpdir, monkeypatch): # of the user's actual $HOME directory. monkeypatch.setenv("HOME", home_dir) # Isolate ourselves from XDG directories - monkeypatch.setenv("XDG_DATA_HOME", os.path.join( - home_dir, ".local", "share", - )) - monkeypatch.setenv("XDG_CONFIG_HOME", os.path.join( - home_dir, ".config", - )) + monkeypatch.setenv( + "XDG_DATA_HOME", + os.path.join( + home_dir, + ".local", + "share", + ), + ) + monkeypatch.setenv( + "XDG_CONFIG_HOME", + os.path.join( + home_dir, + ".config", + ), + ) monkeypatch.setenv("XDG_CACHE_HOME", os.path.join(home_dir, ".cache")) - monkeypatch.setenv("XDG_RUNTIME_DIR", os.path.join( - home_dir, ".runtime", - )) - monkeypatch.setenv("XDG_DATA_DIRS", os.pathsep.join([ - os.path.join(fake_root, "usr", "local", "share"), - os.path.join(fake_root, "usr", "share"), - ])) - monkeypatch.setenv("XDG_CONFIG_DIRS", os.path.join( - fake_root, "etc", "xdg", - )) + monkeypatch.setenv( + "XDG_RUNTIME_DIR", + os.path.join( + home_dir, + ".runtime", + ), + ) + monkeypatch.setenv( + "XDG_DATA_DIRS", + os.pathsep.join( + [ + os.path.join(fake_root, "usr", "local", "share"), + os.path.join(fake_root, "usr", "share"), + ] + ), + ) + monkeypatch.setenv( + "XDG_CONFIG_DIRS", + os.path.join( + fake_root, + "etc", + "xdg", + ), + ) # Configure git, because without an author name/email git will complain # and cause test failures. @@ -229,18 +271,16 @@ def isolate(tmpdir, monkeypatch): monkeypatch.setenv("PIP_DISABLE_PIP_VERSION_CHECK", "true") # Make sure tests don't share a requirements tracker. - monkeypatch.delenv("PIP_REQ_TRACKER", False) + monkeypatch.delenv("PIP_BUILD_TRACKER", False) # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: - fp.write( - b"[user]\n\tname = pip\n\temail = distutils-sig@python.org\n" - ) + fp.write(b"[user]\n\tname = pip\n\temail = distutils-sig@python.org\n") @pytest.fixture(autouse=True) -def scoped_global_tempdir_manager(request): +def scoped_global_tempdir_manager(request: pytest.FixtureRequest) -> Iterator[None]: """Make unit tests with globally-managed tempdirs easier Each test function gets its own individual scope for globally-managed @@ -255,13 +295,15 @@ def scoped_global_tempdir_manager(request): yield -@pytest.fixture(scope='session') -def pip_src(tmpdir_factory): - def not_code_files_and_folders(path, names): +@pytest.fixture(scope="session") +def pip_src(tmpdir_factory: pytest.TempdirFactory) -> Path: + def not_code_files_and_folders(path: str, names: List[str]) -> Iterable[str]: # In the root directory... if path == SRC_DIR: # ignore all folders except "src" - folders = {name for name in names if os.path.isdir(path / name)} + folders = { + name for name in names if os.path.isdir(os.path.join(path, name)) + } to_ignore = folders - {"src"} # and ignore ".git" if present (which may be a file if in a linked # worktree). @@ -275,7 +317,7 @@ def not_code_files_and_folders(path, names): ignored.update(fnmatch.filter(names, pattern)) return ignored - pip_src = Path(str(tmpdir_factory.mktemp('pip_src'))).joinpath('pip_src') + pip_src = Path(str(tmpdir_factory.mktemp("pip_src"))).joinpath("pip_src") # Copy over our source tree so that each use is self contained shutil.copytree( SRC_DIR, @@ -285,84 +327,91 @@ def not_code_files_and_folders(path, names): return pip_src -def _common_wheel_editable_install(tmpdir_factory, common_wheels, package): - wheel_candidates = list( - common_wheels.glob('{package}-*.whl'.format(**locals()))) +def _common_wheel_editable_install( + tmpdir_factory: pytest.TempdirFactory, common_wheels: Path, package: str +) -> Path: + wheel_candidates = list(common_wheels.glob(f"{package}-*.whl")) assert len(wheel_candidates) == 1, wheel_candidates - install_dir = Path(str(tmpdir_factory.mktemp(package))) / 'install' + install_dir = Path(str(tmpdir_factory.mktemp(package))) / "install" Wheel(wheel_candidates[0]).install_as_egg(install_dir) - (install_dir / 'EGG-INFO').rename( - install_dir / '{package}.egg-info'.format(**locals())) + (install_dir / "EGG-INFO").rename(install_dir / f"{package}.egg-info") assert compileall.compile_dir(str(install_dir), quiet=1) return install_dir -@pytest.fixture(scope='session') -def setuptools_install(tmpdir_factory, common_wheels): - return _common_wheel_editable_install(tmpdir_factory, - common_wheels, - 'setuptools') - +@pytest.fixture(scope="session") +def setuptools_install( + tmpdir_factory: pytest.TempdirFactory, common_wheels: Path +) -> Path: + return _common_wheel_editable_install(tmpdir_factory, common_wheels, "setuptools") -@pytest.fixture(scope='session') -def wheel_install(tmpdir_factory, common_wheels): - return _common_wheel_editable_install(tmpdir_factory, - common_wheels, - 'wheel') +@pytest.fixture(scope="session") +def wheel_install(tmpdir_factory: pytest.TempdirFactory, common_wheels: Path) -> Path: + return _common_wheel_editable_install(tmpdir_factory, common_wheels, "wheel") -@pytest.fixture(scope='session') -def coverage_install(tmpdir_factory, common_wheels): - return _common_wheel_editable_install(tmpdir_factory, - common_wheels, - 'coverage') +@pytest.fixture(scope="session") +def coverage_install( + tmpdir_factory: pytest.TempdirFactory, common_wheels: Path +) -> Path: + return _common_wheel_editable_install(tmpdir_factory, common_wheels, "coverage") -def install_egg_link(venv, project_name, egg_info_dir): - with open(venv.site / 'easy-install.pth', 'a') as fp: - fp.write(str(egg_info_dir.resolve()) + '\n') - with open(venv.site / (project_name + '.egg-link'), 'w') as fp: - fp.write(str(egg_info_dir) + '\n.') +def install_egg_link( + venv: VirtualEnvironment, project_name: str, egg_info_dir: Path +) -> None: + with open(venv.site / "easy-install.pth", "a") as fp: + fp.write(str(egg_info_dir.resolve()) + "\n") + with open(venv.site / (project_name + ".egg-link"), "w") as fp: + fp.write(str(egg_info_dir) + "\n.") -@pytest.fixture(scope='session') -def virtualenv_template(request, tmpdir_factory, pip_src, - setuptools_install, coverage_install): - if request.config.getoption('--use-venv'): - venv_type = 'venv' +@pytest.fixture(scope="session") +def virtualenv_template( + request: pytest.FixtureRequest, + tmpdir_factory: pytest.TempdirFactory, + pip_src: Path, + setuptools_install: Path, + coverage_install: Path, +) -> Iterator[VirtualEnvironment]: + + venv_type: VirtualEnvironmentType + if request.config.getoption("--use-venv"): + venv_type = "venv" else: - venv_type = 'virtualenv' + venv_type = "virtualenv" # Create the virtual environment - tmpdir = Path(str(tmpdir_factory.mktemp('virtualenv'))) - venv = VirtualEnvironment( - tmpdir.joinpath("venv_orig"), venv_type=venv_type - ) + tmpdir = Path(str(tmpdir_factory.mktemp("virtualenv"))) + venv = VirtualEnvironment(tmpdir.joinpath("venv_orig"), venv_type=venv_type) # Install setuptools and pip. - install_egg_link(venv, 'setuptools', setuptools_install) - pip_editable = Path(str(tmpdir_factory.mktemp('pip'))) / 'pip' + install_egg_link(venv, "setuptools", setuptools_install) + pip_editable = Path(str(tmpdir_factory.mktemp("pip"))) / "pip" shutil.copytree(pip_src, pip_editable, symlinks=True) # noxfile.py is Python 3 only assert compileall.compile_dir( - str(pip_editable), quiet=1, rx=re.compile("noxfile.py$"), + str(pip_editable), + quiet=1, + rx=re.compile("noxfile.py$"), + ) + subprocess.check_call( + [venv.bin / "python", "setup.py", "-q", "develop"], cwd=pip_editable ) - subprocess.check_call([venv.bin / 'python', 'setup.py', '-q', 'develop'], - cwd=pip_editable) # Install coverage and pth file for executing it in any spawned processes # in this virtual environment. - install_egg_link(venv, 'coverage', coverage_install) + install_egg_link(venv, "coverage", coverage_install) # zz prefix ensures the file is after easy-install.pth. - with open(venv.site / 'zz-coverage-helper.pth', 'a') as f: - f.write('import coverage; coverage.process_startup()') + with open(venv.site / "zz-coverage-helper.pth", "a") as f: + f.write("import coverage; coverage.process_startup()") # Drop (non-relocatable) launchers. for exe in os.listdir(venv.bin): if not ( - exe.startswith('python') or - exe.startswith('libpy') # Don't remove libpypy-c.so... + exe.startswith("python") + or exe.startswith("libpy") # Don't remove libpypy-c.so... ): (venv.bin / exe).unlink() @@ -377,15 +426,19 @@ def virtualenv_template(request, tmpdir_factory, pip_src, @pytest.fixture(scope="session") -def virtualenv_factory(virtualenv_template): - def factory(tmpdir): +def virtualenv_factory( + virtualenv_template: VirtualEnvironment, +) -> Callable[[Path], VirtualEnvironment]: + def factory(tmpdir: Path) -> VirtualEnvironment: return VirtualEnvironment(tmpdir, virtualenv_template) return factory @pytest.fixture -def virtualenv(virtualenv_factory, tmpdir): +def virtualenv( + virtualenv_factory: Callable[[Path], VirtualEnvironment], tmpdir: Path +) -> Iterator[VirtualEnvironment]: """ Return a virtual environment which is unique to each test function invocation created inside of a sub directory of the test function's @@ -396,33 +449,39 @@ def virtualenv(virtualenv_factory, tmpdir): @pytest.fixture -def with_wheel(virtualenv, wheel_install): - install_egg_link(virtualenv, 'wheel', wheel_install) +def with_wheel(virtualenv: VirtualEnvironment, wheel_install: Path) -> None: + install_egg_link(virtualenv, "wheel", wheel_install) + + +class ScriptFactory(Protocol): + def __call__( + self, tmpdir: Path, virtualenv: Optional[VirtualEnvironment] = None + ) -> PipTestEnvironment: + ... @pytest.fixture(scope="session") -def script_factory(virtualenv_factory, deprecated_python): - def factory(tmpdir, virtualenv=None): +def script_factory( + virtualenv_factory: Callable[[Path], VirtualEnvironment], deprecated_python: bool +) -> ScriptFactory: + def factory( + tmpdir: Path, virtualenv: Optional[VirtualEnvironment] = None + ) -> PipTestEnvironment: if virtualenv is None: virtualenv = virtualenv_factory(tmpdir.joinpath("venv")) return PipTestEnvironment( # The base location for our test environment tmpdir, - # Tell the Test Environment where our virtualenv is located virtualenv=virtualenv, - # Do not ignore hidden files, they need to be checked as well ignore_hidden=False, - # We are starting with an already empty directory start_clear=False, - # We want to ensure no temporary files are left behind, so the # PipTestEnvironment needs to capture and assert against temp capture_temp=True, assert_no_temp=True, - # Deprecated python versions produce an extra deprecation warning pip_expect_warning=deprecated_python, ) @@ -431,7 +490,11 @@ def factory(tmpdir, virtualenv=None): @pytest.fixture -def script(tmpdir, virtualenv, script_factory): +def script( + tmpdir: Path, + virtualenv: VirtualEnvironment, + script_factory: Callable[[Path, Optional[VirtualEnvironment]], PipTestEnvironment], +) -> PipTestEnvironment: """ Return a PipTestEnvironment which is unique to each test function and will execute all commands inside of the unique virtual environment for this @@ -442,29 +505,29 @@ def script(tmpdir, virtualenv, script_factory): @pytest.fixture(scope="session") -def common_wheels(): +def common_wheels() -> Path: """Provide a directory with latest setuptools and wheel wheels""" - return DATA_DIR.joinpath('common_wheels') + return DATA_DIR.joinpath("common_wheels") @pytest.fixture(scope="session") -def shared_data(tmpdir_factory): +def shared_data(tmpdir_factory: pytest.TempdirFactory) -> TestData: return TestData.copy(Path(str(tmpdir_factory.mktemp("data")))) @pytest.fixture -def data(tmpdir): +def data(tmpdir: Path) -> TestData: return TestData.copy(tmpdir.joinpath("data")) class InMemoryPipResult: - def __init__(self, returncode, stdout): + def __init__(self, returncode: int, stdout: str) -> None: self.returncode = returncode self.stdout = stdout class InMemoryPip: - def pip(self, *args): + def pip(self, *args: str) -> InMemoryPipResult: orig_stdout = sys.stdout stdout = io.StringIO() sys.stdout = stdout @@ -478,22 +541,28 @@ def pip(self, *args): @pytest.fixture -def in_memory_pip(): +def in_memory_pip() -> InMemoryPip: return InMemoryPip() @pytest.fixture(scope="session") -def deprecated_python(): +def deprecated_python() -> bool: """Used to indicate whether pip deprecated this Python version""" return sys.version_info[:2] in [] +CertFactory = Callable[[], str] + + @pytest.fixture(scope="session") -def cert_factory(tmpdir_factory): - def factory(): - # type: () -> str - """Returns path to cert/key file. - """ +def cert_factory(tmpdir_factory: pytest.TempdirFactory) -> CertFactory: + # Delay the import requiring cryptography in order to make it possible + # to deselect relevant tests on systems where cryptography cannot + # be installed. + from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key + + def factory() -> str: + """Returns path to cert/key file.""" output_path = Path(str(tmpdir_factory.mktemp("certs"))) / "cert.pem" # Must be Text on PY2. cert, key = make_tls_cert("localhost") @@ -507,56 +576,50 @@ def factory(): class MockServer: - def __init__(self, server): - # type: (_MockServer) -> None + def __init__(self, server: _MockServer) -> None: self._server = server self._running = False self.context = ExitStack() @property - def port(self): + def port(self) -> int: return self._server.port @property - def host(self): + def host(self) -> str: return self._server.host - def set_responses(self, responses): - # type: (Iterable[Responder]) -> None + def set_responses(self, responses: Iterable["WSGIApplication"]) -> None: assert not self._running, "responses cannot be set on running server" self._server.mock.side_effect = responses - def start(self): - # type: () -> None + def start(self) -> None: assert not self._running, "running server cannot be started" self.context.enter_context(server_running(self._server)) self.context.enter_context(self._set_running()) @contextmanager - def _set_running(self): + def _set_running(self) -> Iterator[None]: self._running = True try: yield finally: self._running = False - def stop(self): - # type: () -> None + def stop(self) -> None: assert self._running, "idle server cannot be stopped" self.context.close() - def get_requests(self): - # type: () -> Dict[str, str] - """Get environ for each received request. - """ + def get_requests(self) -> List[Dict[str, str]]: + """Get environ for each received request.""" assert not self._running, "cannot get mock from running server" - return [ - call.args[0] for call in self._server.mock.call_args_list - ] + # Legacy: replace call[0][0] with call.args[0] + # when pip drops support for python3.7 + return [call[0][0] for call in self._server.mock.call_args_list] @pytest.fixture -def mock_server(): +def mock_server() -> Iterator[MockServer]: server = make_mock_server() test_server = MockServer(server) with test_server.context: @@ -564,10 +627,15 @@ def mock_server(): @pytest.fixture -def utc(): +def utc() -> Iterator[None]: # time.tzset() is not implemented on some platforms, e.g. Windows. - tzset = getattr(time, 'tzset', lambda: None) - with patch.dict(os.environ, {'TZ': 'UTC'}): + tzset = getattr(time, "tzset", lambda: None) + with patch.dict(os.environ, {"TZ": "UTC"}): tzset() yield tzset() + + +@pytest.fixture +def proxy(request: pytest.FixtureRequest) -> str: + return request.config.getoption("proxy") diff --git a/tests/data/indexes/datarequire/fakepackage/index.html b/tests/data/indexes/datarequire/fakepackage/index.html index 0ca8b9dc3a2..25bf4aa21d5 100644 --- a/tests/data/indexes/datarequire/fakepackage/index.html +++ b/tests/data/indexes/datarequire/fakepackage/index.html @@ -1,3 +1,4 @@ + Links for fakepackage

Links for fakepackage

fakepackage-1.0.0.tar.gz
fakepackage-2.6.0.tar.gz
diff --git a/tests/data/indexes/dev/bar/index.html b/tests/data/indexes/dev/bar/index.html index bcee309212c..c0da6561310 100644 --- a/tests/data/indexes/dev/bar/index.html +++ b/tests/data/indexes/dev/bar/index.html @@ -1,3 +1,4 @@ + bar-1.0.tar.gz diff --git a/tests/data/indexes/in dex/simple/index.html b/tests/data/indexes/in dex/simple/index.html index dba6cc3ebd6..cb078ea7b19 100644 --- a/tests/data/indexes/in dex/simple/index.html +++ b/tests/data/indexes/in dex/simple/index.html @@ -1,3 +1,4 @@ + simple-1.0.tar.gz diff --git a/tests/data/indexes/pre/bar/index.html b/tests/data/indexes/pre/bar/index.html index c50d88bc863..da76454f604 100644 --- a/tests/data/indexes/pre/bar/index.html +++ b/tests/data/indexes/pre/bar/index.html @@ -1,3 +1,4 @@ + bar-1.0.tar.gz diff --git a/tests/data/indexes/simple/simple/index.html b/tests/data/indexes/simple/simple/index.html index dba6cc3ebd6..cb078ea7b19 100644 --- a/tests/data/indexes/simple/simple/index.html +++ b/tests/data/indexes/simple/simple/index.html @@ -1,3 +1,4 @@ + simple-1.0.tar.gz diff --git a/tests/data/indexes/yanked/simple/index.html b/tests/data/indexes/yanked/simple/index.html index bf4994310be..67a2585ae13 100644 --- a/tests/data/indexes/yanked/simple/index.html +++ b/tests/data/indexes/yanked/simple/index.html @@ -1,3 +1,4 @@ + simple-1.0.tar.gz diff --git a/tests/data/indexes/yanked_all/simple/index.html b/tests/data/indexes/yanked_all/simple/index.html new file mode 100644 index 00000000000..060f9904465 --- /dev/null +++ b/tests/data/indexes/yanked_all/simple/index.html @@ -0,0 +1,8 @@ + + + + simple-1.0.tar.gz + simple-2.0.tar.gz + simple-3.0.tar.gz + + diff --git a/tests/data/packages/BrokenEmitsUTF8/setup.py b/tests/data/packages/BrokenEmitsUTF8/setup.py index 81c5baeadba..a40bc60c18f 100644 --- a/tests/data/packages/BrokenEmitsUTF8/setup.py +++ b/tests/data/packages/BrokenEmitsUTF8/setup.py @@ -7,20 +7,32 @@ class FakeError(Exception): pass -if sys.argv[1] == 'install': - if hasattr(sys.stdout, 'buffer'): - sys.stdout.buffer.write('\nThis package prints out UTF-8 stuff like:\n'.encode('utf-8')) - sys.stdout.buffer.write('* return type of ‘main’ is not ‘int’\n'.encode('utf-8')) - sys.stdout.buffer.write('* Björk Guðmundsdóttir [ˈpjœr̥k ˈkvʏðmʏntsˌtoʊhtɪr]'.encode('utf-8')) + +if sys.argv[1] == "install": + if hasattr(sys.stdout, "buffer"): + sys.stdout.buffer.write( + "\nThis package prints out UTF-8 stuff like:\n".encode("utf-8") + ) + sys.stdout.buffer.write( + "* return type of ‘main’ is not ‘int’\n".encode("utf-8") + ) + sys.stdout.buffer.write( + "* Björk Guðmundsdóttir [ˈpjœr̥k ˈkvʏðmʏntsˌtoʊhtɪr]".encode("utf-8") + ) else: pass - sys.stdout.write('\nThis package prints out UTF-8 stuff like:\n') - sys.stdout.write('* return type of \xe2\x80\x98main\xe2\x80\x99 is not \xe2\x80\x98int\xe2\x80\x99\n') - sys.stdout.write('* Bj\xc3\xb6rk Gu\xc3\xb0mundsd\xc3\xb3ttir [\xcb\x88pj\xc5\x93r\xcc\xa5k \xcb\x88kv\xca\x8f\xc3\xb0m\xca\x8fnts\xcb\x8cto\xca\x8aht\xc9\xaar]\n') + sys.stdout.write("\nThis package prints out UTF-8 stuff like:\n") + sys.stdout.write( + "* return type of \xe2\x80\x98main\xe2\x80\x99 is not \xe2\x80\x98int\xe2\x80\x99\n" + ) + sys.stdout.write( + "* Bj\xc3\xb6rk Gu\xc3\xb0mundsd\xc3\xb3ttir [\xcb\x88pj\xc5\x93r\xcc\xa5k \xcb\x88kv\xca\x8f\xc3\xb0m\xca\x8fnts\xcb\x8cto\xca\x8aht\xc9\xaar]\n" + ) - raise FakeError('this package designed to fail on install') + raise FakeError("this package designed to fail on install") -setup(name='broken', - version='0.2', - py_modules=['broken'], - ) +setup( + name="broken", + version="0.2", + py_modules=["broken"], +) diff --git a/tests/data/packages/FSPkg/setup.py b/tests/data/packages/FSPkg/setup.py index d1b725e8cc4..58a78413339 100644 --- a/tests/data/packages/FSPkg/setup.py +++ b/tests/data/packages/FSPkg/setup.py @@ -1,25 +1,26 @@ from setuptools import find_packages, setup -version = '0.1dev' +version = "0.1dev" -setup(name='FSPkg', - version=version, - description="File system test package", - long_description="""\ +setup( + name="FSPkg", + version=version, + description="File system test package", + long_description="""\ File system test package""", - classifiers=[], # Get strings from https://pypi.org/pypi?%3Aaction=list_classifiers - keywords='pip tests', - author='pip', - author_email='pip@openplans.org', - url='http://pip.openplans.org', - license='', - packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), - include_package_data=True, - zip_safe=False, - install_requires=[ - # -*- Extra requirements: -*- - ], - entry_points=""" + classifiers=[], # Get strings from https://pypi.org/pypi?%3Aaction=list_classifiers + keywords="pip tests", + author="pip", + author_email="pip@openplans.org", + url="http://pip.openplans.org", + license="", + packages=find_packages(exclude=["ez_setup", "examples", "tests"]), + include_package_data=True, + zip_safe=False, + install_requires=[ + # -*- Extra requirements: -*- + ], + entry_points=""" # -*- Entry points: -*- """, - ) +) diff --git a/tests/data/packages/HackedEggInfo/setup.py b/tests/data/packages/HackedEggInfo/setup.py index 171f5a2a34b..8f6aad433d4 100644 --- a/tests/data/packages/HackedEggInfo/setup.py +++ b/tests/data/packages/HackedEggInfo/setup.py @@ -1,17 +1,15 @@ -# -*- coding: utf-8 -*- - from setuptools import setup from setuptools.command import egg_info as orig_egg_info -class egg_info (orig_egg_info.egg_info): +class egg_info(orig_egg_info.egg_info): def run(self): orig_egg_info.egg_info.run(self) setup( name="hackedegginfo", - version='0.0.0', - cmdclass={'egg_info':egg_info}, + version="0.0.0", + cmdclass={"egg_info": egg_info}, zip_safe=False, ) diff --git a/tests/data/packages/LocalEnvironMarker/setup.py b/tests/data/packages/LocalEnvironMarker/setup.py index 36ceb214cf7..0ab6c8a5719 100644 --- a/tests/data/packages/LocalEnvironMarker/setup.py +++ b/tests/data/packages/LocalEnvironMarker/setup.py @@ -11,17 +11,17 @@ def path_to_url(path): path = os.path.normpath(os.path.abspath(path)) drive, path = os.path.splitdrive(path) filepath = path.split(os.path.sep) - url = '/'.join(filepath) + url = "/".join(filepath) if drive: - return 'file:///' + drive + url - return 'file://' + url + return "file:///" + drive + url + return "file://" + url setup( - name='LocalEnvironMarker', - version='0.0.1', + name="LocalEnvironMarker", + version="0.0.1", packages=find_packages(), extras_require={ - ":python_version == '2.7'": ['simple'], - } + ":python_version == '2.7'": ["simple"], + }, ) diff --git a/tests/data/packages/LocalExtras-0.0.2/setup.py b/tests/data/packages/LocalExtras-0.0.2/setup.py index cc5c832837f..6e910d1cbb2 100644 --- a/tests/data/packages/LocalExtras-0.0.2/setup.py +++ b/tests/data/packages/LocalExtras-0.0.2/setup.py @@ -11,16 +11,16 @@ def path_to_url(path): path = os.path.normpath(os.path.abspath(path)) drive, path = os.path.splitdrive(path) filepath = path.split(os.path.sep) - url = '/'.join(filepath) + url = "/".join(filepath) if drive: - return 'file:///' + drive + url - return 'file://' + url + return "file:///" + drive + url + return "file://" + url setup( - name='LocalExtras', - version='0.0.2', + name="LocalExtras", + version="0.0.2", packages=find_packages(), - install_requires=['simple==1.0'], - extras_require={'bar': ['simple==2.0'], 'baz': ['singlemodule']} + install_requires=["simple==1.0"], + extras_require={"bar": ["simple==2.0"], "baz": ["singlemodule"]}, ) diff --git a/tests/data/packages/LocalExtras/setup.py b/tests/data/packages/LocalExtras/setup.py index eb390e32e89..4bf2179da78 100644 --- a/tests/data/packages/LocalExtras/setup.py +++ b/tests/data/packages/LocalExtras/setup.py @@ -11,15 +11,15 @@ def path_to_url(path): path = os.path.normpath(os.path.abspath(path)) drive, path = os.path.splitdrive(path) filepath = path.split(os.path.sep) - url = '/'.join(filepath) + url = "/".join(filepath) if drive: - return 'file:///' + drive + url - return 'file://' + url + return "file:///" + drive + url + return "file://" + url setup( - name='LocalExtras', - version='0.0.1', + name="LocalExtras", + version="0.0.1", packages=find_packages(), - extras_require={'bar': ['simple'], 'baz': ['singlemodule']} + extras_require={"bar": ["simple"], "baz": ["singlemodule"]}, ) diff --git a/tests/data/packages/SetupPyLatin1/setup.py b/tests/data/packages/SetupPyLatin1/setup.py index 3ca77c00c96..bc8967c5199 100644 --- a/tests/data/packages/SetupPyLatin1/setup.py +++ b/tests/data/packages/SetupPyLatin1/setup.py @@ -2,6 +2,7 @@ from distutils.core import setup -setup(name="SetupPyUTF8", - author=u"Sal Ibarra Corretg", - ) +setup( + name="SetupPyUTF8", + author=u"Sal Ibarra Corretg", +) diff --git a/tests/data/packages/SetupPyUTF8/setup.py b/tests/data/packages/SetupPyUTF8/setup.py index 9b65f5e79fc..bd9fd2a294b 100644 --- a/tests/data/packages/SetupPyUTF8/setup.py +++ b/tests/data/packages/SetupPyUTF8/setup.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - from distutils.core import setup -setup(name="SetupPyUTF8", - author="Saúl Ibarra Corretgé", - ) +setup( + name="SetupPyUTF8", + author="Saúl Ibarra Corretgé", +) diff --git a/tests/data/packages/corruptwheel-1.0-py2.py3-none-any.whl b/tests/data/packages/corruptwheel-1.0-py2.py3-none-any.whl new file mode 100644 index 00000000000..bf285f13f40 --- /dev/null +++ b/tests/data/packages/corruptwheel-1.0-py2.py3-none-any.whl @@ -0,0 +1 @@ +This is a corrupt wheel which _clearly_ is not a zip file. diff --git a/tests/data/packages/pep517_wrapper_buildsys/mybuildsys.py b/tests/data/packages/pep517_wrapper_buildsys/mybuildsys.py index e2f920ba3f5..08645e8bd5b 100644 --- a/tests/data/packages/pep517_wrapper_buildsys/mybuildsys.py +++ b/tests/data/packages/pep517_wrapper_buildsys/mybuildsys.py @@ -2,9 +2,11 @@ from setuptools.build_meta import build_sdist from setuptools.build_meta import build_wheel as setuptools_build_wheel -from setuptools.build_meta import (get_requires_for_build_sdist, - get_requires_for_build_wheel, - prepare_metadata_for_build_wheel) +from setuptools.build_meta import ( + get_requires_for_build_sdist, + get_requires_for_build_wheel, + prepare_metadata_for_build_wheel, +) def build_wheel(*a, **kw): @@ -12,7 +14,7 @@ def build_wheel(*a, **kw): raise RuntimeError("Failing build_wheel, as requested.") # Create the marker file to record that the hook was called - with open(os.environ['PIP_TEST_MARKER_FILE'], 'wb'): + with open(os.environ["PIP_TEST_MARKER_FILE"], "wb"): pass return setuptools_build_wheel(*a, **kw) diff --git a/tests/data/packages/requiresPaste/requiresPaste.py b/tests/data/packages/requiresPaste/requiresPaste.py index c74209e44fe..84d4ed536d3 100644 --- a/tests/data/packages/requiresPaste/requiresPaste.py +++ b/tests/data/packages/requiresPaste/requiresPaste.py @@ -1,3 +1,3 @@ """Module requiring Paste to test dependencies download of pip wheel.""" -__version__ = '3.1.4' +__version__ = "3.1.4" diff --git a/tests/data/packages/requires_wheelbroken_upper/setup.py b/tests/data/packages/requires_wheelbroken_upper/setup.py index 150f98dfbf7..210b7c67ad8 100644 --- a/tests/data/packages/requires_wheelbroken_upper/setup.py +++ b/tests/data/packages/requires_wheelbroken_upper/setup.py @@ -3,4 +3,5 @@ setuptools.setup( name="requires_wheelbroken_upper", version="0", - install_requires=['wheelbroken', 'upper']) + install_requires=["wheelbroken", "upper"], +) diff --git a/tests/data/packages/symlinks/setup.py b/tests/data/packages/symlinks/setup.py index b71e35f1e1e..8605b6fae95 100644 --- a/tests/data/packages/symlinks/setup.py +++ b/tests/data/packages/symlinks/setup.py @@ -1,8 +1,9 @@ from setuptools import setup -version = '0.1' +version = "0.1" -setup(name='symlinks', - version=version, - packages=["symlinks"], - ) +setup( + name="symlinks", + version=version, + packages=["symlinks"], +) diff --git a/tests/data/packages3/dinner/index.html b/tests/data/packages3/dinner/index.html index e258eb16b40..52a16b11686 100644 --- a/tests/data/packages3/dinner/index.html +++ b/tests/data/packages3/dinner/index.html @@ -1,3 +1,4 @@ + PyPI Mirror

PyPI Mirror

diff --git a/tests/data/packages3/index.html b/tests/data/packages3/index.html index d66e70ec631..262207b6a62 100644 --- a/tests/data/packages3/index.html +++ b/tests/data/packages3/index.html @@ -1,3 +1,4 @@ + PyPI Mirror

PyPI Mirror

diff --git a/tests/data/packages3/requiredinner/index.html b/tests/data/packages3/requiredinner/index.html index 0981c9c7246..52a4e66673c 100644 --- a/tests/data/packages3/requiredinner/index.html +++ b/tests/data/packages3/requiredinner/index.html @@ -1,3 +1,4 @@ + PyPI Mirror

PyPI Mirror

diff --git a/tests/data/src/TopoRequires/setup.py b/tests/data/src/TopoRequires/setup.py index 6cd29a7b656..c4b1029f55c 100644 --- a/tests/data/src/TopoRequires/setup.py +++ b/tests/data/src/TopoRequires/setup.py @@ -1,7 +1,7 @@ from setuptools import setup setup( - name='TopoRequires', - version='0.0.1', - packages=['toporequires'], + name="TopoRequires", + version="0.0.1", + packages=["toporequires"], ) diff --git a/tests/data/src/TopoRequires2/setup.py b/tests/data/src/TopoRequires2/setup.py index 019f43cb231..11d009c4175 100644 --- a/tests/data/src/TopoRequires2/setup.py +++ b/tests/data/src/TopoRequires2/setup.py @@ -1,8 +1,8 @@ from setuptools import setup setup( - name='TopoRequires2', - version='0.0.1', - packages=['toporequires2'], - install_requires=['TopoRequires'], + name="TopoRequires2", + version="0.0.1", + packages=["toporequires2"], + install_requires=["TopoRequires"], ) diff --git a/tests/data/src/TopoRequires3/setup.py b/tests/data/src/TopoRequires3/setup.py index 772ed618e3c..550bb008eb9 100644 --- a/tests/data/src/TopoRequires3/setup.py +++ b/tests/data/src/TopoRequires3/setup.py @@ -1,8 +1,8 @@ from setuptools import setup setup( - name='TopoRequires3', - version='0.0.1', - packages=['toporequires3'], - install_requires=['TopoRequires'], + name="TopoRequires3", + version="0.0.1", + packages=["toporequires3"], + install_requires=["TopoRequires"], ) diff --git a/tests/data/src/TopoRequires4/setup.py b/tests/data/src/TopoRequires4/setup.py index e276f55a240..077eec765a5 100644 --- a/tests/data/src/TopoRequires4/setup.py +++ b/tests/data/src/TopoRequires4/setup.py @@ -1,8 +1,8 @@ from setuptools import setup setup( - name='TopoRequires4', - version='0.0.1', - packages=['toporequires4'], - install_requires=['TopoRequires2', 'TopoRequires', 'TopoRequires3'], + name="TopoRequires4", + version="0.0.1", + packages=["toporequires4"], + install_requires=["TopoRequires2", "TopoRequires", "TopoRequires3"], ) diff --git a/tests/data/src/chattymodule/setup.py b/tests/data/src/chattymodule/setup.py index 01d7720765f..9f411b6fdff 100644 --- a/tests/data/src/chattymodule/setup.py +++ b/tests/data/src/chattymodule/setup.py @@ -5,16 +5,18 @@ from setuptools import setup -print("HELLO FROM CHATTYMODULE {sys.argv[1]}".format(**locals())) -print(os.environ) +print(f"HELLO FROM CHATTYMODULE {sys.argv[1]}") print(sys.argv) +print(sys.executable) +print(sys.version) + if "--fail" in sys.argv: print("I DIE, I DIE") sys.exit(1) setup( name="chattymodule", - version='0.0.1', + version="0.0.1", description="A sample Python project with a single module", - py_modules=['chattymodule'], + py_modules=["chattymodule"], ) diff --git a/tests/data/src/compilewheel/setup.py b/tests/data/src/compilewheel/setup.py index 8319a2a5c20..da994945048 100644 --- a/tests/data/src/compilewheel/setup.py +++ b/tests/data/src/compilewheel/setup.py @@ -1,7 +1,4 @@ #!/usr/bin/env python from setuptools import find_packages, setup -setup(name='compilewheel', - version='1.0', - packages=find_packages() - ) +setup(name="compilewheel", version="1.0", packages=find_packages()) diff --git a/tests/data/src/extension/setup.py b/tests/data/src/extension/setup.py index b26302b0536..83624965de5 100644 --- a/tests/data/src/extension/setup.py +++ b/tests/data/src/extension/setup.py @@ -1,4 +1,4 @@ from setuptools import Extension, setup -module = Extension('extension', sources=['extension.c']) -setup(name='extension', version='0.0.1', ext_modules = [module]) +module = Extension("extension", sources=["extension.c"]) +setup(name="extension", version="0.0.1", ext_modules=[module]) diff --git a/tests/data/src/pep517_setup_cfg_only/setup.cfg b/tests/data/src/pep517_setup_cfg_only/setup.cfg new file mode 100644 index 00000000000..4d62ef58d5a --- /dev/null +++ b/tests/data/src/pep517_setup_cfg_only/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +name = "dummy" +version = "0.1" diff --git a/tests/data/src/pep518-3.0/pep518.py b/tests/data/src/pep518-3.0/pep518.py index 7986d11379a..9ce06a81ea4 100644 --- a/tests/data/src/pep518-3.0/pep518.py +++ b/tests/data/src/pep518-3.0/pep518.py @@ -1 +1 @@ -#dummy +# dummy diff --git a/tests/data/src/pep518-3.0/setup.py b/tests/data/src/pep518-3.0/setup.py index b63f59926f7..587c04fc07d 100644 --- a/tests/data/src/pep518-3.0/setup.py +++ b/tests/data/src/pep518-3.0/setup.py @@ -3,7 +3,8 @@ import simplewheel # ensure dependency is installed -setup(name='pep518', - version='3.0', - py_modules=['pep518'], - ) +setup( + name="pep518", + version="3.0", + py_modules=["pep518"], +) diff --git a/tests/data/src/pep518_conflicting_requires/pep518.py b/tests/data/src/pep518_conflicting_requires/pep518.py index 7986d11379a..9ce06a81ea4 100644 --- a/tests/data/src/pep518_conflicting_requires/pep518.py +++ b/tests/data/src/pep518_conflicting_requires/pep518.py @@ -1 +1 @@ -#dummy +# dummy diff --git a/tests/data/src/pep518_conflicting_requires/setup.py b/tests/data/src/pep518_conflicting_requires/setup.py index 34bdc16b5aa..28f3db53d6e 100644 --- a/tests/data/src/pep518_conflicting_requires/setup.py +++ b/tests/data/src/pep518_conflicting_requires/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( - name='pep518_conflicting_requires', - version='1.0.0', - py_modules=['pep518'], + name="pep518_conflicting_requires", + version="1.0.0", + py_modules=["pep518"], ) diff --git a/tests/data/src/pep518_forkbomb-235/setup.py b/tests/data/src/pep518_forkbomb-235/setup.py index c8bc29287c9..f69346cac82 100644 --- a/tests/data/src/pep518_forkbomb-235/setup.py +++ b/tests/data/src/pep518_forkbomb-235/setup.py @@ -1,5 +1,3 @@ from setuptools import setup -setup(name='pep518_forkbomb', - version='235', - py_modules=['pep518_forkbomb']) +setup(name="pep518_forkbomb", version="235", py_modules=["pep518_forkbomb"]) diff --git a/tests/data/src/pep518_invalid_build_system/pep518.py b/tests/data/src/pep518_invalid_build_system/pep518.py index 7986d11379a..9ce06a81ea4 100644 --- a/tests/data/src/pep518_invalid_build_system/pep518.py +++ b/tests/data/src/pep518_invalid_build_system/pep518.py @@ -1 +1 @@ -#dummy +# dummy diff --git a/tests/data/src/pep518_invalid_build_system/setup.py b/tests/data/src/pep518_invalid_build_system/setup.py index ba23cf24ab2..de4161d3a04 100644 --- a/tests/data/src/pep518_invalid_build_system/setup.py +++ b/tests/data/src/pep518_invalid_build_system/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( - name='pep518_invalid_build_system', - version='1.0.0', - py_modules=['pep518'], + name="pep518_invalid_build_system", + version="1.0.0", + py_modules=["pep518"], ) diff --git a/tests/data/src/pep518_invalid_requires/pep518.py b/tests/data/src/pep518_invalid_requires/pep518.py index 7986d11379a..9ce06a81ea4 100644 --- a/tests/data/src/pep518_invalid_requires/pep518.py +++ b/tests/data/src/pep518_invalid_requires/pep518.py @@ -1 +1 @@ -#dummy +# dummy diff --git a/tests/data/src/pep518_invalid_requires/setup.py b/tests/data/src/pep518_invalid_requires/setup.py index e8a92da312a..ff6ac8b324d 100644 --- a/tests/data/src/pep518_invalid_requires/setup.py +++ b/tests/data/src/pep518_invalid_requires/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( - name='pep518_invalid_requires', - version='1.0.0', - py_modules=['pep518'], + name="pep518_invalid_requires", + version="1.0.0", + py_modules=["pep518"], ) diff --git a/tests/data/src/pep518_missing_requires/pep518.py b/tests/data/src/pep518_missing_requires/pep518.py index 7986d11379a..9ce06a81ea4 100644 --- a/tests/data/src/pep518_missing_requires/pep518.py +++ b/tests/data/src/pep518_missing_requires/pep518.py @@ -1 +1 @@ -#dummy +# dummy diff --git a/tests/data/src/pep518_missing_requires/setup.py b/tests/data/src/pep518_missing_requires/setup.py index cbc5d28af04..4d3c5f3e5f4 100644 --- a/tests/data/src/pep518_missing_requires/setup.py +++ b/tests/data/src/pep518_missing_requires/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( - name='pep518_missing_requires', - version='1.0.0', - py_modules=['pep518'], + name="pep518_missing_requires", + version="1.0.0", + py_modules=["pep518"], ) diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/setup.py b/tests/data/src/pep518_twin_forkbombs_first-234/setup.py index 55e9bbfb17d..acb97e18efc 100644 --- a/tests/data/src/pep518_twin_forkbombs_first-234/setup.py +++ b/tests/data/src/pep518_twin_forkbombs_first-234/setup.py @@ -1,5 +1,7 @@ from setuptools import setup -setup(name='pep518_twin_forkbombs_first', - version='234', - py_modules=['pep518_twin_forkbombs_first']) +setup( + name="pep518_twin_forkbombs_first", + version="234", + py_modules=["pep518_twin_forkbombs_first"], +) diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/setup.py b/tests/data/src/pep518_twin_forkbombs_second-238/setup.py index 985af51df82..c14c1cfb025 100644 --- a/tests/data/src/pep518_twin_forkbombs_second-238/setup.py +++ b/tests/data/src/pep518_twin_forkbombs_second-238/setup.py @@ -1,5 +1,7 @@ from setuptools import setup -setup(name='pep518_twin_forkbombs_second', - version='238', - py_modules=['pep518_twin_forkbombs_second']) +setup( + name="pep518_twin_forkbombs_second", + version="238", + py_modules=["pep518_twin_forkbombs_second"], +) diff --git a/tests/data/src/pep518_with_extra_and_markers-1.0/pep518_with_extra_and_markers.py b/tests/data/src/pep518_with_extra_and_markers-1.0/pep518_with_extra_and_markers.py index 7986d11379a..9ce06a81ea4 100644 --- a/tests/data/src/pep518_with_extra_and_markers-1.0/pep518_with_extra_and_markers.py +++ b/tests/data/src/pep518_with_extra_and_markers-1.0/pep518_with_extra_and_markers.py @@ -1 +1 @@ -#dummy +# dummy diff --git a/tests/data/src/pep518_with_extra_and_markers-1.0/setup.py b/tests/data/src/pep518_with_extra_and_markers-1.0/setup.py index 29a8175e4d1..bfac5b46783 100644 --- a/tests/data/src/pep518_with_extra_and_markers-1.0/setup.py +++ b/tests/data/src/pep518_with_extra_and_markers-1.0/setup.py @@ -1,15 +1,14 @@ #!/usr/bin/env python -import sys - from setuptools import setup # ensure dependencies are installed import simple import simplewheel -assert simplewheel.__version__ == '1.0' if sys.version_info < (3,) else '2.0' +assert simplewheel.__version__ == "2.0" -setup(name='pep518_with_extra_and_markers', - version='1.0', - py_modules=['pep518_with_extra_and_markers'], - ) +setup( + name="pep518_with_extra_and_markers", + version="1.0", + py_modules=["pep518_with_extra_and_markers"], +) diff --git a/tests/data/src/pep518_with_namespace_package-1.0/setup.py b/tests/data/src/pep518_with_namespace_package-1.0/setup.py index 540ede4cf43..263ba19880e 100644 --- a/tests/data/src/pep518_with_namespace_package-1.0/setup.py +++ b/tests/data/src/pep518_with_namespace_package-1.0/setup.py @@ -3,7 +3,7 @@ import simple_namespace.module setup( - name='pep518_with_namespace_package', - version='1.0', - py_modules=['pep518_with_namespace_package'], + name="pep518_with_namespace_package", + version="1.0", + py_modules=["pep518_with_namespace_package"], ) diff --git a/tests/data/src/prjwithdatafile/setup.py b/tests/data/src/prjwithdatafile/setup.py index 94863b57b01..e2c29e6d3a2 100755 --- a/tests/data/src/prjwithdatafile/setup.py +++ b/tests/data/src/prjwithdatafile/setup.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- from setuptools import setup setup( - name='prjwithdatafile', + name="prjwithdatafile", version="1.0", - packages=['prjwithdatafile'], + packages=["prjwithdatafile"], data_files=[ - (r'packages1', ['prjwithdatafile/README.txt']), - (r'packages2', ['prjwithdatafile/README.txt']) - ] + (r"packages1", ["prjwithdatafile/README.txt"]), + (r"packages2", ["prjwithdatafile/README.txt"]), + ], ) diff --git a/tests/data/src/requires_capitalized/setup.py b/tests/data/src/requires_capitalized/setup.py index b3f37b919a6..287704f8628 100644 --- a/tests/data/src/requires_capitalized/setup.py +++ b/tests/data/src/requires_capitalized/setup.py @@ -1,6 +1,3 @@ from setuptools import setup -setup(name='Requires_Capitalized', - version='0.1', - install_requires=['simple==1.0'] - ) +setup(name="Requires_Capitalized", version="0.1", install_requires=["simple==1.0"]) diff --git a/tests/data/src/requires_requires_capitalized/setup.py b/tests/data/src/requires_requires_capitalized/setup.py index d124d072846..8d099fddd21 100644 --- a/tests/data/src/requires_requires_capitalized/setup.py +++ b/tests/data/src/requires_requires_capitalized/setup.py @@ -1,6 +1,7 @@ from setuptools import setup -setup(name='requires_requires_capitalized', - version='1.0', - install_requires=['requires_Capitalized==0.1'] - ) +setup( + name="requires_requires_capitalized", + version="1.0", + install_requires=["requires_Capitalized==0.1"], +) diff --git a/tests/data/src/requires_simple/setup.py b/tests/data/src/requires_simple/setup.py index 57122041aed..5eebde770d2 100644 --- a/tests/data/src/requires_simple/setup.py +++ b/tests/data/src/requires_simple/setup.py @@ -1,6 +1,3 @@ from setuptools import find_packages, setup -setup(name='requires_simple', - version='0.1', - install_requires=['simple==1.0'] - ) +setup(name="requires_simple", version="0.1", install_requires=["simple==1.0"]) diff --git a/tests/data/src/requires_simple_extra/setup.py b/tests/data/src/requires_simple_extra/setup.py index 3378c2ce7d1..5562ebc9541 100644 --- a/tests/data/src/requires_simple_extra/setup.py +++ b/tests/data/src/requires_simple_extra/setup.py @@ -1,9 +1,8 @@ from setuptools import setup -setup(name='requires_simple_extra', - version='0.1', - py_modules=['requires_simple_extra'], - extras_require={ - 'extra': ['simple==1.0'] - } +setup( + name="requires_simple_extra", + version="0.1", + py_modules=["requires_simple_extra"], + extras_require={"extra": ["simple==1.0"]}, ) diff --git a/tests/data/src/setup_error/setup.py b/tests/data/src/setup_error/setup.py new file mode 100644 index 00000000000..d942355ca93 --- /dev/null +++ b/tests/data/src/setup_error/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +# This is to get an error that originates from setuptools, which generates a +# decently sized output. +setup( + cmdclass={ + "egg_info": "", + "install": "", + "bdist_wheel": "", + } +) diff --git a/tests/data/src/simple_namespace/setup.py b/tests/data/src/simple_namespace/setup.py index 9a49d52b757..c6b5978641c 100644 --- a/tests/data/src/simple_namespace/setup.py +++ b/tests/data/src/simple_namespace/setup.py @@ -1,8 +1,8 @@ from setuptools import setup setup( - name='simple_namespace', - version='1.0', - namespace_packages=['simple_namespace'], - packages=['simple_namespace.module'], + name="simple_namespace", + version="1.0", + namespace_packages=["simple_namespace"], + packages=["simple_namespace.module"], ) diff --git a/tests/data/src/simplewheel-1.0/setup.py b/tests/data/src/simplewheel-1.0/setup.py index 461536dce68..28ee850cd48 100644 --- a/tests/data/src/simplewheel-1.0/setup.py +++ b/tests/data/src/simplewheel-1.0/setup.py @@ -3,7 +3,8 @@ import simplewheel -setup(name='simplewheel', - version=simplewheel.__version__, - packages=['simplewheel'], - ) +setup( + name="simplewheel", + version=simplewheel.__version__, + packages=["simplewheel"], +) diff --git a/tests/data/src/simplewheel-1.0/simplewheel/__init__.py b/tests/data/src/simplewheel-1.0/simplewheel/__init__.py index 7e49527e386..4802e90f8ed 100644 --- a/tests/data/src/simplewheel-1.0/simplewheel/__init__.py +++ b/tests/data/src/simplewheel-1.0/simplewheel/__init__.py @@ -1 +1 @@ -__version__ = '1.0' +__version__ = "1.0" diff --git a/tests/data/src/simplewheel-2.0/setup.py b/tests/data/src/simplewheel-2.0/setup.py index 461536dce68..28ee850cd48 100644 --- a/tests/data/src/simplewheel-2.0/setup.py +++ b/tests/data/src/simplewheel-2.0/setup.py @@ -3,7 +3,8 @@ import simplewheel -setup(name='simplewheel', - version=simplewheel.__version__, - packages=['simplewheel'], - ) +setup( + name="simplewheel", + version=simplewheel.__version__, + packages=["simplewheel"], +) diff --git a/tests/data/src/simplewheel-2.0/simplewheel/__init__.py b/tests/data/src/simplewheel-2.0/simplewheel/__init__.py index 3b3dacb9af5..f2dc0e40061 100644 --- a/tests/data/src/simplewheel-2.0/simplewheel/__init__.py +++ b/tests/data/src/simplewheel-2.0/simplewheel/__init__.py @@ -1 +1 @@ -__version__ = '2.0' +__version__ = "2.0" diff --git a/tests/data/src/singlemodule/setup.py b/tests/data/src/singlemodule/setup.py index 622af1f8e99..e6358e2f7ca 100644 --- a/tests/data/src/singlemodule/setup.py +++ b/tests/data/src/singlemodule/setup.py @@ -2,7 +2,7 @@ setup( name="singlemodule", - version='0.0.1', + version="0.0.1", description="A sample Python project with a single module", - py_modules=['singlemodule'], + py_modules=["singlemodule"], ) diff --git a/tests/data/src/withpyproject/setup.py b/tests/data/src/withpyproject/setup.py index 1ea9e3e41c5..af10b3e3f37 100644 --- a/tests/data/src/withpyproject/setup.py +++ b/tests/data/src/withpyproject/setup.py @@ -1,3 +1,3 @@ from setuptools import setup -setup(name='withpyproject', version='0.0.1') +setup(name="withpyproject", version="0.0.1") diff --git a/tests/functional/test_bad_url.py b/tests/functional/test_bad_url.py new file mode 100644 index 00000000000..bc3a987e6f2 --- /dev/null +++ b/tests/functional/test_bad_url.py @@ -0,0 +1,16 @@ +# test the error message returned by pip when +# a bad "file:" URL is passed to it. + +from typing import Any + + +def test_filenotfound_error_message(script: Any) -> None: + # Test the error message returned when using a bad 'file:' URL. + # make pip to fail and get an error message + # by running "pip install -r file:nonexistent_file" + proc = script.pip("install", "-r", "file:unexistent_file", expect_error=True) + assert proc.returncode == 1 + expect = ( + "ERROR: 404 Client Error: FileNotFoundError for url: file:///unexistent_file" + ) + assert proc.stderr.rstrip() == expect diff --git a/tests/functional/test_broken_stdout.py b/tests/functional/test_broken_stdout.py index cb98e31f017..e7313244379 100644 --- a/tests/functional/test_broken_stdout.py +++ b/tests/functional/test_broken_stdout.py @@ -1,25 +1,28 @@ import os import subprocess -import sys +from typing import List, Tuple -if sys.version_info < (3, 6): - _BROKEN_STDOUT_RETURN_CODE = 1 -else: - # The new exit status was added in Python 3.6 as a result of: - # https://bugs.python.org/issue5319 - _BROKEN_STDOUT_RETURN_CODE = 120 +from tests.lib.path import Path +_BROKEN_STDOUT_RETURN_CODE = 120 -def setup_broken_stdout_test(args, deprecated_python): + +def setup_broken_stdout_test( + args: List[str], deprecated_python: bool +) -> Tuple[str, int]: proc = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) # Call close() on stdout to cause a broken pipe. + assert proc.stdout is not None proc.stdout.close() returncode = proc.wait() - stderr = proc.stderr.read().decode('utf-8') + assert proc.stderr is not None + stderr = proc.stderr.read().decode("utf-8") - expected_msg = 'ERROR: Pipe to stdout was broken' + expected_msg = "ERROR: Pipe to stdout was broken" if deprecated_python: assert expected_msg in stderr else: @@ -28,49 +31,51 @@ def setup_broken_stdout_test(args, deprecated_python): return stderr, returncode -def test_broken_stdout_pipe(deprecated_python): +def test_broken_stdout_pipe(deprecated_python: bool) -> None: """ Test a broken pipe to stdout. """ stderr, returncode = setup_broken_stdout_test( - ['pip', 'list'], deprecated_python=deprecated_python, + ["pip", "list"], + deprecated_python=deprecated_python, ) # Check that no traceback occurs. - assert 'raise BrokenStdoutLoggingError()' not in stderr - assert stderr.count('Traceback') == 0 + assert "raise BrokenStdoutLoggingError()" not in stderr + assert stderr.count("Traceback") == 0 assert returncode == _BROKEN_STDOUT_RETURN_CODE -def test_broken_stdout_pipe__log_option(deprecated_python, tmpdir): +def test_broken_stdout_pipe__log_option(deprecated_python: bool, tmpdir: Path) -> None: """ Test a broken pipe to stdout when --log is passed. """ - log_path = os.path.join(str(tmpdir), 'log.txt') + log_path = os.path.join(str(tmpdir), "log.txt") stderr, returncode = setup_broken_stdout_test( - ['pip', '--log', log_path, 'list'], + ["pip", "--log", log_path, "list"], deprecated_python=deprecated_python, ) # Check that no traceback occurs. - assert 'raise BrokenStdoutLoggingError()' not in stderr - assert stderr.count('Traceback') == 0 + assert "raise BrokenStdoutLoggingError()" not in stderr + assert stderr.count("Traceback") == 0 assert returncode == _BROKEN_STDOUT_RETURN_CODE -def test_broken_stdout_pipe__verbose(deprecated_python): +def test_broken_stdout_pipe__verbose(deprecated_python: bool) -> None: """ Test a broken pipe to stdout with verbose logging enabled. """ stderr, returncode = setup_broken_stdout_test( - ['pip', '-v', 'list'], deprecated_python=deprecated_python, + ["pip", "-vv", "list"], + deprecated_python=deprecated_python, ) # Check that a traceback occurs and that it occurs at most once. # We permit up to two because the exception can be chained. - assert 'raise BrokenStdoutLoggingError()' in stderr - assert 1 <= stderr.count('Traceback') <= 2 + assert "raise BrokenStdoutLoggingError()" in stderr + assert 1 <= stderr.count("Traceback") <= 2 assert returncode == _BROKEN_STDOUT_RETURN_CODE diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index 7a392f42646..285f21fda89 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -1,22 +1,30 @@ from textwrap import dedent +from typing import Optional import pytest from pip._internal.build_env import BuildEnvironment -from tests.lib import create_basic_wheel_for_package, make_test_finder +from tests.lib import ( + PipTestEnvironment, + TestPipResult, + create_basic_wheel_for_package, + make_test_finder, +) -def indent(text, prefix): - return '\n'.join((prefix if line else '') + line - for line in text.split('\n')) +def indent(text: str, prefix: str) -> str: + return "\n".join((prefix if line else "") + line for line in text.split("\n")) -def run_with_build_env(script, setup_script_contents, - test_script_contents=None): - build_env_script = script.scratch_path / 'build_env.py' +def run_with_build_env( + script: PipTestEnvironment, + setup_script_contents: str, + test_script_contents: Optional[str] = None, +) -> TestPipResult: + build_env_script = script.scratch_path / "build_env.py" build_env_script.write_text( dedent( - ''' + """ import subprocess import sys @@ -40,68 +48,75 @@ def run_with_build_env(script, setup_script_contents, finder = PackageFinder.create( link_collector=link_collector, selection_prefs=selection_prefs, + use_deprecated_html5lib=False, ) with global_tempdir_manager(): build_env = BuildEnvironment() - '''.format(scratch=str(script.scratch_path))) + - indent(dedent(setup_script_contents), ' ') + - indent( + """.format( + scratch=str(script.scratch_path) + ) + ) + + indent(dedent(setup_script_contents), " ") + + indent( dedent( - ''' + """ if len(sys.argv) > 1: with build_env: subprocess.check_call((sys.executable, sys.argv[1])) - ''' + """ ), - ' ' + " ", ) ) - args = ['python', build_env_script] + args = ["python", build_env_script] if test_script_contents is not None: - test_script = script.scratch_path / 'test.py' + test_script = script.scratch_path / "test.py" test_script.write_text(dedent(test_script_contents)) args.append(test_script) return script.run(*args) -def test_build_env_allow_empty_requirements_install(): +def test_build_env_allow_empty_requirements_install() -> None: + finder = make_test_finder() build_env = BuildEnvironment() - for prefix in ('normal', 'overlay'): - build_env.install_requirements(None, [], prefix, None) + for prefix in ("normal", "overlay"): + build_env.install_requirements( + finder, [], prefix, kind="Installing build dependencies" + ) -def test_build_env_allow_only_one_install(script): - create_basic_wheel_for_package(script, 'foo', '1.0') - create_basic_wheel_for_package(script, 'bar', '1.0') +def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None: + create_basic_wheel_for_package(script, "foo", "1.0") + create_basic_wheel_for_package(script, "bar", "1.0") finder = make_test_finder(find_links=[script.scratch_path]) build_env = BuildEnvironment() - for prefix in ('normal', 'overlay'): + for prefix in ("normal", "overlay"): build_env.install_requirements( - finder, ['foo'], prefix, - 'installing foo in {prefix}'.format(**locals())) + finder, ["foo"], prefix, kind=f"installing foo in {prefix}" + ) with pytest.raises(AssertionError): build_env.install_requirements( - finder, ['bar'], prefix, - 'installing bar in {prefix}'.format(**locals())) + finder, ["bar"], prefix, kind=f"installing bar in {prefix}" + ) with pytest.raises(AssertionError): build_env.install_requirements( - finder, [], prefix, - 'installing in {prefix}'.format(**locals())) + finder, [], prefix, kind=f"installing in {prefix}" + ) -def test_build_env_requirements_check(script): +def test_build_env_requirements_check(script: PipTestEnvironment) -> None: - create_basic_wheel_for_package(script, 'foo', '2.0') - create_basic_wheel_for_package(script, 'bar', '1.0') - create_basic_wheel_for_package(script, 'bar', '3.0') - create_basic_wheel_for_package(script, 'other', '0.5') + create_basic_wheel_for_package(script, "foo", "2.0") + create_basic_wheel_for_package(script, "bar", "1.0") + create_basic_wheel_for_package(script, "bar", "3.0") + create_basic_wheel_for_package(script, "other", "0.5") - script.pip_install_local('-f', script.scratch_path, 'foo', 'bar', 'other') + script.pip_install_local("-f", script.scratch_path, "foo", "bar", "other") run_with_build_env( script, - ''' + """ r = build_env.check_requirements(['foo', 'bar', 'other']) assert r == (set(), {'foo', 'bar', 'other'}), repr(r) @@ -110,13 +125,14 @@ def test_build_env_requirements_check(script): r = build_env.check_requirements(['foo>3.0', 'bar>=2.5']) assert r == (set(), {'foo>3.0', 'bar>=2.5'}), repr(r) - ''') + """, + ) run_with_build_env( script, - ''' + """ build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', - 'installing foo in normal') + kind='installing foo in normal') r = build_env.check_requirements(['foo', 'bar', 'other']) assert r == (set(), {'other'}), repr(r) @@ -126,15 +142,16 @@ def test_build_env_requirements_check(script): r = build_env.check_requirements(['foo>3.0', 'bar>=2.5']) assert r == ({('foo==2.0', 'foo>3.0')}, set()), repr(r) - ''') + """, + ) run_with_build_env( script, - ''' + """ build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', - 'installing foo in normal') + kind='installing foo in normal') build_env.install_requirements(finder, ['bar==1.0'], 'overlay', - 'installing foo in overlay') + kind='installing foo in overlay') r = build_env.check_requirements(['foo', 'bar', 'other']) assert r == (set(), {'other'}), repr(r) @@ -145,53 +162,54 @@ def test_build_env_requirements_check(script): r = build_env.check_requirements(['foo>3.0', 'bar>=2.5']) assert r == ({('bar==1.0', 'bar>=2.5'), ('foo==2.0', 'foo>3.0')}, \ set()), repr(r) - ''') + """, + ) -def test_build_env_overlay_prefix_has_priority(script): - create_basic_wheel_for_package(script, 'pkg', '2.0') - create_basic_wheel_for_package(script, 'pkg', '4.3') +def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> None: + create_basic_wheel_for_package(script, "pkg", "2.0") + create_basic_wheel_for_package(script, "pkg", "4.3") result = run_with_build_env( script, - ''' + """ build_env.install_requirements(finder, ['pkg==2.0'], 'overlay', - 'installing pkg==2.0 in overlay') + kind='installing pkg==2.0 in overlay') build_env.install_requirements(finder, ['pkg==4.3'], 'normal', - 'installing pkg==4.3 in normal') - ''', - ''' + kind='installing pkg==4.3 in normal') + """, + """ print(__import__('pkg').__version__) - ''') - assert result.stdout.strip() == '2.0', str(result) + """, + ) + assert result.stdout.strip() == "2.0", str(result) @pytest.mark.incompatible_with_test_venv -def test_build_env_isolation(script): +def test_build_env_isolation(script: PipTestEnvironment) -> None: # Create dummy `pkg` wheel. - pkg_whl = create_basic_wheel_for_package(script, 'pkg', '1.0') + pkg_whl = create_basic_wheel_for_package(script, "pkg", "1.0") # Install it to site packages. script.pip_install_local(pkg_whl) # And a copy in the user site. - script.pip_install_local('--ignore-installed', '--user', pkg_whl) + script.pip_install_local("--ignore-installed", "--user", pkg_whl) # And to another directory available through a .pth file. - target = script.scratch_path / 'pth_install' - script.pip_install_local('-t', target, pkg_whl) - (script.site_packages_path / 'build_requires.pth').write_text( - str(target) + '\n' - ) + target = script.scratch_path / "pth_install" + script.pip_install_local("-t", target, pkg_whl) + (script.site_packages_path / "build_requires.pth").write_text(str(target) + "\n") # And finally to yet another directory available through PYTHONPATH. - target = script.scratch_path / 'pypath_install' - script.pip_install_local('-t', target, pkg_whl) + target = script.scratch_path / "pypath_install" + script.pip_install_local("-t", target, pkg_whl) script.environ["PYTHONPATH"] = target run_with_build_env( - script, '', - r''' + script, + "", + r""" from distutils.sysconfig import get_python_lib import sys @@ -201,7 +219,7 @@ def test_build_env_isolation(script): pass else: print( - 'imported `pkg` from `{pkg.__file__}`'.format(**locals()), + f'imported `pkg` from `{pkg.__file__}`', file=sys.stderr) print('system sites:\n ' + '\n '.join(sorted({ get_python_lib(plat_specific=0), @@ -209,4 +227,5 @@ def test_build_env_isolation(script): })), file=sys.stderr) print('sys.path:\n ' + '\n '.join(sys.path), file=sys.stderr) sys.exit(1) - ''') + """, + ) diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index 0dc79108182..712e08a85cc 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -1,38 +1,41 @@ import os import shutil from glob import glob +from typing import Callable, List, Tuple import pytest +from tests.lib import PipTestEnvironment, TestPipResult + @pytest.fixture -def cache_dir(script): +def cache_dir(script: PipTestEnvironment) -> str: result = script.run( - 'python', '-c', - 'from pip._internal.locations import USER_CACHE_DIR;' - 'print(USER_CACHE_DIR)' + "python", + "-c", + "from pip._internal.locations import USER_CACHE_DIR;print(USER_CACHE_DIR)", ) return result.stdout.strip() @pytest.fixture -def http_cache_dir(cache_dir): - return os.path.normcase(os.path.join(cache_dir, 'http')) +def http_cache_dir(cache_dir: str) -> str: + return os.path.normcase(os.path.join(cache_dir, "http")) @pytest.fixture -def wheel_cache_dir(cache_dir): - return os.path.normcase(os.path.join(cache_dir, 'wheels')) +def wheel_cache_dir(cache_dir: str) -> str: + return os.path.normcase(os.path.join(cache_dir, "wheels")) @pytest.fixture -def http_cache_files(http_cache_dir): - destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname') +def http_cache_files(http_cache_dir: str) -> List[str]: + destination = os.path.join(http_cache_dir, "arbitrary", "pathname") if not os.path.exists(destination): return [] - filenames = glob(os.path.join(destination, '*')) + filenames = glob(os.path.join(destination, "*")) files = [] for filename in filenames: files.append(os.path.join(destination, filename)) @@ -40,13 +43,13 @@ def http_cache_files(http_cache_dir): @pytest.fixture -def wheel_cache_files(wheel_cache_dir): - destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') +def wheel_cache_files(wheel_cache_dir: str) -> List[str]: + destination = os.path.join(wheel_cache_dir, "arbitrary", "pathname") if not os.path.exists(destination): return [] - filenames = glob(os.path.join(destination, '*.whl')) + filenames = glob(os.path.join(destination, "*.whl")) files = [] for filename in filenames: files.append(os.path.join(destination, filename)) @@ -54,60 +57,60 @@ def wheel_cache_files(wheel_cache_dir): @pytest.fixture -def populate_http_cache(http_cache_dir): - destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname') +def populate_http_cache(http_cache_dir: str) -> List[Tuple[str, str]]: + destination = os.path.join(http_cache_dir, "arbitrary", "pathname") os.makedirs(destination) files = [ - ('aaaaaaaaa', os.path.join(destination, 'aaaaaaaaa')), - ('bbbbbbbbb', os.path.join(destination, 'bbbbbbbbb')), - ('ccccccccc', os.path.join(destination, 'ccccccccc')), + ("aaaaaaaaa", os.path.join(destination, "aaaaaaaaa")), + ("bbbbbbbbb", os.path.join(destination, "bbbbbbbbb")), + ("ccccccccc", os.path.join(destination, "ccccccccc")), ] for _name, filename in files: - with open(filename, 'w'): + with open(filename, "w"): pass return files @pytest.fixture -def populate_wheel_cache(wheel_cache_dir): - destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') +def populate_wheel_cache(wheel_cache_dir: str) -> List[Tuple[str, str]]: + destination = os.path.join(wheel_cache_dir, "arbitrary", "pathname") os.makedirs(destination) files = [ - ('yyy-1.2.3', os.path.join(destination, 'yyy-1.2.3-py3-none-any.whl')), - ('zzz-4.5.6', os.path.join(destination, 'zzz-4.5.6-py3-none-any.whl')), - ('zzz-4.5.7', os.path.join(destination, 'zzz-4.5.7-py3-none-any.whl')), - ('zzz-7.8.9', os.path.join(destination, 'zzz-7.8.9-py3-none-any.whl')), + ("yyy-1.2.3", os.path.join(destination, "yyy-1.2.3-py3-none-any.whl")), + ("zzz-4.5.6", os.path.join(destination, "zzz-4.5.6-py3-none-any.whl")), + ("zzz-4.5.7", os.path.join(destination, "zzz-4.5.7-py3-none-any.whl")), + ("zzz-7.8.9", os.path.join(destination, "zzz-7.8.9-py3-none-any.whl")), ] for _name, filename in files: - with open(filename, 'w'): + with open(filename, "w"): pass return files @pytest.fixture -def empty_wheel_cache(wheel_cache_dir): +def empty_wheel_cache(wheel_cache_dir: str) -> None: if os.path.exists(wheel_cache_dir): shutil.rmtree(wheel_cache_dir) -def list_matches_wheel(wheel_name, result): +def list_matches_wheel(wheel_name: str, result: TestPipResult) -> bool: """Returns True if any line in `result`, which should be the output of a `pip cache list` call, matches `wheel_name`. E.g., If wheel_name is `foo-1.2.3` it searches for a line starting with `- foo-1.2.3-py3-none-any.whl `.""" lines = result.stdout.splitlines() - expected = f' - {wheel_name}-py3-none-any.whl ' + expected = f" - {wheel_name}-py3-none-any.whl " return any(map(lambda l: l.startswith(expected), lines)) -def list_matches_wheel_abspath(wheel_name, result): +def list_matches_wheel_abspath(wheel_name: str, result: TestPipResult) -> bool: """Returns True if any line in `result`, which should be the output of a `pip cache list --format=abspath` call, is a valid path and belongs to `wheel_name`. @@ -115,13 +118,20 @@ def list_matches_wheel_abspath(wheel_name, result): E.g., If wheel_name is `foo-1.2.3` it searches for a line starting with `foo-1.2.3-py3-none-any.whl`.""" lines = result.stdout.splitlines() - expected = f'{wheel_name}-py3-none-any.whl' - return any(map(lambda l: os.path.basename(l).startswith(expected) - and os.path.exists(l), lines)) + expected = f"{wheel_name}-py3-none-any.whl" + return any( + map( + lambda l: os.path.basename(l).startswith(expected) and os.path.exists(l), + lines, + ) + ) + + +RemoveMatches = Callable[[str, TestPipResult], bool] @pytest.fixture -def remove_matches_http(http_cache_dir): +def remove_matches_http(http_cache_dir: str) -> RemoveMatches: """Returns True if any line in `result`, which should be the output of a `pip cache purge` call, matches `http_filename`. @@ -129,22 +139,25 @@ def remove_matches_http(http_cache_dir): `Removed /arbitrary/pathname/aaaaaaaaa`. """ - def _remove_matches_http(http_filename, result): + def _remove_matches_http(http_filename: str, result: TestPipResult) -> bool: lines = result.stdout.splitlines() # The "/arbitrary/pathname/" bit is an implementation detail of how # the `populate_http_cache` fixture is implemented. path = os.path.join( - http_cache_dir, 'arbitrary', 'pathname', http_filename, + http_cache_dir, + "arbitrary", + "pathname", + http_filename, ) - expected = f'Removed {path}' + expected = f"Removed {path}" return expected in lines return _remove_matches_http @pytest.fixture -def remove_matches_wheel(wheel_cache_dir): +def remove_matches_wheel(wheel_cache_dir: str) -> RemoveMatches: """Returns True if any line in `result`, which should be the output of a `pip cache remove`/`pip cache purge` call, matches `wheel_name`. @@ -152,214 +165,240 @@ def remove_matches_wheel(wheel_cache_dir): `Removed /arbitrary/pathname/foo-1.2.3-py3-none-any.whl`. """ - def _remove_matches_wheel(wheel_name, result): + def _remove_matches_wheel(wheel_name: str, result: TestPipResult) -> bool: lines = result.stdout.splitlines() - wheel_filename = f'{wheel_name}-py3-none-any.whl' + wheel_filename = f"{wheel_name}-py3-none-any.whl" # The "/arbitrary/pathname/" bit is an implementation detail of how # the `populate_wheel_cache` fixture is implemented. path = os.path.join( - wheel_cache_dir, 'arbitrary', 'pathname', wheel_filename, + wheel_cache_dir, + "arbitrary", + "pathname", + wheel_filename, ) - expected = f'Removed {path}' + expected = f"Removed {path}" return expected in lines return _remove_matches_wheel -def test_cache_dir(script, cache_dir): - result = script.pip('cache', 'dir') +def test_cache_dir(script: PipTestEnvironment, cache_dir: str) -> None: + result = script.pip("cache", "dir") assert os.path.normcase(cache_dir) == result.stdout.strip() -def test_cache_dir_too_many_args(script, cache_dir): - result = script.pip('cache', 'dir', 'aaa', expect_error=True) +def test_cache_dir_too_many_args(script: PipTestEnvironment, cache_dir: str) -> None: + result = script.pip("cache", "dir", "aaa", expect_error=True) - assert result.stdout == '' + assert result.stdout == "" # This would be `result.stderr == ...`, but pip prints deprecation # warnings on Python 2.7, so we check if the _line_ is in stderr. - assert 'ERROR: Too many arguments' in result.stderr.splitlines() + assert "ERROR: Too many arguments" in result.stderr.splitlines() @pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") def test_cache_info( - script, http_cache_dir, wheel_cache_dir, wheel_cache_files -): - result = script.pip('cache', 'info') - - assert ( - f'Package index page cache location: {http_cache_dir}' - in result.stdout - ) - assert f'Wheels location: {wheel_cache_dir}' in result.stdout + script: PipTestEnvironment, + http_cache_dir: str, + wheel_cache_dir: str, + wheel_cache_files: List[str], +) -> None: + result = script.pip("cache", "info") + + assert f"Package index page cache location: {http_cache_dir}" in result.stdout + assert f"Wheels location: {wheel_cache_dir}" in result.stdout num_wheels = len(wheel_cache_files) - assert f'Number of wheels: {num_wheels}' in result.stdout + assert f"Number of wheels: {num_wheels}" in result.stdout @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_list(script): +def test_cache_list(script: PipTestEnvironment) -> None: """Running `pip cache list` should return exactly what the populate_wheel_cache fixture adds.""" - result = script.pip('cache', 'list') + result = script.pip("cache", "list") - assert list_matches_wheel('yyy-1.2.3', result) - assert list_matches_wheel('zzz-4.5.6', result) - assert list_matches_wheel('zzz-4.5.7', result) - assert list_matches_wheel('zzz-7.8.9', result) + assert list_matches_wheel("yyy-1.2.3", result) + assert list_matches_wheel("zzz-4.5.6", result) + assert list_matches_wheel("zzz-4.5.7", result) + assert list_matches_wheel("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_list_abspath(script): +def test_cache_list_abspath(script: PipTestEnvironment) -> None: """Running `pip cache list --format=abspath` should return full paths of exactly what the populate_wheel_cache fixture adds.""" - result = script.pip('cache', 'list', '--format=abspath') + result = script.pip("cache", "list", "--format=abspath") - assert list_matches_wheel_abspath('yyy-1.2.3', result) - assert list_matches_wheel_abspath('zzz-4.5.6', result) - assert list_matches_wheel_abspath('zzz-4.5.7', result) - assert list_matches_wheel_abspath('zzz-7.8.9', result) + assert list_matches_wheel_abspath("yyy-1.2.3", result) + assert list_matches_wheel_abspath("zzz-4.5.6", result) + assert list_matches_wheel_abspath("zzz-4.5.7", result) + assert list_matches_wheel_abspath("zzz-7.8.9", result) @pytest.mark.usefixtures("empty_wheel_cache") -def test_cache_list_with_empty_cache(script): +def test_cache_list_with_empty_cache(script: PipTestEnvironment) -> None: """Running `pip cache list` with an empty cache should print "Nothing cached." and exit.""" - result = script.pip('cache', 'list') + result = script.pip("cache", "list") assert result.stdout == "Nothing cached.\n" @pytest.mark.usefixtures("empty_wheel_cache") -def test_cache_list_with_empty_cache_abspath(script): +def test_cache_list_with_empty_cache_abspath(script: PipTestEnvironment) -> None: """Running `pip cache list --format=abspath` with an empty cache should not print anything and exit.""" - result = script.pip('cache', 'list', '--format=abspath') + result = script.pip("cache", "list", "--format=abspath") assert result.stdout.strip() == "" -def test_cache_list_too_many_args(script): +@pytest.mark.usefixtures("empty_wheel_cache") +def test_cache_purge_with_empty_cache(script: PipTestEnvironment) -> None: + """Running `pip cache purge` with an empty cache should print a warning + and exit without an error code.""" + result = script.pip("cache", "purge", allow_stderr_warning=True) + assert result.stderr == "WARNING: No matching packages\n" + assert result.stdout == "Files removed: 0\n" + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_remove_with_bad_pattern(script: PipTestEnvironment) -> None: + """Running `pip cache remove` with a bad pattern should print a warning + and exit without an error code.""" + result = script.pip("cache", "remove", "aaa", allow_stderr_warning=True) + assert result.stderr == 'WARNING: No matching packages for pattern "aaa"\n' + assert result.stdout == "Files removed: 0\n" + + +def test_cache_list_too_many_args(script: PipTestEnvironment) -> None: """Passing `pip cache list` too many arguments should cause an error.""" - script.pip('cache', 'list', 'aaa', 'bbb', - expect_error=True) + script.pip("cache", "list", "aaa", "bbb", expect_error=True) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_list_name_match(script): +def test_cache_list_name_match(script: PipTestEnvironment) -> None: """Running `pip cache list zzz` should list zzz-4.5.6, zzz-4.5.7, zzz-7.8.9, but nothing else.""" - result = script.pip('cache', 'list', 'zzz', '--verbose') + result = script.pip("cache", "list", "zzz", "--verbose") - assert not list_matches_wheel('yyy-1.2.3', result) - assert list_matches_wheel('zzz-4.5.6', result) - assert list_matches_wheel('zzz-4.5.7', result) - assert list_matches_wheel('zzz-7.8.9', result) + assert not list_matches_wheel("yyy-1.2.3", result) + assert list_matches_wheel("zzz-4.5.6", result) + assert list_matches_wheel("zzz-4.5.7", result) + assert list_matches_wheel("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_list_name_match_abspath(script): +def test_cache_list_name_match_abspath(script: PipTestEnvironment) -> None: """Running `pip cache list zzz --format=abspath` should list paths of zzz-4.5.6, zzz-4.5.7, zzz-7.8.9, but nothing else.""" - result = script.pip('cache', 'list', 'zzz', '--format=abspath', - '--verbose') + result = script.pip("cache", "list", "zzz", "--format=abspath", "--verbose") - assert not list_matches_wheel_abspath('yyy-1.2.3', result) - assert list_matches_wheel_abspath('zzz-4.5.6', result) - assert list_matches_wheel_abspath('zzz-4.5.7', result) - assert list_matches_wheel_abspath('zzz-7.8.9', result) + assert not list_matches_wheel_abspath("yyy-1.2.3", result) + assert list_matches_wheel_abspath("zzz-4.5.6", result) + assert list_matches_wheel_abspath("zzz-4.5.7", result) + assert list_matches_wheel_abspath("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_list_name_and_version_match(script): +def test_cache_list_name_and_version_match(script: PipTestEnvironment) -> None: """Running `pip cache list zzz-4.5.6` should list zzz-4.5.6, but nothing else.""" - result = script.pip('cache', 'list', 'zzz-4.5.6', '--verbose') + result = script.pip("cache", "list", "zzz-4.5.6", "--verbose") - assert not list_matches_wheel('yyy-1.2.3', result) - assert list_matches_wheel('zzz-4.5.6', result) - assert not list_matches_wheel('zzz-4.5.7', result) - assert not list_matches_wheel('zzz-7.8.9', result) + assert not list_matches_wheel("yyy-1.2.3", result) + assert list_matches_wheel("zzz-4.5.6", result) + assert not list_matches_wheel("zzz-4.5.7", result) + assert not list_matches_wheel("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_list_name_and_version_match_abspath(script): +def test_cache_list_name_and_version_match_abspath(script: PipTestEnvironment) -> None: """Running `pip cache list zzz-4.5.6 --format=abspath` should list path of zzz-4.5.6, but nothing else.""" - result = script.pip('cache', 'list', 'zzz-4.5.6', '--format=abspath', - '--verbose') + result = script.pip("cache", "list", "zzz-4.5.6", "--format=abspath", "--verbose") - assert not list_matches_wheel_abspath('yyy-1.2.3', result) - assert list_matches_wheel_abspath('zzz-4.5.6', result) - assert not list_matches_wheel_abspath('zzz-4.5.7', result) - assert not list_matches_wheel_abspath('zzz-7.8.9', result) + assert not list_matches_wheel_abspath("yyy-1.2.3", result) + assert list_matches_wheel_abspath("zzz-4.5.6", result) + assert not list_matches_wheel_abspath("zzz-4.5.7", result) + assert not list_matches_wheel_abspath("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_remove_no_arguments(script): +def test_cache_remove_no_arguments(script: PipTestEnvironment) -> None: """Running `pip cache remove` with no arguments should cause an error.""" - script.pip('cache', 'remove', expect_error=True) + script.pip("cache", "remove", expect_error=True) -def test_cache_remove_too_many_args(script): +def test_cache_remove_too_many_args(script: PipTestEnvironment) -> None: """Passing `pip cache remove` too many arguments should cause an error.""" - script.pip('cache', 'remove', 'aaa', 'bbb', - expect_error=True) + script.pip("cache", "remove", "aaa", "bbb", expect_error=True) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_remove_name_match(script, remove_matches_wheel): +def test_cache_remove_name_match( + script: PipTestEnvironment, remove_matches_wheel: RemoveMatches +) -> None: """Running `pip cache remove zzz` should remove zzz-4.5.6 and zzz-7.8.9, but nothing else.""" - result = script.pip('cache', 'remove', 'zzz', '--verbose') + result = script.pip("cache", "remove", "zzz", "--verbose") - assert not remove_matches_wheel('yyy-1.2.3', result) - assert remove_matches_wheel('zzz-4.5.6', result) - assert remove_matches_wheel('zzz-4.5.7', result) - assert remove_matches_wheel('zzz-7.8.9', result) + assert not remove_matches_wheel("yyy-1.2.3", result) + assert remove_matches_wheel("zzz-4.5.6", result) + assert remove_matches_wheel("zzz-4.5.7", result) + assert remove_matches_wheel("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_remove_name_and_version_match(script, remove_matches_wheel): +def test_cache_remove_name_and_version_match( + script: PipTestEnvironment, remove_matches_wheel: RemoveMatches +) -> None: """Running `pip cache remove zzz-4.5.6` should remove zzz-4.5.6, but nothing else.""" - result = script.pip('cache', 'remove', 'zzz-4.5.6', '--verbose') + result = script.pip("cache", "remove", "zzz-4.5.6", "--verbose") - assert not remove_matches_wheel('yyy-1.2.3', result) - assert remove_matches_wheel('zzz-4.5.6', result) - assert not remove_matches_wheel('zzz-4.5.7', result) - assert not remove_matches_wheel('zzz-7.8.9', result) + assert not remove_matches_wheel("yyy-1.2.3", result) + assert remove_matches_wheel("zzz-4.5.6", result) + assert not remove_matches_wheel("zzz-4.5.7", result) + assert not remove_matches_wheel("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") -def test_cache_purge(script, remove_matches_http, remove_matches_wheel): +def test_cache_purge( + script: PipTestEnvironment, + remove_matches_http: RemoveMatches, + remove_matches_wheel: RemoveMatches, +) -> None: """Running `pip cache purge` should remove all cached http files and wheels.""" - result = script.pip('cache', 'purge', '--verbose') + result = script.pip("cache", "purge", "--verbose") - assert remove_matches_http('aaaaaaaaa', result) - assert remove_matches_http('bbbbbbbbb', result) - assert remove_matches_http('ccccccccc', result) + assert remove_matches_http("aaaaaaaaa", result) + assert remove_matches_http("bbbbbbbbb", result) + assert remove_matches_http("ccccccccc", result) - assert remove_matches_wheel('yyy-1.2.3', result) - assert remove_matches_wheel('zzz-4.5.6', result) - assert remove_matches_wheel('zzz-4.5.7', result) - assert remove_matches_wheel('zzz-7.8.9', result) + assert remove_matches_wheel("yyy-1.2.3", result) + assert remove_matches_wheel("zzz-4.5.6", result) + assert remove_matches_wheel("zzz-4.5.7", result) + assert remove_matches_wheel("zzz-7.8.9", result) @pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") def test_cache_purge_too_many_args( - script, http_cache_files, wheel_cache_files -): + script: PipTestEnvironment, + http_cache_files: List[str], + wheel_cache_files: List[str], +) -> None: """Running `pip cache purge aaa` should raise an error and remove no cached http files or wheels.""" - result = script.pip('cache', 'purge', 'aaa', '--verbose', - expect_error=True) - assert result.stdout == '' + result = script.pip("cache", "purge", "aaa", "--verbose", expect_error=True) + assert result.stdout == "" # This would be `result.stderr == ...`, but pip prints deprecation # warnings on Python 2.7, so we check if the _line_ is in stderr. - assert 'ERROR: Too many arguments' in result.stderr.splitlines() + assert "ERROR: Too many arguments" in result.stderr.splitlines() # Make sure nothing was deleted. for filename in http_cache_files + wheel_cache_files: @@ -367,12 +406,15 @@ def test_cache_purge_too_many_args( @pytest.mark.parametrize("command", ["info", "list", "remove", "purge"]) -def test_cache_abort_when_no_cache_dir(script, command): +def test_cache_abort_when_no_cache_dir( + script: PipTestEnvironment, command: str +) -> None: """Running any pip cache command when cache is disabled should abort and log an informative error""" - result = script.pip('cache', command, '--no-cache-dir', - expect_error=True) - assert result.stdout == '' + result = script.pip("cache", command, "--no-cache-dir", expect_error=True) + assert result.stdout == "" - assert ('ERROR: pip cache commands can not function' - ' since cache is disabled.' in result.stderr.splitlines()) + assert ( + "ERROR: pip cache commands can not function" + " since cache is disabled." in result.stderr.splitlines() + ) diff --git a/tests/functional/test_check.py b/tests/functional/test_check.py index 5cb41a97e72..e2b1c60ef3a 100644 --- a/tests/functional/test_check.py +++ b/tests/functional/test_check.py @@ -1,146 +1,158 @@ -from tests.lib import create_test_package_with_setup +from typing import Collection +from tests.lib import PipTestEnvironment, create_test_package_with_setup -def matches_expected_lines(string, expected_lines): + +def matches_expected_lines(string: str, expected_lines: Collection[str]) -> bool: # Ignore empty lines output_lines = list(filter(None, string.splitlines())) # We'll match the last n lines, given n lines to match. - last_few_output_lines = output_lines[-len(expected_lines):] + last_few_output_lines = output_lines[-len(expected_lines) :] # And order does not matter return set(last_few_output_lines) == set(expected_lines) -def test_basic_check_clean(script): - """On a clean environment, check should print a helpful message. - - """ - result = script.pip('check') +def test_basic_check_clean(script: PipTestEnvironment) -> None: + """On a clean environment, check should print a helpful message.""" + result = script.pip("check") - expected_lines = ( - "No broken requirements found.", - ) + expected_lines = ("No broken requirements found.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 0 -def test_basic_check_missing_dependency(script): +def test_basic_check_missing_dependency(script: PipTestEnvironment) -> None: # Setup a small project pkga_path = create_test_package_with_setup( script, - name='pkga', version='1.0', install_requires=['missing==0.1'], + name="pkga", + version="1.0", + install_requires=["missing==0.1"], ) # Let's install pkga without its dependency - res = script.pip('install', '--no-index', pkga_path, '--no-deps') + res = script.pip("install", "--no-index", pkga_path, "--no-deps") assert "Successfully installed pkga-1.0" in res.stdout, str(res) - result = script.pip('check', expect_error=True) + result = script.pip("check", expect_error=True) - expected_lines = ( - "pkga 1.0 requires missing, which is not installed.", - ) + expected_lines = ("pkga 1.0 requires missing, which is not installed.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 1 -def test_basic_check_broken_dependency(script): +def test_basic_check_broken_dependency(script: PipTestEnvironment) -> None: # Setup pkga depending on pkgb>=1.0 pkga_path = create_test_package_with_setup( script, - name='pkga', version='1.0', install_requires=['broken>=1.0'], + name="pkga", + version="1.0", + install_requires=["broken>=1.0"], ) # Let's install pkga without its dependency - res = script.pip('install', '--no-index', pkga_path, '--no-deps') + res = script.pip("install", "--no-index", pkga_path, "--no-deps") assert "Successfully installed pkga-1.0" in res.stdout, str(res) # Setup broken==0.1 broken_path = create_test_package_with_setup( script, - name='broken', version='0.1', + name="broken", + version="0.1", ) # Let's install broken==0.1 res = script.pip( - 'install', '--no-index', broken_path, '--no-warn-conflicts', + "install", + "--no-index", + broken_path, + "--no-warn-conflicts", ) assert "Successfully installed broken-0.1" in res.stdout, str(res) - result = script.pip('check', expect_error=True) + result = script.pip("check", expect_error=True) - expected_lines = ( - "pkga 1.0 has requirement broken>=1.0, but you have broken 0.1.", - ) + expected_lines = ("pkga 1.0 has requirement broken>=1.0, but you have broken 0.1.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 1 -def test_basic_check_broken_dependency_and_missing_dependency(script): +def test_basic_check_broken_dependency_and_missing_dependency( + script: PipTestEnvironment, +) -> None: pkga_path = create_test_package_with_setup( script, - name='pkga', version='1.0', install_requires=['broken>=1.0'], + name="pkga", + version="1.0", + install_requires=["broken>=1.0"], ) # Let's install pkga without its dependency - res = script.pip('install', '--no-index', pkga_path, '--no-deps') + res = script.pip("install", "--no-index", pkga_path, "--no-deps") assert "Successfully installed pkga-1.0" in res.stdout, str(res) # Setup broken==0.1 broken_path = create_test_package_with_setup( script, - name='broken', version='0.1', install_requires=['missing'], + name="broken", + version="0.1", + install_requires=["missing"], ) # Let's install broken==0.1 - res = script.pip('install', '--no-index', broken_path, '--no-deps') + res = script.pip("install", "--no-index", broken_path, "--no-deps") assert "Successfully installed broken-0.1" in res.stdout, str(res) - result = script.pip('check', expect_error=True) + result = script.pip("check", expect_error=True) expected_lines = ( "broken 0.1 requires missing, which is not installed.", - "pkga 1.0 has requirement broken>=1.0, but you have broken 0.1." + "pkga 1.0 has requirement broken>=1.0, but you have broken 0.1.", ) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 1 -def test_check_complicated_name_missing(script): +def test_check_complicated_name_missing(script: PipTestEnvironment) -> None: package_a_path = create_test_package_with_setup( script, - name='package_A', version='1.0', - install_requires=['Dependency-B>=1.0'], + name="package_A", + version="1.0", + install_requires=["Dependency-B>=1.0"], ) # Without dependency - result = script.pip('install', '--no-index', package_a_path, '--no-deps') + result = script.pip("install", "--no-index", package_a_path, "--no-deps") assert "Successfully installed package-A-1.0" in result.stdout, str(result) - result = script.pip('check', expect_error=True) - expected_lines = ( - "package-a 1.0 requires dependency-b, which is not installed.", - ) + result = script.pip("check", expect_error=True) + expected_lines = ("package-a 1.0 requires dependency-b, which is not installed.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 1 -def test_check_complicated_name_broken(script): +def test_check_complicated_name_broken(script: PipTestEnvironment) -> None: package_a_path = create_test_package_with_setup( script, - name='package_A', version='1.0', - install_requires=['Dependency-B>=1.0'], + name="package_A", + version="1.0", + install_requires=["Dependency-B>=1.0"], ) dependency_b_path_incompatible = create_test_package_with_setup( script, - name='dependency-b', version='0.1', + name="dependency-b", + version="0.1", ) # With broken dependency - result = script.pip('install', '--no-index', package_a_path, '--no-deps') + result = script.pip("install", "--no-index", package_a_path, "--no-deps") assert "Successfully installed package-A-1.0" in result.stdout, str(result) result = script.pip( - 'install', '--no-index', dependency_b_path_incompatible, '--no-deps', + "install", + "--no-index", + dependency_b_path_incompatible, + "--no-deps", ) assert "Successfully installed dependency-b-0.1" in result.stdout - result = script.pip('check', expect_error=True) + result = script.pip("check", expect_error=True) expected_lines = ( "package-a 1.0 has requirement Dependency-B>=1.0, but you have " "dependency-b 0.1.", @@ -149,101 +161,110 @@ def test_check_complicated_name_broken(script): assert result.returncode == 1 -def test_check_complicated_name_clean(script): +def test_check_complicated_name_clean(script: PipTestEnvironment) -> None: package_a_path = create_test_package_with_setup( script, - name='package_A', version='1.0', - install_requires=['Dependency-B>=1.0'], + name="package_A", + version="1.0", + install_requires=["Dependency-B>=1.0"], ) dependency_b_path = create_test_package_with_setup( script, - name='dependency-b', version='1.0', + name="dependency-b", + version="1.0", ) - result = script.pip('install', '--no-index', package_a_path, '--no-deps') + result = script.pip("install", "--no-index", package_a_path, "--no-deps") assert "Successfully installed package-A-1.0" in result.stdout, str(result) result = script.pip( - 'install', '--no-index', dependency_b_path, '--no-deps', + "install", + "--no-index", + dependency_b_path, + "--no-deps", ) assert "Successfully installed dependency-b-1.0" in result.stdout - result = script.pip('check') - expected_lines = ( - "No broken requirements found.", - ) + result = script.pip("check") + expected_lines = ("No broken requirements found.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 0 -def test_check_considers_conditional_reqs(script): +def test_check_considers_conditional_reqs(script: PipTestEnvironment) -> None: package_a_path = create_test_package_with_setup( script, - name='package_A', version='1.0', + name="package_A", + version="1.0", install_requires=[ "Dependency-B>=1.0; python_version != '2.7'", "Dependency-B>=2.0; python_version == '2.7'", ], ) - result = script.pip('install', '--no-index', package_a_path, '--no-deps') + result = script.pip("install", "--no-index", package_a_path, "--no-deps") assert "Successfully installed package-A-1.0" in result.stdout, str(result) - result = script.pip('check', expect_error=True) - expected_lines = ( - "package-a 1.0 requires dependency-b, which is not installed.", - ) + result = script.pip("check", expect_error=True) + expected_lines = ("package-a 1.0 requires dependency-b, which is not installed.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 1 -def test_check_development_versions_are_also_considered(script): +def test_check_development_versions_are_also_considered( + script: PipTestEnvironment, +) -> None: # Setup pkga depending on pkgb>=1.0 pkga_path = create_test_package_with_setup( script, - name='pkga', version='1.0', install_requires=['depend>=1.0'], + name="pkga", + version="1.0", + install_requires=["depend>=1.0"], ) # Let's install pkga without its dependency - res = script.pip('install', '--no-index', pkga_path, '--no-deps') + res = script.pip("install", "--no-index", pkga_path, "--no-deps") assert "Successfully installed pkga-1.0" in res.stdout, str(res) # Setup depend==1.1.0.dev0 depend_path = create_test_package_with_setup( script, - name='depend', version='1.1.0.dev0', + name="depend", + version="1.1.0.dev0", ) # Let's install depend==1.1.0.dev0 res = script.pip( - 'install', '--no-index', depend_path, '--no-warn-conflicts', + "install", + "--no-index", + depend_path, + "--no-warn-conflicts", ) assert "Successfully installed depend-1.1.0.dev0" in res.stdout, str(res) - result = script.pip('check') - expected_lines = ( - "No broken requirements found.", - ) + result = script.pip("check") + expected_lines = ("No broken requirements found.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 0 -def test_basic_check_broken_metadata(script): +def test_basic_check_broken_metadata(script: PipTestEnvironment) -> None: # Create some corrupt metadata - dist_info_dir = script.site_packages_path / 'pkga-1.0.dist-info' + dist_info_dir = script.site_packages_path / "pkga-1.0.dist-info" dist_info_dir.mkdir() - with open(dist_info_dir / 'METADATA', 'w') as f: - f.write('Metadata-Version: 2.1\n' - 'Name: pkga\n' - 'Version: 1.0\n' - 'Requires-Dist: pip; python_version == "3.4";extra == "test"\n' - ) + with open(dist_info_dir / "METADATA", "w") as f: + f.write( + "Metadata-Version: 2.1\n" + "Name: pkga\n" + "Version: 1.0\n" + 'Requires-Dist: pip; python_version == "3.4";extra == "test"\n' + ) - result = script.pip('check', expect_error=True) + result = script.pip("check", expect_error=True) - assert 'Error parsing requirements' in result.stderr + assert "Error parsing requirements" in result.stderr assert result.returncode == 1 -def test_check_skip_work_dir_pkg(script): +def test_check_skip_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that check should not include package present in working directory @@ -252,23 +273,20 @@ def test_check_skip_work_dir_pkg(script): # Create a test package with dependency missing # and create .egg-info dir pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0', - install_requires=['missing==0.1']) + script, name="simple", version="1.0", install_requires=["missing==0.1"] + ) - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) # Check should not complain about broken requirements # when run from package directory - result = script.pip('check', cwd=pkg_path) - expected_lines = ( - "No broken requirements found.", - ) + result = script.pip("check", cwd=pkg_path) + expected_lines = ("No broken requirements found.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 0 -def test_check_include_work_dir_pkg(script): +def test_check_include_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that check should include package in working directory if working directory is added in PYTHONPATH @@ -277,20 +295,17 @@ def test_check_include_work_dir_pkg(script): # Create a test package with dependency missing # and create .egg-info dir pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0', - install_requires=['missing==0.1']) + script, name="simple", version="1.0", install_requires=["missing==0.1"] + ) - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) - script.environ.update({'PYTHONPATH': pkg_path}) + script.environ.update({"PYTHONPATH": pkg_path}) # Check should mention about missing requirement simple # when run from package directory, when package directory # is in PYTHONPATH - result = script.pip('check', expect_error=True, cwd=pkg_path) - expected_lines = ( - "simple 1.0 requires missing, which is not installed.", - ) + result = script.pip("check", expect_error=True, cwd=pkg_path) + expected_lines = ("simple 1.0 requires missing, which is not installed.",) assert matches_expected_lines(result.stdout, expected_lines) assert result.returncode == 1 diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index e416315125f..3e8570359bb 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -4,16 +4,23 @@ import pytest +from tests.lib import PipTestEnvironment -@pytest.mark.parametrize("entrypoint", [ - ("fake_pip = pip._internal.main:main",), - ("fake_pip = pip._internal:main",), - ("fake_pip = pip:main",), -]) -def test_entrypoints_work(entrypoint, script): + +@pytest.mark.parametrize( + "entrypoint", + [ + ("fake_pip = pip._internal.main:main",), + ("fake_pip = pip._internal:main",), + ("fake_pip = pip:main",), + ], +) +def test_entrypoints_work(entrypoint: str, script: PipTestEnvironment) -> None: fake_pkg = script.temp_path / "fake_pkg" fake_pkg.mkdir() - fake_pkg.joinpath("setup.py").write_text(dedent(""" + fake_pkg.joinpath("setup.py").write_text( + dedent( + """ from setuptools import setup setup( @@ -25,9 +32,14 @@ def test_entrypoints_work(entrypoint, script): ] }} ) - """.format(entrypoint))) + """.format( + entrypoint + ) + ) + ) - script.pip("install", "-vvv", str(fake_pkg)) + # expect_temp because pip install will generate fake_pkg.egg-info + script.pip("install", "-vvv", str(fake_pkg), expect_temp=True) result = script.pip("-V") result2 = script.run("fake_pip", "-V", allow_stderr_warning=True) assert result.stdout == result2.stdout diff --git a/tests/functional/test_completion.py b/tests/functional/test_completion.py index a3986811b6f..27167ccf80d 100644 --- a/tests/functional/test_completion.py +++ b/tests/functional/test_completion.py @@ -1,20 +1,36 @@ import os import sys +from typing import TYPE_CHECKING, Optional, Tuple import pytest +from tests.conftest import ScriptFactory +from tests.lib import PipTestEnvironment, TestData, TestPipResult from tests.lib.path import Path +if TYPE_CHECKING: + from typing import Protocol +else: + # TODO: Protocol was introduced in Python 3.8. Remove this branch when + # dropping support for Python 3.7. + Protocol = object + + COMPLETION_FOR_SUPPORTED_SHELLS_TESTS = ( - ('bash', """\ + ( + "bash", + """\ _pip_completion() { COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ COMP_CWORD=$COMP_CWORD \\ PIP_AUTO_COMPLETE=1 $1 2>/dev/null ) ) } -complete -o default -F _pip_completion pip"""), - ('fish', """\ +complete -o default -F _pip_completion pip""", + ), + ( + "fish", + """\ function __fish_complete_pip set -lx COMP_WORDS (commandline -o) "" set -lx COMP_CWORD ( \\ @@ -23,8 +39,11 @@ set -lx PIP_AUTO_COMPLETE 1 string split \\ -- (eval $COMP_WORDS[1]) end -complete -fa "(__fish_complete_pip)" -c pip"""), - ('zsh', """\ +complete -fa "(__fish_complete_pip)" -c pip""", + ), + ( + "zsh", + """\ function _pip_completion { local words cword read -Ac words @@ -33,56 +52,72 @@ COMP_CWORD=$(( cword-1 )) \\ PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )) } -compctl -K _pip_completion pip"""), +compctl -K _pip_completion pip""", + ), ) @pytest.fixture(scope="session") def script_with_launchers( - tmpdir_factory, script_factory, common_wheels, pip_src -): + tmpdir_factory: pytest.TempdirFactory, + script_factory: ScriptFactory, + common_wheels: Path, + pip_src: Path, +) -> PipTestEnvironment: tmpdir = Path(str(tmpdir_factory.mktemp("script_with_launchers"))) script = script_factory(tmpdir.joinpath("workspace")) # Re-install pip so we get the launchers. - script.pip_install_local('-f', common_wheels, pip_src) + script.pip_install_local("-f", common_wheels, pip_src) return script @pytest.mark.parametrize( - 'shell, completion', + "shell, completion", COMPLETION_FOR_SUPPORTED_SHELLS_TESTS, ids=[t[0] for t in COMPLETION_FOR_SUPPORTED_SHELLS_TESTS], ) def test_completion_for_supported_shells( - script_with_launchers, shell, completion -): + script_with_launchers: PipTestEnvironment, shell: str, completion: str +) -> None: """ Test getting completion for bash shell """ - result = script_with_launchers.pip( - 'completion', '--' + shell, use_module=False - ) + result = script_with_launchers.pip("completion", "--" + shell, use_module=False) assert completion in result.stdout, str(result.stdout) @pytest.fixture(scope="session") -def autocomplete_script(tmpdir_factory, script_factory): +def autocomplete_script( + tmpdir_factory: pytest.TempdirFactory, script_factory: ScriptFactory +) -> PipTestEnvironment: tmpdir = Path(str(tmpdir_factory.mktemp("autocomplete_script"))) return script_factory(tmpdir.joinpath("workspace")) -@pytest.fixture -def autocomplete(autocomplete_script, monkeypatch): - monkeypatch.setattr(autocomplete_script, 'environ', os.environ.copy()) - autocomplete_script.environ['PIP_AUTO_COMPLETE'] = '1' +class DoAutocomplete(Protocol): + def __call__( + self, words: str, cword: str, cwd: Optional[str] = None + ) -> Tuple[TestPipResult, PipTestEnvironment]: + ... - def do_autocomplete(words, cword, cwd=None): - autocomplete_script.environ['COMP_WORDS'] = words - autocomplete_script.environ['COMP_CWORD'] = cword + +@pytest.fixture +def autocomplete( + autocomplete_script: PipTestEnvironment, monkeypatch: pytest.MonkeyPatch +) -> DoAutocomplete: + monkeypatch.setattr(autocomplete_script, "environ", os.environ.copy()) + autocomplete_script.environ["PIP_AUTO_COMPLETE"] = "1" + + def do_autocomplete( + words: str, cword: str, cwd: Optional[str] = None + ) -> Tuple[TestPipResult, PipTestEnvironment]: + autocomplete_script.environ["COMP_WORDS"] = words + autocomplete_script.environ["COMP_CWORD"] = cword result = autocomplete_script.run( - 'python', '-c', - 'from pip._internal.cli.autocompletion import autocomplete;' - 'autocomplete()', + "python", + "-c", + "from pip._internal.cli.autocompletion import autocomplete;" + "autocomplete()", expect_error=True, cwd=cwd, ) @@ -92,225 +127,251 @@ def do_autocomplete(words, cword, cwd=None): return do_autocomplete -def test_completion_for_unknown_shell(autocomplete_script): +def test_completion_for_unknown_shell(autocomplete_script: PipTestEnvironment) -> None: """ Test getting completion for an unknown shell """ - error_msg = 'no such option: --myfooshell' - result = autocomplete_script.pip( - 'completion', '--myfooshell', expect_error=True - ) - assert error_msg in result.stderr, 'tests for an unknown shell failed' + error_msg = "no such option: --myfooshell" + result = autocomplete_script.pip("completion", "--myfooshell", expect_error=True) + assert error_msg in result.stderr, "tests for an unknown shell failed" -def test_completion_alone(autocomplete_script): +def test_completion_alone(autocomplete_script: PipTestEnvironment) -> None: """ Test getting completion for none shell, just pip completion """ - result = autocomplete_script.pip('completion', allow_stderr_error=True) - assert 'ERROR: You must pass --bash or --fish or --zsh' in result.stderr, \ - 'completion alone failed -- ' + result.stderr + result = autocomplete_script.pip("completion", allow_stderr_error=True) + assert "ERROR: You must pass --bash or --fish or --zsh" in result.stderr, ( + "completion alone failed -- " + result.stderr + ) -def test_completion_for_un_snippet(autocomplete): +def test_completion_for_un_snippet(autocomplete: DoAutocomplete) -> None: """ Test getting completion for ``un`` should return uninstall """ - res, env = autocomplete('pip un', '1') - assert res.stdout.strip().split() == ['uninstall'], res.stdout + res, env = autocomplete("pip un", "1") + assert res.stdout.strip().split() == ["uninstall"], res.stdout -def test_completion_for_default_parameters(autocomplete): +def test_completion_for_default_parameters(autocomplete: DoAutocomplete) -> None: """ Test getting completion for ``--`` should contain --help """ - res, env = autocomplete('pip --', '1') - assert '--help' in res.stdout,\ - "autocomplete function could not complete ``--``" + res, env = autocomplete("pip --", "1") + assert "--help" in res.stdout, "autocomplete function could not complete ``--``" -def test_completion_option_for_command(autocomplete): +def test_completion_option_for_command(autocomplete: DoAutocomplete) -> None: """ Test getting completion for ``--`` in command (e.g. ``pip search --``) """ - res, env = autocomplete('pip search --', '2') - assert '--help' in res.stdout,\ - "autocomplete function could not complete ``--``" + res, env = autocomplete("pip search --", "2") + assert "--help" in res.stdout, "autocomplete function could not complete ``--``" -def test_completion_short_option(autocomplete): +def test_completion_short_option(autocomplete: DoAutocomplete) -> None: """ Test getting completion for short options after ``-`` (eg. pip -) """ - res, env = autocomplete('pip -', '1') + res, env = autocomplete("pip -", "1") - assert '-h' in res.stdout.split(),\ - "autocomplete function could not complete short options after ``-``" + assert ( + "-h" in res.stdout.split() + ), "autocomplete function could not complete short options after ``-``" -def test_completion_short_option_for_command(autocomplete): +def test_completion_short_option_for_command(autocomplete: DoAutocomplete) -> None: """ Test getting completion for short options after ``-`` in command (eg. pip search -) """ - res, env = autocomplete('pip search -', '2') + res, env = autocomplete("pip search -", "2") - assert '-h' in res.stdout.split(),\ - "autocomplete function could not complete short options after ``-``" + assert ( + "-h" in res.stdout.split() + ), "autocomplete function could not complete short options after ``-``" -def test_completion_files_after_option(autocomplete, data): +def test_completion_files_after_option( + autocomplete: DoAutocomplete, data: TestData +) -> None: """ Test getting completion for or after options in command (e.g. ``pip install -r``) """ res, env = autocomplete( - words=('pip install -r r'), - cword='3', + words=("pip install -r r"), + cword="3", cwd=data.completion_paths, ) - assert 'requirements.txt' in res.stdout, ( - "autocomplete function could not complete " - "after options in command" - ) - assert os.path.join('resources', '') in res.stdout, ( - "autocomplete function could not complete " - "after options in command" - ) - assert not any(out in res.stdout for out in - (os.path.join('REPLAY', ''), 'README.txt')), ( + assert ( + "requirements.txt" in res.stdout + ), "autocomplete function could not complete after options in command" + assert ( + os.path.join("resources", "") in res.stdout + ), "autocomplete function could not complete after options in command" + assert not any( + out in res.stdout for out in (os.path.join("REPLAY", ""), "README.txt") + ), ( "autocomplete function completed or that " "should not be completed" ) - if sys.platform != 'win32': + if sys.platform != "win32": return - assert 'readme.txt' in res.stdout, ( - "autocomplete function could not complete " - "after options in command" - ) - assert os.path.join('replay', '') in res.stdout, ( - "autocomplete function could not complete " - "after options in command" - ) + assert ( + "readme.txt" in res.stdout + ), "autocomplete function could not complete after options in command" + assert ( + os.path.join("replay", "") in res.stdout + ), "autocomplete function could not complete after options in command" -def test_completion_not_files_after_option(autocomplete, data): +def test_completion_not_files_after_option( + autocomplete: DoAutocomplete, data: TestData +) -> None: """ Test not getting completion files after options which not applicable - (e.g. ``pip install``) + (e.g. ``pip wheel``) """ res, env = autocomplete( - words=('pip install r'), - cword='2', + words=("pip wheel r"), + cword="2", cwd=data.completion_paths, ) - assert not any(out in res.stdout for out in - ('requirements.txt', 'readme.txt',)), ( - "autocomplete function completed when " - "it should not complete" - ) - assert not any(os.path.join(out, '') in res.stdout - for out in ('replay', 'resources')), ( - "autocomplete function completed when " - "it should not complete" + assert not any( + out in res.stdout + for out in ( + "requirements.txt", + "readme.txt", + ) + ), "autocomplete function completed when it should not complete" + assert not any( + os.path.join(out, "") in res.stdout for out in ("replay", "resources") + ), "autocomplete function completed when it should not complete" + + +def test_pip_install_complete_files( + autocomplete: DoAutocomplete, data: TestData +) -> None: + """``pip install`` autocompletes wheel and sdist files.""" + res, env = autocomplete( + words=("pip install r"), + cword="2", + cwd=data.completion_paths, ) + assert all( + out in res.stdout + for out in ( + "requirements.txt", + "resources", + ) + ), "autocomplete function could not complete " @pytest.mark.parametrize("cl_opts", ["-U", "--user", "-h"]) def test_completion_not_files_after_nonexpecting_option( - autocomplete, data, cl_opts -): + autocomplete: DoAutocomplete, data: TestData, cl_opts: str +) -> None: """ Test not getting completion files after options which not applicable (e.g. ``pip install``) """ res, env = autocomplete( - words=('pip install {cl_opts} r'.format(**locals())), - cword='2', + words=(f"pip install {cl_opts} r"), + cword="2", cwd=data.completion_paths, ) - assert not any(out in res.stdout for out in - ('requirements.txt', 'readme.txt',)), ( - "autocomplete function completed when " - "it should not complete" - ) - assert not any(os.path.join(out, '') in res.stdout - for out in ('replay', 'resources')), ( - "autocomplete function completed when " - "it should not complete" - ) + assert not any( + out in res.stdout + for out in ( + "requirements.txt", + "readme.txt", + ) + ), "autocomplete function completed when it should not complete" + assert not any( + os.path.join(out, "") in res.stdout for out in ("replay", "resources") + ), "autocomplete function completed when it should not complete" -def test_completion_directories_after_option(autocomplete, data): +def test_completion_directories_after_option( + autocomplete: DoAutocomplete, data: TestData +) -> None: """ Test getting completion after options in command (e.g. ``pip --cache-dir``) """ res, env = autocomplete( - words=('pip --cache-dir r'), - cword='2', + words=("pip --cache-dir r"), + cword="2", cwd=data.completion_paths, ) - assert os.path.join('resources', '') in res.stdout, ( - "autocomplete function could not complete after options" - ) - assert not any(out in res.stdout for out in ( - 'requirements.txt', 'README.txt', os.path.join('REPLAY', ''))), ( - "autocomplete function completed when " - "it should not complete" - ) - if sys.platform == 'win32': - assert os.path.join('replay', '') in res.stdout, ( - "autocomplete function could not complete after options" - ) - - -def test_completion_subdirectories_after_option(autocomplete, data): + assert ( + os.path.join("resources", "") in res.stdout + ), "autocomplete function could not complete after options" + assert not any( + out in res.stdout + for out in ("requirements.txt", "README.txt", os.path.join("REPLAY", "")) + ), "autocomplete function completed when it should not complete" + if sys.platform == "win32": + assert ( + os.path.join("replay", "") in res.stdout + ), "autocomplete function could not complete after options" + + +def test_completion_subdirectories_after_option( + autocomplete: DoAutocomplete, data: TestData +) -> None: """ Test getting completion after options in command given path of a directory """ res, env = autocomplete( - words=('pip --cache-dir ' + os.path.join('resources', '')), - cword='2', + words=("pip --cache-dir " + os.path.join("resources", "")), + cword="2", cwd=data.completion_paths, ) - assert os.path.join('resources', - os.path.join('images', '')) in res.stdout, ( + assert os.path.join("resources", os.path.join("images", "")) in res.stdout, ( "autocomplete function could not complete " "given path of a directory after options" ) -def test_completion_path_after_option(autocomplete, data): +def test_completion_path_after_option( + autocomplete: DoAutocomplete, data: TestData +) -> None: """ Test getting completion after options in command given absolute path """ res, env = autocomplete( - words=('pip install -e ' + os.path.join(data.completion_paths, 'R')), - cword='3', + words=("pip install -e " + os.path.join(data.completion_paths, "R")), + cword="3", ) - assert all(os.path.normcase(os.path.join(data.completion_paths, out)) - in res.stdout for out in ( - 'README.txt', os.path.join('REPLAY', ''))), ( + assert all( + os.path.normcase(os.path.join(data.completion_paths, out)) in res.stdout + for out in ("README.txt", os.path.join("REPLAY", "")) + ), ( "autocomplete function could not complete " "after options in command given absolute path" ) -@pytest.mark.parametrize('flag', ['--bash', '--zsh', '--fish']) +@pytest.mark.parametrize("flag", ["--bash", "--zsh", "--fish"]) def test_completion_uses_same_executable_name( - autocomplete_script, flag, deprecated_python -): - executable_name = 'pip{}'.format(sys.version_info[0]) + autocomplete_script: PipTestEnvironment, flag: str, deprecated_python: bool +) -> None: + executable_name = "pip{}".format(sys.version_info[0]) # Deprecated python versions produce an extra deprecation warning result = autocomplete_script.run( - executable_name, 'completion', flag, expect_stderr=deprecated_python, + executable_name, + "completion", + flag, + expect_stderr=deprecated_python, ) assert executable_name in result.stdout diff --git a/tests/functional/test_configuration.py b/tests/functional/test_configuration.py index 72c09bd3632..b3de3f697b0 100644 --- a/tests/functional/test_configuration.py +++ b/tests/functional/test_configuration.py @@ -1,36 +1,22 @@ """Tests for the config command """ - import re import textwrap -import pytest - from pip._internal.cli.status_codes import ERROR from pip._internal.configuration import CONFIG_BASENAME, get_configuration_files +from tests.lib import PipTestEnvironment from tests.lib.configuration_helpers import ConfigurationMixin, kinds +from tests.lib.venv import VirtualEnvironment -def test_no_options_passed_should_error(script): - result = script.pip('config', expect_error=True) +def test_no_options_passed_should_error(script: PipTestEnvironment) -> None: + result = script.pip("config", expect_error=True) assert result.returncode == ERROR class TestBasicLoading(ConfigurationMixin): - - @pytest.mark.skip("Can't modify underlying file for any mode") - def test_reads_file_appropriately(self, script): - contents = """ - [test] - hello = 1 - """ - - with self.patched_file(kinds.USER, contents): - result = script.pip("config", "list") - - assert "test.hello=1" in result.stdout - - def test_basic_modification_pipeline(self, script): + def test_basic_modification_pipeline(self, script: PipTestEnvironment) -> None: script.pip("config", "get", "test.blah", expect_error=True) script.pip("config", "set", "test.blah", "1") @@ -40,17 +26,16 @@ def test_basic_modification_pipeline(self, script): script.pip("config", "unset", "test.blah") script.pip("config", "get", "test.blah", expect_error=True) - def test_listing_is_correct(self, script): + def test_listing_is_correct(self, script: PipTestEnvironment) -> None: script.pip("config", "set", "test.listing-beta", "2") script.pip("config", "set", "test.listing-alpha", "1") script.pip("config", "set", "test.listing-gamma", "3") result = script.pip("config", "list") - lines = list(filter( - lambda x: x.startswith("test.listing-"), - result.stdout.splitlines() - )) + lines = list( + filter(lambda x: x.startswith("test.listing-"), result.stdout.splitlines()) + ) expected = """ test.listing-alpha='1' @@ -60,19 +45,18 @@ def test_listing_is_correct(self, script): assert lines == textwrap.dedent(expected).strip().splitlines() - def test_forget_section(self, script): - result = script.pip("config", "set", "isolated", "true", - expect_error=True) + def test_forget_section(self, script: PipTestEnvironment) -> None: + result = script.pip("config", "set", "isolated", "true", expect_error=True) assert "global.isolated" in result.stderr - def test_env_var_values(self, script): + def test_env_var_values(self, script: PipTestEnvironment) -> None: """Test that pip configuration set with environment variables is correctly displayed under "env_var". """ env_vars = { "PIP_DEFAULT_TIMEOUT": "60", - "PIP_FIND_LINKS": "http://mirror.example.com" + "PIP_FIND_LINKS": "http://mirror.example.com", } script.environ.update(env_vars) @@ -81,21 +65,25 @@ def test_env_var_values(self, script): assert "PIP_FIND_LINKS='http://mirror.example.com'" in result.stdout assert re.search(r"env_var:\n( .+\n)+", result.stdout) - def test_env_values(self, script): + def test_env_values(self, script: PipTestEnvironment) -> None: """Test that custom pip configuration using the environment variable PIP_CONFIG_FILE is correctly displayed under "env". This configuration takes place of per-user configuration file displayed under "user". """ config_file = script.scratch_path / "test-pip.cfg" - script.environ['PIP_CONFIG_FILE'] = str(config_file) - config_file.write_text(textwrap.dedent("""\ + script.environ["PIP_CONFIG_FILE"] = str(config_file) + config_file.write_text( + textwrap.dedent( + """\ [global] timeout = 60 [freeze] timeout = 10 - """)) + """ + ) + ) result = script.pip("config", "debug") assert f"{config_file}, exists: True" in result.stdout @@ -103,7 +91,7 @@ def test_env_values(self, script): assert "freeze.timeout: 10" in result.stdout assert re.search(r"env:\n( .+\n)+", result.stdout) - def test_user_values(self, script,): + def test_user_values(self, script: PipTestEnvironment) -> None: """Test that the user pip configuration set using --user is correctly displayed under "user". This configuration takes place of custom path location using the environment variable PIP_CONFIG_FILE @@ -122,7 +110,9 @@ def test_user_values(self, script,): assert "freeze.timeout: 10" in result.stdout assert re.search(r"user:\n( .+\n)+", result.stdout) - def test_site_values(self, script, virtualenv): + def test_site_values( + self, script: PipTestEnvironment, virtualenv: VirtualEnvironment + ) -> None: """Test that the current environment configuration set using --site is correctly displayed under "site". """ @@ -139,7 +129,7 @@ def test_site_values(self, script, virtualenv): assert "freeze.timeout: 10" in result.stdout assert re.search(r"site:\n( .+\n)+", result.stdout) - def test_global_config_file(self, script): + def test_global_config_file(self, script: PipTestEnvironment) -> None: """Test that the system-wide configuration can be identified""" # We cannot write to system-wide files which might have permissions @@ -150,3 +140,10 @@ def test_global_config_file(self, script): global_config_file = get_configuration_files()[kinds.GLOBAL][0] result = script.pip("config", "debug") assert f"{global_config_file}, exists:" in result.stdout + + def test_editor_does_not_exist(self, script: PipTestEnvironment) -> None: + """Ensure that FileNotFoundError sets filename correctly""" + result = script.pip( + "config", "edit", "--editor", "notrealeditor", expect_error=True + ) + assert "notrealeditor" in result.stderr diff --git a/tests/functional/test_debug.py b/tests/functional/test_debug.py index 0e2261e1ae0..41374f8cb88 100644 --- a/tests/functional/test_debug.py +++ b/tests/functional/test_debug.py @@ -1,86 +1,93 @@ +from typing import List + import pytest from pip._internal.commands.debug import create_vendor_txt_map from pip._internal.utils import compatibility_tags +from tests.lib import PipTestEnvironment -@pytest.mark.parametrize('expected_text', [ - 'sys.executable: ', - 'sys.getdefaultencoding: ', - 'sys.getfilesystemencoding: ', - 'locale.getpreferredencoding: ', - 'sys.platform: ', - 'sys.implementation:', - '\'cert\' config value: ', - 'REQUESTS_CA_BUNDLE: ', - 'CURL_CA_BUNDLE: ', - 'pip._vendor.certifi.where(): ', - 'pip._vendor.DEBUNDLED: ', - 'vendored library versions:', - -]) -def test_debug(script, expected_text): +@pytest.mark.parametrize( + "expected_text", + [ + "sys.executable: ", + "sys.getdefaultencoding: ", + "sys.getfilesystemencoding: ", + "locale.getpreferredencoding: ", + "sys.platform: ", + "sys.implementation:", + "'cert' config value: ", + "REQUESTS_CA_BUNDLE: ", + "CURL_CA_BUNDLE: ", + "pip._vendor.certifi.where(): ", + "pip._vendor.DEBUNDLED: ", + "vendored library versions:", + ], +) +def test_debug(script: PipTestEnvironment, expected_text: str) -> None: """ Check that certain strings are present in the output. """ - args = ['debug'] + args = ["debug"] result = script.pip(*args, allow_stderr_warning=True) stdout = result.stdout assert expected_text in stdout -def test_debug__library_versions(script): +def test_debug__library_versions(script: PipTestEnvironment) -> None: """ Check the library versions normal output. """ - args = ['debug'] + args = ["debug"] result = script.pip(*args, allow_stderr_warning=True) print(result.stdout) vendored_versions = create_vendor_txt_map() for name, value in vendored_versions.items(): - assert f'{name}=={value}' in result.stdout + assert f"{name}=={value}" in result.stdout @pytest.mark.parametrize( - 'args', + "args", [ [], - ['--verbose'], - ] + ["--verbose"], + ], ) -def test_debug__tags(script, args): +def test_debug__tags(script: PipTestEnvironment, args: List[str]) -> None: """ Check the compatible tag output. """ - args = ['debug'] + args + args = ["debug"] + args result = script.pip(*args, allow_stderr_warning=True) stdout = result.stdout tags = compatibility_tags.get_supported() - expected_tag_header = 'Compatible tags: {}'.format(len(tags)) + expected_tag_header = "Compatible tags: {}".format(len(tags)) assert expected_tag_header in stdout - show_verbose_note = '--verbose' not in args + show_verbose_note = "--verbose" not in args assert ( - '...\n [First 10 tags shown. Pass --verbose to show all.]' in stdout + "...\n [First 10 tags shown. Pass --verbose to show all.]" in stdout ) == show_verbose_note @pytest.mark.parametrize( - 'args, expected', + "args, expected", [ - (['--python-version', '3.7'], "(target: version_info='3.7')"), - ] + (["--python-version", "3.7"], "(target: version_info='3.7')"), + ], ) -def test_debug__target_options(script, args, expected): +def test_debug__target_options( + script: PipTestEnvironment, args: List[str], expected: str +) -> None: """ Check passing target-related options. """ - args = ['debug'] + args + args = ["debug"] + args result = script.pip(*args, allow_stderr_warning=True) stdout = result.stdout - assert 'Compatible tags: ' in stdout + assert "Compatible tags: " in stdout assert expected in stdout diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 95f1b63cf83..ace2ff74c5b 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -2,17 +2,19 @@ import shutil import textwrap from hashlib import sha256 +from typing import List import pytest from pip._internal.cli.status_codes import ERROR from pip._internal.utils.urls import path_to_url -from tests.lib import create_really_basic_wheel +from tests.conftest import MockServer, ScriptFactory +from tests.lib import PipTestEnvironment, TestData, create_really_basic_wheel from tests.lib.path import Path from tests.lib.server import file_response -def fake_wheel(data, wheel_path): +def fake_wheel(data: TestData, wheel_path: str) -> None: wheel_name = os.path.basename(wheel_path) name, version, rest = wheel_name.split("-", 2) wheel_data = create_really_basic_wheel(name, version) @@ -20,308 +22,376 @@ def fake_wheel(data, wheel_path): @pytest.mark.network -def test_download_if_requested(script): +def test_download_if_requested(script: PipTestEnvironment) -> None: """ It should download (in the scratch path) and not install if requested. """ - result = script.pip( - 'download', '-d', 'pip_downloads', 'INITools==0.1' - ) - result.did_create( - Path('scratch') / 'pip_downloads' / 'INITools-0.1.tar.gz' - ) - result.did_not_create(script.site_packages / 'initools') + result = script.pip("download", "-d", "pip_downloads", "INITools==0.1") + result.did_create(Path("scratch") / "pip_downloads" / "INITools-0.1.tar.gz") + result.did_not_create(script.site_packages / "initools") @pytest.mark.network -def test_basic_download_setuptools(script): +def test_basic_download_setuptools(script: PipTestEnvironment) -> None: """ It should download (in the scratch path) and not install if requested. """ - result = script.pip('download', 'setuptools') - setuptools_prefix = str(Path('scratch') / 'setuptools') - assert any( - path.startswith(setuptools_prefix) for path in result.files_created - ) + result = script.pip("download", "setuptools") + setuptools_prefix = str(Path("scratch") / "setuptools") + assert any(path.startswith(setuptools_prefix) for path in result.files_created) -def test_download_wheel(script, data): +def test_download_wheel(script: PipTestEnvironment, data: TestData) -> None: """ Test using "pip download" to download a *.whl archive. """ result = script.pip( - 'download', - '--no-index', - '-f', data.packages, - '-d', '.', 'meta' + "download", "--no-index", "-f", data.packages, "-d", ".", "meta" ) - result.did_create(Path('scratch') / 'meta-1.0-py2.py3-none-any.whl') - result.did_not_create(script.site_packages / 'piptestpackage') + result.did_create(Path("scratch") / "meta-1.0-py2.py3-none-any.whl") + result.did_not_create(script.site_packages / "piptestpackage") @pytest.mark.network -def test_single_download_from_requirements_file(script): +def test_single_download_from_requirements_file(script: PipTestEnvironment) -> None: """ It should support download (in the scratch path) from PyPI from a requirements file """ - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ INITools==0.1 - """)) + """ + ) + ) result = script.pip( - 'download', '-r', script.scratch_path / 'test-req.txt', '-d', '.', + "download", + "-r", + script.scratch_path / "test-req.txt", + "-d", + ".", ) - result.did_create(Path('scratch') / 'INITools-0.1.tar.gz') - result.did_not_create(script.site_packages / 'initools') + result.did_create(Path("scratch") / "INITools-0.1.tar.gz") + result.did_not_create(script.site_packages / "initools") @pytest.mark.network -def test_basic_download_should_download_dependencies(script): +def test_basic_download_should_download_dependencies( + script: PipTestEnvironment, +) -> None: """ It should download dependencies (in the scratch path) """ - result = script.pip( - 'download', 'Paste[openid]==1.7.5.1', '-d', '.' - ) - result.did_create(Path('scratch') / 'Paste-1.7.5.1.tar.gz') - openid_tarball_prefix = str(Path('scratch') / 'python-openid-') - assert any( - path.startswith(openid_tarball_prefix) for path in result.files_created - ) - result.did_not_create(script.site_packages / 'openid') + result = script.pip("download", "Paste[openid]==1.7.5.1", "-d", ".") + result.did_create(Path("scratch") / "Paste-1.7.5.1.tar.gz") + openid_tarball_prefix = str(Path("scratch") / "python-openid-") + assert any(path.startswith(openid_tarball_prefix) for path in result.files_created) + result.did_not_create(script.site_packages / "openid") -def test_download_wheel_archive(script, data): +def test_download_wheel_archive(script: PipTestEnvironment, data: TestData) -> None: """ It should download a wheel archive path """ - wheel_filename = 'colander-0.9.9-py2.py3-none-any.whl' - wheel_path = '/'.join((data.find_links, wheel_filename)) - result = script.pip( - 'download', wheel_path, - '-d', '.', '--no-deps' - ) - result.did_create(Path('scratch') / wheel_filename) + wheel_filename = "colander-0.9.9-py2.py3-none-any.whl" + wheel_path = "/".join((data.find_links, wheel_filename)) + result = script.pip("download", wheel_path, "-d", ".", "--no-deps") + result.did_create(Path("scratch") / wheel_filename) -def test_download_should_download_wheel_deps(script, data): +def test_download_should_download_wheel_deps( + script: PipTestEnvironment, data: TestData +) -> None: """ It should download dependencies for wheels(in the scratch path) """ - wheel_filename = 'colander-0.9.9-py2.py3-none-any.whl' - dep_filename = 'translationstring-1.1.tar.gz' - wheel_path = '/'.join((data.find_links, wheel_filename)) + wheel_filename = "colander-0.9.9-py2.py3-none-any.whl" + dep_filename = "translationstring-1.1.tar.gz" + wheel_path = "/".join((data.find_links, wheel_filename)) result = script.pip( - 'download', wheel_path, - '-d', '.', '--find-links', data.find_links, '--no-index' + "download", wheel_path, "-d", ".", "--find-links", data.find_links, "--no-index" ) - result.did_create(Path('scratch') / wheel_filename) - result.did_create(Path('scratch') / dep_filename) + result.did_create(Path("scratch") / wheel_filename) + result.did_create(Path("scratch") / dep_filename) @pytest.mark.network -def test_download_should_skip_existing_files(script): +def test_download_should_skip_existing_files(script: PipTestEnvironment) -> None: """ It should not download files already existing in the scratch dir """ - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ INITools==0.1 - """)) + """ + ) + ) result = script.pip( - 'download', '-r', script.scratch_path / 'test-req.txt', '-d', '.', + "download", + "-r", + script.scratch_path / "test-req.txt", + "-d", + ".", ) - result.did_create(Path('scratch') / 'INITools-0.1.tar.gz') - result.did_not_create(script.site_packages / 'initools') + result.did_create(Path("scratch") / "INITools-0.1.tar.gz") + result.did_not_create(script.site_packages / "initools") # adding second package to test-req.txt - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ INITools==0.1 python-openid==2.2.5 - """)) + """ + ) + ) # only the second package should be downloaded result = script.pip( - 'download', '-r', script.scratch_path / 'test-req.txt', '-d', '.', - ) - openid_tarball_prefix = str(Path('scratch') / 'python-openid-') - assert any( - path.startswith(openid_tarball_prefix) for path in result.files_created + "download", + "-r", + script.scratch_path / "test-req.txt", + "-d", + ".", ) - result.did_not_create(Path('scratch') / 'INITools-0.1.tar.gz') - result.did_not_create(script.site_packages / 'initools') - result.did_not_create(script.site_packages / 'openid') + openid_tarball_prefix = str(Path("scratch") / "python-openid-") + assert any(path.startswith(openid_tarball_prefix) for path in result.files_created) + result.did_not_create(Path("scratch") / "INITools-0.1.tar.gz") + result.did_not_create(script.site_packages / "initools") + result.did_not_create(script.site_packages / "openid") @pytest.mark.network -def test_download_vcs_link(script): +def test_download_vcs_link(script: PipTestEnvironment) -> None: """ It should allow -d flag for vcs links, regression test for issue #798. """ result = script.pip( - 'download', '-d', '.', 'git+git://github.com/pypa/pip-test-package.git' + "download", "-d", ".", "git+https://github.com/pypa/pip-test-package.git" ) - result.did_create(Path('scratch') / 'pip-test-package-0.1.1.zip') - result.did_not_create(script.site_packages / 'piptestpackage') + result.did_create(Path("scratch") / "pip-test-package-0.1.1.zip") + result.did_not_create(script.site_packages / "piptestpackage") -def test_only_binary_set_then_download_specific_platform(script, data): +def test_only_binary_set_then_download_specific_platform( + script: PipTestEnvironment, data: TestData +) -> None: """ Confirm that specifying an interpreter/platform constraint is allowed when ``--only-binary=:all:`` is set. """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") -def test_no_deps_set_then_download_specific_platform(script, data): +def test_no_deps_set_then_download_specific_platform( + script: PipTestEnvironment, data: TestData +) -> None: """ Confirm that specifying an interpreter/platform constraint is allowed when ``--no-deps`` is set. """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--no-deps', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--no-deps", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") -def test_download_specific_platform_fails(script, data): +def test_download_specific_platform_fails( + script: PipTestEnvironment, data: TestData +) -> None: """ Confirm that specifying an interpreter/platform constraint enforces that ``--no-deps`` or ``--only-binary=:all:`` is set. """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake", expect_error=True, ) - assert '--only-binary=:all:' in result.stderr + assert "--only-binary=:all:" in result.stderr -def test_no_binary_set_then_download_specific_platform_fails(script, data): +def test_no_binary_set_then_download_specific_platform_fails( + script: PipTestEnvironment, data: TestData +) -> None: """ Confirm that specifying an interpreter/platform constraint enforces that ``--only-binary=:all:`` is set without ``--no-binary``. """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--no-binary=fake', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--no-binary=fake", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake", expect_error=True, ) - assert '--only-binary=:all:' in result.stderr + assert "--only-binary=:all:" in result.stderr -def test_download_specify_platform(script, data): +def test_download_specify_platform(script: PipTestEnvironment, data: TestData) -> None: """ Test using "pip download --platform" to download a .whl archive supported for a specific platform """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") # Confirm that universal wheels are returned even for specific # platforms. result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'macosx_10_9_x86_64', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "macosx_10_9_x86_64", + "fake", ) data.reset() - fake_wheel(data, 'fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl') - fake_wheel(data, 'fake-2.0-py2.py3-none-linux_x86_64.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl") + fake_wheel(data, "fake-2.0-py2.py3-none-linux_x86_64.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'macosx_10_10_x86_64', - 'fake' - ) - result.did_create( - Path('scratch') / - 'fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "macosx_10_10_x86_64", + "fake", ) + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-macosx_10_9_x86_64.whl") # OSX platform wheels are not backward-compatible. result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'macosx_10_8_x86_64', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "macosx_10_8_x86_64", + "fake", expect_error=True, ) # No linux wheel provided for this version. result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake==1', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake==1", expect_error=True, ) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake==2' - ) - result.did_create( - Path('scratch') / 'fake-2.0-py2.py3-none-linux_x86_64.whl' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake==2", ) + result.did_create(Path("scratch") / "fake-2.0-py2.py3-none-linux_x86_64.whl") # Test with multiple supported platforms specified. data.reset() - fake_wheel(data, 'fake-3.0-py2.py3-none-linux_x86_64.whl') + fake_wheel(data, "fake-3.0-py2.py3-none-linux_x86_64.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'manylinux1_x86_64', '--platform', 'linux_x86_64', - '--platform', 'any', - 'fake==3' - ) - result.did_create( - Path('scratch') / 'fake-3.0-py2.py3-none-linux_x86_64.whl' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "manylinux1_x86_64", + "--platform", + "linux_x86_64", + "--platform", + "any", + "fake==3", ) + result.did_create(Path("scratch") / "fake-3.0-py2.py3-none-linux_x86_64.whl") class TestDownloadPlatformManylinuxes: @@ -330,148 +400,219 @@ class TestDownloadPlatformManylinuxes: manylinux platforms. """ - @pytest.mark.parametrize("platform", [ - "linux_x86_64", - "manylinux1_x86_64", - "manylinux2010_x86_64", - "manylinux2014_x86_64", - ]) - def test_download_universal(self, platform, script, data): + @pytest.mark.parametrize( + "platform", + [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + ], + ) + def test_download_universal( + self, platform: str, script: PipTestEnvironment, data: TestData + ) -> None: """ Universal wheels are returned even for specific platforms. """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', platform, - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + platform, + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') - - @pytest.mark.parametrize("wheel_abi,platform", [ - ("manylinux1_x86_64", "manylinux1_x86_64"), - ("manylinux1_x86_64", "manylinux2010_x86_64"), - ("manylinux2010_x86_64", "manylinux2010_x86_64"), - ("manylinux1_x86_64", "manylinux2014_x86_64"), - ("manylinux2010_x86_64", "manylinux2014_x86_64"), - ("manylinux2014_x86_64", "manylinux2014_x86_64"), - ]) + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") + + @pytest.mark.parametrize( + "wheel_abi,platform", + [ + ("manylinux1_x86_64", "manylinux1_x86_64"), + ("manylinux1_x86_64", "manylinux2010_x86_64"), + ("manylinux2010_x86_64", "manylinux2010_x86_64"), + ("manylinux1_x86_64", "manylinux2014_x86_64"), + ("manylinux2010_x86_64", "manylinux2014_x86_64"), + ("manylinux2014_x86_64", "manylinux2014_x86_64"), + ], + ) def test_download_compatible_manylinuxes( - self, wheel_abi, platform, script, data, - ): + self, + wheel_abi: str, + platform: str, + script: PipTestEnvironment, + data: TestData, + ) -> None: """ Earlier manylinuxes are compatible with later manylinuxes. """ - wheel = f'fake-1.0-py2.py3-none-{wheel_abi}.whl' + wheel = f"fake-1.0-py2.py3-none-{wheel_abi}.whl" fake_wheel(data, wheel) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', platform, - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + platform, + "fake", ) - result.did_create(Path('scratch') / wheel) + result.did_create(Path("scratch") / wheel) - def test_explicit_platform_only(self, data, script): + def test_explicit_platform_only( + self, data: TestData, script: PipTestEnvironment + ) -> None: """ When specifying the platform, manylinux1 needs to be the explicit platform--it won't ever be added to the compatible tags. """ - fake_wheel(data, 'fake-1.0-py2.py3-none-linux_x86_64.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-linux_x86_64.whl") script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--platform', 'linux_x86_64', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--platform", + "linux_x86_64", + "fake", ) -def test_download__python_version(script, data): +def test_download__python_version(script: PipTestEnvironment, data: TestData) -> None: """ Test using "pip download --python-version" to download a .whl archive supported for a specific interpreter """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '2', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "2", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '3', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "3", + "fake", ) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '27', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "27", + "fake", ) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '33', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "33", + "fake", ) data.reset() - fake_wheel(data, 'fake-1.0-py2-none-any.whl') - fake_wheel(data, 'fake-2.0-py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2-none-any.whl") + fake_wheel(data, "fake-2.0-py3-none-any.whl") # No py3 provided for version 1. result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '3', - 'fake==1.0', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "3", + "fake==1.0", expect_error=True, ) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '2', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "2", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '26', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "26", + "fake", ) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '3', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "3", + "fake", ) - result.did_create(Path('scratch') / 'fake-2.0-py3-none-any.whl') + result.did_create(Path("scratch") / "fake-2.0-py3-none-any.whl") -def make_wheel_with_python_requires(script, package_name, python_requires): +def make_wheel_with_python_requires( + script: PipTestEnvironment, package_name: str, python_requires: str +) -> Path: """ Create a wheel using the given python_requires. @@ -480,58 +621,72 @@ def make_wheel_with_python_requires(script, package_name, python_requires): package_dir = script.scratch_path / package_name package_dir.mkdir() - text = textwrap.dedent("""\ + text = textwrap.dedent( + """\ from setuptools import setup setup(name='{}', python_requires='{}', version='1.0') - """).format(package_name, python_requires) - package_dir.joinpath('setup.py').write_text(text) + """ + ).format(package_name, python_requires) + package_dir.joinpath("setup.py").write_text(text) script.run( - 'python', 'setup.py', 'bdist_wheel', '--universal', cwd=package_dir, + "python", + "setup.py", + "bdist_wheel", + "--universal", + cwd=package_dir, ) - file_name = f'{package_name}-1.0-py2.py3-none-any.whl' - return package_dir / 'dist' / file_name + file_name = f"{package_name}-1.0-py2.py3-none-any.whl" + return package_dir / "dist" / file_name +@pytest.mark.usefixtures("with_wheel") def test_download__python_version_used_for_python_requires( - script, data, with_wheel, -): + script: PipTestEnvironment, data: TestData +) -> None: """ Test that --python-version is used for the Requires-Python check. """ wheel_path = make_wheel_with_python_requires( - script, 'mypackage', python_requires='==3.2', + script, + "mypackage", + python_requires="==3.2", ) wheel_dir = os.path.dirname(wheel_path) - def make_args(python_version): + def make_args(python_version: str) -> List[str]: return [ - 'download', '--no-index', '--find-links', wheel_dir, - '--only-binary=:all:', - '--dest', '.', - '--python-version', python_version, - 'mypackage==1.0', + "download", + "--no-index", + "--find-links", + wheel_dir, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + python_version, + "mypackage==1.0", ] - args = make_args('33') + args = make_args("33") result = script.pip(*args, expect_error=True) expected_err = ( "ERROR: Package 'mypackage' requires a different Python: " "3.3.0 not in '==3.2'" ) - assert expected_err in result.stderr, f'stderr: {result.stderr}' + assert expected_err in result.stderr, f"stderr: {result.stderr}" # Now try with a --python-version that satisfies the Requires-Python. - args = make_args('32') + args = make_args("32") script.pip(*args) # no exception +@pytest.mark.usefixtures("with_wheel") def test_download_ignore_requires_python_dont_fail_with_wrong_python( - script, - with_wheel, -): + script: PipTestEnvironment, +) -> None: """ Test that --ignore-requires-python ignores Requires-Python check. """ @@ -553,255 +708,366 @@ def test_download_ignore_requires_python_dont_fail_with_wrong_python( ".", "mypackage==1.0", ) - result.did_create(Path('scratch') / 'mypackage-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "mypackage-1.0-py2.py3-none-any.whl") -def test_download_specify_abi(script, data): +def test_download_specify_abi(script: PipTestEnvironment, data: TestData) -> None: """ Test using "pip download --abi" to download a .whl archive supported for a specific abi """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--implementation', 'fk', - '--abi', 'fake_abi', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--implementation", + "fk", + "--abi", + "fake_abi", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--implementation', 'fk', - '--abi', 'none', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--implementation", + "fk", + "--abi", + "none", + "fake", ) result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--abi', 'cp27m', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--abi", + "cp27m", + "fake", ) data.reset() - fake_wheel(data, 'fake-1.0-fk2-fakeabi-fake_platform.whl') + fake_wheel(data, "fake-1.0-fk2-fakeabi-fake_platform.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '2', - '--implementation', 'fk', - '--platform', 'fake_platform', - '--abi', 'fakeabi', - 'fake' - ) - result.did_create( - Path('scratch') / 'fake-1.0-fk2-fakeabi-fake_platform.whl' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "2", + "--implementation", + "fk", + "--platform", + "fake_platform", + "--abi", + "fakeabi", + "fake", ) + result.did_create(Path("scratch") / "fake-1.0-fk2-fakeabi-fake_platform.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--implementation', 'fk', - '--platform', 'fake_platform', - '--abi', 'none', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--implementation", + "fk", + "--platform", + "fake_platform", + "--abi", + "none", + "fake", expect_error=True, ) data.reset() - fake_wheel(data, 'fake-1.0-fk2-otherabi-fake_platform.whl') + fake_wheel(data, "fake-1.0-fk2-otherabi-fake_platform.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--python-version', '2', - '--implementation', 'fk', - '--platform', 'fake_platform', - '--abi', 'fakeabi', '--abi', 'otherabi', '--abi', 'none', - 'fake' - ) - result.did_create( - Path('scratch') / 'fake-1.0-fk2-otherabi-fake_platform.whl' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--python-version", + "2", + "--implementation", + "fk", + "--platform", + "fake_platform", + "--abi", + "fakeabi", + "--abi", + "otherabi", + "--abi", + "none", + "fake", ) + result.did_create(Path("scratch") / "fake-1.0-fk2-otherabi-fake_platform.whl") -def test_download_specify_implementation(script, data): +def test_download_specify_implementation( + script: PipTestEnvironment, data: TestData +) -> None: """ Test using "pip download --abi" to download a .whl archive supported for a specific abi """ - fake_wheel(data, 'fake-1.0-py2.py3-none-any.whl') + fake_wheel(data, "fake-1.0-py2.py3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--implementation', 'fk', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--implementation", + "fk", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-py2.py3-none-any.whl") data.reset() - fake_wheel(data, 'fake-1.0-fk3-none-any.whl') + fake_wheel(data, "fake-1.0-fk3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--implementation', 'fk', - '--python-version', '3', - 'fake' + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--implementation", + "fk", + "--python-version", + "3", + "fake", ) - result.did_create(Path('scratch') / 'fake-1.0-fk3-none-any.whl') + result.did_create(Path("scratch") / "fake-1.0-fk3-none-any.whl") result = script.pip( - 'download', '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--dest', '.', - '--implementation', 'fk', - '--python-version', '2', - 'fake', + "download", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--dest", + ".", + "--implementation", + "fk", + "--python-version", + "2", + "fake", expect_error=True, ) -def test_download_exit_status_code_when_no_requirements(script): +def test_download_exit_status_code_when_no_requirements( + script: PipTestEnvironment, +) -> None: """ Test download exit status code when no requirements specified """ - result = script.pip('download', expect_error=True) - assert ( - "You must give at least one requirement to download" in result.stderr - ) + result = script.pip("download", expect_error=True) + assert "You must give at least one requirement to download" in result.stderr assert result.returncode == ERROR -def test_download_exit_status_code_when_blank_requirements_file(script): +def test_download_exit_status_code_when_blank_requirements_file( + script: PipTestEnvironment, +) -> None: """ Test download exit status code when blank requirements file specified """ script.scratch_path.joinpath("blank.txt").write_text("\n") - script.pip('download', '-r', 'blank.txt') + script.pip("download", "-r", "blank.txt") -def test_download_prefer_binary_when_tarball_higher_than_wheel(script, data): - fake_wheel(data, 'source-0.8-py2.py3-none-any.whl') +def test_download_prefer_binary_when_tarball_higher_than_wheel( + script: PipTestEnvironment, data: TestData +) -> None: + fake_wheel(data, "source-0.8-py2.py3-none-any.whl") result = script.pip( - 'download', - '--prefer-binary', - '--no-index', - '-f', data.packages, - '-d', '.', 'source' + "download", + "--prefer-binary", + "--no-index", + "-f", + data.packages, + "-d", + ".", + "source", ) - result.did_create(Path('scratch') / 'source-0.8-py2.py3-none-any.whl') - result.did_not_create(Path('scratch') / 'source-1.0.tar.gz') + result.did_create(Path("scratch") / "source-0.8-py2.py3-none-any.whl") + result.did_not_create(Path("scratch") / "source-1.0.tar.gz") -def test_prefer_binary_tarball_higher_than_wheel_req_file(script, data): - fake_wheel(data, 'source-0.8-py2.py3-none-any.whl') - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" +def test_prefer_binary_tarball_higher_than_wheel_req_file( + script: PipTestEnvironment, data: TestData +) -> None: + fake_wheel(data, "source-0.8-py2.py3-none-any.whl") + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ --prefer-binary source - """)) + """ + ) + ) result = script.pip( - 'download', - '-r', script.scratch_path / 'test-req.txt', - '--no-index', - '-f', data.packages, - '-d', '.' + "download", + "-r", + script.scratch_path / "test-req.txt", + "--no-index", + "-f", + data.packages, + "-d", + ".", ) - result.did_create(Path('scratch') / 'source-0.8-py2.py3-none-any.whl') - result.did_not_create(Path('scratch') / 'source-1.0.tar.gz') + result.did_create(Path("scratch") / "source-0.8-py2.py3-none-any.whl") + result.did_not_create(Path("scratch") / "source-1.0.tar.gz") -def test_download_prefer_binary_when_wheel_doesnt_satisfy_req(script, data): - fake_wheel(data, 'source-0.8-py2.py3-none-any.whl') - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" +def test_download_prefer_binary_when_wheel_doesnt_satisfy_req( + script: PipTestEnvironment, data: TestData +) -> None: + fake_wheel(data, "source-0.8-py2.py3-none-any.whl") + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ source>0.9 - """)) + """ + ) + ) result = script.pip( - 'download', - '--prefer-binary', - '--no-index', - '-f', data.packages, - '-d', '.', - '-r', script.scratch_path / 'test-req.txt' + "download", + "--prefer-binary", + "--no-index", + "-f", + data.packages, + "-d", + ".", + "-r", + script.scratch_path / "test-req.txt", ) - result.did_create(Path('scratch') / 'source-1.0.tar.gz') - result.did_not_create(Path('scratch') / 'source-0.8-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "source-1.0.tar.gz") + result.did_not_create(Path("scratch") / "source-0.8-py2.py3-none-any.whl") -def test_prefer_binary_when_wheel_doesnt_satisfy_req_req_file(script, data): - fake_wheel(data, 'source-0.8-py2.py3-none-any.whl') - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" +def test_prefer_binary_when_wheel_doesnt_satisfy_req_req_file( + script: PipTestEnvironment, data: TestData +) -> None: + fake_wheel(data, "source-0.8-py2.py3-none-any.whl") + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ --prefer-binary source>0.9 - """)) + """ + ) + ) result = script.pip( - 'download', - '--no-index', - '-f', data.packages, - '-d', '.', - '-r', script.scratch_path / 'test-req.txt' + "download", + "--no-index", + "-f", + data.packages, + "-d", + ".", + "-r", + script.scratch_path / "test-req.txt", ) - result.did_create(Path('scratch') / 'source-1.0.tar.gz') - result.did_not_create(Path('scratch') / 'source-0.8-py2.py3-none-any.whl') + result.did_create(Path("scratch") / "source-1.0.tar.gz") + result.did_not_create(Path("scratch") / "source-0.8-py2.py3-none-any.whl") -def test_download_prefer_binary_when_only_tarball_exists(script, data): +def test_download_prefer_binary_when_only_tarball_exists( + script: PipTestEnvironment, data: TestData +) -> None: result = script.pip( - 'download', - '--prefer-binary', - '--no-index', - '-f', data.packages, - '-d', '.', 'source' + "download", + "--prefer-binary", + "--no-index", + "-f", + data.packages, + "-d", + ".", + "source", ) - result.did_create(Path('scratch') / 'source-1.0.tar.gz') + result.did_create(Path("scratch") / "source-1.0.tar.gz") -def test_prefer_binary_when_only_tarball_exists_req_file(script, data): - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" +def test_prefer_binary_when_only_tarball_exists_req_file( + script: PipTestEnvironment, data: TestData +) -> None: + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ --prefer-binary source - """)) + """ + ) + ) result = script.pip( - 'download', - '--no-index', - '-f', data.packages, - '-d', '.', - '-r', script.scratch_path / 'test-req.txt' + "download", + "--no-index", + "-f", + data.packages, + "-d", + ".", + "-r", + script.scratch_path / "test-req.txt", ) - result.did_create(Path('scratch') / 'source-1.0.tar.gz') + result.did_create(Path("scratch") / "source-1.0.tar.gz") @pytest.fixture(scope="session") -def shared_script(tmpdir_factory, script_factory): +def shared_script( + tmpdir_factory: pytest.TempdirFactory, script_factory: ScriptFactory +) -> PipTestEnvironment: tmpdir = Path(str(tmpdir_factory.mktemp("download_shared_script"))) script = script_factory(tmpdir.joinpath("workspace")) return script -def test_download_file_url(shared_script, shared_data, tmpdir): - download_dir = tmpdir / 'download' +def test_download_file_url( + shared_script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: + download_dir = tmpdir / "download" download_dir.mkdir() - downloaded_path = download_dir / 'simple-1.0.tar.gz' + downloaded_path = download_dir / "simple-1.0.tar.gz" - simple_pkg = shared_data.packages / 'simple-1.0.tar.gz' + simple_pkg = shared_data.packages / "simple-1.0.tar.gz" shared_script.pip( - 'download', - '-d', + "download", + "-d", str(download_dir), - '--no-index', + "--no-index", path_to_url(str(simple_pkg)), ) @@ -810,83 +1076,89 @@ def test_download_file_url(shared_script, shared_data, tmpdir): def test_download_file_url_existing_ok_download( - shared_script, shared_data, tmpdir -): - download_dir = tmpdir / 'download' + shared_script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: + download_dir = tmpdir / "download" download_dir.mkdir() - downloaded_path = download_dir / 'simple-1.0.tar.gz' - fake_existing_package = shared_data.packages / 'simple-2.0.tar.gz' + downloaded_path = download_dir / "simple-1.0.tar.gz" + fake_existing_package = shared_data.packages / "simple-2.0.tar.gz" shutil.copy(str(fake_existing_package), str(downloaded_path)) downloaded_path_bytes = downloaded_path.read_bytes() digest = sha256(downloaded_path_bytes).hexdigest() - simple_pkg = shared_data.packages / 'simple-1.0.tar.gz' + simple_pkg = shared_data.packages / "simple-1.0.tar.gz" url = "{}#sha256={}".format(path_to_url(simple_pkg), digest) - shared_script.pip('download', '-d', str(download_dir), url) + shared_script.pip("download", "-d", str(download_dir), url) assert downloaded_path_bytes == downloaded_path.read_bytes() def test_download_file_url_existing_bad_download( - shared_script, shared_data, tmpdir -): - download_dir = tmpdir / 'download' + shared_script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: + download_dir = tmpdir / "download" download_dir.mkdir() - downloaded_path = download_dir / 'simple-1.0.tar.gz' - fake_existing_package = shared_data.packages / 'simple-2.0.tar.gz' + downloaded_path = download_dir / "simple-1.0.tar.gz" + fake_existing_package = shared_data.packages / "simple-2.0.tar.gz" shutil.copy(str(fake_existing_package), str(downloaded_path)) - simple_pkg = shared_data.packages / 'simple-1.0.tar.gz' + simple_pkg = shared_data.packages / "simple-1.0.tar.gz" simple_pkg_bytes = simple_pkg.read_bytes() digest = sha256(simple_pkg_bytes).hexdigest() url = "{}#sha256={}".format(path_to_url(simple_pkg), digest) - shared_script.pip('download', '-d', str(download_dir), url) + shared_script.pip("download", "-d", str(download_dir), url) assert simple_pkg_bytes == downloaded_path.read_bytes() def test_download_http_url_bad_hash( - shared_script, shared_data, tmpdir, mock_server -): - download_dir = tmpdir / 'download' + shared_script: PipTestEnvironment, + shared_data: TestData, + tmpdir: Path, + mock_server: MockServer, +) -> None: + """ + If already-downloaded file has bad checksum, re-download. + """ + download_dir = tmpdir / "download" download_dir.mkdir() - downloaded_path = download_dir / 'simple-1.0.tar.gz' - fake_existing_package = shared_data.packages / 'simple-2.0.tar.gz' + downloaded_path = download_dir / "simple-1.0.tar.gz" + fake_existing_package = shared_data.packages / "simple-2.0.tar.gz" shutil.copy(str(fake_existing_package), str(downloaded_path)) - simple_pkg = shared_data.packages / 'simple-1.0.tar.gz' + simple_pkg = shared_data.packages / "simple-1.0.tar.gz" simple_pkg_bytes = simple_pkg.read_bytes() digest = sha256(simple_pkg_bytes).hexdigest() - mock_server.set_responses([ - file_response(simple_pkg) - ]) + mock_server.set_responses([file_response(simple_pkg)]) mock_server.start() - base_address = f'http://{mock_server.host}:{mock_server.port}' + base_address = f"http://{mock_server.host}:{mock_server.port}" url = f"{base_address}/simple-1.0.tar.gz#sha256={digest}" - shared_script.pip('download', '-d', str(download_dir), url) + shared_script.pip("download", "-d", str(download_dir), url) assert simple_pkg_bytes == downloaded_path.read_bytes() mock_server.stop() requests = mock_server.get_requests() assert len(requests) == 1 - assert requests[0]['PATH_INFO'] == '/simple-1.0.tar.gz' - assert requests[0]['HTTP_ACCEPT_ENCODING'] == 'identity' + assert requests[0]["PATH_INFO"] == "/simple-1.0.tar.gz" + assert requests[0]["HTTP_ACCEPT_ENCODING"] == "identity" -def test_download_editable(script, data, tmpdir): +def test_download_editable( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test 'pip download' of editables in requirement file. """ - editable_path = str(data.src / 'simplewheel-1.0').replace(os.path.sep, "/") + editable_path = str(data.src / "simplewheel-1.0").replace(os.path.sep, "/") requirements_path = tmpdir / "requirements.txt" requirements_path.write_text("-e " + editable_path + "\n") download_dir = tmpdir / "download_dir" script.pip( - 'download', '--no-deps', '-r', str(requirements_path), '-d', str(download_dir) + "download", "--no-deps", "-r", str(requirements_path), "-d", str(download_dir) ) downloads = os.listdir(download_dir) assert len(downloads) == 1 diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py index e82641986ac..87c070c78c5 100644 --- a/tests/functional/test_fast_deps.py +++ b/tests/functional/test_fast_deps.py @@ -1,79 +1,102 @@ import fnmatch import json +import pathlib from os.path import basename +from typing import Iterable from pip._vendor.packaging.utils import canonicalize_name from pytest import mark +from tests.lib import PipTestEnvironment, TestData, TestPipResult -def pip(script, command, requirement): + +def pip(script: PipTestEnvironment, command: str, requirement: str) -> TestPipResult: return script.pip( - command, '--prefer-binary', '--no-cache-dir', - '--use-feature=fast-deps', requirement, + command, + "--prefer-binary", + "--no-cache-dir", + "--use-feature=fast-deps", + requirement, allow_stderr_warning=True, ) -def assert_installed(script, names): - list_output = json.loads(script.pip('list', '--format=json').stdout) - installed = {canonicalize_name(item['name']) for item in list_output} +def assert_installed(script: PipTestEnvironment, names: str) -> None: + list_output = json.loads(script.pip("list", "--format=json").stdout) + installed = {canonicalize_name(item["name"]) for item in list_output} assert installed.issuperset(map(canonicalize_name, names)) @mark.network -@mark.parametrize(('requirement', 'expected'), ( - ('Paste==3.4.2', ('Paste', 'six')), - ('Paste[flup]==3.4.2', ('Paste', 'six', 'flup')), -)) -def test_install_from_pypi(requirement, expected, script): - pip(script, 'install', requirement) +@mark.parametrize( + ("requirement", "expected"), + ( + ("Paste==3.4.2", ("Paste", "six")), + ("Paste[flup]==3.4.2", ("Paste", "six", "flup")), + ), +) +def test_install_from_pypi( + requirement: str, expected: str, script: PipTestEnvironment +) -> None: + pip(script, "install", requirement) assert_installed(script, expected) @mark.network -@mark.parametrize(('requirement', 'expected'), ( - ('Paste==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl')), - ('Paste[flup]==3.4.2', ('Paste-3.4.2-*.whl', 'six-*.whl', 'flup-*')), -)) -def test_download_from_pypi(requirement, expected, script): - result = pip(script, 'download', requirement) - created = list(map(basename, result.files_created)) +@mark.parametrize( + ("requirement", "expected"), + ( + ("Paste==3.4.2", ("Paste-3.4.2-*.whl", "six-*.whl")), + ("Paste[flup]==3.4.2", ("Paste-3.4.2-*.whl", "six-*.whl", "flup-*")), + ), +) +def test_download_from_pypi( + requirement: str, expected: Iterable[str], script: PipTestEnvironment +) -> None: + result = pip(script, "download", requirement) + created = [basename(f) for f in result.files_created] assert all(fnmatch.filter(created, f) for f in expected) @mark.network -def test_build_wheel_with_deps(data, script): - result = pip(script, 'wheel', data.packages/'requiresPaste') - created = list(map(basename, result.files_created)) - assert fnmatch.filter(created, 'requiresPaste-3.1.4-*.whl') - assert fnmatch.filter(created, 'Paste-3.4.2-*.whl') - assert fnmatch.filter(created, 'six-*.whl') +def test_build_wheel_with_deps(data: TestData, script: PipTestEnvironment) -> None: + result = pip(script, "wheel", data.packages / "requiresPaste") + created = [basename(f) for f in result.files_created] + assert fnmatch.filter(created, "requirespaste-3.1.4-*.whl") + assert fnmatch.filter(created, "Paste-3.4.2-*.whl") + assert fnmatch.filter(created, "six-*.whl") @mark.network -def test_require_hash(script, tmp_path): - reqs = tmp_path / 'requirements.txt' +def test_require_hash(script: PipTestEnvironment, tmp_path: pathlib.Path) -> None: + reqs = tmp_path / "requirements.txt" reqs.write_text( - 'idna==2.10' - ' --hash=sha256:' - 'b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0' - ' --hash=sha256:' - 'b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6' + "idna==2.10" + " --hash=sha256:" + "b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + " --hash=sha256:" + "b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6" ) result = script.pip( - 'download', '--use-feature=fast-deps', '-r', str(reqs), + "download", + "--use-feature=fast-deps", + "-r", + str(reqs), allow_stderr_warning=True, ) - created = list(map(basename, result.files_created)) - assert fnmatch.filter(created, 'idna-2.10*') + created = [basename(f) for f in result.files_created] + assert fnmatch.filter(created, "idna-2.10*") @mark.network -def test_hash_mismatch(script, tmp_path): - reqs = tmp_path / 'requirements.txt' - reqs.write_text('idna==2.10 --hash=sha256:irna') +def test_hash_mismatch(script: PipTestEnvironment, tmp_path: pathlib.Path) -> None: + reqs = tmp_path / "requirements.txt" + reqs.write_text("idna==2.10 --hash=sha256:irna") result = script.pip( - 'download', '--use-feature=fast-deps', '-r', str(reqs), + "download", + "--use-feature=fast-deps", + "-r", + str(reqs), expect_error=True, ) - assert 'DO NOT MATCH THE HASHES' in result.stderr + assert "DO NOT MATCH THE HASHES" in result.stderr diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 5d3d4968619..bae9eadbd30 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -5,8 +5,12 @@ from doctest import ELLIPSIS, OutputChecker import pytest +from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.models.direct_url import DirectUrl, DirInfo from tests.lib import ( + PipTestEnvironment, + TestData, _create_test_package, _create_test_package_with_srcdir, _git_commit, @@ -18,11 +22,14 @@ path_to_url, wheel, ) +from tests.lib.direct_url import get_created_direct_url_path +from tests.lib.path import Path +from tests.lib.venv import VirtualEnvironment -distribute_re = re.compile('^distribute==[0-9.]+\n', re.MULTILINE) +distribute_re = re.compile("^distribute==[0-9.]+\n", re.MULTILINE) -def _check_output(result, expected): +def _check_output(result: str, expected: str) -> None: checker = OutputChecker() actual = str(result) @@ -34,23 +41,22 @@ def _check_output(result, expected): # but then you have to remember to upcase . The right # thing to do in the end is probably to find out how to report # the proper fully-cased package name in our error message. - if sys.platform == 'win32': - actual = actual.replace('initools', 'INITools') + if sys.platform == "win32": + actual = actual.replace("initools", "INITools") # This allows our existing tests to work when run in a context # with distribute installed. - actual = distribute_re.sub('', actual) + actual = distribute_re.sub("", actual) - def banner(msg): - return '\n========== {msg} ==========\n'.format(**locals()) + def banner(msg: str) -> str: + return f"\n========== {msg} ==========\n" assert checker.check_output(expected, actual, ELLIPSIS), ( - banner('EXPECTED') + expected + banner('ACTUAL') + actual + - banner(6 * '=') + banner("EXPECTED") + expected + banner("ACTUAL") + actual + banner(6 * "=") ) -def test_basic_freeze(script): +def test_basic_freeze(script: PipTestEnvironment) -> None: """ Some tests of freeze, first we have to install some stuff. Note that the test is a little crude at the end because Python 2.5+ adds egg @@ -59,154 +65,177 @@ def test_basic_freeze(script): currently it is not). """ - script.scratch_path.joinpath("initools-req.txt").write_text(textwrap.dedent("""\ + script.scratch_path.joinpath("initools-req.txt").write_text( + textwrap.dedent( + """\ simple==2.0 # and something else to test out: simple2<=3.0 - """)) + """ + ) + ) script.pip_install_local( - '-r', script.scratch_path / 'initools-req.txt', + "-r", + script.scratch_path / "initools-req.txt", ) - result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent("""\ + result = script.pip("freeze", expect_stderr=True) + expected = textwrap.dedent( + """\ ...simple==2.0 simple2==3.0... - """) + """ + ) _check_output(result.stdout, expected) -def test_freeze_with_pip(script): +def test_freeze_with_pip(script: PipTestEnvironment) -> None: """Test pip shows itself""" - result = script.pip('freeze', '--all') - assert 'pip==' in result.stdout + result = script.pip("freeze", "--all") + assert "pip==" in result.stdout -def test_exclude_and_normalization(script, tmpdir): - req_path = wheel.make_wheel( - name="Normalizable_Name", version="1.0").save_to_dir(tmpdir) +def test_exclude_and_normalization(script: PipTestEnvironment, tmpdir: Path) -> None: + req_path = wheel.make_wheel(name="Normalizable_Name", version="1.0").save_to_dir( + tmpdir + ) script.pip("install", "--no-index", req_path) result = script.pip("freeze") - assert "Normalizable-Name" in result.stdout + assert "Normalizable_Name" in result.stdout result = script.pip("freeze", "--exclude", "normalizablE-namE") - assert "Normalizable-Name" not in result.stdout + assert "Normalizable_Name" not in result.stdout -def test_freeze_multiple_exclude_with_all(script, with_wheel): - result = script.pip('freeze', '--all') - assert 'pip==' in result.stdout - assert 'wheel==' in result.stdout - result = script.pip('freeze', '--all', '--exclude', 'pip', '--exclude', 'wheel') - assert 'pip==' not in result.stdout - assert 'wheel==' not in result.stdout +@pytest.mark.usefixtures("with_wheel") +def test_freeze_multiple_exclude_with_all(script: PipTestEnvironment) -> None: + result = script.pip("freeze", "--all") + assert "pip==" in result.stdout + assert "wheel==" in result.stdout + result = script.pip("freeze", "--all", "--exclude", "pip", "--exclude", "wheel") + assert "pip==" not in result.stdout + assert "wheel==" not in result.stdout -def test_freeze_with_invalid_names(script): +def test_freeze_with_invalid_names(script: PipTestEnvironment) -> None: """ Test that invalid names produce warnings and are passed over gracefully. """ - def fake_install(pkgname, dest): + def fake_install(pkgname: str, dest: str) -> None: egg_info_path = os.path.join( - dest, '{}-1.0-py{}.{}.egg-info'.format( - pkgname.replace('-', '_'), - sys.version_info[0], - sys.version_info[1] - ) + dest, + "{}-1.0-py{}.{}.egg-info".format( + pkgname.replace("-", "_"), sys.version_info[0], sys.version_info[1] + ), ) - with open(egg_info_path, 'w') as egg_info_file: - egg_info_file.write(textwrap.dedent("""\ + with open(egg_info_path, "w") as egg_info_file: + egg_info_file.write( + textwrap.dedent( + """\ Metadata-Version: 1.0 Name: {} Version: 1.0 - """.format(pkgname) - )) + """.format( + pkgname + ) + ) + ) - valid_pkgnames = ('middle-dash', 'middle_underscore', 'middle.dot') + valid_pkgnames = ("middle-dash", "middle_underscore", "middle.dot") invalid_pkgnames = ( - '-leadingdash', '_leadingunderscore', '.leadingdot', - 'trailingdash-', 'trailingunderscore_', 'trailingdot.' + "-leadingdash", + "_leadingunderscore", + ".leadingdot", + "trailingdash-", + "trailingunderscore_", + "trailingdot.", ) for pkgname in valid_pkgnames + invalid_pkgnames: fake_install(pkgname, script.site_packages_path) - result = script.pip('freeze', expect_stderr=True) - for pkgname in valid_pkgnames: - _check_output( - result.stdout, - '...{}==1.0...'.format(pkgname.replace('_', '-')) - ) - for pkgname in invalid_pkgnames: - # Check that the full distribution repr is present. - dist_repr = '{} 1.0 ('.format(pkgname.replace('_', '-')) - expected = ( - '...Could not generate requirement for ' - 'distribution {}...'.format(dist_repr) - ) - _check_output(result.stderr, expected) - # Also check that the parse error details occur at least once. - # We only need to find one occurrence to know that exception details - # are logged. - expected = '...site-packages): Parse error at "...' - _check_output(result.stderr, expected) + result = script.pip("freeze", expect_stderr=True) + + # Check all valid names are in the output. + output_lines = {line.strip() for line in result.stdout.splitlines()} + for name in valid_pkgnames: + assert f"{name}==1.0" in output_lines + + # Check all invalid names are excluded from the output. + canonical_invalid_names = {canonicalize_name(n) for n in invalid_pkgnames} + for line in output_lines: + output_name, _, _ = line.partition("=") + assert canonicalize_name(output_name) not in canonical_invalid_names + + # The invalid names should be logged. + for name in canonical_invalid_names: + assert f"Ignoring invalid distribution {name} (" in result.stderr @pytest.mark.git -def test_freeze_editable_not_vcs(script, tmpdir): +def test_freeze_editable_not_vcs(script: PipTestEnvironment) -> None: """ Test an editable install that is not version controlled. """ pkg_path = _create_test_package(script) # Rename the .git directory so the directory is no longer recognized # as a VCS directory. - os.rename(os.path.join(pkg_path, '.git'), os.path.join(pkg_path, '.bak')) - script.pip('install', '-e', pkg_path) - result = script.pip('freeze') + os.rename(os.path.join(pkg_path, ".git"), os.path.join(pkg_path, ".bak")) + script.pip("install", "-e", pkg_path) + result = script.pip("freeze") # We need to apply os.path.normcase() to the path since that is what # the freeze code does. - expected = textwrap.dedent("""\ + expected = textwrap.dedent( + """\ ...# Editable install with no version control (version-pkg==0.1) -e {} - ...""".format(os.path.normcase(pkg_path))) + ...""".format( + os.path.normcase(pkg_path) + ) + ) _check_output(result.stdout, expected) @pytest.mark.git -def test_freeze_editable_git_with_no_remote(script, tmpdir, deprecated_python): +def test_freeze_editable_git_with_no_remote( + script: PipTestEnvironment, deprecated_python: bool +) -> None: """ Test an editable Git install with no remote url. """ pkg_path = _create_test_package(script) - script.pip('install', '-e', pkg_path) - result = script.pip('freeze') + script.pip("install", "-e", pkg_path) + result = script.pip("freeze") if not deprecated_python: - assert result.stderr == '' + assert result.stderr == "" # We need to apply os.path.normcase() to the path since that is what # the freeze code does. - expected = textwrap.dedent("""\ + expected = textwrap.dedent( + """\ ...# Editable Git install with no remote (version-pkg==0.1) -e {} - ...""".format(os.path.normcase(pkg_path))) + ...""".format( + os.path.normcase(pkg_path) + ) + ) _check_output(result.stdout, expected) @need_svn -def test_freeze_svn(script, tmpdir): +def test_freeze_svn(script: PipTestEnvironment) -> None: """Test freezing a svn checkout""" - checkout_path = _create_test_package(script, vcs='svn') + checkout_path = _create_test_package(script, vcs="svn") # Install with develop - script.run( - 'python', 'setup.py', 'develop', - cwd=checkout_path, expect_stderr=True - ) - result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent("""\ + script.run("python", "setup.py", "develop", cwd=checkout_path, expect_stderr=True) + result = script.pip("freeze", expect_stderr=True) + expected = textwrap.dedent( + """\ ...-e svn+...#egg=version_pkg - ...""") + ...""" + ) _check_output(result.stdout, expected) @@ -217,7 +246,7 @@ def test_freeze_svn(script, tmpdir): run=True, strict=True, ) -def test_freeze_exclude_editable(script, tmpdir): +def test_freeze_exclude_editable(script: PipTestEnvironment) -> None: """ Test excluding editable from freezing list. """ @@ -225,16 +254,21 @@ def test_freeze_exclude_editable(script, tmpdir): pkg_version = _create_test_package(script) result = script.run( - 'git', 'clone', pkg_version, 'pip-test-package', + "git", + "clone", + pkg_version, + "pip-test-package", expect_stderr=True, ) - repo_dir = script.scratch_path / 'pip-test-package' + repo_dir = script.scratch_path / "pip-test-package" result = script.run( - 'python', 'setup.py', 'develop', + "python", + "setup.py", + "develop", cwd=repo_dir, expect_stderr=True, ) - result = script.pip('freeze', '--exclude-editable', expect_stderr=True) + result = script.pip("freeze", "--exclude-editable", expect_stderr=True) expected = textwrap.dedent( """ ...-e git+...#egg=version_pkg @@ -245,7 +279,7 @@ def test_freeze_exclude_editable(script, tmpdir): @pytest.mark.git -def test_freeze_git_clone(script, tmpdir): +def test_freeze_git_clone(script: PipTestEnvironment) -> None: """ Test freezing a Git clone. """ @@ -253,16 +287,21 @@ def test_freeze_git_clone(script, tmpdir): pkg_version = _create_test_package(script) result = script.run( - 'git', 'clone', pkg_version, 'pip-test-package', + "git", + "clone", + pkg_version, + "pip-test-package", expect_stderr=True, ) - repo_dir = script.scratch_path / 'pip-test-package' + repo_dir = script.scratch_path / "pip-test-package" result = script.run( - 'python', 'setup.py', 'develop', + "python", + "setup.py", + "develop", cwd=repo_dir, expect_stderr=True, ) - result = script.pip('freeze', expect_stderr=True) + result = script.pip("freeze", expect_stderr=True) expected = textwrap.dedent( """ ...-e git+...#egg=version_pkg @@ -271,33 +310,23 @@ def test_freeze_git_clone(script, tmpdir): ).strip() _check_output(result.stdout, expected) - result = script.pip( - 'freeze', '-f', '{repo_dir}#egg=pip_test_package'.format(**locals()), - expect_stderr=True, - ) - expected = textwrap.dedent( - """ - -f {repo}#egg=pip_test_package... - -e git+...#egg=version_pkg - ... - """.format(repo=repo_dir), - ).strip() - _check_output(result.stdout, expected) - # Check that slashes in branch or tag names are translated. # See also issue #1083: https://github.com/pypa/pip/issues/1083 script.run( - 'git', 'checkout', '-b', 'branch/name/with/slash', + "git", + "checkout", + "-b", + "branch/name/with/slash", cwd=repo_dir, expect_stderr=True, ) # Create a new commit to ensure that the commit has only one branch # or tag name associated to it (to avoid the non-determinism reported # in issue #1867). - (repo_dir / 'newfile').touch() - script.run('git', 'add', 'newfile', cwd=repo_dir) - _git_commit(script, repo_dir, message='...') - result = script.pip('freeze', expect_stderr=True) + (repo_dir / "newfile").touch() + script.run("git", "add", "newfile", cwd=repo_dir) + _git_commit(script, repo_dir, message="...") + result = script.pip("freeze", expect_stderr=True) expected = textwrap.dedent( """ ...-e ...@...#egg=version_pkg @@ -308,7 +337,7 @@ def test_freeze_git_clone(script, tmpdir): @pytest.mark.git -def test_freeze_git_clone_srcdir(script, tmpdir): +def test_freeze_git_clone_srcdir(script: PipTestEnvironment) -> None: """ Test freezing a Git clone where setup.py is in a subdirectory relative the repo root and the source code is in a subdirectory @@ -318,16 +347,21 @@ def test_freeze_git_clone_srcdir(script, tmpdir): pkg_version = _create_test_package_with_srcdir(script) result = script.run( - 'git', 'clone', pkg_version, 'pip-test-package', + "git", + "clone", + pkg_version, + "pip-test-package", expect_stderr=True, ) - repo_dir = script.scratch_path / 'pip-test-package' + repo_dir = script.scratch_path / "pip-test-package" result = script.run( - 'python', 'setup.py', 'develop', - cwd=repo_dir / 'subdir', + "python", + "setup.py", + "develop", + cwd=repo_dir / "subdir", expect_stderr=True, ) - result = script.pip('freeze', expect_stderr=True) + result = script.pip("freeze", expect_stderr=True) expected = textwrap.dedent( """ ...-e git+...#egg=version_pkg&subdirectory=subdir @@ -336,39 +370,21 @@ def test_freeze_git_clone_srcdir(script, tmpdir): ).strip() _check_output(result.stdout, expected) - result = script.pip( - 'freeze', '-f', '{repo_dir}#egg=pip_test_package'.format(**locals()), - expect_stderr=True, - ) - expected = textwrap.dedent( - """ - -f {repo}#egg=pip_test_package... - -e git+...#egg=version_pkg&subdirectory=subdir - ... - """.format(repo=repo_dir), - ).strip() - _check_output(result.stdout, expected) - @need_mercurial -def test_freeze_mercurial_clone_srcdir(script, tmpdir): +def test_freeze_mercurial_clone_srcdir(script: PipTestEnvironment) -> None: """ Test freezing a Mercurial clone where setup.py is in a subdirectory relative to the repo root and the source code is in a subdirectory relative to setup.py. """ # Returns path to a generated package called "version_pkg" - pkg_version = _create_test_package_with_srcdir(script, vcs='hg') + pkg_version = _create_test_package_with_srcdir(script, vcs="hg") - result = script.run( - 'hg', 'clone', pkg_version, 'pip-test-package' - ) - repo_dir = script.scratch_path / 'pip-test-package' - result = script.run( - 'python', 'setup.py', 'develop', - cwd=repo_dir / 'subdir' - ) - result = script.pip('freeze') + result = script.run("hg", "clone", pkg_version, "pip-test-package") + repo_dir = script.scratch_path / "pip-test-package" + result = script.run("python", "setup.py", "develop", cwd=repo_dir / "subdir") + result = script.pip("freeze") expected = textwrap.dedent( """ ...-e hg+...#egg=version_pkg&subdirectory=subdir @@ -377,22 +393,9 @@ def test_freeze_mercurial_clone_srcdir(script, tmpdir): ).strip() _check_output(result.stdout, expected) - result = script.pip( - 'freeze', '-f', '{repo_dir}#egg=pip_test_package'.format(**locals()), - expect_stderr=True, - ) - expected = textwrap.dedent( - """ - -f {repo}#egg=pip_test_package... - -e hg+...#egg=version_pkg&subdirectory=subdir - ... - """.format(repo=repo_dir), - ).strip() - _check_output(result.stdout, expected) - @pytest.mark.git -def test_freeze_git_remote(script, tmpdir): +def test_freeze_git_remote(script: PipTestEnvironment) -> None: """ Test freezing a Git clone. """ @@ -400,70 +403,105 @@ def test_freeze_git_remote(script, tmpdir): pkg_version = _create_test_package(script) result = script.run( - 'git', 'clone', pkg_version, 'pip-test-package', + "git", + "clone", + pkg_version, + "pip-test-package", expect_stderr=True, ) - repo_dir = script.scratch_path / 'pip-test-package' + repo_dir = script.scratch_path / "pip-test-package" result = script.run( - 'python', 'setup.py', 'develop', + "python", + "setup.py", + "develop", cwd=repo_dir, expect_stderr=True, ) origin_remote = pkg_version - other_remote = pkg_version + '-other' # check frozen remote after clone - result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent( - """ + result = script.pip("freeze", expect_stderr=True) + expected = ( + textwrap.dedent( + """ ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=origin_remote).strip() + ) + .format(remote=path_to_url(origin_remote)) + .strip() + ) _check_output(result.stdout, expected) # check frozen remote when there is no remote named origin - script.run('git', 'remote', 'remove', 'origin', cwd=repo_dir) - script.run('git', 'remote', 'add', 'other', other_remote, cwd=repo_dir) - result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent( - """ + script.run("git", "remote", "rename", "origin", "other", cwd=repo_dir) + result = script.pip("freeze", expect_stderr=True) + expected = ( + textwrap.dedent( + """ ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=other_remote).strip() + ) + .format(remote=path_to_url(origin_remote)) + .strip() + ) _check_output(result.stdout, expected) + # When the remote is a local path, it must exist. + # If it doesn't, it gets flagged as invalid. + other_remote = pkg_version + "-other" + script.run("git", "remote", "set-url", "other", other_remote, cwd=repo_dir) + result = script.pip("freeze", expect_stderr=True) + expected = os.path.normcase( + textwrap.dedent( + f""" + ...# Editable Git...(version-pkg...)... + # '{other_remote}' + -e {repo_dir}... + """ + ).strip() + ) + _check_output(os.path.normcase(result.stdout), expected) # when there are more than one origin, priority is given to the # remote named origin - script.run('git', 'remote', 'add', 'origin', origin_remote, cwd=repo_dir) - result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent( - """ + script.run("git", "remote", "add", "origin", origin_remote, cwd=repo_dir) + result = script.pip("freeze", expect_stderr=True) + expected = ( + textwrap.dedent( + """ ...-e git+{remote}@...#egg=version_pkg ... """ - ).format(remote=origin_remote).strip() + ) + .format(remote=path_to_url(origin_remote)) + .strip() + ) _check_output(result.stdout, expected) @need_mercurial -def test_freeze_mercurial_clone(script, tmpdir): +def test_freeze_mercurial_clone(script: PipTestEnvironment) -> None: """ Test freezing a Mercurial clone. """ # Returns path to a generated package called "version_pkg" - pkg_version = _create_test_package(script, vcs='hg') + pkg_version = _create_test_package(script, vcs="hg") result = script.run( - 'hg', 'clone', pkg_version, 'pip-test-package', + "hg", + "clone", + pkg_version, + "pip-test-package", expect_stderr=True, ) - repo_dir = script.scratch_path / 'pip-test-package' + repo_dir = script.scratch_path / "pip-test-package" result = script.run( - 'python', 'setup.py', 'develop', + "python", + "setup.py", + "develop", cwd=repo_dir, expect_stderr=True, ) - result = script.pip('freeze', expect_stderr=True) + result = script.pip("freeze", expect_stderr=True) expected = textwrap.dedent( """ ...-e hg+...#egg=version_pkg @@ -472,54 +510,32 @@ def test_freeze_mercurial_clone(script, tmpdir): ).strip() _check_output(result.stdout, expected) - result = script.pip( - 'freeze', '-f', '{repo_dir}#egg=pip_test_package'.format(**locals()), - expect_stderr=True, - ) - expected = textwrap.dedent( - """ - -f {repo}#egg=pip_test_package... - ...-e hg+...#egg=version_pkg - ... - """.format(repo=repo_dir), - ).strip() - _check_output(result.stdout, expected) - @need_bzr -def test_freeze_bazaar_clone(script, tmpdir): +def test_freeze_bazaar_clone(script: PipTestEnvironment) -> None: """ Test freezing a Bazaar clone. """ try: - checkout_path = _create_test_package(script, vcs='bazaar') + checkout_path = _create_test_package(script, vcs="bazaar") except OSError as e: - pytest.fail(f'Invoking `bzr` failed: {e}') + pytest.fail(f"Invoking `bzr` failed: {e}") + result = script.run("bzr", "checkout", checkout_path, "bzr-package") result = script.run( - 'bzr', 'checkout', checkout_path, 'bzr-package' - ) - result = script.run( - 'python', 'setup.py', 'develop', - cwd=script.scratch_path / 'bzr-package', + "python", + "setup.py", + "develop", + cwd=script.scratch_path / "bzr-package", expect_stderr=True, ) - result = script.pip('freeze', expect_stderr=True) - expected = textwrap.dedent("""\ + result = script.pip("freeze", expect_stderr=True) + expected = textwrap.dedent( + """\ ...-e bzr+file://...@1#egg=version_pkg - ...""") - _check_output(result.stdout, expected) - - result = script.pip( - 'freeze', '-f', - '{checkout_path}/#egg=django-wikiapp'.format(**locals()), - expect_stderr=True, + ...""" ) - expected = textwrap.dedent("""\ - -f {repo}/#egg=django-wikiapp - ...-e bzr+file://...@...#egg=version_pkg - ...""".format(repo=checkout_path)) _check_output(result.stdout, expected) @@ -529,9 +545,10 @@ def test_freeze_bazaar_clone(script, tmpdir): "outer_vcs, inner_vcs", [("hg", "git"), ("git", "hg")], ) -def test_freeze_nested_vcs(script, outer_vcs, inner_vcs): - """Test VCS can be correctly freezed when resides inside another VCS repo. - """ +def test_freeze_nested_vcs( + script: PipTestEnvironment, outer_vcs: str, inner_vcs: str +) -> None: + """Test VCS can be correctly freezed when resides inside another VCS repo.""" # Create Python package. pkg_path = _create_test_package(script, vcs=inner_vcs) @@ -557,7 +574,8 @@ def test_freeze_nested_vcs(script, outer_vcs, inner_vcs): # used by the test_freeze_with_requirement_* tests below -_freeze_req_opts = textwrap.dedent("""\ +_freeze_req_opts = textwrap.dedent( + """\ # Unchanged requirements below this line -r ignore.txt --requirement ignore.txt @@ -570,25 +588,30 @@ def test_freeze_nested_vcs(script, outer_vcs, inner_vcs): --find-links http://ignore --index-url http://ignore --use-feature 2020-resolver -""") +""" +) def test_freeze_with_requirement_option_file_url_egg_not_installed( - script, deprecated_python): + script: PipTestEnvironment, deprecated_python: bool +) -> None: """ Test "freeze -r requirements.txt" with a local file URL whose egg name is not installed. """ - url = path_to_url('my-package.tar.gz') + '#egg=Does.Not-Exist' - requirements_path = script.scratch_path.joinpath('requirements.txt') - requirements_path.write_text(url + '\n') + url = path_to_url("my-package.tar.gz") + "#egg=Does.Not-Exist" + requirements_path = script.scratch_path.joinpath("requirements.txt") + requirements_path.write_text(url + "\n") result = script.pip( - 'freeze', '--requirement', 'requirements.txt', expect_stderr=True, + "freeze", + "--requirement", + "requirements.txt", + expect_stderr=True, ) expected_err = ( - 'WARNING: Requirement file [requirements.txt] contains {}, ' + "WARNING: Requirement file [requirements.txt] contains {}, " "but package 'Does.Not-Exist' is not installed\n" ).format(url) if deprecated_python: @@ -597,32 +620,46 @@ def test_freeze_with_requirement_option_file_url_egg_not_installed( assert expected_err == result.stderr -def test_freeze_with_requirement_option(script): +def test_freeze_with_requirement_option(script: PipTestEnvironment) -> None: """ Test that new requirements are created correctly with --requirement hints """ - script.scratch_path.joinpath("hint1.txt").write_text(textwrap.dedent("""\ + script.scratch_path.joinpath("hint1.txt").write_text( + textwrap.dedent( + """\ INITools==0.1 NoExist==4.2 # A comment that ensures end of line comments work. simple==3.0; python_version > '1.0' - """) + _freeze_req_opts) - script.scratch_path.joinpath("hint2.txt").write_text(textwrap.dedent("""\ + """ + ) + + _freeze_req_opts + ) + script.scratch_path.joinpath("hint2.txt").write_text( + textwrap.dedent( + """\ iniTools==0.1 Noexist==4.2 # A comment that ensures end of line comments work. Simple==3.0; python_version > '1.0' - """) + _freeze_req_opts) - result = script.pip_install_local('initools==0.2') - result = script.pip_install_local('simple') + """ + ) + + _freeze_req_opts + ) + result = script.pip_install_local("initools==0.2") + result = script.pip_install_local("simple") result = script.pip( - 'freeze', '--requirement', 'hint1.txt', + "freeze", + "--requirement", + "hint1.txt", expect_stderr=True, ) - expected = textwrap.dedent("""\ + expected = textwrap.dedent( + """\ INITools==0.2 simple==3.0 - """) + """ + ) expected += _freeze_req_opts expected += "## The following requirements were added by pip freeze:..." _check_output(result.stdout, expected) @@ -631,7 +668,9 @@ def test_freeze_with_requirement_option(script): "'NoExist' is not installed" ) in result.stderr result = script.pip( - 'freeze', '--requirement', 'hint2.txt', + "freeze", + "--requirement", + "hint2.txt", expect_stderr=True, ) _check_output(result.stdout, expected) @@ -641,41 +680,61 @@ def test_freeze_with_requirement_option(script): ) in result.stderr -def test_freeze_with_requirement_option_multiple(script): +def test_freeze_with_requirement_option_multiple(script: PipTestEnvironment) -> None: """ Test that new requirements are created correctly with multiple --requirement hints """ - script.scratch_path.joinpath('hint1.txt').write_text(textwrap.dedent("""\ + script.scratch_path.joinpath("hint1.txt").write_text( + textwrap.dedent( + """\ INITools==0.1 NoExist==4.2 simple==3.0; python_version > '1.0' - """) + _freeze_req_opts) - script.scratch_path.joinpath('hint2.txt').write_text(textwrap.dedent("""\ + """ + ) + + _freeze_req_opts + ) + script.scratch_path.joinpath("hint2.txt").write_text( + textwrap.dedent( + """\ NoExist2==2.0 simple2==1.0 - """) + _freeze_req_opts) - result = script.pip_install_local('initools==0.2') - result = script.pip_install_local('simple') - result = script.pip_install_local('simple2==1.0') - result = script.pip_install_local('meta') + """ + ) + + _freeze_req_opts + ) + result = script.pip_install_local("initools==0.2") + result = script.pip_install_local("simple") + result = script.pip_install_local("simple2==1.0") + result = script.pip_install_local("meta") result = script.pip( - 'freeze', '--requirement', 'hint1.txt', '--requirement', 'hint2.txt', + "freeze", + "--requirement", + "hint1.txt", + "--requirement", + "hint2.txt", expect_stderr=True, ) - expected = textwrap.dedent("""\ + expected = textwrap.dedent( + """\ INITools==0.2 simple==1.0 - """) + """ + ) expected += _freeze_req_opts - expected += textwrap.dedent("""\ + expected += textwrap.dedent( + """\ simple2==1.0 - """) + """ + ) expected += "## The following requirements were added by pip freeze:" - expected += '\n' + textwrap.dedent("""\ + expected += "\n" + textwrap.dedent( + """\ ...meta==1.0... - """) + """ + ) _check_output(result.stdout, expected) assert ( "Requirement file [hint1.txt] contains NoExist==4.2, but package " @@ -690,136 +749,184 @@ def test_freeze_with_requirement_option_multiple(script): assert result.stdout.count("--index-url http://ignore") == 1 -def test_freeze_with_requirement_option_package_repeated_one_file(script): +def test_freeze_with_requirement_option_package_repeated_one_file( + script: PipTestEnvironment, +) -> None: """ Test freezing with single requirements file that contains a package multiple times """ - script.scratch_path.joinpath('hint1.txt').write_text(textwrap.dedent("""\ + script.scratch_path.joinpath("hint1.txt").write_text( + textwrap.dedent( + """\ simple2 simple2 NoExist - """) + _freeze_req_opts) - result = script.pip_install_local('simple2==1.0') - result = script.pip_install_local('meta') + """ + ) + + _freeze_req_opts + ) + result = script.pip_install_local("simple2==1.0") + result = script.pip_install_local("meta") result = script.pip( - 'freeze', '--requirement', 'hint1.txt', + "freeze", + "--requirement", + "hint1.txt", expect_stderr=True, ) - expected_out = textwrap.dedent("""\ + expected_out = textwrap.dedent( + """\ simple2==1.0 - """) + """ + ) expected_out += _freeze_req_opts expected_out += "## The following requirements were added by pip freeze:" - expected_out += '\n' + textwrap.dedent("""\ + expected_out += "\n" + textwrap.dedent( + """\ ...meta==1.0... - """) + """ + ) _check_output(result.stdout, expected_out) - err1 = ("Requirement file [hint1.txt] contains NoExist, " - "but package 'NoExist' is not installed\n") + err1 = ( + "Requirement file [hint1.txt] contains NoExist, " + "but package 'NoExist' is not installed\n" + ) err2 = "Requirement simple2 included multiple times [hint1.txt]\n" assert err1 in result.stderr assert err2 in result.stderr # there shouldn't be any other 'is not installed' warnings - assert result.stderr.count('is not installed') == 1 + assert result.stderr.count("is not installed") == 1 -def test_freeze_with_requirement_option_package_repeated_multi_file(script): +def test_freeze_with_requirement_option_package_repeated_multi_file( + script: PipTestEnvironment, +) -> None: """ Test freezing with multiple requirements file that contain a package """ - script.scratch_path.joinpath('hint1.txt').write_text(textwrap.dedent("""\ + script.scratch_path.joinpath("hint1.txt").write_text( + textwrap.dedent( + """\ simple - """) + _freeze_req_opts) - script.scratch_path.joinpath('hint2.txt').write_text(textwrap.dedent("""\ + """ + ) + + _freeze_req_opts + ) + script.scratch_path.joinpath("hint2.txt").write_text( + textwrap.dedent( + """\ simple NoExist - """) + _freeze_req_opts) - result = script.pip_install_local('simple==1.0') - result = script.pip_install_local('meta') + """ + ) + + _freeze_req_opts + ) + result = script.pip_install_local("simple==1.0") + result = script.pip_install_local("meta") result = script.pip( - 'freeze', '--requirement', 'hint1.txt', - '--requirement', 'hint2.txt', + "freeze", + "--requirement", + "hint1.txt", + "--requirement", + "hint2.txt", expect_stderr=True, ) - expected_out = textwrap.dedent("""\ + expected_out = textwrap.dedent( + """\ simple==1.0 - """) + """ + ) expected_out += _freeze_req_opts expected_out += "## The following requirements were added by pip freeze:" - expected_out += '\n' + textwrap.dedent("""\ + expected_out += "\n" + textwrap.dedent( + """\ ...meta==1.0... - """) + """ + ) _check_output(result.stdout, expected_out) - err1 = ("Requirement file [hint2.txt] contains NoExist, but package " - "'NoExist' is not installed\n") - err2 = ("Requirement simple included multiple times " - "[hint1.txt, hint2.txt]\n") + err1 = ( + "Requirement file [hint2.txt] contains NoExist, but package " + "'NoExist' is not installed\n" + ) + err2 = "Requirement simple included multiple times [hint1.txt, hint2.txt]\n" assert err1 in result.stderr assert err2 in result.stderr # there shouldn't be any other 'is not installed' warnings - assert result.stderr.count('is not installed') == 1 + assert result.stderr.count("is not installed") == 1 @pytest.mark.network @pytest.mark.incompatible_with_test_venv -def test_freeze_user(script, virtualenv, data): +def test_freeze_user( + script: PipTestEnvironment, virtualenv: VirtualEnvironment, data: TestData +) -> None: """ Testing freeze with --user, first we have to install some stuff. """ - script.pip('download', 'setuptools', 'wheel', '-d', data.packages) - script.pip_install_local('--find-links', data.find_links, - '--user', 'simple==2.0') - script.pip_install_local('--find-links', data.find_links, - 'simple2==3.0') - result = script.pip('freeze', '--user', expect_stderr=True) - expected = textwrap.dedent("""\ + script.pip("download", "setuptools", "wheel", "-d", data.packages) + script.pip_install_local("--find-links", data.find_links, "--user", "simple==2.0") + script.pip_install_local("--find-links", data.find_links, "simple2==3.0") + result = script.pip("freeze", "--user", expect_stderr=True) + expected = textwrap.dedent( + """\ simple==2.0 - """) + """ + ) _check_output(result.stdout, expected) - assert 'simple2' not in result.stdout + assert "simple2" not in result.stdout @pytest.mark.network -def test_freeze_path(tmpdir, script, data): +def test_freeze_path(tmpdir: Path, script: PipTestEnvironment, data: TestData) -> None: """ Test freeze with --path. """ - script.pip('install', '--find-links', data.find_links, - '--target', tmpdir, 'simple==2.0') - result = script.pip('freeze', '--path', tmpdir) - expected = textwrap.dedent("""\ + script.pip( + "install", "--find-links", data.find_links, "--target", tmpdir, "simple==2.0" + ) + result = script.pip("freeze", "--path", tmpdir) + expected = textwrap.dedent( + """\ simple==2.0 - """) + """ + ) _check_output(result.stdout, expected) @pytest.mark.network @pytest.mark.incompatible_with_test_venv -def test_freeze_path_exclude_user(tmpdir, script, data): +def test_freeze_path_exclude_user( + tmpdir: Path, script: PipTestEnvironment, data: TestData +) -> None: """ Test freeze with --path and make sure packages from --user are not picked up. """ - script.pip_install_local('--find-links', data.find_links, - '--user', 'simple2') - script.pip('install', '--find-links', data.find_links, - '--target', tmpdir, 'simple==1.0') - result = script.pip('freeze', '--user') - expected = textwrap.dedent("""\ + script.pip_install_local("--find-links", data.find_links, "--user", "simple2") + script.pip( + "install", "--find-links", data.find_links, "--target", tmpdir, "simple==1.0" + ) + result = script.pip("freeze", "--user") + expected = textwrap.dedent( + """\ simple2==3.0 - """) + """ + ) _check_output(result.stdout, expected) - result = script.pip('freeze', '--path', tmpdir) - expected = textwrap.dedent("""\ + result = script.pip("freeze", "--path", tmpdir) + expected = textwrap.dedent( + """\ simple==1.0 - """) + """ + ) _check_output(result.stdout, expected) @pytest.mark.network -def test_freeze_path_multiple(tmpdir, script, data): +def test_freeze_path_multiple( + tmpdir: Path, script: PipTestEnvironment, data: TestData +) -> None: """ Test freeze with multiple --path arguments. """ @@ -827,24 +934,33 @@ def test_freeze_path_multiple(tmpdir, script, data): os.mkdir(path1) path2 = tmpdir / "path2" os.mkdir(path2) - script.pip('install', '--find-links', data.find_links, - '--target', path1, 'simple==2.0') - script.pip('install', '--find-links', data.find_links, - '--target', path2, 'simple2==3.0') - result = script.pip('freeze', '--path', path1) - expected = textwrap.dedent("""\ + script.pip( + "install", "--find-links", data.find_links, "--target", path1, "simple==2.0" + ) + script.pip( + "install", "--find-links", data.find_links, "--target", path2, "simple2==3.0" + ) + result = script.pip("freeze", "--path", path1) + expected = textwrap.dedent( + """\ simple==2.0 - """) + """ + ) _check_output(result.stdout, expected) - result = script.pip('freeze', '--path', path1, '--path', path2) - expected = textwrap.dedent("""\ + result = script.pip("freeze", "--path", path1, "--path", path2) + expected = textwrap.dedent( + """\ simple==2.0 simple2==3.0 - """) + """ + ) _check_output(result.stdout, expected) -def test_freeze_direct_url_archive(script, shared_data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_freeze_direct_url_archive( + script: PipTestEnvironment, shared_data: TestData +) -> None: req = "simple @ " + path_to_url(shared_data.packages / "simple-2.0.tar.gz") assert req.startswith("simple @ file://") script.pip("install", req) @@ -852,38 +968,55 @@ def test_freeze_direct_url_archive(script, shared_data, with_wheel): assert req in result.stdout -def test_freeze_skip_work_dir_pkg(script): +def test_freeze_skip_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that freeze should not include package present in working directory """ # Create a test package and create .egg-info dir - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) # Freeze should not include package simple when run from package directory - result = script.pip('freeze', cwd=pkg_path) - assert 'simple' not in result.stdout + result = script.pip("freeze", cwd=pkg_path) + assert "simple" not in result.stdout -def test_freeze_include_work_dir_pkg(script): +def test_freeze_include_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that freeze should include package in working directory if working directory is added in PYTHONPATH """ # Create a test package and create .egg-info dir - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) - script.environ.update({'PYTHONPATH': pkg_path}) + script.environ.update({"PYTHONPATH": pkg_path}) # Freeze should include package simple when run from package directory, # when package directory is in PYTHONPATH - result = script.pip('freeze', cwd=pkg_path) - assert 'simple==1.0' in result.stdout + result = script.pip("freeze", cwd=pkg_path) + assert "simple==1.0" in result.stdout + + +@pytest.mark.usefixtures("with_wheel") +def test_freeze_pep610_editable(script: PipTestEnvironment) -> None: + """ + Test that a package installed with a direct_url.json with editable=true + is correctly frozeon as editable. + """ + pkg_path = _create_test_package(script, name="testpkg") + result = script.pip("install", pkg_path) + direct_url_path = get_created_direct_url_path(result, "testpkg") + assert direct_url_path + # patch direct_url.json to simulate an editable install + with open(direct_url_path) as f: + direct_url = DirectUrl.from_json(f.read()) + assert isinstance(direct_url.info, DirInfo) + direct_url.info.editable = True + with open(direct_url_path, "w") as f: + f.write(direct_url.to_json()) + result = script.pip("freeze") + assert "# Editable Git install with no remote (testpkg==0.1)" in result.stdout diff --git a/tests/functional/test_hash.py b/tests/functional/test_hash.py index 5d7bd975e18..7734ce58bb1 100644 --- a/tests/functional/test_hash.py +++ b/tests/functional/test_hash.py @@ -1,32 +1,39 @@ """Tests for the ``pip hash`` command""" +from tests.lib import PipTestEnvironment +from tests.lib.path import Path -def test_basic_hash(script, tmpdir): +def test_basic_hash(script: PipTestEnvironment, tmpdir: Path) -> None: """Run 'pip hash' through its default behavior.""" - expected = ('--hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425' - 'e73043362938b9824') - result = script.pip('hash', _hello_file(tmpdir)) + expected = ( + "--hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425" + "e73043362938b9824" + ) + result = script.pip("hash", _hello_file(tmpdir)) assert expected in str(result) -def test_good_algo_option(script, tmpdir): +def test_good_algo_option(script: PipTestEnvironment, tmpdir: Path) -> None: """Make sure the -a option works.""" - expected = ('--hash=sha512:9b71d224bd62f3785d96d46ad3ea3d73319bfbc2890caad' - 'ae2dff72519673ca72323c3d99ba5c11d7c7acc6e14b8c5da0c4663475c2e' - '5c3adef46f73bcdec043') - result = script.pip('hash', '-a', 'sha512', _hello_file(tmpdir)) + expected = ( + "--hash=sha512:9b71d224bd62f3785d96d46ad3ea3d73319bfbc2890caad" + "ae2dff72519673ca72323c3d99ba5c11d7c7acc6e14b8c5da0c4663475c2e" + "5c3adef46f73bcdec043" + ) + result = script.pip("hash", "-a", "sha512", _hello_file(tmpdir)) assert expected in str(result) -def test_bad_algo_option(script, tmpdir): +def test_bad_algo_option(script: PipTestEnvironment, tmpdir: Path) -> None: """Make sure the -a option raises an error when given a bad operand.""" - result = script.pip('hash', '-a', 'invalidname', _hello_file(tmpdir), - expect_error=True) + result = script.pip( + "hash", "-a", "invalidname", _hello_file(tmpdir), expect_error=True + ) assert "invalid choice: 'invalidname'" in str(result) -def _hello_file(tmpdir): +def _hello_file(tmpdir: Path) -> Path: """Return a temp file to hash containing "hello".""" - file = tmpdir / 'hashable' - file.write_text('hello') + file = tmpdir / "hashable" + file.write_text("hello") return file diff --git a/tests/functional/test_help.py b/tests/functional/test_help.py index a660cdf520d..dba41af5f79 100644 --- a/tests/functional/test_help.py +++ b/tests/functional/test_help.py @@ -1,107 +1,118 @@ +from unittest.mock import Mock + import pytest -from mock import Mock from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.commands import commands_dict, create_command from pip._internal.exceptions import CommandError +from tests.conftest import InMemoryPip +from tests.lib import PipTestEnvironment -def test_run_method_should_return_success_when_finds_command_name(): +def test_run_method_should_return_success_when_finds_command_name() -> None: """ Test HelpCommand.run for existing command """ options_mock = Mock() - args = ('freeze',) - help_cmd = create_command('help') + args = ["freeze"] + help_cmd = create_command("help") status = help_cmd.run(options_mock, args) assert status == SUCCESS -def test_run_method_should_return_success_when_command_name_not_specified(): +def test_run_method_should_return_success_when_command_name_not_specified() -> None: """ Test HelpCommand.run when there are no args """ options_mock = Mock() - args = () - help_cmd = create_command('help') - status = help_cmd.run(options_mock, args) + help_cmd = create_command("help") + status = help_cmd.run(options_mock, []) assert status == SUCCESS -def test_run_method_should_raise_command_error_when_command_does_not_exist(): +def test_run_method_should_raise_command_error_when_command_does_not_exist() -> None: """ Test HelpCommand.run for non-existing command """ options_mock = Mock() - args = ('mycommand',) - help_cmd = create_command('help') + args = ["mycommand"] + help_cmd = create_command("help") with pytest.raises(CommandError): help_cmd.run(options_mock, args) -def test_help_command_should_exit_status_ok_when_command_exists(script): +def test_help_command_should_exit_status_ok_when_command_exists( + script: PipTestEnvironment, +) -> None: """ Test `help` command for existing command """ - result = script.pip('help', 'freeze') + result = script.pip("help", "freeze") assert result.returncode == SUCCESS -def test_help_command_should_exit_status_ok_when_no_cmd_is_specified(script): +def test_help_command_should_exit_status_ok_when_no_cmd_is_specified( + script: PipTestEnvironment, +) -> None: """ Test `help` command for no command """ - result = script.pip('help') + result = script.pip("help") assert result.returncode == SUCCESS -def test_help_command_should_exit_status_error_when_cmd_does_not_exist(script): +def test_help_command_should_exit_status_error_when_cmd_does_not_exist( + script: PipTestEnvironment, +) -> None: """ Test `help` command for non-existing command """ - result = script.pip('help', 'mycommand', expect_error=True) + result = script.pip("help", "mycommand", expect_error=True) assert result.returncode == ERROR -def test_help_command_redact_auth_from_url(script): +def test_help_command_redact_auth_from_url(script: PipTestEnvironment) -> None: """ Test `help` on various subcommands redact auth from url """ - script.environ['PIP_INDEX_URL'] = 'https://user:secret@example.com' - result = script.pip('install', '--help') + script.environ["PIP_INDEX_URL"] = "https://user:secret@example.com" + result = script.pip("install", "--help") assert result.returncode == SUCCESS - assert 'secret' not in result.stdout + assert "secret" not in result.stdout -def test_help_command_redact_auth_from_url_with_extra_index_url(script): +def test_help_command_redact_auth_from_url_with_extra_index_url( + script: PipTestEnvironment, +) -> None: """ Test `help` on various subcommands redact auth from url with extra index url """ - script.environ['PIP_INDEX_URL'] = 'https://user:secret@example.com' - script.environ['PIP_EXTRA_INDEX_URL'] = 'https://user:secret@example2.com' - result = script.pip('install', '--help') + script.environ["PIP_INDEX_URL"] = "https://user:secret@example.com" + script.environ["PIP_EXTRA_INDEX_URL"] = "https://user:secret@example2.com" + result = script.pip("install", "--help") assert result.returncode == SUCCESS - assert 'secret' not in result.stdout + assert "secret" not in result.stdout -def test_help_commands_equally_functional(in_memory_pip): +def test_help_commands_equally_functional(in_memory_pip: InMemoryPip) -> None: """ Test if `pip help` and 'pip --help' behave the same way. """ - results = list(map(in_memory_pip.pip, ('help', '--help'))) + results = list(map(in_memory_pip.pip, ("help", "--help"))) results.append(in_memory_pip.pip()) out = map(lambda x: x.stdout, results) ret = map(lambda x: x.returncode, results) msg = '"pip --help" != "pip help" != "pip"' - assert len(set(out)) == 1, 'output of: ' + msg - assert sum(ret) == 0, 'exit codes of: ' + msg + assert len(set(out)) == 1, "output of: " + msg + assert sum(ret) == 0, "exit codes of: " + msg assert all(len(o) > 0 for o in out) for name in commands_dict: assert ( - in_memory_pip.pip('help', name).stdout == - in_memory_pip.pip(name, '--help').stdout != "" + in_memory_pip.pip("help", name).stdout + == in_memory_pip.pip(name, "--help").stdout + != "" ) diff --git a/tests/functional/test_index.py b/tests/functional/test_index.py new file mode 100644 index 00000000000..43b8f09c311 --- /dev/null +++ b/tests/functional/test_index.py @@ -0,0 +1,75 @@ +import pytest + +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.commands import create_command +from tests.lib import PipTestEnvironment + + +@pytest.mark.network +def test_list_all_versions_basic_search(script: PipTestEnvironment) -> None: + """ + End to end test of index versions command. + """ + output = script.pip("index", "versions", "pip", allow_stderr_warning=True) + assert "Available versions:" in output.stdout + assert ( + "20.2.3, 20.2.2, 20.2.1, 20.2, 20.1.1, 20.1, 20.0.2" + ", 20.0.1, 19.3.1, 19.3, 19.2.3, 19.2.2, 19.2.1, 19.2, 19.1.1" + ", 19.1, 19.0.3, 19.0.2, 19.0.1, 19.0, 18.1, 18.0, 10.0.1, 10.0.0, " + "9.0.3, 9.0.2, 9.0.1, 9.0.0, 8.1.2, 8.1.1, " + "8.1.0, 8.0.3, 8.0.2, 8.0.1, 8.0.0, 7.1.2, 7.1.1, 7.1.0, 7.0.3, " + "7.0.2, 7.0.1, 7.0.0, 6.1.1, 6.1.0, 6.0.8, 6.0.7, 6.0.6, 6.0.5, " + "6.0.4, 6.0.3, 6.0.2, 6.0.1, 6.0, 1.5.6, 1.5.5, 1.5.4, 1.5.3, " + "1.5.2, 1.5.1, 1.5, 1.4.1, 1.4, 1.3.1, 1.3, 1.2.1, 1.2, 1.1, 1.0.2," + " 1.0.1, 1.0, 0.8.3, 0.8.2, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.3, " + "0.6.2, 0.6.1, 0.6, 0.5.1, 0.5, 0.4, 0.3.1, " + "0.3, 0.2.1, 0.2" in output.stdout + ) + + +@pytest.mark.network +def test_list_all_versions_search_with_pre(script: PipTestEnvironment) -> None: + """ + See that adding the --pre flag adds pre-releases + """ + output = script.pip("index", "versions", "pip", "--pre", allow_stderr_warning=True) + assert "Available versions:" in output.stdout + assert ( + "20.2.3, 20.2.2, 20.2.1, 20.2, 20.2b1, 20.1.1, 20.1, 20.1b1, 20.0.2" + ", 20.0.1, 19.3.1, 19.3, 19.2.3, 19.2.2, 19.2.1, 19.2, 19.1.1" + ", 19.1, 19.0.3, 19.0.2, 19.0.1, 19.0, 18.1, 18.0, 10.0.1, 10.0.0, " + "10.0.0b2, 10.0.0b1, 9.0.3, 9.0.2, 9.0.1, 9.0.0, 8.1.2, 8.1.1, " + "8.1.0, 8.0.3, 8.0.2, 8.0.1, 8.0.0, 7.1.2, 7.1.1, 7.1.0, 7.0.3, " + "7.0.2, 7.0.1, 7.0.0, 6.1.1, 6.1.0, 6.0.8, 6.0.7, 6.0.6, 6.0.5, " + "6.0.4, 6.0.3, 6.0.2, 6.0.1, 6.0, 1.5.6, 1.5.5, 1.5.4, 1.5.3, " + "1.5.2, 1.5.1, 1.5, 1.4.1, 1.4, 1.3.1, 1.3, 1.2.1, 1.2, 1.1, 1.0.2," + " 1.0.1, 1.0, 0.8.3, 0.8.2, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.3, " + "0.6.2, 0.6.1, 0.6, 0.5.1, 0.5, 0.4, 0.3.1, " + "0.3, 0.2.1, 0.2" in output.stdout + ) + + +@pytest.mark.network +def test_list_all_versions_returns_no_matches_found_when_name_not_exact() -> None: + """ + Test that non exact name do not match + """ + command = create_command("index") + cmdline = "versions pand" + with command.main_context(): + options, args = command.parse_args(cmdline.split()) + status = command.run(options, args) + assert status == ERROR + + +@pytest.mark.network +def test_list_all_versions_returns_matches_found_when_name_is_exact() -> None: + """ + Test that exact name matches + """ + command = create_command("index") + cmdline = "versions pandas" + with command.main_context(): + options, args = command.parse_args(cmdline.split()) + status = command.run(options, args) + assert status == SUCCESS diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 9c36fef0ecb..e089a8f6932 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -7,13 +7,18 @@ import sys import textwrap from os.path import curdir, join, pardir +from typing import Dict, List, Tuple import pytest from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.models.index import PyPI, TestPyPI from pip._internal.utils.misc import rmtree +from tests.conftest import CertFactory from tests.lib import ( + PipTestEnvironment, + ResolverVariant, + TestData, _create_svn_repo, _create_test_package, create_basic_wheel_for_package, @@ -23,9 +28,7 @@ need_svn, path_to_url, pyversion, - pyversion_tuple, requirements_file, - windows_workaround_7667, ) from tests.lib.filesystem import make_socket_file from tests.lib.local_repos import local_checkout @@ -38,82 +41,131 @@ ) -@pytest.mark.parametrize('command', ('install', 'wheel')) -@pytest.mark.parametrize('variant', ('missing_setuptools', 'bad_setuptools')) -def test_pep518_uses_build_env(script, data, common_wheels, command, variant): - if variant == 'missing_setuptools': +@pytest.mark.parametrize("command", ("install", "wheel")) +@pytest.mark.parametrize("variant", ("missing_setuptools", "bad_setuptools")) +def test_pep518_uses_build_env( + script: PipTestEnvironment, + data: TestData, + common_wheels: Path, + command: str, + variant: str, +) -> None: + if variant == "missing_setuptools": script.pip("uninstall", "-y", "setuptools") - elif variant == 'bad_setuptools': + elif variant == "bad_setuptools": setuptools_mod = script.site_packages_path.joinpath("setuptools.py") - with open(setuptools_mod, 'a') as f: + with open(setuptools_mod, "a") as f: f.write('\nraise ImportError("toto")') else: raise ValueError(variant) script.pip( - command, '--no-index', '-f', common_wheels, '-f', data.packages, + command, + "--no-index", + "-f", + common_wheels, + "-f", + data.packages, data.src.joinpath("pep518-3.0"), ) def test_pep518_build_env_uses_same_pip( - script, data, pip_src, common_wheels, deprecated_python): + script: PipTestEnvironment, + data: TestData, + pip_src: Path, + common_wheels: Path, + deprecated_python: bool, +) -> None: """Ensure the subprocess call to pip for installing the build dependencies is using the same version of pip. """ - with open(script.scratch_path / 'pip.py', 'w') as fp: - fp.write('raise ImportError') + with open(script.scratch_path / "pip.py", "w") as fp: + fp.write("raise ImportError") script.run( - 'python', pip_src / 'src/pip', 'install', '--no-index', - '-f', common_wheels, '-f', data.packages, + "python", + pip_src / "src/pip", + "install", + "--no-index", + "-f", + common_wheels, + "-f", + data.packages, data.src.joinpath("pep518-3.0"), expect_stderr=deprecated_python, ) -def test_pep518_refuses_conflicting_requires(script, data): - create_basic_wheel_for_package(script, 'setuptools', '1.0') - create_basic_wheel_for_package(script, 'wheel', '1.0') +def test_pep518_refuses_conflicting_requires( + script: PipTestEnvironment, data: TestData +) -> None: + create_basic_wheel_for_package(script, "setuptools", "1.0") + create_basic_wheel_for_package(script, "wheel", "1.0") project_dir = data.src.joinpath("pep518_conflicting_requires") - result = script.pip_install_local('-f', script.scratch_path, - project_dir, expect_error=True) + result = script.pip_install_local( + "-f", script.scratch_path, project_dir, expect_error=True + ) assert ( - result.returncode != 0 and ( - 'Some build dependencies for {url} conflict ' - 'with PEP 517/518 supported ' - 'requirements: setuptools==1.0 is incompatible with ' - 'setuptools>=40.8.0.' - .format(url=path_to_url(project_dir))) in result.stderr + result.returncode != 0 + and ( + "Some build dependencies for {url} conflict " + "with PEP 517/518 supported " + "requirements: setuptools==1.0 is incompatible with " + "setuptools>=40.8.0.".format(url=path_to_url(project_dir)) + ) + in result.stderr ), str(result) -def test_pep518_refuses_invalid_requires(script, data, common_wheels): +def test_pep518_refuses_invalid_requires( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: result = script.pip( - 'install', '-f', common_wheels, + "install", + "-f", + common_wheels, data.src.joinpath("pep518_invalid_requires"), - expect_error=True + expect_error=True, ) assert result.returncode == 1 - assert "does not comply with PEP 518" in result.stderr + + # Ensure the relevant things are mentioned. + assert "PEP 518" in result.stderr + assert "not a list of strings" in result.stderr + assert "build-system.requires" in result.stderr + assert "pyproject.toml" in result.stderr -def test_pep518_refuses_invalid_build_system(script, data, common_wheels): +def test_pep518_refuses_invalid_build_system( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: result = script.pip( - 'install', '-f', common_wheels, + "install", + "-f", + common_wheels, data.src.joinpath("pep518_invalid_build_system"), - expect_error=True + expect_error=True, ) assert result.returncode == 1 - assert "does not comply with PEP 518" in result.stderr + # Ensure the relevant things are mentioned. + assert "PEP 518" in result.stderr + assert "mandatory `requires` key" in result.stderr + assert "[build-system] table" in result.stderr + assert "pyproject.toml" in result.stderr -def test_pep518_allows_missing_requires(script, data, common_wheels): + +def test_pep518_allows_missing_requires( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: result = script.pip( - 'install', '-f', common_wheels, + "install", + "-f", + common_wheels, data.src.joinpath("pep518_missing_requires"), - expect_stderr=True + expect_stderr=True, ) # Make sure we don't warn when this occurs. - assert "does not comply with PEP 518" not in result.stderr + assert "PEP 518" not in result.stderr # We want it to go through isolation for now. assert "Installing build dependencies" in result.stdout, result.stdout @@ -123,7 +175,9 @@ def test_pep518_allows_missing_requires(script, data, common_wheels): @pytest.mark.incompatible_with_test_venv -def test_pep518_with_user_pip(script, pip_src, data, common_wheels): +def test_pep518_with_user_pip( + script: PipTestEnvironment, pip_src: Path, data: TestData, common_wheels: Path +) -> None: """ Check that build dependencies are installed into the build environment without using build isolation for the pip invocation. @@ -133,113 +187,139 @@ def test_pep518_with_user_pip(script, pip_src, data, common_wheels): non-isolated environment, and break pip in the system site-packages, so that isolated uses of pip will fail. """ - script.pip("install", "--ignore-installed", - "-f", common_wheels, "--user", pip_src) - system_pip_dir = script.site_packages_path / 'pip' + script.pip("install", "--ignore-installed", "-f", common_wheels, "--user", pip_src) + system_pip_dir = script.site_packages_path / "pip" assert not system_pip_dir.exists() system_pip_dir.mkdir() - with open(system_pip_dir / '__init__.py', 'w') as fp: - fp.write('raise ImportError\n') + with open(system_pip_dir / "__init__.py", "w") as fp: + fp.write("raise ImportError\n") script.pip( - 'wheel', '--no-index', '-f', common_wheels, '-f', data.packages, + "wheel", + "--no-index", + "-f", + common_wheels, + "-f", + data.packages, data.src.joinpath("pep518-3.0"), ) -def test_pep518_with_extra_and_markers(script, data, common_wheels): +def test_pep518_with_extra_and_markers( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: script.pip( - 'wheel', '--no-index', - '-f', common_wheels, - '-f', data.find_links, + "wheel", + "--no-index", + "-f", + common_wheels, + "-f", + data.find_links, data.src.joinpath("pep518_with_extra_and_markers-1.0"), ) -def test_pep518_with_namespace_package(script, data, common_wheels): +def test_pep518_with_namespace_package( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: script.pip( - 'wheel', '--no-index', - '-f', common_wheels, - '-f', data.find_links, + "wheel", + "--no-index", + "-f", + common_wheels, + "-f", + data.find_links, data.src.joinpath("pep518_with_namespace_package-1.0"), use_module=True, ) -@pytest.mark.timeout(60) -@pytest.mark.parametrize('command', ('install', 'wheel')) -@pytest.mark.parametrize('package', ('pep518_forkbomb', - 'pep518_twin_forkbombs_first', - 'pep518_twin_forkbombs_second')) -def test_pep518_forkbombs(script, data, common_wheels, command, package): - package_source = next(data.packages.glob(package + '-[0-9]*.tar.gz')) +@pytest.mark.parametrize("command", ("install", "wheel")) +@pytest.mark.parametrize( + "package", + ("pep518_forkbomb", "pep518_twin_forkbombs_first", "pep518_twin_forkbombs_second"), +) +def test_pep518_forkbombs( + script: PipTestEnvironment, + data: TestData, + common_wheels: Path, + command: str, + package: str, +) -> None: + package_source = next(data.packages.glob(package + "-[0-9]*.tar.gz")) result = script.pip( - command, '--no-index', '-v', - '-f', common_wheels, - '-f', data.find_links, + command, + "--no-index", + "-v", + "-f", + common_wheels, + "-f", + data.find_links, package, expect_error=True, ) - assert '{1} is already being built: {0} from {1}'.format( - package, path_to_url(package_source), - ) in result.stderr, str(result) + assert ( + "{1} is already being built: {0} from {1}".format( + package, + path_to_url(package_source), + ) + in result.stderr + ), str(result) @pytest.mark.network +@pytest.mark.usefixtures("with_wheel") def test_pip_second_command_line_interface_works( - script, pip_src, data, common_wheels, deprecated_python, with_wheel -): + script: PipTestEnvironment, + pip_src: Path, + data: TestData, + common_wheels: Path, + deprecated_python: bool, +) -> None: """ Check if ``pip`` commands behaves equally """ # Re-install pip so we get the launchers. - script.pip_install_local('-f', common_wheels, pip_src) - # On old versions of Python, urllib3/requests will raise a warning about - # the lack of an SSLContext. - kwargs = {'expect_stderr': deprecated_python} - if pyversion_tuple < (2, 7, 9): - kwargs['expect_stderr'] = True - - args = ['pip{pyversion}'.format(**globals())] - args.extend(['install', 'INITools==0.2']) - args.extend(['-f', data.packages]) - result = script.run(*args, **kwargs) - dist_info_folder = ( - script.site_packages / - 'INITools-0.2.dist-info' - ) - initools_folder = script.site_packages / 'initools' + script.pip_install_local("-f", common_wheels, pip_src) + args = [f"pip{pyversion}"] + args.extend(["install", "INITools==0.2"]) + args.extend(["-f", data.packages]) + result = script.run(*args) + dist_info_folder = script.site_packages / "INITools-0.2.dist-info" + initools_folder = script.site_packages / "initools" result.did_create(dist_info_folder) result.did_create(initools_folder) -def test_install_exit_status_code_when_no_requirements(script): +def test_install_exit_status_code_when_no_requirements( + script: PipTestEnvironment, +) -> None: """ Test install exit status code when no requirements specified """ - result = script.pip('install', expect_error=True) + result = script.pip("install", expect_error=True) assert "You must give at least one requirement to install" in result.stderr assert result.returncode == ERROR -def test_install_exit_status_code_when_blank_requirements_file(script): +def test_install_exit_status_code_when_blank_requirements_file( + script: PipTestEnvironment, +) -> None: """ Test install exit status code when blank requirements file specified """ script.scratch_path.joinpath("blank.txt").write_text("\n") - script.pip('install', '-r', 'blank.txt') + script.pip("install", "-r", "blank.txt") @pytest.mark.network -def test_basic_install_from_pypi(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_basic_install_from_pypi(script: PipTestEnvironment) -> None: """ Test installing a package from PyPI. """ - result = script.pip('install', 'INITools==0.2') - dist_info_folder = ( - script.site_packages / - 'INITools-0.2.dist-info' - ) - initools_folder = script.site_packages / 'initools' + result = script.pip("install", "INITools==0.2") + dist_info_folder = script.site_packages / "INITools-0.2.dist-info" + initools_folder = script.site_packages / "initools" result.did_create(dist_info_folder) result.did_create(initools_folder) @@ -255,54 +335,51 @@ def test_basic_install_from_pypi(script, with_wheel): assert "https://" not in result.stdout -def test_basic_editable_install(script): +def test_basic_editable_install(script: PipTestEnvironment) -> None: """ Test editable installation. """ - result = script.pip('install', '-e', 'INITools==0.2', expect_error=True) - assert ( - "INITools==0.2 is not a valid editable requirement" - in result.stderr - ) + result = script.pip("install", "-e", "INITools==0.2", expect_error=True) + assert "INITools==0.2 is not a valid editable requirement" in result.stderr assert not result.files_created @need_svn -def test_basic_install_editable_from_svn(script): +def test_basic_install_editable_from_svn(script: PipTestEnvironment) -> None: """ Test checking out from svn. """ checkout_path = _create_test_package(script) repo_url = _create_svn_repo(script, checkout_path) - result = script.pip( - 'install', - '-e', 'svn+' + repo_url + '#egg=version-pkg' - ) - result.assert_installed('version-pkg', with_files=['.svn']) + result = script.pip("install", "-e", "svn+" + repo_url + "#egg=version-pkg") + result.assert_installed("version-pkg", with_files=[".svn"]) -def _test_install_editable_from_git(script, tmpdir): +def _test_install_editable_from_git(script: PipTestEnvironment) -> None: """Test cloning from Git.""" - pkg_path = _create_test_package(script, name='testpackage', vcs='git') + pkg_path = _create_test_package(script, name="testpackage", vcs="git") args = [ - 'install', '-e', - 'git+{url}#egg=testpackage'.format(url=path_to_url(pkg_path)), + "install", + "-e", + "git+{url}#egg=testpackage".format(url=path_to_url(pkg_path)), ] result = script.pip(*args) - result.assert_installed('testpackage', with_files=['.git']) + result.assert_installed("testpackage", with_files=[".git"]) -def test_basic_install_editable_from_git(script, tmpdir): - _test_install_editable_from_git(script, tmpdir) +def test_basic_install_editable_from_git(script: PipTestEnvironment) -> None: + _test_install_editable_from_git(script) -def test_install_editable_from_git_autobuild_wheel( - script, tmpdir, with_wheel): - _test_install_editable_from_git(script, tmpdir) +@pytest.mark.usefixtures("with_wheel") +def test_install_editable_from_git_autobuild_wheel(script: PipTestEnvironment) -> None: + _test_install_editable_from_git(script) @pytest.mark.network -def test_install_editable_uninstalls_existing(data, script, tmpdir): +def test_install_editable_uninstalls_existing( + data: TestData, script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test that installing an editable uninstalls a previously installed non-editable version. @@ -311,103 +388,118 @@ def test_install_editable_uninstalls_existing(data, script, tmpdir): """ to_install = data.packages.joinpath("pip-test-package-0.1.tar.gz") result = script.pip_install_local(to_install) - assert 'Successfully installed pip-test-package' in result.stdout - result.assert_installed('piptestpackage', editable=False) + assert "Successfully installed pip-test-package" in result.stdout + result.assert_installed("piptestpackage", editable=False) result = script.pip( - 'install', '-e', - '{dir}#egg=pip-test-package'.format( + "install", + "-e", + "{dir}#egg=pip-test-package".format( dir=local_checkout( - 'git+https://github.com/pypa/pip-test-package.git', tmpdir, - )), + "git+https://github.com/pypa/pip-test-package.git", + tmpdir, + ) + ), ) - result.assert_installed('pip-test-package', with_files=['.git']) - assert 'Found existing installation: pip-test-package 0.1' in result.stdout - assert 'Uninstalling pip-test-package-' in result.stdout - assert 'Successfully uninstalled pip-test-package' in result.stdout + result.assert_installed("pip-test-package", with_files=[".git"]) + assert "Found existing installation: pip-test-package 0.1" in result.stdout + assert "Uninstalling pip-test-package-" in result.stdout + assert "Successfully uninstalled pip-test-package" in result.stdout -def test_install_editable_uninstalls_existing_from_path(script, data): +def test_install_editable_uninstalls_existing_from_path( + script: PipTestEnvironment, data: TestData +) -> None: """ Test that installing an editable uninstalls a previously installed non-editable version from path """ - to_install = data.src.joinpath('simplewheel-1.0') + to_install = data.src.joinpath("simplewheel-1.0") result = script.pip_install_local(to_install) - assert 'Successfully installed simplewheel' in result.stdout - simple_folder = script.site_packages / 'simplewheel' - result.assert_installed('simplewheel', editable=False) + assert "Successfully installed simplewheel" in result.stdout + simple_folder = script.site_packages / "simplewheel" + result.assert_installed("simplewheel", editable=False) result.did_create(simple_folder) result = script.pip( - 'install', '-e', + "install", + "-e", to_install, ) - install_path = script.site_packages / 'simplewheel.egg-link' + install_path = script.site_packages / "simplewheel.egg-link" result.did_create(install_path) - assert 'Found existing installation: simplewheel 1.0' in result.stdout - assert 'Uninstalling simplewheel-' in result.stdout - assert 'Successfully uninstalled simplewheel' in result.stdout + assert "Found existing installation: simplewheel 1.0" in result.stdout + assert "Uninstalling simplewheel-" in result.stdout + assert "Successfully uninstalled simplewheel" in result.stdout assert simple_folder in result.files_deleted, str(result.stdout) @need_mercurial -def test_basic_install_editable_from_hg(script, tmpdir): +def test_basic_install_editable_from_hg(script: PipTestEnvironment) -> None: """Test cloning and hg+file install from Mercurial.""" - pkg_path = _create_test_package(script, name='testpackage', vcs='hg') - url = 'hg+{}#egg=testpackage'.format(path_to_url(pkg_path)) - assert url.startswith('hg+file') - args = ['install', '-e', url] + pkg_path = _create_test_package(script, name="testpackage", vcs="hg") + url = "hg+{}#egg=testpackage".format(path_to_url(pkg_path)) + assert url.startswith("hg+file") + args = ["install", "-e", url] result = script.pip(*args) - result.assert_installed('testpackage', with_files=['.hg']) + result.assert_installed("testpackage", with_files=[".hg"]) @need_mercurial -def test_vcs_url_final_slash_normalization(script, tmpdir): +def test_vcs_url_final_slash_normalization(script: PipTestEnvironment) -> None: """ Test that presence or absence of final slash in VCS URL is normalized. """ - pkg_path = _create_test_package(script, name='testpackage', vcs='hg') + pkg_path = _create_test_package(script, name="testpackage", vcs="hg") args = [ - 'install', - '-e', 'hg+{url}/#egg=testpackage'.format(url=path_to_url(pkg_path))] + "install", + "-e", + "hg+{url}/#egg=testpackage".format(url=path_to_url(pkg_path)), + ] result = script.pip(*args) - result.assert_installed('testpackage', with_files=['.hg']) + result.assert_installed("testpackage", with_files=[".hg"]) @need_bzr -def test_install_editable_from_bazaar(script, tmpdir): +def test_install_editable_from_bazaar(script: PipTestEnvironment) -> None: """Test checking out from Bazaar.""" - pkg_path = _create_test_package(script, name='testpackage', vcs='bazaar') + pkg_path = _create_test_package(script, name="testpackage", vcs="bazaar") args = [ - 'install', - '-e', 'bzr+{url}/#egg=testpackage'.format(url=path_to_url(pkg_path))] + "install", + "-e", + "bzr+{url}/#egg=testpackage".format(url=path_to_url(pkg_path)), + ] result = script.pip(*args) - result.assert_installed('testpackage', with_files=['.bzr']) + result.assert_installed("testpackage", with_files=[".bzr"]) @pytest.mark.network @need_bzr -def test_vcs_url_urlquote_normalization(script, tmpdir): +def test_vcs_url_urlquote_normalization( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test that urlquoted characters are normalized for repo URL comparison. """ script.pip( - 'install', '-e', - '{url}/#egg=django-wikiapp'.format( + "install", + "-e", + "{url}/#egg=django-wikiapp".format( url=local_checkout( - 'bzr+http://bazaar.launchpad.net/' - '%7Edjango-wikiapp/django-wikiapp' - '/release-0.1', + "bzr+http://bazaar.launchpad.net/" + "%7Edjango-wikiapp/django-wikiapp" + "/release-0.1", tmpdir, - )), + ) + ), ) @pytest.mark.parametrize("resolver", ["", "--use-deprecated=legacy-resolver"]) +@pytest.mark.usefixtures("with_wheel") def test_basic_install_from_local_directory( - script, data, resolver, with_wheel -): + script: PipTestEnvironment, data: TestData, resolver: str +) -> None: """ Test installing from a local directory. """ @@ -417,45 +509,39 @@ def test_basic_install_from_local_directory( to_install = data.packages.joinpath("FSPkg") args.append(to_install) result = script.pip(*args) - fspkg_folder = script.site_packages / 'fspkg' - dist_info_folder = ( - script.site_packages / - 'FSPkg-0.1.dev0.dist-info' - ) + fspkg_folder = script.site_packages / "fspkg" + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" result.did_create(fspkg_folder) result.did_create(dist_info_folder) -@pytest.mark.parametrize("test_type,editable", [ - ("rel_path", False), - ("rel_path", True), - ("rel_url", False), - ("rel_url", True), - ("embedded_rel_path", False), - ("embedded_rel_path", True), -]) +@pytest.mark.parametrize( + "test_type,editable", + [ + ("rel_path", False), + ("rel_path", True), + ("rel_url", False), + ("rel_url", True), + ("embedded_rel_path", False), + ("embedded_rel_path", True), + ], +) +@pytest.mark.usefixtures("with_wheel") def test_basic_install_relative_directory( - script, data, test_type, editable, with_wheel -): + script: PipTestEnvironment, data: TestData, test_type: str, editable: bool +) -> None: """ Test installing a requirement using a relative path. """ - dist_info_folder = ( - script.site_packages / - 'FSPkg-0.1.dev0.dist-info' - ) - egg_link_file = ( - script.site_packages / 'FSPkg.egg-link' - ) - package_folder = script.site_packages / 'fspkg' + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" + egg_link_file = script.site_packages / "FSPkg.egg-link" + package_folder = script.site_packages / "fspkg" # Compute relative install path to FSPkg from scratch path. full_rel_path = Path( - os.path.relpath(data.packages.joinpath('FSPkg'), script.scratch_path) - ) - full_rel_url = ( - 'file:' + full_rel_path.replace(os.path.sep, '/') + '#egg=FSPkg' + os.path.relpath(data.packages.joinpath("FSPkg"), script.scratch_path) ) + full_rel_url = "file:" + full_rel_path.replace(os.path.sep, "/") + "#egg=FSPkg" embedded_rel_path = script.scratch_path.joinpath(full_rel_path) req_path = { @@ -466,18 +552,16 @@ def test_basic_install_relative_directory( # Install as either editable or not. if not editable: - result = script.pip('install', req_path, - cwd=script.scratch_path) + result = script.pip("install", req_path, cwd=script.scratch_path) result.did_create(dist_info_folder) result.did_create(package_folder) else: # Editable install. - result = script.pip('install', '-e' + req_path, - cwd=script.scratch_path) + result = script.pip("install", "-e" + req_path, cwd=script.scratch_path) result.did_create(egg_link_file) -def test_install_quiet(script, data): +def test_install_quiet(script: PipTestEnvironment, data: TestData) -> None: """ Test that install -q is actually quiet. """ @@ -486,12 +570,14 @@ def test_install_quiet(script, data): # https://github.com/pypa/pip/issues/3418 # https://github.com/docker-library/python/issues/83 to_install = data.packages.joinpath("FSPkg") - result = script.pip('install', '-qqq', to_install) + result = script.pip("install", "-qqq", to_install) assert result.stdout == "" assert result.stderr == "" -def test_hashed_install_success(script, data, tmpdir): +def test_hashed_install_success( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test that installing various sorts of requirements with correct hashes works. @@ -500,18 +586,18 @@ def test_hashed_install_success(script, data, tmpdir): scenes). """ - file_url = path_to_url( - (data.packages / 'simple-1.0.tar.gz').resolve()) + file_url = path_to_url((data.packages / "simple-1.0.tar.gz").resolve()) with requirements_file( - 'simple2==1.0 --hash=sha256:9336af72ca661e6336eb87bc7de3e8844d853e' - '3848c2b9bbd2e8bf01db88c2c7\n' - '{simple} --hash=sha256:393043e672415891885c9a2a0929b1af95fb866d6c' - 'a016b42d2e6ce53619b653'.format(simple=file_url), - tmpdir) as reqs_file: - script.pip_install_local('-r', reqs_file.resolve()) + "simple2==1.0 --hash=sha256:9336af72ca661e6336eb87bc7de3e8844d853e" + "3848c2b9bbd2e8bf01db88c2c7\n" + "{simple} --hash=sha256:393043e672415891885c9a2a0929b1af95fb866d6c" + "a016b42d2e6ce53619b653".format(simple=file_url), + tmpdir, + ) as reqs_file: + script.pip_install_local("-r", reqs_file.resolve()) -def test_hashed_install_failure(script, tmpdir): +def test_hashed_install_failure(script: PipTestEnvironment, tmpdir: Path) -> None: """Test that wrong hashes stop installation. This makes sure prepare_files() is called in the course of installation @@ -519,24 +605,24 @@ def test_hashed_install_failure(script, tmpdir): kinds of hashes are in test_req.py. """ - with requirements_file('simple2==1.0 --hash=sha256:9336af72ca661e6336eb87b' - 'c7de3e8844d853e3848c2b9bbd2e8bf01db88c2c\n', - tmpdir) as reqs_file: - result = script.pip_install_local('-r', - reqs_file.resolve(), - expect_error=True) + with requirements_file( + "simple2==1.0 --hash=sha256:9336af72ca661e6336eb87b" + "c7de3e8844d853e3848c2b9bbd2e8bf01db88c2c\n", + tmpdir, + ) as reqs_file: + result = script.pip_install_local("-r", reqs_file.resolve(), expect_error=True) assert len(result.files_created) == 0 -def assert_re_match(pattern, text): - assert re.search(pattern, text), ( - f"Could not find {pattern!r} in {text!r}" - ) +def assert_re_match(pattern: str, text: str) -> None: + assert re.search(pattern, text), f"Could not find {pattern!r} in {text!r}" @pytest.mark.network @pytest.mark.skip("Fails on new resolver") -def test_hashed_install_failure_later_flag(script, tmpdir): +def test_hashed_install_failure_later_flag( + script: PipTestEnvironment, tmpdir: Path +) -> None: with requirements_file( "blessings==1.0\n" "tracefront==0.1 --hash=sha256:somehash\n" @@ -546,52 +632,73 @@ def test_hashed_install_failure_later_flag(script, tmpdir): "packages/source/p/peep/peep-3.1.1.tar.gz\n", tmpdir, ) as reqs_file: - result = script.pip( - "install", "-r", reqs_file.resolve(), expect_error=True - ) + result = script.pip("install", "-r", reqs_file.resolve(), expect_error=True) assert_re_match( - r'Hashes are required in --require-hashes mode, but they are ' - r'missing .*\n' - r' https://files\.pythonhosted\.org/packages/source/p/peep/peep' - r'-3\.1\.1\.tar\.gz --hash=sha256:[0-9a-f]+\n' - r' blessings==1.0 --hash=sha256:[0-9a-f]+\n' - r'THESE PACKAGES DO NOT MATCH THE HASHES.*\n' - r' tracefront==0.1 .*:\n' - r' Expected sha256 somehash\n' - r' Got [0-9a-f]+', + r"Hashes are required in --require-hashes mode, but they are " + r"missing .*\n" + r" https://files\.pythonhosted\.org/packages/source/p/peep/peep" + r"-3\.1\.1\.tar\.gz --hash=sha256:[0-9a-f]+\n" + r" blessings==1.0 --hash=sha256:[0-9a-f]+\n" + r"THESE PACKAGES DO NOT MATCH THE HASHES.*\n" + r" tracefront==0.1 .*:\n" + r" Expected sha256 somehash\n" + r" Got [0-9a-f]+", result.stderr, ) +@pytest.mark.usefixtures("with_wheel") def test_install_from_local_directory_with_symlinks_to_directories( - script, data, with_wheel -): + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing from a local directory containing symlinks to directories. """ to_install = data.packages.joinpath("symlinks") - result = script.pip('install', to_install) - pkg_folder = script.site_packages / 'symlinks' - dist_info_folder = ( - script.site_packages / - 'symlinks-0.1.dev0.dist-info' + result = script.pip( + "install", + "--use-deprecated=out-of-tree-build", + to_install, + allow_stderr_warning=True, # TODO: set to False when removing out-of-tree-build ) + pkg_folder = script.site_packages / "symlinks" + dist_info_folder = script.site_packages / "symlinks-0.1.dev0.dist-info" result.did_create(pkg_folder) result.did_create(dist_info_folder) -@pytest.mark.skipif("sys.platform == 'win32' or sys.version_info < (3,)") +@pytest.mark.usefixtures("with_wheel") +def test_install_from_local_directory_with_in_tree_build( + script: PipTestEnvironment, data: TestData +) -> None: + """ + Test installing from a local directory with default in tree build. + """ + to_install = data.packages.joinpath("FSPkg") + args = ["install", to_install] + + in_tree_build_dir = to_install / "build" + assert not in_tree_build_dir.exists() + result = script.pip(*args) + fspkg_folder = script.site_packages / "fspkg" + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" + result.did_create(fspkg_folder) + result.did_create(dist_info_folder) + assert in_tree_build_dir.exists() + + +@pytest.mark.skipif("sys.platform == 'win32'") +@pytest.mark.usefixtures("with_wheel") def test_install_from_local_directory_with_socket_file( - script, data, tmpdir, with_wheel -): + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test installing from a local directory containing a socket file. """ - dist_info_folder = ( - script.site_packages / - "FSPkg-0.1.dev0.dist-info" - ) + # TODO: remove this test when removing out-of-tree-build support, + # it is only meant to test the copy of socket files + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" package_folder = script.site_packages / "fspkg" to_copy = data.packages.joinpath("FSPkg") to_install = tmpdir.joinpath("src") @@ -601,70 +708,97 @@ def test_install_from_local_directory_with_socket_file( socket_file_path = os.path.join(to_install, "example") make_socket_file(socket_file_path) - result = script.pip("install", "--verbose", to_install) + result = script.pip( + "install", + "--use-deprecated=out-of-tree-build", + "--verbose", + to_install, + allow_stderr_warning=True, # because of the out-of-tree deprecation warning + ) result.did_create(package_folder) result.did_create(dist_info_folder) assert str(socket_file_path) in result.stderr -def test_install_from_local_directory_with_no_setup_py(script, data): +def test_install_from_local_directory_with_no_setup_py( + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing from a local directory with no 'setup.py'. """ - result = script.pip('install', data.root, expect_error=True) + result = script.pip("install", data.root, expect_error=True) assert not result.files_created - assert "is not installable." in result.stderr assert "Neither 'setup.py' nor 'pyproject.toml' found." in result.stderr def test_editable_install__local_dir_no_setup_py( - script, data, deprecated_python): + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing in editable mode from a local directory with no setup.py. """ - result = script.pip('install', '-e', data.root, expect_error=True) + result = script.pip("install", "-e", data.root, expect_error=True) assert not result.files_created - - msg = result.stderr - if deprecated_python: - assert 'File "setup.py" not found. ' in msg - else: - assert msg.startswith('ERROR: File "setup.py" not found. ') - assert 'pyproject.toml' not in msg + assert ( + "does not appear to be a Python project: " + "neither 'setup.py' nor 'pyproject.toml' found" in result.stderr + ) def test_editable_install__local_dir_no_setup_py_with_pyproject( - script, deprecated_python): + script: PipTestEnvironment, +) -> None: """ Test installing in editable mode from a local directory with no setup.py but that does have pyproject.toml. """ - local_dir = script.scratch_path.joinpath('temp') + local_dir = script.scratch_path.joinpath("temp") local_dir.mkdir() - pyproject_path = local_dir.joinpath('pyproject.toml') - pyproject_path.write_text('') + pyproject_path = local_dir.joinpath("pyproject.toml") + pyproject_path.write_text("") - result = script.pip('install', '-e', local_dir, expect_error=True) + result = script.pip("install", "-e", local_dir, expect_error=True) assert not result.files_created msg = result.stderr - if deprecated_python: - assert 'File "setup.py" not found. ' in msg - else: - assert msg.startswith('ERROR: File "setup.py" not found. ') - assert 'A "pyproject.toml" file was found' in msg + assert "has a 'pyproject.toml'" in msg + assert "does not have a 'setup.py' nor a 'setup.cfg'" in msg + assert "cannot be installed in editable mode" in msg + + +def test_editable_install__local_dir_setup_requires_with_pyproject( + script: PipTestEnvironment, shared_data: TestData +) -> None: + """ + Test installing in editable mode from a local directory with a setup.py + that has setup_requires and a pyproject.toml. + + https://github.com/pypa/pip/issues/10573 + """ + local_dir = script.scratch_path.joinpath("temp") + local_dir.mkdir() + pyproject_path = local_dir.joinpath("pyproject.toml") + pyproject_path.write_text("") + setup_py_path = local_dir.joinpath("setup.py") + setup_py_path.write_text( + "from setuptools import setup\n" + "setup(name='dummy', setup_requires=['simplewheel'])\n" + ) + + script.pip("install", "--find-links", shared_data.find_links, "-e", local_dir) @pytest.mark.network -def test_upgrade_argparse_shadowed(script): +def test_upgrade_argparse_shadowed(script: PipTestEnvironment) -> None: # If argparse is installed - even if shadowed for imported - we support # upgrading it and properly remove the older versions files. - script.pip('install', 'argparse==1.3') - result = script.pip('install', 'argparse>=1.4') + script.pip("install", "argparse==1.3") + result = script.pip("install", "argparse>=1.4") assert "Not uninstalling argparse" not in result.stdout -def test_install_curdir(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_curdir(script: PipTestEnvironment, data: TestData) -> None: """ Test installing current directory ('.'). """ @@ -673,69 +807,70 @@ def test_install_curdir(script, data, with_wheel): egg_info = join(run_from, "FSPkg.egg-info") if os.path.isdir(egg_info): rmtree(egg_info) - result = script.pip('install', curdir, cwd=run_from) - fspkg_folder = script.site_packages / 'fspkg' - dist_info_folder = ( - script.site_packages / - 'FSPkg-0.1.dev0.dist-info' - ) + result = script.pip("install", curdir, cwd=run_from) + fspkg_folder = script.site_packages / "fspkg" + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" result.did_create(fspkg_folder) result.did_create(dist_info_folder) -def test_install_pardir(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_pardir(script: PipTestEnvironment, data: TestData) -> None: """ Test installing parent directory ('..'). """ run_from = data.packages.joinpath("FSPkg", "fspkg") - result = script.pip('install', pardir, cwd=run_from) - fspkg_folder = script.site_packages / 'fspkg' - dist_info_folder = ( - script.site_packages / - 'FSPkg-0.1.dev0.dist-info' - ) + result = script.pip("install", pardir, cwd=run_from) + fspkg_folder = script.site_packages / "fspkg" + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" result.did_create(fspkg_folder) result.did_create(dist_info_folder) @pytest.mark.network -def test_install_global_option(script): +def test_install_global_option(script: PipTestEnvironment) -> None: """ Test using global distutils options. (In particular those that disable the actual install action) """ result = script.pip( - 'install', '--global-option=--version', "INITools==0.1", - expect_stderr=True) - assert 'INITools==0.1\n' in result.stdout + "install", "--global-option=--version", "INITools==0.1", expect_stderr=True + ) + assert "INITools==0.1\n" in result.stdout assert not result.files_created -def test_install_with_hacked_egg_info(script, data): +def test_install_with_hacked_egg_info( + script: PipTestEnvironment, data: TestData +) -> None: """ test installing a package which defines its own egg_info class """ run_from = data.packages.joinpath("HackedEggInfo") - result = script.pip('install', '.', cwd=run_from) - assert 'Successfully installed hackedegginfo-0.0.0\n' in result.stdout + result = script.pip("install", ".", cwd=run_from) + assert "Successfully installed hackedegginfo-0.0.0\n" in result.stdout @pytest.mark.network -def test_install_using_install_option_and_editable(script, tmpdir): +def test_install_using_install_option_and_editable( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test installing a tool using -e and --install-option """ - folder = 'script_folder' + folder = "script_folder" script.scratch_path.joinpath(folder).mkdir() - url = 'git+git://github.com/pypa/pip-test-package' + url = local_checkout("git+https://github.com/pypa/pip-test-package", tmpdir) result = script.pip( - 'install', '-e', '{url}#egg=pip-test-package' - .format(url=local_checkout(url, tmpdir)), - '--install-option=--script-dir={folder}'.format(**locals()), - expect_stderr=True) + "install", + "-e", + f"{url}#egg=pip-test-package", + f"--install-option=--script-dir={folder}", + expect_stderr=True, + ) script_file = ( - script.venv / 'src' / 'pip-test-package' / - folder / 'pip-test-package' + script.exe + script.venv / "src" / "pip-test-package" / folder / "pip-test-package" + + script.exe ) result.did_create(script_file) @@ -743,197 +878,207 @@ def test_install_using_install_option_and_editable(script, tmpdir): @pytest.mark.xfail @pytest.mark.network @need_mercurial -@windows_workaround_7667 -def test_install_global_option_using_editable(script, tmpdir): +def test_install_global_option_using_editable( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test using global distutils options, but in an editable installation """ - url = 'hg+http://bitbucket.org/runeh/anyjson' + url = "hg+http://bitbucket.org/runeh/anyjson" result = script.pip( - 'install', '--global-option=--version', '-e', - '{url}@0.2.5#egg=anyjson'.format(url=local_checkout(url, tmpdir)), - expect_stderr=True) - assert 'Successfully installed anyjson' in result.stdout + "install", + "--global-option=--version", + "-e", + "{url}@0.2.5#egg=anyjson".format(url=local_checkout(url, tmpdir)), + expect_stderr=True, + ) + assert "Successfully installed anyjson" in result.stdout @pytest.mark.network -def test_install_package_with_same_name_in_curdir(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_package_with_same_name_in_curdir(script: PipTestEnvironment) -> None: """ Test installing a package with the same name of a local folder """ script.scratch_path.joinpath("mock==0.6").mkdir() - result = script.pip('install', 'mock==0.6') - dist_info_folder = ( - script.site_packages / - 'mock-0.6.0.dist-info' - ) + result = script.pip("install", "mock==0.6") + dist_info_folder = script.site_packages / "mock-0.6.0.dist-info" result.did_create(dist_info_folder) -mock100_setup_py = textwrap.dedent('''\ +mock100_setup_py = textwrap.dedent( + """\ from setuptools import setup setup(name='mock', - version='100.1')''') + version='100.1')""" +) -def test_install_folder_using_dot_slash(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_folder_using_dot_slash(script: PipTestEnvironment) -> None: """ Test installing a folder using pip install ./foldername """ script.scratch_path.joinpath("mock").mkdir() - pkg_path = script.scratch_path / 'mock' + pkg_path = script.scratch_path / "mock" pkg_path.joinpath("setup.py").write_text(mock100_setup_py) - result = script.pip('install', './mock') - dist_info_folder = ( - script.site_packages / - 'mock-100.1.dist-info' - ) + result = script.pip("install", "./mock") + dist_info_folder = script.site_packages / "mock-100.1.dist-info" result.did_create(dist_info_folder) -def test_install_folder_using_slash_in_the_end(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_folder_using_slash_in_the_end(script: PipTestEnvironment) -> None: r""" Test installing a folder using pip install foldername/ or foldername\ """ script.scratch_path.joinpath("mock").mkdir() - pkg_path = script.scratch_path / 'mock' + pkg_path = script.scratch_path / "mock" pkg_path.joinpath("setup.py").write_text(mock100_setup_py) - result = script.pip('install', 'mock' + os.path.sep) - dist_info_folder = ( - script.site_packages / - 'mock-100.1.dist-info' - ) + result = script.pip("install", "mock" + os.path.sep) + dist_info_folder = script.site_packages / "mock-100.1.dist-info" result.did_create(dist_info_folder) -def test_install_folder_using_relative_path(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_folder_using_relative_path(script: PipTestEnvironment) -> None: """ Test installing a folder using pip install folder1/folder2 """ script.scratch_path.joinpath("initools").mkdir() script.scratch_path.joinpath("initools", "mock").mkdir() - pkg_path = script.scratch_path / 'initools' / 'mock' + pkg_path = script.scratch_path / "initools" / "mock" pkg_path.joinpath("setup.py").write_text(mock100_setup_py) - result = script.pip('install', Path('initools') / 'mock') - dist_info_folder = ( - script.site_packages / - 'mock-100.1.dist-info'.format(**globals()) - ) + result = script.pip("install", Path("initools") / "mock") + dist_info_folder = script.site_packages / "mock-100.1.dist-info" result.did_create(dist_info_folder) @pytest.mark.network -def test_install_package_which_contains_dev_in_name(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_package_which_contains_dev_in_name(script: PipTestEnvironment) -> None: """ Test installing package from PyPI which contains 'dev' in name """ - result = script.pip('install', 'django-devserver==0.0.4') - devserver_folder = script.site_packages / 'devserver' - dist_info_folder = ( - script.site_packages / - 'django_devserver-0.0.4.dist-info' - ) + result = script.pip("install", "django-devserver==0.0.4") + devserver_folder = script.site_packages / "devserver" + dist_info_folder = script.site_packages / "django_devserver-0.0.4.dist-info" result.did_create(devserver_folder) result.did_create(dist_info_folder) -def test_install_package_with_target(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_package_with_target(script: PipTestEnvironment) -> None: """ Test installing a package using pip install --target """ - target_dir = script.scratch_path / 'target' - result = script.pip_install_local('-t', target_dir, "simple==1.0") - result.did_create(Path('scratch') / 'target' / 'simple') + target_dir = script.scratch_path / "target" + result = script.pip_install_local("-t", target_dir, "simple==1.0") + result.did_create(Path("scratch") / "target" / "simple") # Test repeated call without --upgrade, no files should have changed result = script.pip_install_local( - '-t', target_dir, "simple==1.0", expect_stderr=True, + "-t", + target_dir, + "simple==1.0", + expect_stderr=True, ) - result.did_not_update(Path('scratch') / 'target' / 'simple') + result.did_not_update(Path("scratch") / "target" / "simple") # Test upgrade call, check that new version is installed - result = script.pip_install_local('--upgrade', '-t', - target_dir, "simple==2.0") - result.did_update(Path('scratch') / 'target' / 'simple') - dist_info_folder = ( - Path('scratch') / 'target' / - 'simple-2.0.dist-info' - ) + result = script.pip_install_local("--upgrade", "-t", target_dir, "simple==2.0") + result.did_update(Path("scratch") / "target" / "simple") + dist_info_folder = Path("scratch") / "target" / "simple-2.0.dist-info" result.did_create(dist_info_folder) # Test install and upgrade of single-module package - result = script.pip_install_local('-t', target_dir, 'singlemodule==0.0.0') - singlemodule_py = Path('scratch') / 'target' / 'singlemodule.py' + result = script.pip_install_local("-t", target_dir, "singlemodule==0.0.0") + singlemodule_py = Path("scratch") / "target" / "singlemodule.py" result.did_create(singlemodule_py) - result = script.pip_install_local('-t', target_dir, 'singlemodule==0.0.1', - '--upgrade') + result = script.pip_install_local( + "-t", target_dir, "singlemodule==0.0.1", "--upgrade" + ) result.did_update(singlemodule_py) -@pytest.mark.parametrize("target_option", ['--target', '-t']) -def test_install_package_to_usersite_with_target_must_fail(script, - target_option): +@pytest.mark.parametrize("target_option", ["--target", "-t"]) +def test_install_package_to_usersite_with_target_must_fail( + script: PipTestEnvironment, target_option: str +) -> None: """ Test that installing package to usersite with target must raise error """ - target_dir = script.scratch_path / 'target' + target_dir = script.scratch_path / "target" result = script.pip_install_local( - '--user', target_option, target_dir, "simple==1.0", expect_error=True - ) - assert "Can not combine '--user' and '--target'" in result.stderr, ( - str(result) + "--user", target_option, target_dir, "simple==1.0", expect_error=True ) + assert "Can not combine '--user' and '--target'" in result.stderr, str(result) -def test_install_nonlocal_compatible_wheel(script, data): - target_dir = script.scratch_path / 'target' +def test_install_nonlocal_compatible_wheel( + script: PipTestEnvironment, data: TestData +) -> None: + target_dir = script.scratch_path / "target" # Test install with --target result = script.pip( - 'install', - '-t', target_dir, - '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--python', '3', - '--platform', 'fakeplat', - '--abi', 'fakeabi', - 'simplewheel', + "install", + "-t", + target_dir, + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--python", + "3", + "--platform", + "fakeplat", + "--abi", + "fakeabi", + "simplewheel", ) assert result.returncode == SUCCESS - distinfo = Path('scratch') / 'target' / 'simplewheel-2.0-1.dist-info' + distinfo = Path("scratch") / "target" / "simplewheel-2.0-1.dist-info" result.did_create(distinfo) # Test install without --target result = script.pip( - 'install', - '--no-index', '--find-links', data.find_links, - '--only-binary=:all:', - '--python', '3', - '--platform', 'fakeplat', - '--abi', 'fakeabi', - 'simplewheel', - expect_error=True + "install", + "--no-index", + "--find-links", + data.find_links, + "--only-binary=:all:", + "--python", + "3", + "--platform", + "fakeplat", + "--abi", + "fakeabi", + "simplewheel", + expect_error=True, ) assert result.returncode == ERROR def test_install_nonlocal_compatible_wheel_path( - script, - data, - resolver_variant, -): - target_dir = script.scratch_path / 'target' + script: PipTestEnvironment, + data: TestData, + resolver_variant: ResolverVariant, +) -> None: + target_dir = script.scratch_path / "target" # Test a full path requirement result = script.pip( - 'install', - '-t', target_dir, - '--no-index', - '--only-binary=:all:', - Path(data.packages) / 'simplewheel-2.0-py3-fakeabi-fakeplat.whl', + "install", + "-t", + target_dir, + "--no-index", + "--only-binary=:all:", + Path(data.packages) / "simplewheel-2.0-py3-fakeabi-fakeplat.whl", expect_error=(resolver_variant == "2020-resolver"), ) if resolver_variant == "2020-resolver": @@ -941,29 +1086,35 @@ def test_install_nonlocal_compatible_wheel_path( else: assert result.returncode == SUCCESS - distinfo = Path('scratch') / 'target' / 'simplewheel-2.0.dist-info' + distinfo = Path("scratch") / "target" / "simplewheel-2.0.dist-info" result.did_create(distinfo) # Test a full path requirement (without --target) result = script.pip( - 'install', - '--no-index', - '--only-binary=:all:', - Path(data.packages) / 'simplewheel-2.0-py3-fakeabi-fakeplat.whl', - expect_error=True + "install", + "--no-index", + "--only-binary=:all:", + Path(data.packages) / "simplewheel-2.0-py3-fakeabi-fakeplat.whl", + expect_error=True, ) assert result.returncode == ERROR -def test_install_with_target_and_scripts_no_warning(script, with_wheel): +@pytest.mark.parametrize("opt", ("--target", "--prefix")) +@pytest.mark.usefixtures("with_wheel") +def test_install_with_target_or_prefix_and_scripts_no_warning( + opt: str, script: PipTestEnvironment +) -> None: """ Test that installing with --target does not trigger the "script not in PATH" warning (issue #5201) """ - target_dir = script.scratch_path / 'target' - pkga_path = script.scratch_path / 'pkga' + target_dir = script.scratch_path / "target" + pkga_path = script.scratch_path / "pkga" pkga_path.mkdir() - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', version='0.1', @@ -972,36 +1123,45 @@ def test_install_with_target_and_scripts_no_warning(script, with_wheel): 'console_scripts': ['pkga=pkga:main'] } ) - """)) - pkga_path.joinpath("pkga.py").write_text(textwrap.dedent(""" + """ + ) + ) + pkga_path.joinpath("pkga.py").write_text( + textwrap.dedent( + """ def main(): pass - """)) - result = script.pip('install', '--target', target_dir, pkga_path) + """ + ) + ) + result = script.pip("install", opt, target_dir, pkga_path) # This assertion isn't actually needed, if we get the script warning # the script.pip() call will fail with "stderr not expected". But we # leave the assertion to make the intention of the code clearer. assert "--no-warn-script-location" not in result.stderr, str(result) -def test_install_package_with_root(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_package_with_root(script: PipTestEnvironment, data: TestData) -> None: """ Test installing a package using pip install --root """ - root_dir = script.scratch_path / 'root' + root_dir = script.scratch_path / "root" result = script.pip( - 'install', '--root', root_dir, '-f', data.find_links, '--no-index', - 'simple==1.0', + "install", + "--root", + root_dir, + "-f", + data.find_links, + "--no-index", + "simple==1.0", ) normal_install_path = ( - script.base_path / script.site_packages / - 'simple-1.0.dist-info' + script.base_path / script.site_packages / "simple-1.0.dist-info" ) # use distutils to change the root exactly how the --root option does it from distutils.util import change_root - root_path = change_root( - os.path.join(script.scratch, 'root'), - normal_install_path - ) + + root_path = change_root(os.path.join(script.scratch, "root"), normal_install_path) result.did_create(root_path) # Should show find-links location in output @@ -1009,40 +1169,50 @@ def test_install_package_with_root(script, data, with_wheel): assert "Looking in links: " in result.stdout -def test_install_package_with_prefix(script, data): +def test_install_package_with_prefix( + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing a package using pip install --prefix """ - prefix_path = script.scratch_path / 'prefix' + prefix_path = script.scratch_path / "prefix" result = script.pip( - 'install', '--prefix', prefix_path, '-f', data.find_links, - '--no-binary', 'simple', '--no-index', 'simple==1.0', - ) - - rel_prefix_path = script.scratch / 'prefix' - install_path = ( - distutils.sysconfig.get_python_lib(prefix=rel_prefix_path) / + "install", + "--prefix", + prefix_path, + "-f", + data.find_links, + "--no-binary", + "simple", + "--no-index", + "simple==1.0", + ) + + rel_prefix_path = script.scratch / "prefix" + install_path = join( + distutils.sysconfig.get_python_lib(prefix=rel_prefix_path), # we still test for egg-info because no-binary implies setup.py install - f'simple-1.0-py{pyversion}.egg-info' + f"simple-1.0-py{pyversion}.egg-info", ) result.did_create(install_path) -def test_install_editable_with_prefix(script): +def _test_install_editable_with_prefix( + script: PipTestEnvironment, files: Dict[str, str] +) -> None: # make a dummy project - pkga_path = script.scratch_path / 'pkga' + pkga_path = script.scratch_path / "pkga" pkga_path.mkdir() - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" - from setuptools import setup - setup(name='pkga', - version='0.1') - """)) + + for fn, contents in files.items(): + pkga_path.joinpath(fn).write_text(textwrap.dedent(contents)) if hasattr(sys, "pypy_version_info"): site_packages = os.path.join( - 'prefix', 'lib', f'python{pyversion}', 'site-packages') + "prefix", "lib", f"python{pyversion}", "site-packages" + ) else: - site_packages = distutils.sysconfig.get_python_lib(prefix='prefix') + site_packages = distutils.sysconfig.get_python_lib(prefix="prefix") # make sure target path is in PYTHONPATH pythonpath = script.scratch_path / site_packages @@ -1050,31 +1220,85 @@ def test_install_editable_with_prefix(script): script.environ["PYTHONPATH"] = pythonpath # install pkga package into the absolute prefix directory - prefix_path = script.scratch_path / 'prefix' - result = script.pip( - 'install', '--editable', pkga_path, '--prefix', prefix_path) + prefix_path = script.scratch_path / "prefix" + result = script.pip("install", "--editable", pkga_path, "--prefix", prefix_path) # assert pkga is installed at correct location - install_path = script.scratch / site_packages / 'pkga.egg-link' + install_path = script.scratch / site_packages / "pkga.egg-link" result.did_create(install_path) -def test_install_package_conflict_prefix_and_user(script, data): +@pytest.mark.network +def test_install_editable_with_target(script: PipTestEnvironment) -> None: + pkg_path = script.scratch_path / "pkg" + pkg_path.mkdir() + pkg_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ + from setuptools import setup + setup( + name='pkg', + install_requires=['watching_testrunner'] + ) + """ + ) + ) + + target = script.scratch_path / "target" + target.mkdir() + result = script.pip("install", "--editable", pkg_path, "--target", target) + + result.did_create(script.scratch / "target" / "pkg.egg-link") + result.did_create(script.scratch / "target" / "watching_testrunner.py") + + +def test_install_editable_with_prefix_setup_py(script: PipTestEnvironment) -> None: + setup_py = """ +from setuptools import setup +setup(name='pkga', version='0.1') +""" + _test_install_editable_with_prefix(script, {"setup.py": setup_py}) + + +def test_install_editable_with_prefix_setup_cfg(script: PipTestEnvironment) -> None: + setup_cfg = """[metadata] +name = pkga +version = 0.1 +""" + pyproject_toml = """[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" +""" + _test_install_editable_with_prefix( + script, {"setup.cfg": setup_cfg, "pyproject.toml": pyproject_toml} + ) + + +def test_install_package_conflict_prefix_and_user( + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing a package using pip install --prefix --user errors out """ - prefix_path = script.scratch_path / 'prefix' + prefix_path = script.scratch_path / "prefix" result = script.pip( - 'install', '-f', data.find_links, '--no-index', '--user', - '--prefix', prefix_path, 'simple==1.0', - expect_error=True, quiet=True, - ) - assert ( - "Can not combine '--user' and '--prefix'" in result.stderr + "install", + "-f", + data.find_links, + "--no-index", + "--user", + "--prefix", + prefix_path, + "simple==1.0", + expect_error=True, + quiet=True, ) + assert "Can not combine '--user' and '--prefix'" in result.stderr -def test_install_package_that_emits_unicode(script, data): +def test_install_package_that_emits_unicode( + script: PipTestEnvironment, data: TestData +) -> None: """ Install a package with a setup.py that emits UTF-8 output and then fails. @@ -1082,28 +1306,39 @@ def test_install_package_that_emits_unicode(script, data): """ to_install = data.packages.joinpath("BrokenEmitsUTF8") result = script.pip( - 'install', to_install, expect_error=True, expect_temp=True, quiet=True, + "install", + to_install, + expect_error=True, + expect_temp=True, + quiet=True, ) assert ( - 'FakeError: this package designed to fail on install' in result.stderr - ), f'stderr: {result.stderr}' - assert 'UnicodeDecodeError' not in result.stderr - assert 'UnicodeDecodeError' not in result.stdout + "FakeError: this package designed to fail on install" in result.stderr + ), f"stderr: {result.stderr}" + assert "UnicodeDecodeError" not in result.stderr + assert "UnicodeDecodeError" not in result.stdout -def test_install_package_with_utf8_setup(script, data): +def test_install_package_with_utf8_setup( + script: PipTestEnvironment, data: TestData +) -> None: """Install a package with a setup.py that declares a utf-8 encoding.""" to_install = data.packages.joinpath("SetupPyUTF8") - script.pip('install', to_install) + script.pip("install", to_install) -def test_install_package_with_latin1_setup(script, data): +def test_install_package_with_latin1_setup( + script: PipTestEnvironment, data: TestData +) -> None: """Install a package with a setup.py that declares a latin-1 encoding.""" to_install = data.packages.joinpath("SetupPyLatin1") - script.pip('install', to_install) + script.pip("install", to_install) -def test_url_req_case_mismatch_no_index(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_url_req_case_mismatch_no_index( + script: PipTestEnvironment, data: TestData +) -> None: """ tar ball url requirements (with no egg fragment), that happen to have upper case project names, should be considered equal to later requirements that @@ -1112,21 +1347,22 @@ def test_url_req_case_mismatch_no_index(script, data, with_wheel): tests/data/packages contains Upper-1.0.tar.gz and Upper-2.0.tar.gz 'requiresupper' has install_requires = ['upper'] """ - Upper = '/'.join((data.find_links, 'Upper-1.0.tar.gz')) + Upper = "/".join((data.find_links, "Upper-1.0.tar.gz")) result = script.pip( - 'install', '--no-index', '-f', data.find_links, Upper, 'requiresupper' + "install", "--no-index", "-f", data.find_links, Upper, "requiresupper" ) # only Upper-1.0.tar.gz should get installed. - dist_info_folder = script.site_packages / \ - 'Upper-1.0.dist-info' + dist_info_folder = script.site_packages / "Upper-1.0.dist-info" result.did_create(dist_info_folder) - dist_info_folder = script.site_packages / \ - 'Upper-2.0.dist-info' + dist_info_folder = script.site_packages / "Upper-2.0.dist-info" result.did_not_create(dist_info_folder) -def test_url_req_case_mismatch_file_index(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_url_req_case_mismatch_file_index( + script: PipTestEnvironment, data: TestData +) -> None: """ tar ball url requirements (with no egg fragment), that happen to have upper case project names, should be considered equal to later requirements that @@ -1141,56 +1377,63 @@ def test_url_req_case_mismatch_file_index(script, data, with_wheel): set of packages as it requires a prepared index.html file and subdirectory-per-package structure. """ - Dinner = '/'.join((data.find_links3, 'dinner', 'Dinner-1.0.tar.gz')) + Dinner = "/".join((data.find_links3, "dinner", "Dinner-1.0.tar.gz")) result = script.pip( - 'install', '--index-url', data.find_links3, Dinner, 'requiredinner' + "install", "--index-url", data.find_links3, Dinner, "requiredinner" ) # only Upper-1.0.tar.gz should get installed. - dist_info_folder = script.site_packages / \ - 'Dinner-1.0.dist-info' + dist_info_folder = script.site_packages / "Dinner-1.0.dist-info" result.did_create(dist_info_folder) - dist_info_folder = script.site_packages / \ - 'Dinner-2.0.dist-info' + dist_info_folder = script.site_packages / "Dinner-2.0.dist-info" result.did_not_create(dist_info_folder) -def test_url_incorrect_case_no_index(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_url_incorrect_case_no_index( + script: PipTestEnvironment, data: TestData +) -> None: """ Same as test_url_req_case_mismatch_no_index, except testing for the case where the incorrect case is given in the name of the package to install rather than in a requirements file. """ result = script.pip( - 'install', '--no-index', '-f', data.find_links, "upper", + "install", + "--no-index", + "-f", + data.find_links, + "upper", ) # only Upper-2.0.tar.gz should get installed. - dist_info_folder = script.site_packages / \ - 'Upper-1.0.dist-info' + dist_info_folder = script.site_packages / "Upper-1.0.dist-info" result.did_not_create(dist_info_folder) - dist_info_folder = script.site_packages / \ - 'Upper-2.0.dist-info' + dist_info_folder = script.site_packages / "Upper-2.0.dist-info" result.did_create(dist_info_folder) -def test_url_incorrect_case_file_index(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_url_incorrect_case_file_index( + script: PipTestEnvironment, data: TestData +) -> None: """ Same as test_url_req_case_mismatch_file_index, except testing for the case where the incorrect case is given in the name of the package to install rather than in a requirements file. """ result = script.pip( - 'install', '--index-url', data.find_links3, "dinner", + "install", + "--index-url", + data.find_links3, + "dinner", expect_stderr=True, ) # only Upper-2.0.tar.gz should get installed. - dist_info_folder = script.site_packages / \ - 'Dinner-1.0.dist-info' + dist_info_folder = script.site_packages / "Dinner-1.0.dist-info" result.did_not_create(dist_info_folder) - dist_info_folder = script.site_packages / \ - 'Dinner-2.0.dist-info' + dist_info_folder = script.site_packages / "Dinner-2.0.dist-info" result.did_create(dist_info_folder) # Should show index-url location in output @@ -1199,7 +1442,7 @@ def test_url_incorrect_case_file_index(script, data, with_wheel): @pytest.mark.network -def test_compiles_pyc(script): +def test_compiles_pyc(script: PipTestEnvironment) -> None: """ Test installing with --compile on """ @@ -1210,17 +1453,14 @@ def test_compiles_pyc(script): # any of them exists = [ os.path.exists(script.site_packages_path / "initools/__init__.pyc"), + *glob.glob(script.site_packages_path / "initools/__pycache__/__init__*.pyc"), ] - exists += glob.glob( - script.site_packages_path / "initools/__pycache__/__init__*.pyc" - ) - assert any(exists) @pytest.mark.network -def test_no_compiles_pyc(script): +def test_no_compiles_pyc(script: PipTestEnvironment) -> None: """ Test installing from wheel with --compile on """ @@ -1231,42 +1471,51 @@ def test_no_compiles_pyc(script): # any of them exists = [ os.path.exists(script.site_packages_path / "initools/__init__.pyc"), + *glob.glob(script.site_packages_path / "initools/__pycache__/__init__*.pyc"), ] - exists += glob.glob( - script.site_packages_path / "initools/__pycache__/__init__*.pyc" - ) - assert not any(exists) -def test_install_upgrade_editable_depending_on_other_editable(script): +def test_install_upgrade_editable_depending_on_other_editable( + script: PipTestEnvironment, +) -> None: script.scratch_path.joinpath("pkga").mkdir() - pkga_path = script.scratch_path / 'pkga' - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path = script.scratch_path / "pkga" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', version='0.1') - """)) - script.pip('install', '--editable', pkga_path) - result = script.pip('list', '--format=freeze') + """ + ) + ) + script.pip("install", "--editable", pkga_path) + result = script.pip("list", "--format=freeze") assert "pkga==0.1" in result.stdout script.scratch_path.joinpath("pkgb").mkdir() - pkgb_path = script.scratch_path / 'pkgb' - pkgb_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkgb_path = script.scratch_path / "pkgb" + pkgb_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkgb', version='0.1', install_requires=['pkga']) - """)) - script.pip('install', '--upgrade', '--editable', pkgb_path, '--no-index') - result = script.pip('list', '--format=freeze') + """ + ) + ) + script.pip("install", "--upgrade", "--editable", pkgb_path, "--no-index") + result = script.pip("list", "--format=freeze") assert "pkgb==0.1" in result.stdout -def test_install_subprocess_output_handling(script, data): - args = ['install', data.src.joinpath('chattymodule')] +def test_install_subprocess_output_handling( + script: PipTestEnvironment, data: TestData +) -> None: + args = ["install", data.src.joinpath("chattymodule")] # Regular install should not show output from the chatty setup.py result = script.pip(*args) @@ -1282,68 +1531,77 @@ def test_install_subprocess_output_handling(script, data): # If the install fails, then we *should* show the output... but only once, # even if --verbose is given. - result = script.pip(*(args + ["--global-option=--fail"]), - expect_error=True) + result = script.pip(*(args + ["--global-option=--fail"]), expect_error=True) assert 1 == result.stderr.count("I DIE, I DIE") - result = script.pip(*(args + ["--global-option=--fail", "--verbose"]), - expect_error=True) + result = script.pip( + *(args + ["--global-option=--fail", "--verbose"]), expect_error=True + ) assert 1 == result.stderr.count("I DIE, I DIE") -def test_install_log(script, data, tmpdir): +def test_install_log(script: PipTestEnvironment, data: TestData, tmpdir: Path) -> None: # test that verbose logs go to "--log" file f = tmpdir.joinpath("log.txt") - args = ['--log={f}'.format(**locals()), - 'install', data.src.joinpath('chattymodule')] + args = [f"--log={f}", "install", data.src.joinpath("chattymodule")] result = script.pip(*args) assert 0 == result.stdout.count("HELLO FROM CHATTYMODULE") - with open(f, 'r') as fp: + with open(f) as fp: # one from egg_info, one from install assert 2 == fp.read().count("HELLO FROM CHATTYMODULE") -def test_install_topological_sort(script, data): - args = ['install', 'TopoRequires4', '--no-index', '-f', data.packages] +def test_install_topological_sort(script: PipTestEnvironment, data: TestData) -> None: + args = ["install", "TopoRequires4", "--no-index", "-f", data.packages] res = str(script.pip(*args)) - order1 = 'TopoRequires, TopoRequires2, TopoRequires3, TopoRequires4' - order2 = 'TopoRequires, TopoRequires3, TopoRequires2, TopoRequires4' + order1 = "TopoRequires, TopoRequires2, TopoRequires3, TopoRequires4" + order2 = "TopoRequires, TopoRequires3, TopoRequires2, TopoRequires4" assert order1 in res or order2 in res, res -def test_install_wheel_broken(script, with_wheel): - res = script.pip_install_local('wheelbroken', expect_stderr=True) +@pytest.mark.usefixtures("with_wheel") +def test_install_wheel_broken(script: PipTestEnvironment) -> None: + res = script.pip_install_local("wheelbroken", expect_stderr=True) assert "Successfully installed wheelbroken-0.1" in str(res), str(res) -def test_cleanup_after_failed_wheel(script, with_wheel): - res = script.pip_install_local('wheelbrokenafter', expect_stderr=True) +@pytest.mark.usefixtures("with_wheel") +def test_cleanup_after_failed_wheel(script: PipTestEnvironment) -> None: + res = script.pip_install_local("wheelbrokenafter", expect_stderr=True) # One of the effects of not cleaning up is broken scripts: script_py = script.bin_path / "script.py" assert script_py.exists(), script_py - shebang = open(script_py, 'r').readline().strip() - assert shebang != '#!python', shebang + with open(script_py) as f: + shebang = f.readline().strip() + assert shebang != "#!python", shebang # OK, assert that we *said* we were cleaning up: # /!\ if in need to change this, also change test_pep517_no_legacy_cleanup assert "Running setup.py clean for wheelbrokenafter" in str(res), str(res) -def test_install_builds_wheels(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_builds_wheels(script: PipTestEnvironment, data: TestData) -> None: # We need to use a subprocess to get the right value on Windows. - res = script.run('python', '-c', ( - 'from pip._internal.utils import appdirs; ' - 'print(appdirs.user_cache_dir("pip"))' - )) - wheels_cache = os.path.join(res.stdout.rstrip('\n'), 'wheels') + res = script.run( + "python", + "-c", + ( + "from pip._internal.utils import appdirs; " + 'print(appdirs.user_cache_dir("pip"))' + ), + ) + wheels_cache = os.path.join(res.stdout.rstrip("\n"), "wheels") # NB This incidentally tests a local tree + tarball inputs # see test_install_editable_from_git_autobuild_wheel for editable # vcs coverage. - to_install = data.packages.joinpath('requires_wheelbroken_upper') + to_install = data.packages.joinpath("requires_wheelbroken_upper") res = script.pip( - 'install', '--no-index', '-f', data.find_links, - to_install, expect_stderr=True) - expected = ("Successfully installed requires-wheelbroken-upper-0" - " upper-2.0 wheelbroken-0.1") + "install", "--no-index", "-f", data.find_links, to_install, expect_stderr=True + ) + expected = ( + "Successfully installed requires-wheelbroken-upper-0" + " upper-2.0 wheelbroken-0.1" + ) # Must have installed it all assert expected in str(res), str(res) wheels = [] @@ -1369,13 +1627,24 @@ def test_install_builds_wheels(script, data, with_wheel): ] -def test_install_no_binary_disables_building_wheels(script, data, with_wheel): - to_install = data.packages.joinpath('requires_wheelbroken_upper') +@pytest.mark.usefixtures("with_wheel") +def test_install_no_binary_disables_building_wheels( + script: PipTestEnvironment, data: TestData +) -> None: + to_install = data.packages.joinpath("requires_wheelbroken_upper") res = script.pip( - 'install', '--no-index', '--no-binary=upper', '-f', data.find_links, - to_install, expect_stderr=True) - expected = ("Successfully installed requires-wheelbroken-upper-0" - " upper-2.0 wheelbroken-0.1") + "install", + "--no-index", + "--no-binary=upper", + "-f", + data.find_links, + to_install, + expect_stderr=True, + ) + expected = ( + "Successfully installed requires-wheelbroken-upper-0" + " upper-2.0 wheelbroken-0.1" + ) # Must have installed it all assert expected in str(res), str(res) # and built wheels for wheelbroken only @@ -1392,13 +1661,13 @@ def test_install_no_binary_disables_building_wheels(script, data, with_wheel): @pytest.mark.network -@windows_workaround_7667 -def test_install_no_binary_builds_pep_517_wheel(script, data, with_wheel): - to_install = data.packages.joinpath('pep517_setup_and_pyproject') - res = script.pip( - 'install', '--no-binary=:all:', '-f', data.find_links, to_install - ) - expected = ("Successfully installed pep517-setup-and-pyproject") +@pytest.mark.usefixtures("with_wheel") +def test_install_no_binary_builds_pep_517_wheel( + script: PipTestEnvironment, data: TestData +) -> None: + to_install = data.packages.joinpath("pep517_setup_and_pyproject") + res = script.pip("install", "--no-binary=:all:", "-f", data.find_links, to_install) + expected = "Successfully installed pep517-setup-and-pyproject" # Must have installed the package assert expected in str(res), str(res) @@ -1407,14 +1676,13 @@ def test_install_no_binary_builds_pep_517_wheel(script, data, with_wheel): @pytest.mark.network -@windows_workaround_7667 +@pytest.mark.usefixtures("with_wheel") def test_install_no_binary_uses_local_backend( - script, data, with_wheel, tmpdir): - to_install = data.packages.joinpath('pep517_wrapper_buildsys') - script.environ['PIP_TEST_MARKER_FILE'] = marker = str(tmpdir / 'marker') - res = script.pip( - 'install', '--no-binary=:all:', '-f', data.find_links, to_install - ) + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: + to_install = data.packages.joinpath("pep517_wrapper_buildsys") + script.environ["PIP_TEST_MARKER_FILE"] = marker = str(tmpdir / "marker") + res = script.pip("install", "--no-binary=:all:", "-f", data.find_links, to_install) expected = "Successfully installed pep517-wrapper-buildsys" # Must have installed the package assert expected in str(res), str(res) @@ -1422,15 +1690,22 @@ def test_install_no_binary_uses_local_backend( assert os.path.isfile(marker), "Local PEP 517 backend not used" -def test_install_no_binary_disables_cached_wheels(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_no_binary_disables_cached_wheels( + script: PipTestEnvironment, data: TestData +) -> None: # Seed the cache - script.pip( - 'install', '--no-index', '-f', data.find_links, - 'upper') - script.pip('uninstall', 'upper', '-y') + script.pip("install", "--no-index", "-f", data.find_links, "upper") + script.pip("uninstall", "upper", "-y") res = script.pip( - 'install', '--no-index', '--no-binary=:all:', '-f', data.find_links, - 'upper', expect_stderr=True) + "install", + "--no-index", + "--no-binary=:all:", + "-f", + data.find_links, + "upper", + expect_stderr=True, + ) assert "Successfully installed upper-2.0" in str(res), str(res) # No wheel building for upper, which was blacklisted assert "Building wheel for upper" not in str(res), str(res) @@ -1438,173 +1713,210 @@ def test_install_no_binary_disables_cached_wheels(script, data, with_wheel): assert "Running setup.py install for upper" in str(res), str(res) -def test_install_editable_with_wrong_egg_name(script, resolver_variant): +def test_install_editable_with_wrong_egg_name( + script: PipTestEnvironment, resolver_variant: ResolverVariant +) -> None: script.scratch_path.joinpath("pkga").mkdir() - pkga_path = script.scratch_path / 'pkga' - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path = script.scratch_path / "pkga" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', version='0.1') - """)) + """ + ) + ) result = script.pip( - 'install', '--editable', - 'file://{pkga_path}#egg=pkgb'.format(**locals()), + "install", + "--editable", + f"file://{pkga_path}#egg=pkgb", expect_error=(resolver_variant == "2020-resolver"), ) - assert ("Generating metadata for package pkgb produced metadata " - "for project name pkga. Fix your #egg=pkgb " - "fragments.") in result.stderr + assert ( + "Generating metadata for package pkgb produced metadata " + "for project name pkga. Fix your #egg=pkgb " + "fragments." + ) in result.stderr if resolver_variant == "2020-resolver": - assert "has different name in metadata" in result.stderr, str(result) + assert "has inconsistent" in result.stdout, str(result) else: assert "Successfully installed pkga" in str(result), str(result) -def test_install_tar_xz(script, data): +def test_install_tar_xz(script: PipTestEnvironment, data: TestData) -> None: try: import lzma # noqa except ImportError: pytest.skip("No lzma support") - res = script.pip('install', data.packages / 'singlemodule-0.0.1.tar.xz') + res = script.pip("install", data.packages / "singlemodule-0.0.1.tar.xz") assert "Successfully installed singlemodule-0.0.1" in res.stdout, res -def test_install_tar_lzma(script, data): +def test_install_tar_lzma(script: PipTestEnvironment, data: TestData) -> None: try: import lzma # noqa except ImportError: pytest.skip("No lzma support") - res = script.pip('install', data.packages / 'singlemodule-0.0.1.tar.lzma') + res = script.pip("install", data.packages / "singlemodule-0.0.1.tar.lzma") assert "Successfully installed singlemodule-0.0.1" in res.stdout, res -def test_double_install(script): +def test_double_install(script: PipTestEnvironment) -> None: """ Test double install passing with two same version requirements """ - result = script.pip('install', 'pip', 'pip') + result = script.pip("install", "pip", "pip") msg = "Double requirement given: pip (already in pip, name='pip')" assert msg not in result.stderr -def test_double_install_fail(script, resolver_variant): +def test_double_install_fail( + script: PipTestEnvironment, resolver_variant: ResolverVariant +) -> None: """ Test double install failing with two different version requirements """ result = script.pip( - 'install', - 'pip==7.*', - 'pip==7.1.2', + "install", + "pip==7.*", + "pip==7.1.2", # The new resolver is perfectly capable of handling this expect_error=(resolver_variant == "legacy"), ) if resolver_variant == "legacy": - msg = ("Double requirement given: pip==7.1.2 (already in pip==7.*, " - "name='pip')") + msg = "Double requirement given: pip==7.1.2 (already in pip==7.*, name='pip')" assert msg in result.stderr -def _get_expected_error_text(): - return ( - "Package 'pkga' requires a different Python: {} not in '<1.0'" - ).format('.'.join(map(str, sys.version_info[:3]))) +def _get_expected_error_text() -> str: + return ("Package 'pkga' requires a different Python: {} not in '<1.0'").format( + ".".join(map(str, sys.version_info[:3])) + ) -def test_install_incompatible_python_requires(script): +def test_install_incompatible_python_requires(script: PipTestEnvironment) -> None: script.scratch_path.joinpath("pkga").mkdir() - pkga_path = script.scratch_path / 'pkga' - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path = script.scratch_path / "pkga" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', python_requires='<1.0', version='0.1') - """)) - result = script.pip('install', pkga_path, expect_error=True) + """ + ) + ) + result = script.pip("install", pkga_path, expect_error=True) assert _get_expected_error_text() in result.stderr, str(result) -def test_install_incompatible_python_requires_editable(script): +def test_install_incompatible_python_requires_editable( + script: PipTestEnvironment, +) -> None: script.scratch_path.joinpath("pkga").mkdir() - pkga_path = script.scratch_path / 'pkga' - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path = script.scratch_path / "pkga" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', python_requires='<1.0', version='0.1') - """)) - result = script.pip( - 'install', - '--editable={pkga_path}'.format(**locals()), - expect_error=True) + """ + ) + ) + result = script.pip("install", f"--editable={pkga_path}", expect_error=True) assert _get_expected_error_text() in result.stderr, str(result) -def test_install_incompatible_python_requires_wheel(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_incompatible_python_requires_wheel(script: PipTestEnvironment) -> None: script.scratch_path.joinpath("pkga").mkdir() - pkga_path = script.scratch_path / 'pkga' - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path = script.scratch_path / "pkga" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', python_requires='<1.0', version='0.1') - """)) + """ + ) + ) script.run( - 'python', 'setup.py', 'bdist_wheel', '--universal', + "python", + "setup.py", + "bdist_wheel", + "--universal", cwd=pkga_path, ) - result = script.pip('install', './pkga/dist/pkga-0.1-py2.py3-none-any.whl', - expect_error=True) + result = script.pip( + "install", "./pkga/dist/pkga-0.1-py2.py3-none-any.whl", expect_error=True + ) assert _get_expected_error_text() in result.stderr, str(result) -def test_install_compatible_python_requires(script): +def test_install_compatible_python_requires(script: PipTestEnvironment) -> None: script.scratch_path.joinpath("pkga").mkdir() - pkga_path = script.scratch_path / 'pkga' - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path = script.scratch_path / "pkga" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', python_requires='>1.0', version='0.1') - """)) - res = script.pip('install', pkga_path) + """ + ) + ) + res = script.pip("install", pkga_path) assert "Successfully installed pkga-0.1" in res.stdout, res @pytest.mark.network -def test_install_pep508_with_url(script): +def test_install_pep508_with_url(script: PipTestEnvironment) -> None: res = script.pip( - 'install', '--no-index', - 'packaging@https://files.pythonhosted.org/packages/2f/2b/' - 'c681de3e1dbcd469537aefb15186b800209aa1f299d933d23b48d85c9d56/' - 'packaging-15.3-py2.py3-none-any.whl#sha256=' - 'ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4' + "install", + "--no-index", + "packaging@https://files.pythonhosted.org/packages/2f/2b/" + "c681de3e1dbcd469537aefb15186b800209aa1f299d933d23b48d85c9d56/" + "packaging-15.3-py2.py3-none-any.whl#sha256=" + "ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4", ) assert "Successfully installed packaging-15.3" in str(res), str(res) @pytest.mark.network -def test_install_pep508_with_url_in_install_requires(script): +def test_install_pep508_with_url_in_install_requires( + script: PipTestEnvironment, +) -> None: pkga_path = create_test_package_with_setup( - script, name='pkga', version='1.0', + script, + name="pkga", + version="1.0", install_requires=[ - 'packaging@https://files.pythonhosted.org/packages/2f/2b/' - 'c681de3e1dbcd469537aefb15186b800209aa1f299d933d23b48d85c9d56/' - 'packaging-15.3-py2.py3-none-any.whl#sha256=' - 'ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4' + "packaging@https://files.pythonhosted.org/packages/2f/2b/" + "c681de3e1dbcd469537aefb15186b800209aa1f299d933d23b48d85c9d56/" + "packaging-15.3-py2.py3-none-any.whl#sha256=" + "ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4" ], ) - res = script.pip('install', pkga_path) + res = script.pip("install", pkga_path) assert "Successfully installed packaging-15.3" in str(res), str(res) @pytest.mark.network -@pytest.mark.parametrize('index', (PyPI.simple_url, TestPyPI.simple_url)) -def test_install_from_test_pypi_with_ext_url_dep_is_blocked(script, index): +@pytest.mark.parametrize("index", (PyPI.simple_url, TestPyPI.simple_url)) +def test_install_from_test_pypi_with_ext_url_dep_is_blocked( + script: PipTestEnvironment, index: str +) -> None: res = script.pip( - 'install', - '--index-url', + "install", + "--index-url", index, - 'pep-508-url-deps', + "pep-508-url-deps", expect_error=True, ) error_message = ( @@ -1620,106 +1932,129 @@ def test_install_from_test_pypi_with_ext_url_dep_is_blocked(script, index): assert error_cause in res.stderr, str(res) -def test_installing_scripts_outside_path_prints_warning(script): - result = script.pip_install_local( - "--prefix", script.scratch_path, "script_wheel1" - ) - assert "Successfully installed script-wheel1" in result.stdout, str(result) +@pytest.mark.xfail( + reason="No longer possible to trigger the warning with either --prefix or --target" +) +def test_installing_scripts_outside_path_prints_warning( + script: PipTestEnvironment, +) -> None: + result = script.pip_install_local("--prefix", script.scratch_path, "script_wheel1") + assert "Successfully installed script_wheel1" in result.stdout, str(result) assert "--no-warn-script-location" in result.stderr -def test_installing_scripts_outside_path_can_suppress_warning(script): +def test_installing_scripts_outside_path_can_suppress_warning( + script: PipTestEnvironment, +) -> None: result = script.pip_install_local( - "--prefix", script.scratch_path, "--no-warn-script-location", - "script_wheel1" + "--prefix", script.scratch_path, "--no-warn-script-location", "script_wheel1" ) - assert "Successfully installed script-wheel1" in result.stdout, str(result) + assert "Successfully installed script_wheel1" in result.stdout, str(result) assert "--no-warn-script-location" not in result.stderr -def test_installing_scripts_on_path_does_not_print_warning(script): +def test_installing_scripts_on_path_does_not_print_warning( + script: PipTestEnvironment, +) -> None: result = script.pip_install_local("script_wheel1") - assert "Successfully installed script-wheel1" in result.stdout, str(result) + assert "Successfully installed script_wheel1" in result.stdout, str(result) assert "--no-warn-script-location" not in result.stderr -def test_installed_files_recorded_in_deterministic_order(script, data): +def test_installed_files_recorded_in_deterministic_order( + script: PipTestEnvironment, data: TestData +) -> None: """ Ensure that we record the files installed by a package in a deterministic order, to make installs reproducible. """ to_install = data.packages.joinpath("FSPkg") - result = script.pip('install', to_install) - fspkg_folder = script.site_packages / 'fspkg' - egg_info = 'FSPkg-0.1.dev0-py{pyversion}.egg-info'.format(**globals()) - installed_files_path = ( - script.site_packages / egg_info / 'installed-files.txt' - ) + result = script.pip("install", to_install) + fspkg_folder = script.site_packages / "fspkg" + egg_info = f"FSPkg-0.1.dev0-py{pyversion}.egg-info" + installed_files_path = script.site_packages / egg_info / "installed-files.txt" result.did_create(fspkg_folder) result.did_create(installed_files_path) installed_files_path = result.files_created[installed_files_path].full installed_files_lines = [ - p for p in Path(installed_files_path).read_text().split('\n') if p + p for p in Path(installed_files_path).read_text().split("\n") if p ] assert installed_files_lines == sorted(installed_files_lines) -def test_install_conflict_results_in_warning(script, data): +def test_install_conflict_results_in_warning( + script: PipTestEnvironment, data: TestData +) -> None: pkgA_path = create_test_package_with_setup( script, - name='pkgA', version='1.0', install_requires=['pkgb == 1.0'], + name="pkgA", + version="1.0", + install_requires=["pkgb == 1.0"], ) pkgB_path = create_test_package_with_setup( script, - name='pkgB', version='2.0', + name="pkgB", + version="2.0", ) # Install pkgA without its dependency - result1 = script.pip('install', '--no-index', pkgA_path, '--no-deps') + result1 = script.pip("install", "--no-index", pkgA_path, "--no-deps") assert "Successfully installed pkgA-1.0" in result1.stdout, str(result1) # Then install an incorrect version of the dependency result2 = script.pip( - 'install', '--no-index', pkgB_path, allow_stderr_error=True, + "install", + "--no-index", + pkgB_path, + allow_stderr_error=True, ) assert "pkga 1.0 requires pkgb==1.0" in result2.stderr, str(result2) assert "Successfully installed pkgB-2.0" in result2.stdout, str(result2) -def test_install_conflict_warning_can_be_suppressed(script, data): +def test_install_conflict_warning_can_be_suppressed( + script: PipTestEnvironment, data: TestData +) -> None: pkgA_path = create_test_package_with_setup( script, - name='pkgA', version='1.0', install_requires=['pkgb == 1.0'], + name="pkgA", + version="1.0", + install_requires=["pkgb == 1.0"], ) pkgB_path = create_test_package_with_setup( script, - name='pkgB', version='2.0', + name="pkgB", + version="2.0", ) # Install pkgA without its dependency - result1 = script.pip('install', '--no-index', pkgA_path, '--no-deps') + result1 = script.pip("install", "--no-index", pkgA_path, "--no-deps") assert "Successfully installed pkgA-1.0" in result1.stdout, str(result1) # Then install an incorrect version of the dependency; suppressing warning - result2 = script.pip( - 'install', '--no-index', pkgB_path, '--no-warn-conflicts' - ) + result2 = script.pip("install", "--no-index", pkgB_path, "--no-warn-conflicts") assert "Successfully installed pkgB-2.0" in result2.stdout, str(result2) -def test_target_install_ignores_distutils_config_install_prefix(script): - prefix = script.scratch_path / 'prefix' - distutils_config = Path(os.path.expanduser('~'), - 'pydistutils.cfg' if sys.platform == 'win32' - else '.pydistutils.cfg') - distutils_config.write_text(textwrap.dedent( - ''' +def test_target_install_ignores_distutils_config_install_prefix( + script: PipTestEnvironment, +) -> None: + prefix = script.scratch_path / "prefix" + distutils_config = Path( + os.path.expanduser("~"), + "pydistutils.cfg" if sys.platform == "win32" else ".pydistutils.cfg", + ) + distutils_config.write_text( + textwrap.dedent( + f""" [install] prefix={prefix} - '''.format(**locals()))) - target = script.scratch_path / 'target' - result = script.pip_install_local('simplewheel', '-t', target) + """ + ) + ) + target = script.scratch_path / "target" + result = script.pip_install_local("simplewheel", "-t", target) assert "Successfully installed simplewheel" in result.stdout @@ -1730,78 +2065,94 @@ def test_target_install_ignores_distutils_config_install_prefix(script): @pytest.mark.incompatible_with_test_venv -def test_user_config_accepted(script): +def test_user_config_accepted(script: PipTestEnvironment) -> None: # user set in the config file is parsed as 0/1 instead of True/False. # Check that this doesn't cause a problem. - config_file = script.scratch_path / 'pip.conf' - script.environ['PIP_CONFIG_FILE'] = str(config_file) + config_file = script.scratch_path / "pip.conf" + script.environ["PIP_CONFIG_FILE"] = str(config_file) config_file.write_text("[install]\nuser = true") - result = script.pip_install_local('simplewheel') + result = script.pip_install_local("simplewheel") assert "Successfully installed simplewheel" in result.stdout relative_user = os.path.relpath(script.user_site_path, script.base_path) - result.did_create(join(relative_user, 'simplewheel')) + result.did_create(join(relative_user, "simplewheel")) @pytest.mark.parametrize( - 'install_args, expected_message', [ - ([], 'Requirement already satisfied: pip'), - (['--upgrade'], 'Requirement already {}: pip in'), - ] + "install_args, expected_message", + [ + ([], "Requirement already satisfied: pip"), + (["--upgrade"], "Requirement already {}: pip in"), + ], ) @pytest.mark.parametrize("use_module", [True, False]) def test_install_pip_does_not_modify_pip_when_satisfied( - script, install_args, expected_message, use_module, resolver_variant): + script: PipTestEnvironment, + install_args: List[str], + expected_message: str, + use_module: bool, + resolver_variant: ResolverVariant, +) -> None: """ Test it doesn't upgrade the pip if it already satisfies the requirement. """ variation = "satisfied" if resolver_variant else "up-to-date" expected_message = expected_message.format(variation) - result = script.pip_install_local( - 'pip', *install_args, use_module=use_module - ) + result = script.pip_install_local("pip", *install_args, use_module=use_module) assert expected_message in result.stdout, str(result) -def test_ignore_yanked_file(script, data): +def test_ignore_yanked_file(script: PipTestEnvironment, data: TestData) -> None: """ Test ignore a "yanked" file. """ result = script.pip( - 'install', 'simple', - '--index-url', data.index_url('yanked'), + "install", + "simple", + "--index-url", + data.index_url("yanked"), ) # Make sure a "yanked" release is ignored - assert 'Successfully installed simple-2.0\n' in result.stdout, str(result) + assert "Successfully installed simple-2.0\n" in result.stdout, str(result) -def test_invalid_index_url_argument(script, shared_data): +def test_invalid_index_url_argument( + script: PipTestEnvironment, shared_data: TestData +) -> None: """ Test the behaviour of an invalid --index-url argument """ - result = script.pip('install', '--index-url', '--user', - shared_data.find_links3, "Dinner", - expect_error=True) + result = script.pip( + "install", + "--index-url", + "--user", + shared_data.find_links3, + "Dinner", + expect_error=True, + ) - assert 'WARNING: The index url "--user" seems invalid, ' \ - 'please provide a scheme.' in result.stderr, str(result) + assert ( + 'WARNING: The index url "--user" seems invalid, please provide a scheme.' + ) in result.stderr, str(result) -def test_valid_index_url_argument(script, shared_data): +def test_valid_index_url_argument( + script: PipTestEnvironment, shared_data: TestData +) -> None: """ Test the behaviour of an valid --index-url argument """ - result = script.pip('install', '--index-url', - shared_data.find_links3, - "Dinner") + result = script.pip("install", "--index-url", shared_data.find_links3, "Dinner") - assert 'Successfully installed Dinner' in result.stdout, str(result) + assert "Successfully installed Dinner" in result.stdout, str(result) -def test_install_yanked_file_and_print_warning(script, data): +def test_install_yanked_file_and_print_warning( + script: PipTestEnvironment, data: TestData +) -> None: """ Test install a "yanked" file and print a warning. @@ -1809,21 +2160,51 @@ def test_install_yanked_file_and_print_warning(script, data): matches a version specifier that "pins" to an exact version (PEP 592). """ result = script.pip( - 'install', 'simple==3.0', - '--index-url', data.index_url('yanked'), + "install", + "simple==3.0", + "--index-url", + data.index_url("yanked"), expect_stderr=True, ) - expected_warning = 'Reason for being yanked: test reason message' + expected_warning = "Reason for being yanked: test reason message" assert expected_warning in result.stderr, str(result) # Make sure a "yanked" release is installed - assert 'Successfully installed simple-3.0\n' in result.stdout, str(result) + assert "Successfully installed simple-3.0\n" in result.stdout, str(result) -@pytest.mark.parametrize("install_args", [ - (), - ("--trusted-host", "localhost"), -]) -def test_install_sends_client_cert(install_args, script, cert_factory, data): +def test_error_all_yanked_files_and_no_pin( + script: PipTestEnvironment, data: TestData +) -> None: + """ + Test raising an error if there are only "yanked" files available and no pin + """ + result = script.pip( + "install", + "simple", + "--index-url", + data.index_url("yanked_all"), + expect_error=True, + ) + # Make sure an error is raised + assert ( + result.returncode == 1 + and "ERROR: No matching distribution found for simple\n" in result.stderr + ), str(result) + + +@pytest.mark.parametrize( + "install_args", + [ + (), + ("--trusted-host", "localhost"), + ], +) +def test_install_sends_client_cert( + install_args: Tuple[str, ...], + script: PipTestEnvironment, + cert_factory: CertFactory, + data: TestData, +) -> None: cert_path = cert_factory() ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ctx.load_cert_chain(cert_path, cert_path) @@ -1832,9 +2213,11 @@ def test_install_sends_client_cert(install_args, script, cert_factory, data): server = make_mock_server(ssl_context=ctx) server.mock.side_effect = [ - package_page({ - "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", - }), + package_page( + { + "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", + } + ), file_response(str(data.packages / "simple-3.0.tar.gz")), ] @@ -1850,12 +2233,14 @@ def test_install_sends_client_cert(install_args, script, cert_factory, data): assert server.mock.call_count == 2 for call_args in server.mock.call_args_list: - environ, _ = call_args.args + # Legacy: replace call_args[0] with call_args.args + # when pip drops support for python3.7 + environ, _ = call_args[0] assert "SSL_CLIENT_CERT" in environ assert environ["SSL_CLIENT_CERT"] -def test_install_skip_work_dir_pkg(script, data): +def test_install_skip_work_dir_pkg(script: PipTestEnvironment, data: TestData) -> None: """ Test that install of a package in working directory should pass on the second attempt after an install @@ -1863,26 +2248,32 @@ def test_install_skip_work_dir_pkg(script, data): """ # Create a test package, install it and then uninstall it - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.pip('install', '-e', '.', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.pip("install", "-e", ".", expect_stderr=True, cwd=pkg_path) - script.pip('uninstall', 'simple', '-y') + script.pip("uninstall", "simple", "-y") # Running the install command again from the working directory # will install the package as it was uninstalled earlier - result = script.pip('install', '--find-links', - data.find_links, 'simple', - expect_stderr=True, cwd=pkg_path) + result = script.pip( + "install", + "--find-links", + data.find_links, + "simple", + expect_stderr=True, + cwd=pkg_path, + ) - assert 'Requirement already satisfied: simple' not in result.stdout - assert 'Successfully installed simple' in result.stdout + assert "Requirement already satisfied: simple" not in result.stdout + assert "Successfully installed simple" in result.stdout -@pytest.mark.parametrize('package_name', ('simple-package', 'simple_package', - 'simple.package')) -def test_install_verify_package_name_normalization(script, package_name): +@pytest.mark.parametrize( + "package_name", ("simple-package", "simple_package", "simple.package") +) +def test_install_verify_package_name_normalization( + script: PipTestEnvironment, package_name: str +) -> None: """ Test that install of a package again using a name which @@ -1890,18 +2281,19 @@ def test_install_verify_package_name_normalization(script, package_name): since the package is already installed """ pkg_path = create_test_package_with_setup( - script, name='simple-package', version='1.0') - result = script.pip('install', '-e', '.', - expect_stderr=True, cwd=pkg_path) - assert 'Successfully installed simple-package' in result.stdout + script, name="simple-package", version="1.0" + ) + result = script.pip("install", "-e", ".", expect_stderr=True, cwd=pkg_path) + assert "Successfully installed simple-package" in result.stdout - result = script.pip('install', package_name) - assert 'Requirement already satisfied: {}'.format( - package_name) in result.stdout + result = script.pip("install", package_name) + assert "Requirement already satisfied: {}".format(package_name) in result.stdout -def test_install_logs_pip_version_in_debug(script, shared_data): - fake_package = shared_data.packages / 'simple-2.0.tar.gz' - result = script.pip('install', '-v', fake_package) +def test_install_logs_pip_version_in_debug( + script: PipTestEnvironment, shared_data: TestData +) -> None: + fake_package = shared_data.packages / "simple-2.0.tar.gz" + result = script.pip("install", "-v", fake_package) pattern = "Using pip .* from .*" assert_re_match(pattern, result.stdout) diff --git a/tests/functional/test_install_check.py b/tests/functional/test_install_check.py index 56ac7daf65a..8a8a7c93a80 100644 --- a/tests/functional/test_install_check.py +++ b/tests/functional/test_install_check.py @@ -1,38 +1,42 @@ -from tests.lib import create_test_package_with_setup +from typing import Iterable +from tests.lib import PipTestEnvironment, create_test_package_with_setup -def assert_contains_expected_lines(string, expected_lines): + +def assert_contains_expected_lines(string: str, expected_lines: Iterable[str]) -> None: for expected_line in expected_lines: - assert (expected_line + '\n') in string + assert (expected_line + "\n") in string -def test_check_install_canonicalization(script): +def test_check_install_canonicalization(script: PipTestEnvironment) -> None: pkga_path = create_test_package_with_setup( script, - name='pkgA', - version='1.0', - install_requires=['normal-missing', 'SPECIAL.missing'], + name="pkgA", + version="1.0", + install_requires=["normal-missing", "SPECIAL.missing"], ) normal_path = create_test_package_with_setup( script, - name='normal-missing', version='0.1', + name="normal-missing", + version="0.1", ) special_path = create_test_package_with_setup( script, - name='SPECIAL.missing', version='0.1', + name="SPECIAL.missing", + version="0.1", ) # Let's install pkgA without its dependency - result = script.pip('install', '--no-index', pkga_path, '--no-deps') + result = script.pip("install", "--no-index", pkga_path, "--no-deps") assert "Successfully installed pkgA-1.0" in result.stdout, str(result) # Install the first missing dependency. Only an error for the # second dependency should remain. result = script.pip( - 'install', - '--no-index', + "install", + "--no-index", normal_path, - '--quiet', + "--quiet", allow_stderr_error=True, ) expected_lines = [ @@ -46,13 +50,16 @@ def test_check_install_canonicalization(script): # during the installation. This is special as the package name requires # name normalization (as in https://github.com/pypa/pip/issues/5134) result = script.pip( - 'install', '--no-index', special_path, '--quiet', + "install", + "--no-index", + special_path, + "--quiet", ) assert "requires" not in result.stderr assert result.returncode == 0 # Double check that all errors are resolved in the end - result = script.pip('check') + result = script.pip("check") expected_lines = [ "No broken requirements found.", ] @@ -60,46 +67,57 @@ def test_check_install_canonicalization(script): assert result.returncode == 0 -def test_check_install_does_not_warn_for_out_of_graph_issues(script): +def test_check_install_does_not_warn_for_out_of_graph_issues( + script: PipTestEnvironment, +) -> None: pkg_broken_path = create_test_package_with_setup( script, - name='broken', - version='1.0', - install_requires=['missing', 'conflict < 1.0'], + name="broken", + version="1.0", + install_requires=["missing", "conflict < 1.0"], ) pkg_unrelated_path = create_test_package_with_setup( script, - name='unrelated', - version='1.0', + name="unrelated", + version="1.0", ) pkg_conflict_path = create_test_package_with_setup( script, - name='conflict', - version='1.0', + name="conflict", + version="1.0", ) # Install a package without it's dependencies - result = script.pip('install', '--no-index', pkg_broken_path, '--no-deps') + result = script.pip("install", "--no-index", pkg_broken_path, "--no-deps") assert "requires" not in result.stderr # Install conflict package result = script.pip( - 'install', '--no-index', pkg_conflict_path, allow_stderr_error=True, + "install", + "--no-index", + pkg_conflict_path, + allow_stderr_error=True, + ) + assert_contains_expected_lines( + result.stderr, + [ + "broken 1.0 requires missing, which is not installed.", + "broken 1.0 requires conflict<1.0, " + "but you have conflict 1.0 which is incompatible.", + ], ) - assert_contains_expected_lines(result.stderr, [ - "broken 1.0 requires missing, which is not installed.", - "broken 1.0 requires conflict<1.0, " - "but you have conflict 1.0 which is incompatible." - ]) # Install unrelated package result = script.pip( - 'install', '--no-index', pkg_unrelated_path, '--quiet', + "install", + "--no-index", + pkg_unrelated_path, + "--quiet", ) # should not warn about broken's deps when installing unrelated package assert "requires" not in result.stderr - result = script.pip('check', expect_error=True) + result = script.pip("check", expect_error=True) expected_lines = [ "broken 1.0 requires missing, which is not installed.", "broken 1.0 has requirement conflict<1.0, but you have conflict 1.0.", diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index 7b64ed4b5e9..c0ea5a425b9 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -2,19 +2,26 @@ import pytest +from tests.lib import PipTestEnvironment, TestData + @pytest.mark.network -@pytest.mark.xfail( - reason="The --build option was removed" -) -def test_no_clean_option_blocks_cleaning_after_install(script, data): +@pytest.mark.xfail(reason="The --build option was removed") +def test_no_clean_option_blocks_cleaning_after_install( + script: PipTestEnvironment, data: TestData +) -> None: """ Test --no-clean option blocks cleaning after install """ - build = script.base_path / 'pip-build' + build = script.base_path / "pip-build" script.pip( - 'install', '--no-clean', '--no-index', '--build', build, - f'--find-links={data.find_links}', 'simple', + "install", + "--no-clean", + "--no-index", + "--build", + build, + f"--find-links={data.find_links}", + "simple", expect_temp=True, # TODO: allow_stderr_warning is used for the --build deprecation, # remove it when removing support for --build @@ -24,14 +31,12 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data): @pytest.mark.network -def test_pep517_no_legacy_cleanup(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_pep517_no_legacy_cleanup(script: PipTestEnvironment, data: TestData) -> None: """Test a PEP 517 failed build does not attempt a legacy cleanup""" - to_install = data.packages.joinpath('pep517_wrapper_buildsys') + to_install = data.packages.joinpath("pep517_wrapper_buildsys") script.environ["PIP_TEST_FAIL_BUILD_WHEEL"] = "1" - res = script.pip( - 'install', '-f', data.find_links, to_install, - expect_error=True - ) + res = script.pip("install", "-f", data.find_links, to_install, expect_error=True) # Must not have built the package expected = "Failed building wheel for pep517-wrapper-buildsys" assert expected in str(res) diff --git a/tests/functional/test_install_compat.py b/tests/functional/test_install_compat.py index a5a0df65218..4b6b46b02df 100644 --- a/tests/functional/test_install_compat.py +++ b/tests/functional/test_install_compat.py @@ -7,11 +7,11 @@ import pytest from tests.lib import pyversion # noqa: F401 -from tests.lib import assert_all_changes +from tests.lib import PipTestEnvironment, TestData, assert_all_changes @pytest.mark.network -def test_debian_egg_name_workaround(script): +def test_debian_egg_name_workaround(script: PipTestEnvironment) -> None: """ We can uninstall packages installed with the pyversion removed from the egg-info metadata directory name. @@ -22,27 +22,21 @@ def test_debian_egg_name_workaround(script): https://bitbucket.org/ianb/pip/issue/104/pip-uninstall-on-ubuntu-linux """ - result = script.pip('install', 'INITools==0.2') + result = script.pip("install", "INITools==0.2") egg_info = os.path.join( - script.site_packages, - "INITools-0.2-py{pyversion}.egg-info".format(**globals())) + script.site_packages, f"INITools-0.2-py{pyversion}.egg-info" + ) # Debian only removes pyversion for global installs, not inside a venv # so even if this test runs on a Debian/Ubuntu system with broken # setuptools, since our test runs inside a venv we'll still have the normal # .egg-info - result.did_create( - egg_info, - message="Couldn't find {egg_info}".format(**locals()) - ) + result.did_create(egg_info, message=f"Couldn't find {egg_info}") # The Debian no-pyversion version of the .egg-info mangled = os.path.join(script.site_packages, "INITools-0.2.egg-info") - result.did_not_create( - mangled, - message="Found unexpected {mangled}".format(**locals()) - ) + result.did_not_create(mangled, message=f"Found unexpected {mangled}") # Simulate a Debian install by copying the .egg-info to their name for it full_egg_info = os.path.join(script.base_path, egg_info) @@ -53,14 +47,16 @@ def test_debian_egg_name_workaround(script): # Try the uninstall and verify that everything is removed. result2 = script.pip("uninstall", "INITools", "-y") - assert_all_changes(result, result2, [script.venv / 'build', 'cache']) + assert_all_changes(result, result2, [script.venv / "build", "cache"]) -def test_setup_py_with_dos_line_endings(script, data): +def test_setup_py_with_dos_line_endings( + script: PipTestEnvironment, data: TestData +) -> None: """ It doesn't choke on a setup.py file that uses DOS line endings (\\r\\n). Refs https://github.com/pypa/pip/issues/237 """ to_install = data.packages.joinpath("LineEndings") - script.pip('install', to_install) + script.pip("install", to_install) diff --git a/tests/functional/test_install_config.py b/tests/functional/test_install_config.py index 41be6fbbbb6..2e4bb742785 100644 --- a/tests/functional/test_install_config.py +++ b/tests/functional/test_install_config.py @@ -5,6 +5,8 @@ import pytest +from tests.conftest import CertFactory, MockServer +from tests.lib import PipTestEnvironment, TestData from tests.lib.server import ( authorization_response, file_response, @@ -12,36 +14,41 @@ package_page, server_running, ) +from tests.lib.venv import VirtualEnvironment +TEST_PYPI_INITOOLS = "https://test.pypi.org/simple/initools/" -def test_options_from_env_vars(script): + +def test_options_from_env_vars(script: PipTestEnvironment) -> None: """ Test if ConfigOptionParser reads env vars (e.g. not using PyPI here) """ - script.environ['PIP_NO_INDEX'] = '1' - result = script.pip('install', '-vvv', 'INITools', expect_error=True) + script.environ["PIP_NO_INDEX"] = "1" + result = script.pip("install", "-vvv", "INITools", expect_error=True) assert "Ignoring indexes:" in result.stdout, str(result) msg = "DistributionNotFound: No matching distribution found for INITools" - # Case insensitive as the new resolver canonicalises the project name + # Case insensitive as the new resolver canonicalizes the project name assert msg.lower() in result.stdout.lower(), str(result) -def test_command_line_options_override_env_vars(script, virtualenv): +def test_command_line_options_override_env_vars( + script: PipTestEnvironment, virtualenv: VirtualEnvironment +) -> None: """ Test that command line options override environmental variables. """ - script.environ['PIP_INDEX_URL'] = 'https://example.com/simple/' - result = script.pip('install', '-vvv', 'INITools', expect_error=True) - assert ( - "Getting page https://example.com/simple/initools" - in result.stdout - ) + script.environ["PIP_INDEX_URL"] = "https://example.com/simple/" + result = script.pip("install", "-vvv", "INITools", expect_error=True) + assert "Getting page https://example.com/simple/initools" in result.stdout virtualenv.clear() result = script.pip( - 'install', '-vvv', '--index-url', 'https://download.zope.org/ppix', - 'INITools', + "install", + "-vvv", + "--index-url", + "https://download.zope.org/ppix", + "INITools", expect_error=True, ) assert "example.com" not in result.stdout @@ -49,42 +56,53 @@ def test_command_line_options_override_env_vars(script, virtualenv): @pytest.mark.network -def test_env_vars_override_config_file(script, virtualenv): +def test_env_vars_override_config_file( + script: PipTestEnvironment, virtualenv: VirtualEnvironment +) -> None: """ Test that environmental variables override settings in config files. """ config_file = script.scratch_path / "test-pip.cfg" # set this to make pip load it - script.environ['PIP_CONFIG_FILE'] = str(config_file) + script.environ["PIP_CONFIG_FILE"] = str(config_file) # It's important that we test this particular config value ('no-index') # because there is/was a bug which only shows up in cases in which # 'config-item' and 'config_item' hash to the same value modulo the size # of the config dictionary. - config_file.write_text(textwrap.dedent("""\ + config_file.write_text( + textwrap.dedent( + """\ [global] no-index = 1 - """)) - result = script.pip('install', '-vvv', 'INITools', expect_error=True) + """ + ) + ) + result = script.pip("install", "-vvv", "INITools", expect_error=True) msg = "DistributionNotFound: No matching distribution found for INITools" - # Case insensitive as the new resolver canonicalises the project name + # Case insensitive as the new resolver canonicalizes the project name assert msg.lower() in result.stdout.lower(), str(result) - script.environ['PIP_NO_INDEX'] = '0' + script.environ["PIP_NO_INDEX"] = "0" virtualenv.clear() - result = script.pip('install', '-vvv', 'INITools') + result = script.pip("install", "-vvv", "INITools") assert "Successfully installed INITools" in result.stdout @pytest.mark.network -def test_command_line_append_flags(script, virtualenv, data): +def test_command_line_append_flags( + script: PipTestEnvironment, virtualenv: VirtualEnvironment, data: TestData +) -> None: """ Test command line flags that append to defaults set by environmental variables. """ - script.environ['PIP_FIND_LINKS'] = 'https://test.pypi.org' + script.environ["PIP_FIND_LINKS"] = TEST_PYPI_INITOOLS result = script.pip( - 'install', '-vvv', 'INITools', '--trusted-host', - 'test.pypi.org', + "install", + "-vvv", + "INITools", + "--trusted-host", + "test.pypi.org", ) assert ( "Fetching project page and analyzing links: https://test.pypi.org" @@ -92,31 +110,38 @@ def test_command_line_append_flags(script, virtualenv, data): ), str(result) virtualenv.clear() result = script.pip( - 'install', '-vvv', '--find-links', data.find_links, 'INITools', - '--trusted-host', 'test.pypi.org', + "install", + "-vvv", + "--find-links", + data.find_links, + "INITools", + "--trusted-host", + "test.pypi.org", ) assert ( "Fetching project page and analyzing links: https://test.pypi.org" in result.stdout ) assert ( - f'Skipping link: not a file: {data.find_links}' in - result.stdout - ), f'stdout: {result.stdout}' + f"Skipping link: not a file: {data.find_links}" in result.stdout + ), f"stdout: {result.stdout}" @pytest.mark.network -def test_command_line_appends_correctly(script, data): +def test_command_line_appends_correctly( + script: PipTestEnvironment, data: TestData +) -> None: """ Test multiple appending options set by environmental variables. """ - script.environ['PIP_FIND_LINKS'] = ( - 'https://test.pypi.org {data.find_links}'.format(**locals()) - ) + script.environ["PIP_FIND_LINKS"] = f"{TEST_PYPI_INITOOLS} {data.find_links}" result = script.pip( - 'install', '-vvv', 'INITools', '--trusted-host', - 'test.pypi.org', + "install", + "-vvv", + "INITools", + "--trusted-host", + "test.pypi.org", ) assert ( @@ -124,50 +149,68 @@ def test_command_line_appends_correctly(script, data): in result.stdout ), result.stdout assert ( - f'Skipping link: not a file: {data.find_links}' in - result.stdout - ), f'stdout: {result.stdout}' + f"Skipping link: not a file: {data.find_links}" in result.stdout + ), f"stdout: {result.stdout}" def test_config_file_override_stack( - script, virtualenv, mock_server, shared_data -): + script: PipTestEnvironment, + virtualenv: VirtualEnvironment, + mock_server: MockServer, + shared_data: TestData, +) -> None: """ Test config files (global, overriding a global config with a local, overriding all with a command line flag). """ - mock_server.set_responses([ - package_page({}), - package_page({}), - package_page({"INITools-0.2.tar.gz": "/files/INITools-0.2.tar.gz"}), - file_response(shared_data.packages.joinpath("INITools-0.2.tar.gz")), - ]) + mock_server.set_responses( + [ + package_page({}), + package_page({}), + package_page({"INITools-0.2.tar.gz": "/files/INITools-0.2.tar.gz"}), + file_response(shared_data.packages.joinpath("INITools-0.2.tar.gz")), + ] + ) mock_server.start() base_address = f"http://{mock_server.host}:{mock_server.port}" config_file = script.scratch_path / "test-pip.cfg" # set this to make pip load it - script.environ['PIP_CONFIG_FILE'] = str(config_file) + script.environ["PIP_CONFIG_FILE"] = str(config_file) - config_file.write_text(textwrap.dedent("""\ + config_file.write_text( + textwrap.dedent( + """\ [global] index-url = {}/simple1 - """.format(base_address))) - script.pip('install', '-vvv', 'INITools', expect_error=True) + """.format( + base_address + ) + ) + ) + script.pip("install", "-vvv", "INITools", expect_error=True) virtualenv.clear() - config_file.write_text(textwrap.dedent("""\ + config_file.write_text( + textwrap.dedent( + """\ [global] index-url = {address}/simple1 [install] index-url = {address}/simple2 - """.format(address=base_address)) + """.format( + address=base_address + ) + ) ) - script.pip('install', '-vvv', 'INITools', expect_error=True) + script.pip("install", "-vvv", "INITools", expect_error=True) script.pip( - 'install', '-vvv', '--index-url', f"{base_address}/simple3", - 'INITools', + "install", + "-vvv", + "--index-url", + f"{base_address}/simple3", + "INITools", ) mock_server.stop() @@ -179,36 +222,45 @@ def test_config_file_override_stack( assert requests[3]["PATH_INFO"] == "/files/INITools-0.2.tar.gz" -def test_options_from_venv_config(script, virtualenv): +def test_options_from_venv_config( + script: PipTestEnvironment, virtualenv: VirtualEnvironment +) -> None: """ Test if ConfigOptionParser reads a virtualenv-local config file """ from pip._internal.configuration import CONFIG_BASENAME + conf = "[global]\nno-index = true" ini = virtualenv.location / CONFIG_BASENAME - with open(ini, 'w') as f: + with open(ini, "w") as f: f.write(conf) - result = script.pip('install', '-vvv', 'INITools', expect_error=True) + result = script.pip("install", "-vvv", "INITools", expect_error=True) assert "Ignoring indexes:" in result.stdout, str(result) msg = "DistributionNotFound: No matching distribution found for INITools" - # Case insensitive as the new resolver canonicalises the project name + # Case insensitive as the new resolver canonicalizes the project name assert msg.lower() in result.stdout.lower(), str(result) +@pytest.mark.usefixtures("with_wheel") def test_install_no_binary_via_config_disables_cached_wheels( - script, data, with_wheel): - config_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) + script: PipTestEnvironment, data: TestData +) -> None: + config_file = tempfile.NamedTemporaryFile(mode="wt", delete=False) try: - script.environ['PIP_CONFIG_FILE'] = config_file.name - config_file.write(textwrap.dedent("""\ + script.environ["PIP_CONFIG_FILE"] = config_file.name + config_file.write( + textwrap.dedent( + """\ [global] no-binary = :all: - """)) + """ + ) + ) config_file.close() res = script.pip( - 'install', '--no-index', '-f', data.find_links, - 'upper', expect_stderr=True) + "install", "--no-index", "-f", data.find_links, "upper", expect_stderr=True + ) finally: os.unlink(config_file.name) assert "Successfully installed upper-2.0" in str(res), str(res) @@ -218,7 +270,9 @@ def test_install_no_binary_via_config_disables_cached_wheels( assert "Running setup.py install for upper" in str(res), str(res) -def test_prompt_for_authentication(script, data, cert_factory): +def test_prompt_for_authentication( + script: PipTestEnvironment, data: TestData, cert_factory: CertFactory +) -> None: """Test behaviour while installing from a index url requiring authentication """ @@ -230,24 +284,35 @@ def test_prompt_for_authentication(script, data, cert_factory): server = make_mock_server(ssl_context=ctx) server.mock.side_effect = [ - package_page({ - "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", - }), + package_page( + { + "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", + } + ), authorization_response(str(data.packages / "simple-3.0.tar.gz")), ] url = f"https://{server.host}:{server.port}/simple" with server_running(server): - result = script.pip('install', "--index-url", url, - "--cert", cert_path, "--client-cert", cert_path, - 'simple', expect_error=True) - - assert f'User for {server.host}:{server.port}' in \ - result.stdout, str(result) - - -def test_do_not_prompt_for_authentication(script, data, cert_factory): + result = script.pip( + "install", + "--index-url", + url, + "--cert", + cert_path, + "--client-cert", + cert_path, + "simple", + expect_error=True, + ) + + assert f"User for {server.host}:{server.port}" in result.stdout, str(result) + + +def test_do_not_prompt_for_authentication( + script: PipTestEnvironment, data: TestData, cert_factory: CertFactory +) -> None: """Test behaviour if --no-input option is given while installing from a index url requiring authentication """ @@ -260,17 +325,93 @@ def test_do_not_prompt_for_authentication(script, data, cert_factory): server = make_mock_server(ssl_context=ctx) server.mock.side_effect = [ - package_page({ - "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", - }), + package_page( + { + "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", + } + ), authorization_response(str(data.packages / "simple-3.0.tar.gz")), ] url = f"https://{server.host}:{server.port}/simple" with server_running(server): - result = script.pip('install', "--index-url", url, - "--cert", cert_path, "--client-cert", cert_path, - '--no-input', 'simple', expect_error=True) + result = script.pip( + "install", + "--index-url", + url, + "--cert", + cert_path, + "--client-cert", + cert_path, + "--no-input", + "simple", + expect_error=True, + ) assert "ERROR: HTTP error 401" in result.stderr + + +@pytest.mark.parametrize("auth_needed", (True, False)) +def test_prompt_for_keyring_if_needed( + script: PipTestEnvironment, + data: TestData, + cert_factory: CertFactory, + auth_needed: bool, +) -> None: + """Test behaviour while installing from a index url + requiring authentication and keyring is possible. + """ + cert_path = cert_factory() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.load_cert_chain(cert_path, cert_path) + ctx.load_verify_locations(cafile=cert_path) + ctx.verify_mode = ssl.CERT_REQUIRED + + response = authorization_response if auth_needed else file_response + + server = make_mock_server(ssl_context=ctx) + server.mock.side_effect = [ + package_page( + { + "simple-3.0.tar.gz": "/files/simple-3.0.tar.gz", + } + ), + response(str(data.packages / "simple-3.0.tar.gz")), + response(str(data.packages / "simple-3.0.tar.gz")), + ] + + url = f"https://{server.host}:{server.port}/simple" + + keyring_content = textwrap.dedent( + """\ + import os + import sys + from collections import namedtuple + + Cred = namedtuple("Cred", ["username", "password"]) + + def get_credential(url, username): + sys.stderr.write("get_credential was called" + os.linesep) + return Cred("USERNAME", "PASSWORD") + """ + ) + keyring_path = script.site_packages_path / "keyring.py" + keyring_path.write_text(keyring_content) + + with server_running(server): + result = script.pip( + "install", + "--index-url", + url, + "--cert", + cert_path, + "--client-cert", + cert_path, + "simple", + ) + + if auth_needed: + assert "get_credential was called" in result.stderr + else: + assert "get_credential was called" not in result.stderr diff --git a/tests/functional/test_install_direct_url.py b/tests/functional/test_install_direct_url.py index 23273774d16..9d5e0612ea9 100644 --- a/tests/functional/test_install_direct_url.py +++ b/tests/functional/test_install_direct_url.py @@ -1,48 +1,65 @@ -import re +import pytest -from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl -from tests.lib import _create_test_package, path_to_url +from pip._internal.models.direct_url import VcsInfo +from tests.lib import PipTestEnvironment, TestData, _create_test_package, path_to_url +from tests.lib.direct_url import get_created_direct_url -def _get_created_direct_url(result, pkg): - direct_url_metadata_re = re.compile( - pkg + r"-[\d\.]+\.dist-info." + DIRECT_URL_METADATA_NAME + r"$" - ) - for filename in result.files_created: - if direct_url_metadata_re.search(filename): - direct_url_path = result.test_env.base_path / filename - with open(direct_url_path) as f: - return DirectUrl.from_json(f.read()) - return None - - -def test_install_find_links_no_direct_url(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_find_links_no_direct_url(script: PipTestEnvironment) -> None: result = script.pip_install_local("simple") - assert not _get_created_direct_url(result, "simple") + assert not get_created_direct_url(result, "simple") -def test_install_vcs_editable_no_direct_url(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_vcs_editable_no_direct_url(script: PipTestEnvironment) -> None: pkg_path = _create_test_package(script, name="testpkg") args = ["install", "-e", "git+%s#egg=testpkg" % path_to_url(pkg_path)] result = script.pip(*args) # legacy editable installs do not generate .dist-info, # hence no direct_url.json - assert not _get_created_direct_url(result, "testpkg") + assert not get_created_direct_url(result, "testpkg") -def test_install_vcs_non_editable_direct_url(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_vcs_non_editable_direct_url(script: PipTestEnvironment) -> None: pkg_path = _create_test_package(script, name="testpkg") url = path_to_url(pkg_path) args = ["install", f"git+{url}#egg=testpkg"] result = script.pip(*args) - direct_url = _get_created_direct_url(result, "testpkg") + direct_url = get_created_direct_url(result, "testpkg") assert direct_url assert direct_url.url == url + assert isinstance(direct_url.info, VcsInfo) assert direct_url.info.vcs == "git" -def test_install_archive_direct_url(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_archive_direct_url(script: PipTestEnvironment, data: TestData) -> None: req = "simple @ " + path_to_url(data.packages / "simple-2.0.tar.gz") assert req.startswith("simple @ file://") result = script.pip("install", req) - assert _get_created_direct_url(result, "simple") + assert get_created_direct_url(result, "simple") + + +@pytest.mark.network +@pytest.mark.usefixtures("with_wheel") +def test_install_vcs_constraint_direct_url(script: PipTestEnvironment) -> None: + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text( + "git+https://github.com/pypa/pip-test-package" + "@5547fa909e83df8bd743d3978d6667497983a4b7" + "#egg=pip-test-package" + ) + result = script.pip("install", "pip-test-package", "-c", constraints_file) + assert get_created_direct_url(result, "pip_test_package") + + +@pytest.mark.usefixtures("with_wheel") +def test_install_vcs_constraint_direct_file_url(script: PipTestEnvironment) -> None: + pkg_path = _create_test_package(script, name="testpkg") + url = path_to_url(pkg_path) + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text(f"git+{url}#egg=testpkg") + result = script.pip("install", "testpkg", "-c", constraints_file) + assert get_created_direct_url(result, "testpkg") diff --git a/tests/functional/test_install_extras.py b/tests/functional/test_install_extras.py index 0ec42940630..c6cef00fa9c 100644 --- a/tests/functional/test_install_extras.py +++ b/tests/functional/test_install_extras.py @@ -4,55 +4,71 @@ import pytest +from tests.lib import PipTestEnvironment, ResolverVariant, TestData + @pytest.mark.network -def test_simple_extras_install_from_pypi(script): +def test_simple_extras_install_from_pypi(script: PipTestEnvironment) -> None: """ Test installing a package from PyPI using extras dependency Paste[openid]. """ result = script.pip( - 'install', 'Paste[openid]==1.7.5.1', expect_stderr=True, + "install", + "Paste[openid]==1.7.5.1", + expect_stderr=True, ) - initools_folder = script.site_packages / 'openid' + initools_folder = script.site_packages / "openid" result.did_create(initools_folder) -def test_extras_after_wheel(script, data): +def test_extras_after_wheel(script: PipTestEnvironment, data: TestData) -> None: """ Test installing a package with extras after installing from a wheel. """ - simple = script.site_packages / 'simple' + simple = script.site_packages / "simple" no_extra = script.pip( - 'install', '--no-index', '-f', data.find_links, - 'requires_simple_extra', expect_stderr=True, + "install", + "--no-index", + "-f", + data.find_links, + "requires_simple_extra", + expect_stderr=True, ) no_extra.did_not_create(simple) extra = script.pip( - 'install', '--no-index', '-f', data.find_links, - 'requires_simple_extra[extra]', expect_stderr=True, + "install", + "--no-index", + "-f", + data.find_links, + "requires_simple_extra[extra]", + expect_stderr=True, ) extra.did_create(simple) @pytest.mark.network -def test_no_extras_uninstall(script): +def test_no_extras_uninstall(script: PipTestEnvironment) -> None: """ No extras dependency gets uninstalled when the root package is uninstalled """ result = script.pip( - 'install', 'Paste[openid]==1.7.5.1', expect_stderr=True, + "install", + "Paste[openid]==1.7.5.1", + expect_stderr=True, ) - result.did_create(join(script.site_packages, 'paste')) - result.did_create(join(script.site_packages, 'openid')) - result2 = script.pip('uninstall', 'Paste', '-y') + result.did_create(join(script.site_packages, "paste")) + result.did_create(join(script.site_packages, "openid")) + result2 = script.pip("uninstall", "Paste", "-y") # openid should not be uninstalled - initools_folder = script.site_packages / 'openid' + initools_folder = script.site_packages / "openid" assert initools_folder not in result2.files_deleted, result.files_deleted -def test_nonexistent_extra_warns_user_no_wheel(script, data): +def test_nonexistent_extra_warns_user_no_wheel( + script: PipTestEnvironment, data: TestData +) -> None: """ A warning is logged telling the user that the extra option they requested does not exist in the project they are wishing to install. @@ -60,17 +76,21 @@ def test_nonexistent_extra_warns_user_no_wheel(script, data): This exercises source installs. """ result = script.pip( - 'install', '--no-binary=:all:', '--no-index', - '--find-links=' + data.find_links, - 'simple[nonexistent]', expect_stderr=True, + "install", + "--no-binary=:all:", + "--no-index", + "--find-links=" + data.find_links, + "simple[nonexistent]", + expect_stderr=True, + ) + assert "simple 3.0 does not provide the extra 'nonexistent'" in result.stderr, str( + result ) - assert ( - "simple 3.0 does not provide the extra 'nonexistent'" - in result.stderr - ), str(result) -def test_nonexistent_extra_warns_user_with_wheel(script, data): +def test_nonexistent_extra_warns_user_with_wheel( + script: PipTestEnvironment, data: TestData +) -> None: """ A warning is logged telling the user that the extra option they requested does not exist in the project they are wishing to install. @@ -78,35 +98,39 @@ def test_nonexistent_extra_warns_user_with_wheel(script, data): This exercises wheel installs. """ result = script.pip( - 'install', '--no-index', - '--find-links=' + data.find_links, - 'simplewheel[nonexistent]', expect_stderr=True, - ) - assert ( - "simplewheel 2.0 does not provide the extra 'nonexistent'" - in result.stderr + "install", + "--no-index", + "--find-links=" + data.find_links, + "simplewheel[nonexistent]", + expect_stderr=True, ) + assert "simplewheel 2.0 does not provide the extra 'nonexistent'" in result.stderr -def test_nonexistent_options_listed_in_order(script, data): +def test_nonexistent_options_listed_in_order( + script: PipTestEnvironment, data: TestData +) -> None: """ Warn the user for each extra that doesn't exist. """ result = script.pip( - 'install', '--no-index', - '--find-links=' + data.find_links, - 'simplewheel[nonexistent, nope]', expect_stderr=True, + "install", + "--no-index", + "--find-links=" + data.find_links, + "simplewheel[nonexistent, nope]", + expect_stderr=True, ) matches = re.findall( - "WARNING: simplewheel 2.0 does not provide the extra '([a-z]*)'", - result.stderr + "WARNING: simplewheel 2.0 does not provide the extra '([a-z]*)'", result.stderr ) - assert matches == ['nonexistent', 'nope'] + assert matches == ["nonexistent", "nope"] -def test_install_deprecated_extra(script, data): +def test_install_fails_if_extra_at_end( + script: PipTestEnvironment, data: TestData +) -> None: """ - Warn about deprecated order of specifiers and extras. + Fail if order of specifiers and extras is incorrect. Test uses a requirements file to avoid a testing issue where the specifier gets interpreted as shell redirect. @@ -114,50 +138,72 @@ def test_install_deprecated_extra(script, data): script.scratch_path.joinpath("requirements.txt").write_text( "requires_simple_extra>=0.1[extra]" ) - simple = script.site_packages / 'simple' result = script.pip( - 'install', '--no-index', '--find-links=' + data.find_links, - '-r', script.scratch_path / 'requirements.txt', expect_stderr=True, + "install", + "--no-index", + "--find-links=" + data.find_links, + "-r", + script.scratch_path / "requirements.txt", + expect_error=True, ) - - result.did_create(simple) - assert ("DEPRECATION: Extras after version" in result.stderr) + assert "Extras after version" in result.stderr -def test_install_special_extra(script): +def test_install_special_extra(script: PipTestEnvironment) -> None: # Check that uppercase letters and '-' are dealt with # make a dummy project - pkga_path = script.scratch_path / 'pkga' + pkga_path = script.scratch_path / "pkga" pkga_path.mkdir() - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', version='0.1', extras_require={'Hop_hOp-hoP': ['missing_pkg']}, ) - """)) + """ + ) + ) result = script.pip( - 'install', '--no-index', '{pkga_path}[Hop_hOp-hoP]'.format(**locals()), - expect_error=True) + "install", "--no-index", f"{pkga_path}[Hop_hOp-hoP]", expect_error=True + ) assert ( "Could not find a version that satisfies the requirement missing_pkg" ) in result.stderr, str(result) +def test_install_requirements_no_r_flag(script: PipTestEnvironment) -> None: + """Beginners sometimes forget the -r and this leads to confusion""" + result = script.pip("install", "requirements.txt", expect_error=True) + assert 'literally named "requirements.txt"' in result.stdout + + @pytest.mark.parametrize( - "extra_to_install, simple_version", [ - ['', '3.0'], - pytest.param('[extra1]', '2.0', marks=pytest.mark.xfail), - pytest.param('[extra2]', '1.0', marks=pytest.mark.xfail), - pytest.param('[extra1,extra2]', '1.0', marks=pytest.mark.xfail), - ]) -def test_install_extra_merging(script, data, extra_to_install, simple_version): + "extra_to_install, simple_version, fails_on_legacy", + [ + ("", "3.0", False), + ("[extra1]", "2.0", True), + ("[extra2]", "1.0", True), + ("[extra1,extra2]", "1.0", True), + ], +) +@pytest.mark.usefixtures("data") +def test_install_extra_merging( + script: PipTestEnvironment, + resolver_variant: ResolverVariant, + extra_to_install: str, + simple_version: str, + fails_on_legacy: bool, +) -> None: # Check that extra specifications in the extras section are honoured. - pkga_path = script.scratch_path / 'pkga' + pkga_path = script.scratch_path / "pkga" pkga_path.mkdir() - pkga_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkga_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from setuptools import setup setup(name='pkga', version='0.1', @@ -165,10 +211,15 @@ def test_install_extra_merging(script, data, extra_to_install, simple_version): extras_require={'extra1': ['simple<3'], 'extra2': ['simple==1.*']}, ) - """)) + """ + ) + ) result = script.pip_install_local( - '{pkga_path}{extra_to_install}'.format(**locals()), + f"{pkga_path}{extra_to_install}", + expect_error=(fails_on_legacy and resolver_variant == "legacy"), ) - assert f'Successfully installed pkga-0.1 simple-{simple_version}' in result.stdout + if not fails_on_legacy or resolver_variant == "2020-resolver": + expected = f"Successfully installed pkga-0.1 simple-{simple_version}" + assert expected in result.stdout diff --git a/tests/functional/test_install_force_reinstall.py b/tests/functional/test_install_force_reinstall.py index 265c52b20db..9c49d58c4b7 100644 --- a/tests/functional/test_install_force_reinstall.py +++ b/tests/functional/test_install_force_reinstall.py @@ -1,55 +1,61 @@ import os -from tests.lib import assert_all_changes +from tests.lib import PipTestEnvironment, assert_all_changes -def check_installed_version(script, package, expected): - result = script.pip('show', package) +def check_installed_version( + script: PipTestEnvironment, package: str, expected: str +) -> None: + result = script.pip("show", package) lines = result.stdout.splitlines() version = None for line in lines: - if line.startswith('Version: '): + if line.startswith("Version: "): version = line.split()[-1] break - assert version == expected, f'version {version} != {expected}' + assert version == expected, f"version {version} != {expected}" -def check_force_reinstall(script, specifier, expected): +def check_force_reinstall( + script: PipTestEnvironment, specifier: str, expected: str +) -> None: """ Args: specifier: the requirement specifier to force-reinstall. expected: the expected version after force-reinstalling. """ - result = script.pip_install_local('simplewheel==1.0') - check_installed_version(script, 'simplewheel', '1.0') + result = script.pip_install_local("simplewheel==1.0") + check_installed_version(script, "simplewheel", "1.0") # Remove an installed file to test whether --force-reinstall fixes it. to_fix = script.site_packages_path.joinpath("simplewheel", "__init__.py") to_fix.unlink() - result2 = script.pip_install_local('--force-reinstall', specifier) - check_installed_version(script, 'simplewheel', expected) + result2 = script.pip_install_local("--force-reinstall", specifier) + check_installed_version(script, "simplewheel", expected) # site_packages_path is absolute, but files_created mapping uses # relative paths as key. fixed_key = os.path.relpath(to_fix, script.base_path) - result2.did_create(fixed_key, message='force-reinstall failed') + result2.did_create(fixed_key, message="force-reinstall failed") - result3 = script.pip('uninstall', 'simplewheel', '-y') - assert_all_changes(result, result3, [script.venv / 'build', 'cache']) + result3 = script.pip("uninstall", "simplewheel", "-y") + assert_all_changes(result, result3, [script.venv / "build", "cache"]) -def test_force_reinstall_with_no_version_specifier(script): +def test_force_reinstall_with_no_version_specifier(script: PipTestEnvironment) -> None: """ Check --force-reinstall when there is no version specifier and the installed version is not the newest version. """ - check_force_reinstall(script, 'simplewheel', '2.0') + check_force_reinstall(script, "simplewheel", "2.0") -def test_force_reinstall_with_same_version_specifier(script): +def test_force_reinstall_with_same_version_specifier( + script: PipTestEnvironment, +) -> None: """ Check --force-reinstall when the version specifier equals the installed version and the installed version is not the newest version. """ - check_force_reinstall(script, 'simplewheel==1.0', '1.0') + check_force_reinstall(script, "simplewheel==1.0", "1.0") diff --git a/tests/functional/test_install_index.py b/tests/functional/test_install_index.py index 42d2f7a675e..a492863b542 100644 --- a/tests/functional/test_install_index.py +++ b/tests/functional/test_install_index.py @@ -1,68 +1,92 @@ import os +import shutil import textwrap import urllib.parse +import pytest -def test_find_links_relative_path(script, data, with_wheel): +from tests.lib import PipTestEnvironment, TestData + + +@pytest.mark.usefixtures("with_wheel") +def test_find_links_relative_path(script: PipTestEnvironment, data: TestData) -> None: """Test find-links as a relative path.""" result = script.pip( - 'install', - 'parent==0.1', - '--no-index', - '--find-links', - 'packages/', + "install", + "parent==0.1", + "--no-index", + "--find-links", + "packages/", cwd=data.root, ) - dist_info_folder = ( - script.site_packages / 'parent-0.1.dist-info' - ) - initools_folder = script.site_packages / 'parent' + dist_info_folder = script.site_packages / "parent-0.1.dist-info" + initools_folder = script.site_packages / "parent" result.did_create(dist_info_folder) result.did_create(initools_folder) -def test_find_links_requirements_file_relative_path(script, data, with_wheel): +def test_find_links_no_doctype(script: PipTestEnvironment, data: TestData) -> None: + shutil.copy(data.packages / "simple-1.0.tar.gz", script.scratch_path) + html = script.scratch_path.joinpath("index.html") + html.write_text('') + result = script.pip( + "install", + "simple==1.0", + "--no-index", + "--find-links", + script.scratch_path, + expect_stderr=True, + ) + assert not result.stderr + + +@pytest.mark.usefixtures("with_wheel") +def test_find_links_requirements_file_relative_path( + script: PipTestEnvironment, data: TestData +) -> None: """Test find-links as a relative path to a reqs file.""" - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent(""" + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """ --no-index --find-links={} parent==0.1 - """ .format(data.packages.replace(os.path.sep, '/')))) + """.format( + data.packages.replace(os.path.sep, "/") + ) + ) + ) result = script.pip( - 'install', - '-r', + "install", + "-r", script.scratch_path / "test-req.txt", cwd=data.root, ) - dist_info_folder = ( - script.site_packages / 'parent-0.1.dist-info' - ) - initools_folder = script.site_packages / 'parent' + dist_info_folder = script.site_packages / "parent-0.1.dist-info" + initools_folder = script.site_packages / "parent" result.did_create(dist_info_folder) result.did_create(initools_folder) -def test_install_from_file_index_hash_link(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_from_file_index_hash_link( + script: PipTestEnvironment, data: TestData +) -> None: """ Test that a pkg can be installed from a file:// index using a link with a hash """ - result = script.pip('install', '-i', data.index_url(), 'simple==1.0') - dist_info_folder = ( - script.site_packages / 'simple-1.0.dist-info' - ) + result = script.pip("install", "-i", data.index_url(), "simple==1.0") + dist_info_folder = script.site_packages / "simple-1.0.dist-info" result.did_create(dist_info_folder) -def test_file_index_url_quoting(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_file_index_url_quoting(script: PipTestEnvironment, data: TestData) -> None: """ Test url quoting of file index url with a space """ index_url = data.index_url(urllib.parse.quote("in dex")) - result = script.pip( - 'install', '-vvv', '--index-url', index_url, 'simple' - ) - result.did_create(script.site_packages / 'simple') - result.did_create( - script.site_packages / 'simple-1.0.dist-info' - ) + result = script.pip("install", "-vvv", "--index-url", index_url, "simple") + result.did_create(script.site_packages / "simple") + result.did_create(script.site_packages / "simple-1.0.dist-info") diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 9c35aee8320..a7f2f46be94 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -1,10 +1,14 @@ import json import os import textwrap +from typing import Any, Callable import pytest from tests.lib import ( + PipTestEnvironment, + ResolverVariant, + TestData, _create_test_package_with_subdirectory, create_basic_sdist_for_package, create_basic_wheel_for_package, @@ -17,16 +21,18 @@ class ArgRecordingSdist: - def __init__(self, sdist_path, args_path): + def __init__(self, sdist_path: Path, args_path: Path) -> None: self.sdist_path = sdist_path self._args_path = args_path - def args(self): + def args(self) -> Any: return json.loads(self._args_path.read_text()) @pytest.fixture() -def arg_recording_sdist_maker(script): +def arg_recording_sdist_maker( + script: PipTestEnvironment, +) -> Callable[[str], ArgRecordingSdist]: arg_writing_setup_py = textwrap.dedent( """ import io @@ -43,18 +49,13 @@ def arg_recording_sdist_maker(script): setup(name={name!r}, version="0.1.0") """ ) - output_dir = script.scratch_path.joinpath( - "args_recording_sdist_maker_output" - ) + output_dir = script.scratch_path.joinpath("args_recording_sdist_maker_output") output_dir.mkdir(parents=True) script.environ["OUTPUT_DIR"] = str(output_dir) - def _arg_recording_sdist_maker(name): - # type: (str) -> ArgRecordingSdist + def _arg_recording_sdist_maker(name: str) -> ArgRecordingSdist: extra_files = {"setup.py": arg_writing_setup_py.format(name=name)} - sdist_path = create_basic_sdist_for_package( - script, name, "0.1.0", extra_files - ) + sdist_path = create_basic_sdist_for_package(script, name, "0.1.0", extra_files) args_path = output_dir / f"{name}.json" return ArgRecordingSdist(sdist_path, args_path) @@ -62,31 +63,31 @@ def _arg_recording_sdist_maker(name): @pytest.mark.network -def test_requirements_file(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_requirements_file(script: PipTestEnvironment) -> None: """ Test installing from a requirements file. """ - other_lib_name, other_lib_version = 'anyjson', '0.3' - script.scratch_path.joinpath("initools-req.txt").write_text(textwrap.dedent("""\ + other_lib_name, other_lib_version = "peppercorn", "0.6" + script.scratch_path.joinpath("initools-req.txt").write_text( + textwrap.dedent( + f"""\ INITools==0.2 # and something else to test out: {other_lib_name}<={other_lib_version} - """.format(**locals()))) - result = script.pip( - 'install', '-r', script.scratch_path / 'initools-req.txt' - ) - result.did_create( - script.site_packages / 'INITools-0.2.dist-info' + """ + ) ) - result.did_create(script.site_packages / 'initools') + result = script.pip("install", "-r", script.scratch_path / "initools-req.txt") + result.did_create(script.site_packages / "INITools-0.2.dist-info") + result.did_create(script.site_packages / "initools") assert result.files_created[script.site_packages / other_lib_name].dir - fn = '{}-{}.dist-info'.format( - other_lib_name, other_lib_version) + fn = "{}-{}.dist-info".format(other_lib_name, other_lib_version) assert result.files_created[script.site_packages / fn].dir -def test_schema_check_in_requirements_file(script): +def test_schema_check_in_requirements_file(script: PipTestEnvironment) -> None: """ Test installing from a requirements file with an invalid vcs schema.. @@ -99,41 +100,38 @@ def test_schema_check_in_requirements_file(script): ) with pytest.raises(AssertionError): - script.pip( - "install", "-vvv", "-r", script.scratch_path / "file-egg-req.txt" - ) - - -@pytest.mark.parametrize("test_type,editable", [ - ("rel_path", False), - ("rel_path", True), - ("rel_url", False), - ("rel_url", True), - ("embedded_rel_path", False), - ("embedded_rel_path", True), -]) + script.pip("install", "-vvv", "-r", script.scratch_path / "file-egg-req.txt") + + +@pytest.mark.parametrize( + "test_type,editable", + [ + ("rel_path", False), + ("rel_path", True), + ("rel_url", False), + ("rel_url", True), + ("embedded_rel_path", False), + ("embedded_rel_path", True), + ], +) +@pytest.mark.usefixtures("with_wheel") def test_relative_requirements_file( - script, data, test_type, editable, with_wheel -): + script: PipTestEnvironment, data: TestData, test_type: str, editable: bool +) -> None: """ Test installing from a requirements file with a relative path. For path URLs, use an egg= definition. """ - dist_info_folder = ( - script.site_packages / - 'FSPkg-0.1.dev0.dist-info' - ) - egg_link_file = ( - script.site_packages / 'FSPkg.egg-link' - ) - package_folder = script.site_packages / 'fspkg' + dist_info_folder = script.site_packages / "FSPkg-0.1.dev0.dist-info" + egg_link_file = script.site_packages / "FSPkg.egg-link" + package_folder = script.site_packages / "fspkg" # Compute relative install path to FSPkg from scratch path. - full_rel_path = Path( - os.path.relpath(data.packages.joinpath('FSPkg'), script.scratch_path) + full_rel_path = os.path.relpath( + data.packages.joinpath("FSPkg"), script.scratch_path ) - full_rel_url = 'file:' + full_rel_path + '#egg=FSPkg' + full_rel_url = "file:" + full_rel_path + "#egg=FSPkg" embedded_rel_path = script.scratch_path.joinpath(full_rel_path) req_path = { @@ -142,311 +140,384 @@ def test_relative_requirements_file( "embedded_rel_path": embedded_rel_path, }[test_type] - req_path = req_path.replace(os.path.sep, '/') + req_path = req_path.replace(os.path.sep, "/") # Install as either editable or not. if not editable: - with requirements_file(req_path + '\n', - script.scratch_path) as reqs_file: - result = script.pip('install', '-vvv', '-r', reqs_file.name, - cwd=script.scratch_path) + with requirements_file(req_path + "\n", script.scratch_path) as reqs_file: + result = script.pip( + "install", "-vvv", "-r", reqs_file.name, cwd=script.scratch_path + ) result.did_create(dist_info_folder) result.did_create(package_folder) else: - with requirements_file('-e ' + req_path + '\n', - script.scratch_path) as reqs_file: - result = script.pip('install', '-vvv', '-r', reqs_file.name, - cwd=script.scratch_path) + with requirements_file( + "-e " + req_path + "\n", script.scratch_path + ) as reqs_file: + result = script.pip( + "install", "-vvv", "-r", reqs_file.name, cwd=script.scratch_path + ) result.did_create(egg_link_file) @pytest.mark.xfail @pytest.mark.network @need_svn -def test_multiple_requirements_files(script, tmpdir, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_multiple_requirements_files(script: PipTestEnvironment, tmpdir: Path) -> None: """ Test installing from multiple nested requirements files. """ - other_lib_name, other_lib_version = 'anyjson', '0.3' + other_lib_name, other_lib_version = "six", "1.16.0" script.scratch_path.joinpath("initools-req.txt").write_text( - textwrap.dedent(""" + textwrap.dedent( + """ -e {}@10#egg=INITools -r {}-req.txt - """).format - ( - local_checkout('svn+http://svn.colorstudy.com/INITools', tmpdir), - other_lib_name + """ + ).format( + local_checkout("svn+http://svn.colorstudy.com/INITools", tmpdir), + other_lib_name, ), ) - script.scratch_path.joinpath( - "{other_lib_name}-req.txt".format(**locals())).write_text( - "{other_lib_name}<={other_lib_version}".format(**locals()) - ) - result = script.pip( - 'install', '-r', script.scratch_path / 'initools-req.txt' + script.scratch_path.joinpath(f"{other_lib_name}-req.txt").write_text( + f"{other_lib_name}<={other_lib_version}" ) + result = script.pip("install", "-r", script.scratch_path / "initools-req.txt") assert result.files_created[script.site_packages / other_lib_name].dir - fn = '{other_lib_name}-{other_lib_version}.dist-info'.format(**locals()) + fn = f"{other_lib_name}-{other_lib_version}.dist-info" assert result.files_created[script.site_packages / fn].dir - result.did_create(script.venv / 'src' / 'initools') + result.did_create(script.venv / "src" / "initools") -def test_package_in_constraints_and_dependencies(script, data): +def test_package_in_constraints_and_dependencies( + script: PipTestEnvironment, data: TestData +) -> None: script.scratch_path.joinpath("constraints.txt").write_text( "TopoRequires2==0.0.1\nTopoRequires==0.0.1" ) - result = script.pip('install', '--no-index', '-f', - data.find_links, '-c', script.scratch_path / - 'constraints.txt', 'TopoRequires2') - assert 'installed TopoRequires-0.0.1' in result.stdout + result = script.pip( + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + "TopoRequires2", + ) + assert "installed TopoRequires-0.0.1" in result.stdout -def test_multiple_constraints_files(script, data): +def test_multiple_constraints_files(script: PipTestEnvironment, data: TestData) -> None: script.scratch_path.joinpath("outer.txt").write_text("-c inner.txt") - script.scratch_path.joinpath("inner.txt").write_text( - "Upper==1.0") + script.scratch_path.joinpath("inner.txt").write_text("Upper==1.0") result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'outer.txt', 'Upper') - assert 'installed Upper-1.0' in result.stdout - - -@pytest.mark.xfail(reason="Unclear what this guarantee is for.") -def test_respect_order_in_requirements_file(script, data): - script.scratch_path.joinpath("frameworks-req.txt").write_text(textwrap.dedent("""\ + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "outer.txt", + "Upper", + ) + assert "installed Upper-1.0" in result.stdout + + +# FIXME: Unclear what this guarantee is for. +def test_respect_order_in_requirements_file( + script: PipTestEnvironment, data: TestData +) -> None: + script.scratch_path.joinpath("frameworks-req.txt").write_text( + textwrap.dedent( + """\ parent child simple - """)) - - result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-r', - script.scratch_path / 'frameworks-req.txt' - ) - - downloaded = [line for line in result.stdout.split('\n') - if 'Processing' in line] - - assert 'parent' in downloaded[0], ( - 'First download should be "parent" but was "{}"'.format(downloaded[0]) - ) - assert 'child' in downloaded[1], ( - 'Second download should be "child" but was "{}"'.format(downloaded[1]) - ) - assert 'simple' in downloaded[2], ( - 'Third download should be "simple" but was "{}"'.format(downloaded[2]) + """ + ) ) - -def test_install_local_editable_with_extras(script, data): + result = script.pip( + "install", + "--no-index", + "-f", + data.find_links, + "-r", + script.scratch_path / "frameworks-req.txt", + ) + + downloaded = [line for line in result.stdout.split("\n") if "Processing" in line] + + assert ( + "parent" in downloaded[0] + ), 'First download should be "parent" but was "{}"'.format(downloaded[0]) + assert ( + "child" in downloaded[1] + ), 'Second download should be "child" but was "{}"'.format(downloaded[1]) + assert ( + "simple" in downloaded[2] + ), 'Third download should be "simple" but was "{}"'.format(downloaded[2]) + + +def test_install_local_editable_with_extras( + script: PipTestEnvironment, data: TestData +) -> None: to_install = data.packages.joinpath("LocalExtras") res = script.pip_install_local( - '-e', to_install + '[bar]', allow_stderr_warning=True + "-e", to_install + "[bar]", allow_stderr_warning=True ) - res.did_update(script.site_packages / 'easy-install.pth') - res.did_create(script.site_packages / 'LocalExtras.egg-link') - res.did_create(script.site_packages / 'simple') + res.did_update(script.site_packages / "easy-install.pth") + res.did_create(script.site_packages / "LocalExtras.egg-link") + res.did_create(script.site_packages / "simple") -def test_install_collected_dependencies_first(script): +def test_install_collected_dependencies_first(script: PipTestEnvironment) -> None: result = script.pip_install_local( - 'toporequires2', + "toporequires2", ) - text = [line for line in result.stdout.split('\n') - if 'Installing' in line][0] - assert text.endswith('toporequires2') + text = [line for line in result.stdout.split("\n") if "Installing" in line][0] + assert text.endswith("toporequires2") @pytest.mark.network -def test_install_local_editable_with_subdirectory(script): - version_pkg_path = _create_test_package_with_subdirectory(script, - 'version_subdir') +def test_install_local_editable_with_subdirectory(script: PipTestEnvironment) -> None: + version_pkg_path = _create_test_package_with_subdirectory(script, "version_subdir") result = script.pip( - 'install', '-e', - '{uri}#egg=version_subpkg&subdirectory=version_subdir'.format( - uri='git+' + path_to_url(version_pkg_path), + "install", + "-e", + "{uri}#egg=version_subpkg&subdirectory=version_subdir".format( + uri="git+" + path_to_url(version_pkg_path), ), ) - result.assert_installed('version-subpkg', sub_dir='version_subdir') + result.assert_installed("version-subpkg", sub_dir="version_subdir") @pytest.mark.network -def test_install_local_with_subdirectory(script): - version_pkg_path = _create_test_package_with_subdirectory(script, - 'version_subdir') +def test_install_local_with_subdirectory(script: PipTestEnvironment) -> None: + version_pkg_path = _create_test_package_with_subdirectory(script, "version_subdir") result = script.pip( - 'install', - '{uri}#egg=version_subpkg&subdirectory=version_subdir'.format( - uri='git+' + path_to_url(version_pkg_path), + "install", + "{uri}#egg=version_subpkg&subdirectory=version_subdir".format( + uri="git+" + path_to_url(version_pkg_path), ), ) - result.assert_installed('version_subpkg.py', editable=False) + result.assert_installed("version_subpkg.py", editable=False) @pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("with_wheel") def test_wheel_user_with_prefix_in_pydistutils_cfg( - script, data, with_wheel): - if os.name == 'posix': + script: PipTestEnvironment, data: TestData +) -> None: + if os.name == "posix": user_filename = ".pydistutils.cfg" else: user_filename = "pydistutils.cfg" - user_cfg = os.path.join(os.path.expanduser('~'), user_filename) + user_cfg = os.path.join(os.path.expanduser("~"), user_filename) script.scratch_path.joinpath("bin").mkdir() with open(user_cfg, "w") as cfg: - cfg.write(textwrap.dedent(""" + cfg.write( + textwrap.dedent( + f""" [install] - prefix={script.scratch_path}""".format(**locals()))) + prefix={script.scratch_path}""" + ) + ) result = script.pip( - 'install', '--user', '--no-index', - '-f', data.find_links, - 'requiresupper') + "install", "--user", "--no-index", "-f", data.find_links, "requiresupper" + ) # Check that we are really installing a wheel - assert 'Running setup.py install for requiresupper' not in result.stdout - assert 'installed requiresupper' in result.stdout + assert "Running setup.py install for requiresupper" not in result.stdout + assert "installed requiresupper" in result.stdout def test_install_option_in_requirements_file_overrides_cli( - script, arg_recording_sdist_maker -): + script: PipTestEnvironment, + arg_recording_sdist_maker: Callable[[str], ArgRecordingSdist], +) -> None: simple_sdist = arg_recording_sdist_maker("simple") reqs_file = script.scratch_path.joinpath("reqs.txt") reqs_file.write_text("simple --install-option='-O0'") script.pip( - 'install', '--no-index', '-f', str(simple_sdist.sdist_path.parent), - '-r', str(reqs_file), '--install-option=-O1', + "install", + "--no-index", + "-f", + str(simple_sdist.sdist_path.parent), + "-r", + str(reqs_file), + "--install-option=-O1", + allow_stderr_warning=True, ) simple_args = simple_sdist.args() - assert 'install' in simple_args - assert simple_args.index('-O1') < simple_args.index('-O0') + assert "install" in simple_args + assert simple_args.index("-O1") < simple_args.index("-O0") -def test_constraints_not_installed_by_default(script, data): +def test_constraints_not_installed_by_default( + script: PipTestEnvironment, data: TestData +) -> None: script.scratch_path.joinpath("c.txt").write_text("requiresupper") result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'c.txt', 'Upper') - assert 'requiresupper' not in result.stdout + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "c.txt", + "Upper", + ) + assert "requiresupper" not in result.stdout -def test_constraints_only_causes_error(script, data): +def test_constraints_only_causes_error( + script: PipTestEnvironment, data: TestData +) -> None: script.scratch_path.joinpath("c.txt").write_text("requiresupper") result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'c.txt', expect_error=True) - assert 'installed requiresupper' not in result.stdout + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "c.txt", + expect_error=True, + ) + assert "installed requiresupper" not in result.stdout def test_constraints_local_editable_install_causes_error( - script, - data, - resolver_variant, -): - script.scratch_path.joinpath("constraints.txt").write_text( - "singlemodule==0.0.0" - ) + script: PipTestEnvironment, + data: TestData, + resolver_variant: ResolverVariant, +) -> None: + script.scratch_path.joinpath("constraints.txt").write_text("singlemodule==0.0.0") to_install = data.src.joinpath("singlemodule") result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'constraints.txt', '-e', - to_install, expect_error=True) + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + "-e", + to_install, + expect_error=True, + ) if resolver_variant == "legacy-resolver": - assert 'Could not satisfy constraints' in result.stderr, str(result) + assert "Could not satisfy constraints" in result.stderr, str(result) else: # Because singlemodule only has 0.0.1 available. - assert 'No matching distribution found' in result.stderr, str(result) + assert "Cannot install singlemodule 0.0.1" in result.stderr, str(result) @pytest.mark.network -def test_constraints_local_editable_install_pep518(script, data): +def test_constraints_local_editable_install_pep518( + script: PipTestEnvironment, data: TestData +) -> None: to_install = data.src.joinpath("pep518-3.0") - script.pip('download', 'setuptools', 'wheel', '-d', data.packages) - script.pip( - 'install', '--no-index', '-f', data.find_links, '-e', to_install) + script.pip("download", "setuptools", "wheel", "-d", data.packages) + script.pip("install", "--no-index", "-f", data.find_links, "-e", to_install) def test_constraints_local_install_causes_error( - script, - data, - resolver_variant, -): - script.scratch_path.joinpath("constraints.txt").write_text( - "singlemodule==0.0.0" - ) + script: PipTestEnvironment, + data: TestData, + resolver_variant: ResolverVariant, +) -> None: + script.scratch_path.joinpath("constraints.txt").write_text("singlemodule==0.0.0") to_install = data.src.joinpath("singlemodule") result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'constraints.txt', - to_install, expect_error=True) + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + to_install, + expect_error=True, + ) if resolver_variant == "legacy-resolver": - assert 'Could not satisfy constraints' in result.stderr, str(result) + assert "Could not satisfy constraints" in result.stderr, str(result) else: # Because singlemodule only has 0.0.1 available. - assert 'No matching distribution found' in result.stderr, str(result) + assert "Cannot install singlemodule 0.0.1" in result.stderr, str(result) def test_constraints_constrain_to_local_editable( - script, - data, - resolver_variant, -): + script: PipTestEnvironment, + data: TestData, + resolver_variant: ResolverVariant, +) -> None: to_install = data.src.joinpath("singlemodule") script.scratch_path.joinpath("constraints.txt").write_text( "-e {url}#egg=singlemodule".format(url=path_to_url(to_install)) ) result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'constraints.txt', 'singlemodule', + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + "singlemodule", allow_stderr_warning=True, expect_error=(resolver_variant == "2020-resolver"), ) if resolver_variant == "2020-resolver": - assert 'Links are not allowed as constraints' in result.stderr + assert "Editable requirements are not allowed as constraints" in result.stderr else: - assert 'Running setup.py develop for singlemodule' in result.stdout + assert "Running setup.py develop for singlemodule" in result.stdout -def test_constraints_constrain_to_local(script, data, resolver_variant): +def test_constraints_constrain_to_local( + script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant +) -> None: to_install = data.src.joinpath("singlemodule") script.scratch_path.joinpath("constraints.txt").write_text( "{url}#egg=singlemodule".format(url=path_to_url(to_install)) ) result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'constraints.txt', 'singlemodule', + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + "singlemodule", allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), ) - if resolver_variant == "2020-resolver": - assert 'Links are not allowed as constraints' in result.stderr - else: - assert 'Running setup.py install for singlemodule' in result.stdout + assert "Running setup.py install for singlemodule" in result.stdout -def test_constrained_to_url_install_same_url(script, data, resolver_variant): +def test_constrained_to_url_install_same_url( + script: PipTestEnvironment, data: TestData +) -> None: to_install = data.src.joinpath("singlemodule") constraints = path_to_url(to_install) + "#egg=singlemodule" script.scratch_path.joinpath("constraints.txt").write_text(constraints) result = script.pip( - 'install', '--no-index', '-f', data.find_links, '-c', - script.scratch_path / 'constraints.txt', to_install, + "install", + "--no-index", + "-f", + data.find_links, + "-c", + script.scratch_path / "constraints.txt", + to_install, allow_stderr_warning=True, - expect_error=(resolver_variant == "2020-resolver"), ) - if resolver_variant == "2020-resolver": - assert 'Links are not allowed as constraints' in result.stderr - else: - assert ('Running setup.py install for singlemodule' - in result.stdout), str(result) + assert "Running setup.py install for singlemodule" in result.stdout, str(result) +@pytest.mark.usefixtures("with_wheel") def test_double_install_spurious_hash_mismatch( - script, tmpdir, data, with_wheel): + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: """Make sure installing the same hashed sdist twice doesn't throw hash mismatch errors. @@ -457,139 +528,164 @@ def test_double_install_spurious_hash_mismatch( """ # Install wheel package, otherwise, it won't try to build wheels. - with requirements_file('simple==1.0 --hash=sha256:393043e672415891885c9a2a' - '0929b1af95fb866d6ca016b42d2e6ce53619b653', - tmpdir) as reqs_file: + with requirements_file( + "simple==1.0 --hash=sha256:393043e672415891885c9a2a" + "0929b1af95fb866d6ca016b42d2e6ce53619b653", + tmpdir, + ) as reqs_file: # Install a package (and build its wheel): result = script.pip_install_local( - '--find-links', data.find_links, - '-r', reqs_file.resolve(), + "--find-links", + data.find_links, + "-r", + reqs_file.resolve(), ) - assert 'Successfully installed simple-1.0' in str(result) + assert "Successfully installed simple-1.0" in str(result) # Uninstall it: - script.pip('uninstall', '-y', 'simple') + script.pip("uninstall", "-y", "simple") # Then install it again. We should not hit a hash mismatch, and the # package should install happily. result = script.pip_install_local( - '--find-links', data.find_links, - '-r', reqs_file.resolve(), + "--find-links", + data.find_links, + "-r", + reqs_file.resolve(), ) - assert 'Successfully installed simple-1.0' in str(result) + assert "Successfully installed simple-1.0" in str(result) -def test_install_with_extras_from_constraints(script, data, resolver_variant): +def test_install_with_extras_from_constraints( + script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant +) -> None: to_install = data.packages.joinpath("LocalExtras") script.scratch_path.joinpath("constraints.txt").write_text( "{url}#egg=LocalExtras[bar]".format(url=path_to_url(to_install)) ) result = script.pip_install_local( - '-c', script.scratch_path / 'constraints.txt', 'LocalExtras', + "-c", + script.scratch_path / "constraints.txt", + "LocalExtras", allow_stderr_warning=True, expect_error=(resolver_variant == "2020-resolver"), ) if resolver_variant == "2020-resolver": - assert 'Links are not allowed as constraints' in result.stderr + assert "Constraints cannot have extras" in result.stderr else: - result.did_create(script.site_packages / 'simple') + result.did_create(script.site_packages / "simple") -def test_install_with_extras_from_install(script): +def test_install_with_extras_from_install(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, name="LocalExtras", version="0.0.1", - extras={"bar": "simple", "baz": ["singlemodule"]}, + extras={"bar": ["simple"], "baz": ["singlemodule"]}, ) script.scratch_path.joinpath("constraints.txt").write_text("LocalExtras") result = script.pip_install_local( - '--find-links', script.scratch_path, - '-c', script.scratch_path / 'constraints.txt', - 'LocalExtras[baz]', + "--find-links", + script.scratch_path, + "-c", + script.scratch_path / "constraints.txt", + "LocalExtras[baz]", ) - result.did_create(script.site_packages / 'singlemodule.py') + result.did_create(script.site_packages / "singlemodule.py") -def test_install_with_extras_joined(script, data, resolver_variant): +def test_install_with_extras_joined( + script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant +) -> None: to_install = data.packages.joinpath("LocalExtras") script.scratch_path.joinpath("constraints.txt").write_text( "{url}#egg=LocalExtras[bar]".format(url=path_to_url(to_install)) ) result = script.pip_install_local( - '-c', script.scratch_path / 'constraints.txt', 'LocalExtras[baz]', + "-c", + script.scratch_path / "constraints.txt", + "LocalExtras[baz]", allow_stderr_warning=True, expect_error=(resolver_variant == "2020-resolver"), ) if resolver_variant == "2020-resolver": - assert 'Links are not allowed as constraints' in result.stderr + assert "Constraints cannot have extras" in result.stderr else: - result.did_create(script.site_packages / 'simple') - result.did_create(script.site_packages / 'singlemodule.py') + result.did_create(script.site_packages / "simple") + result.did_create(script.site_packages / "singlemodule.py") -def test_install_with_extras_editable_joined(script, data, resolver_variant): +def test_install_with_extras_editable_joined( + script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant +) -> None: to_install = data.packages.joinpath("LocalExtras") script.scratch_path.joinpath("constraints.txt").write_text( "-e {url}#egg=LocalExtras[bar]".format(url=path_to_url(to_install)) ) result = script.pip_install_local( - '-c', script.scratch_path / 'constraints.txt', 'LocalExtras[baz]', + "-c", + script.scratch_path / "constraints.txt", + "LocalExtras[baz]", allow_stderr_warning=True, expect_error=(resolver_variant == "2020-resolver"), ) if resolver_variant == "2020-resolver": - assert 'Links are not allowed as constraints' in result.stderr + assert "Editable requirements are not allowed as constraints" in result.stderr else: - result.did_create(script.site_packages / 'simple') - result.did_create(script.site_packages / 'singlemodule.py') + result.did_create(script.site_packages / "simple") + result.did_create(script.site_packages / "singlemodule.py") -def test_install_distribution_full_union(script, data): +def test_install_distribution_full_union( + script: PipTestEnvironment, data: TestData +) -> None: to_install = data.packages.joinpath("LocalExtras") result = script.pip_install_local( - to_install, to_install + "[bar]", to_install + "[baz]") - assert 'Running setup.py install for LocalExtras' in result.stdout - result.did_create(script.site_packages / 'simple') - result.did_create(script.site_packages / 'singlemodule.py') + to_install, to_install + "[bar]", to_install + "[baz]" + ) + assert "Running setup.py install for LocalExtras" in result.stdout + result.did_create(script.site_packages / "simple") + result.did_create(script.site_packages / "singlemodule.py") -def test_install_distribution_duplicate_extras(script, data): +def test_install_distribution_duplicate_extras( + script: PipTestEnvironment, data: TestData +) -> None: to_install = data.packages.joinpath("LocalExtras") package_name = to_install + "[bar]" with pytest.raises(AssertionError): result = script.pip_install_local(package_name, package_name) - expected = ( - 'Double requirement given: {package_name}'.format(**locals())) + expected = f"Double requirement given: {package_name}" assert expected in result.stderr def test_install_distribution_union_with_constraints( - script, - data, - resolver_variant, -): + script: PipTestEnvironment, + data: TestData, + resolver_variant: ResolverVariant, +) -> None: to_install = data.packages.joinpath("LocalExtras") - script.scratch_path.joinpath("constraints.txt").write_text( - "{to_install}[bar]".format(**locals())) + script.scratch_path.joinpath("constraints.txt").write_text(f"{to_install}[bar]") result = script.pip_install_local( - '-c', script.scratch_path / 'constraints.txt', to_install + '[baz]', + "-c", + script.scratch_path / "constraints.txt", + to_install + "[baz]", allow_stderr_warning=True, expect_error=(resolver_variant == "2020-resolver"), ) if resolver_variant == "2020-resolver": - msg = 'Unnamed requirements are not allowed as constraints' + msg = "Unnamed requirements are not allowed as constraints" assert msg in result.stderr else: - assert 'Running setup.py install for LocalExtras' in result.stdout - result.did_create(script.site_packages / 'singlemodule.py') + assert "Running setup.py install for LocalExtras" in result.stdout + result.did_create(script.site_packages / "singlemodule.py") def test_install_distribution_union_with_versions( - script, - data, - resolver_variant, -): + script: PipTestEnvironment, + data: TestData, + resolver_variant: ResolverVariant, +) -> None: to_install_001 = data.packages.joinpath("LocalExtras") to_install_002 = data.packages.joinpath("LocalExtras-0.0.2") result = script.pip_install_local( @@ -599,68 +695,75 @@ def test_install_distribution_union_with_versions( ) if resolver_variant == "2020-resolver": assert "Cannot install localextras[bar]" in result.stderr - assert ( - "localextras[bar] 0.0.1 depends on localextras 0.0.1" - ) in result.stdout - assert ( - "localextras[baz] 0.0.2 depends on localextras 0.0.2" - ) in result.stdout + assert ("localextras[bar] 0.0.1 depends on localextras 0.0.1") in result.stdout + assert ("localextras[baz] 0.0.2 depends on localextras 0.0.2") in result.stdout else: assert ( - "Successfully installed LocalExtras-0.0.1 simple-3.0 " - "singlemodule-0.0.1" + "Successfully installed LocalExtras-0.0.1 simple-3.0 singlemodule-0.0.1" ) in result.stdout @pytest.mark.xfail -def test_install_distribution_union_conflicting_extras(script, data): +def test_install_distribution_union_conflicting_extras( + script: PipTestEnvironment, data: TestData +) -> None: # LocalExtras requires simple==1.0, LocalExtras[bar] requires simple==2.0; # without a resolver, pip does not detect the conflict between simple==1.0 # and simple==2.0. Once a resolver is added, this conflict should be # detected. to_install = data.packages.joinpath("LocalExtras-0.0.2") - result = script.pip_install_local(to_install, to_install + "[bar]", - expect_error=True) - assert 'installed' not in result.stdout + result = script.pip_install_local( + to_install, to_install + "[bar]", expect_error=True + ) + assert "installed" not in result.stdout assert "Conflict" in result.stderr -def test_install_unsupported_wheel_link_with_marker(script): +def test_install_unsupported_wheel_link_with_marker(script: PipTestEnvironment) -> None: script.scratch_path.joinpath("with-marker.txt").write_text( - textwrap.dedent("""\ + textwrap.dedent( + """\ {url}; {req} - """).format( - url='https://github.com/a/b/c/asdf-1.5.2-cp27-none-xyz.whl', + """ + ).format( + url="https://github.com/a/b/c/asdf-1.5.2-cp27-none-xyz.whl", req='sys_platform == "xyz"', ) ) - result = script.pip( - 'install', '-r', script.scratch_path / 'with-marker.txt' - ) + result = script.pip("install", "-r", script.scratch_path / "with-marker.txt") - assert ("Ignoring asdf: markers 'sys_platform == \"xyz\"' don't match " - "your environment") in result.stdout + assert ( + "Ignoring asdf: markers 'sys_platform == \"xyz\"' don't match " + "your environment" + ) in result.stdout assert len(result.files_created) == 0 -def test_install_unsupported_wheel_file(script, data): +def test_install_unsupported_wheel_file( + script: PipTestEnvironment, data: TestData +) -> None: # Trying to install a local wheel with an incompatible version/type # should fail. path = data.packages.joinpath("simple.dist-0.1-py1-none-invalid.whl") - script.scratch_path.joinpath("wheel-file.txt").write_text(textwrap.dedent("""\ - {path} - """.format(**locals()))) + script.scratch_path.joinpath("wheel-file.txt").write_text(path + "\n") result = script.pip( - 'install', '-r', script.scratch_path / 'wheel-file.txt', + "install", + "-r", + script.scratch_path / "wheel-file.txt", expect_error=True, expect_stderr=True, ) - assert ("simple.dist-0.1-py1-none-invalid.whl is not a supported " + - "wheel on this platform" in result.stderr) + assert ( + "simple.dist-0.1-py1-none-invalid.whl is not a supported wheel on this platform" + in result.stderr + ) assert len(result.files_created) == 0 -def test_install_options_local_to_package(script, arg_recording_sdist_maker): +def test_install_options_local_to_package( + script: PipTestEnvironment, + arg_recording_sdist_maker: Callable[[str], ArgRecordingSdist], +) -> None: """Make sure --install-options does not leak across packages. A requirements.txt file can have per-package --install-options; these @@ -682,27 +785,34 @@ def test_install_options_local_to_package(script, arg_recording_sdist_maker): ) ) script.pip( - 'install', - '--no-index', '-f', str(simple1_sdist.sdist_path.parent), - '-r', reqs_file, + "install", + "--no-index", + "-f", + str(simple1_sdist.sdist_path.parent), + "-r", + reqs_file, + allow_stderr_warning=True, ) simple1_args = simple1_sdist.args() - assert 'install' in simple1_args - assert '-O0' in simple1_args + assert "install" in simple1_args + assert "-O0" in simple1_args simple2_args = simple2_sdist.args() - assert 'install' in simple2_args - assert '-O0' not in simple2_args + assert "install" in simple2_args + assert "-O0" not in simple2_args -def test_location_related_install_option_fails(script): +def test_location_related_install_option_fails(script: PipTestEnvironment) -> None: simple_sdist = create_basic_sdist_for_package(script, "simple", "0.1.0") reqs_file = script.scratch_path.joinpath("reqs.txt") reqs_file.write_text("simple --install-option='--home=/tmp'") result = script.pip( - 'install', - '--no-index', '-f', str(simple_sdist.parent), - '-r', reqs_file, - expect_error=True + "install", + "--no-index", + "-f", + str(simple_sdist.parent), + "-r", + reqs_file, + expect_error=True, ) assert "['--home'] from simple" in result.stderr diff --git a/tests/functional/test_install_requested.py b/tests/functional/test_install_requested.py index 6caec988eb5..0b5d5f18e75 100644 --- a/tests/functional/test_install_requested.py +++ b/tests/functional/test_install_requested.py @@ -1,18 +1,28 @@ -def _assert_requested_present(script, result, name, version): +import pytest + +from tests.lib import PipTestEnvironment, TestData, TestPipResult + + +def _assert_requested_present( + script: PipTestEnvironment, result: TestPipResult, name: str, version: str +) -> None: dist_info = script.site_packages / name + "-" + version + ".dist-info" requested = dist_info / "REQUESTED" assert dist_info in result.files_created assert requested in result.files_created -def _assert_requested_absent(script, result, name, version): +def _assert_requested_absent( + script: PipTestEnvironment, result: TestPipResult, name: str, version: str +) -> None: dist_info = script.site_packages / name + "-" + version + ".dist-info" requested = dist_info / "REQUESTED" assert dist_info in result.files_created assert requested not in result.files_created -def test_install_requested_basic(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_requested_basic(script: PipTestEnvironment, data: TestData) -> None: result = script.pip( "install", "--no-index", "-f", data.find_links, "require_simple" ) @@ -21,10 +31,11 @@ def test_install_requested_basic(script, data, with_wheel): _assert_requested_absent(script, result, "simple", "3.0") -def test_install_requested_requirements(script, data, with_wheel): - script.scratch_path.joinpath("requirements.txt").write_text( - "require_simple\n" - ) +@pytest.mark.usefixtures("with_wheel") +def test_install_requested_requirements( + script: PipTestEnvironment, data: TestData +) -> None: + script.scratch_path.joinpath("requirements.txt").write_text("require_simple\n") result = script.pip( "install", "--no-index", @@ -37,7 +48,10 @@ def test_install_requested_requirements(script, data, with_wheel): _assert_requested_absent(script, result, "simple", "3.0") -def test_install_requested_dep_in_requirements(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_requested_dep_in_requirements( + script: PipTestEnvironment, data: TestData +) -> None: script.scratch_path.joinpath("requirements.txt").write_text( "require_simple\nsimple<3\n" ) @@ -54,10 +68,11 @@ def test_install_requested_dep_in_requirements(script, data, with_wheel): _assert_requested_present(script, result, "simple", "2.0") -def test_install_requested_reqs_and_constraints(script, data, with_wheel): - script.scratch_path.joinpath("requirements.txt").write_text( - "require_simple\n" - ) +@pytest.mark.usefixtures("with_wheel") +def test_install_requested_reqs_and_constraints( + script: PipTestEnvironment, data: TestData +) -> None: + script.scratch_path.joinpath("requirements.txt").write_text("require_simple\n") script.scratch_path.joinpath("constraints.txt").write_text("simple<3\n") result = script.pip( "install", @@ -74,7 +89,10 @@ def test_install_requested_reqs_and_constraints(script, data, with_wheel): _assert_requested_absent(script, result, "simple", "2.0") -def test_install_requested_in_reqs_and_constraints(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_requested_in_reqs_and_constraints( + script: PipTestEnvironment, data: TestData +) -> None: script.scratch_path.joinpath("requirements.txt").write_text( "require_simple\nsimple\n" ) diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index 46aac8f9d26..3dab9161e80 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -6,60 +6,57 @@ import pytest from tests.lib import pyversion # noqa: F401 -from tests.lib import assert_all_changes +from tests.lib import PipTestEnvironment, ResolverVariant, TestData, assert_all_changes from tests.lib.local_repos import local_checkout +from tests.lib.path import Path from tests.lib.wheel import make_wheel @pytest.mark.network -def test_no_upgrade_unless_requested(script): +def test_no_upgrade_unless_requested(script: PipTestEnvironment) -> None: """ No upgrade if not specifically requested. """ - script.pip('install', 'INITools==0.1') - result = script.pip('install', 'INITools') - assert not result.files_created, ( - 'pip install INITools upgraded when it should not have' - ) + script.pip("install", "INITools==0.1") + result = script.pip("install", "INITools") + assert ( + not result.files_created + ), "pip install INITools upgraded when it should not have" -def test_invalid_upgrade_strategy_causes_error(script): +def test_invalid_upgrade_strategy_causes_error(script: PipTestEnvironment) -> None: """ It errors out when the upgrade-strategy is an invalid/unrecognised one """ result = script.pip_install_local( - '--upgrade', '--upgrade-strategy=bazinga', 'simple', - expect_error=True + "--upgrade", "--upgrade-strategy=bazinga", "simple", expect_error=True ) assert result.returncode assert "invalid choice" in result.stderr +@pytest.mark.usefixtures("with_wheel") def test_only_if_needed_does_not_upgrade_deps_when_satisfied( - script, - resolver_variant, - with_wheel -): + script: PipTestEnvironment, resolver_variant: ResolverVariant +) -> None: """ It doesn't upgrade a dependency if it already satisfies the requirements. """ - script.pip_install_local('simple==2.0') + script.pip_install_local("simple==2.0") result = script.pip_install_local( - '--upgrade', '--upgrade-strategy=only-if-needed', 'require_simple' + "--upgrade", "--upgrade-strategy=only-if-needed", "require_simple" ) assert ( - (script.site_packages / 'require_simple-1.0.dist-info') - not in result.files_deleted - ), "should have installed require_simple==1.0" + script.site_packages / "require_simple-1.0.dist-info" + ) not in result.files_deleted, "should have installed require_simple==1.0" assert ( - (script.site_packages / 'simple-2.0.dist-info') - not in result.files_deleted - ), "should not have uninstalled simple==2.0" + script.site_packages / "simple-2.0.dist-info" + ) not in result.files_deleted, "should not have uninstalled simple==2.0" msg = "Requirement already satisfied" if resolver_variant == "legacy": @@ -69,131 +66,113 @@ def test_only_if_needed_does_not_upgrade_deps_when_satisfied( ), "did not print correct message for not-upgraded requirement" +@pytest.mark.usefixtures("with_wheel") def test_only_if_needed_does_upgrade_deps_when_no_longer_satisfied( - script, with_wheel -): + script: PipTestEnvironment, +) -> None: """ It does upgrade a dependency if it no longer satisfies the requirements. """ - script.pip_install_local('simple==1.0') + script.pip_install_local("simple==1.0") result = script.pip_install_local( - '--upgrade', '--upgrade-strategy=only-if-needed', 'require_simple' + "--upgrade", "--upgrade-strategy=only-if-needed", "require_simple" ) assert ( - (script.site_packages / 'require_simple-1.0.dist-info') - not in result.files_deleted - ), "should have installed require_simple==1.0" - expected = ( - script.site_packages / - 'simple-3.0.dist-info' - ) + script.site_packages / "require_simple-1.0.dist-info" + ) not in result.files_deleted, "should have installed require_simple==1.0" + expected = script.site_packages / "simple-3.0.dist-info" result.did_create(expected, message="should have installed simple==3.0") - expected = ( - script.site_packages / - 'simple-1.0.dist-info' - ) - assert ( - expected in result.files_deleted - ), "should have uninstalled simple==1.0" + expected = script.site_packages / "simple-1.0.dist-info" + assert expected in result.files_deleted, "should have uninstalled simple==1.0" -def test_eager_does_upgrade_dependecies_when_currently_satisfied( - script, with_wheel -): +@pytest.mark.usefixtures("with_wheel") +def test_eager_does_upgrade_dependencies_when_currently_satisfied( + script: PipTestEnvironment, +) -> None: """ It does upgrade a dependency even if it already satisfies the requirements. """ - script.pip_install_local('simple==2.0') + script.pip_install_local("simple==2.0") result = script.pip_install_local( - '--upgrade', '--upgrade-strategy=eager', 'require_simple' + "--upgrade", "--upgrade-strategy=eager", "require_simple" ) assert ( - (script.site_packages / - 'require_simple-1.0.dist-info') - not in result.files_deleted - ), "should have installed require_simple==1.0" + script.site_packages / "require_simple-1.0.dist-info" + ) not in result.files_deleted, "should have installed require_simple==1.0" assert ( - (script.site_packages / - 'simple-2.0.dist-info') - in result.files_deleted - ), "should have uninstalled simple==2.0" + script.site_packages / "simple-2.0.dist-info" + ) in result.files_deleted, "should have uninstalled simple==2.0" -def test_eager_does_upgrade_dependecies_when_no_longer_satisfied( - script, with_wheel -): +@pytest.mark.usefixtures("with_wheel") +def test_eager_does_upgrade_dependencies_when_no_longer_satisfied( + script: PipTestEnvironment, +) -> None: """ It does upgrade a dependency if it no longer satisfies the requirements. """ - script.pip_install_local('simple==1.0') + script.pip_install_local("simple==1.0") result = script.pip_install_local( - '--upgrade', '--upgrade-strategy=eager', 'require_simple' + "--upgrade", "--upgrade-strategy=eager", "require_simple" ) assert ( - (script.site_packages / 'require_simple-1.0.dist-info') - not in result.files_deleted - ), "should have installed require_simple==1.0" + script.site_packages / "require_simple-1.0.dist-info" + ) not in result.files_deleted, "should have installed require_simple==1.0" result.did_create( - script.site_packages / 'simple-3.0.dist-info', - message="should have installed simple==3.0" + script.site_packages / "simple-3.0.dist-info", + message="should have installed simple==3.0", ) assert ( - script.site_packages / 'simple-1.0.dist-info' - in result.files_deleted + script.site_packages / "simple-1.0.dist-info" in result.files_deleted ), "should have uninstalled simple==1.0" @pytest.mark.network -def test_upgrade_to_specific_version(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_upgrade_to_specific_version(script: PipTestEnvironment) -> None: """ It does upgrade to specific version requested. """ - script.pip('install', 'INITools==0.1') - result = script.pip('install', 'INITools==0.2') - assert result.files_created, ( - 'pip install with specific version did not upgrade' - ) - assert ( - script.site_packages / 'INITools-0.1.dist-info' - in result.files_deleted - ) - result.did_create( - script.site_packages / 'INITools-0.2.dist-info' - ) + script.pip("install", "INITools==0.1") + result = script.pip("install", "INITools==0.2") + assert result.files_created, "pip install with specific version did not upgrade" + assert script.site_packages / "INITools-0.1.dist-info" in result.files_deleted + result.did_create(script.site_packages / "INITools-0.2.dist-info") @pytest.mark.network -def test_upgrade_if_requested(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_upgrade_if_requested(script: PipTestEnvironment) -> None: """ And it does upgrade if requested. """ - script.pip('install', 'INITools==0.1') - result = script.pip('install', '--upgrade', 'INITools') - assert result.files_created, 'pip install --upgrade did not upgrade' - result.did_not_create( - script.site_packages / - 'INITools-0.1.dist-info' - ) + script.pip("install", "INITools==0.1") + result = script.pip("install", "--upgrade", "INITools") + assert result.files_created, "pip install --upgrade did not upgrade" + result.did_not_create(script.site_packages / "INITools-0.1.dist-info") -def test_upgrade_with_newest_already_installed(script, data, resolver_variant): +def test_upgrade_with_newest_already_installed( + script: PipTestEnvironment, data: TestData, resolver_variant: ResolverVariant +) -> None: """ If the newest version of a package is already installed, the package should not be reinstalled and the user should be informed. """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple') + script.pip("install", "-f", data.find_links, "--no-index", "simple") result = script.pip( - 'install', '--upgrade', '-f', data.find_links, '--no-index', 'simple' + "install", "--upgrade", "-f", data.find_links, "--no-index", "simple" ) - assert not result.files_created, 'simple upgraded when it should not have' + assert not result.files_created, "simple upgraded when it should not have" if resolver_variant == "2020-resolver": msg = "Requirement already satisfied" else: @@ -202,175 +181,173 @@ def test_upgrade_with_newest_already_installed(script, data, resolver_variant): @pytest.mark.network -def test_upgrade_force_reinstall_newest(script): +def test_upgrade_force_reinstall_newest(script: PipTestEnvironment) -> None: """ Force reinstallation of a package even if it is already at its newest version if --force-reinstall is supplied. """ - result = script.pip('install', 'INITools') - result.did_create(script.site_packages / 'initools') - result2 = script.pip( - 'install', '--upgrade', '--force-reinstall', 'INITools' - ) - assert result2.files_updated, 'upgrade to INITools 0.3 failed' - result3 = script.pip('uninstall', 'initools', '-y') - assert_all_changes(result, result3, [script.venv / 'build', 'cache']) + result = script.pip("install", "INITools") + result.did_create(script.site_packages / "initools") + result2 = script.pip("install", "--upgrade", "--force-reinstall", "INITools") + assert result2.files_updated, "upgrade to INITools 0.3 failed" + result3 = script.pip("uninstall", "initools", "-y") + assert_all_changes(result, result3, [script.venv / "build", "cache"]) @pytest.mark.network -def test_uninstall_before_upgrade(script): +def test_uninstall_before_upgrade(script: PipTestEnvironment) -> None: """ Automatic uninstall-before-upgrade. """ - result = script.pip('install', 'INITools==0.2') - result.did_create(script.site_packages / 'initools') - result2 = script.pip('install', 'INITools==0.3') - assert result2.files_created, 'upgrade to INITools 0.3 failed' - result3 = script.pip('uninstall', 'initools', '-y') - assert_all_changes(result, result3, [script.venv / 'build', 'cache']) + result = script.pip("install", "INITools==0.2") + result.did_create(script.site_packages / "initools") + result2 = script.pip("install", "INITools==0.3") + assert result2.files_created, "upgrade to INITools 0.3 failed" + result3 = script.pip("uninstall", "initools", "-y") + assert_all_changes(result, result3, [script.venv / "build", "cache"]) @pytest.mark.network -def test_uninstall_before_upgrade_from_url(script): +def test_uninstall_before_upgrade_from_url(script: PipTestEnvironment) -> None: """ Automatic uninstall-before-upgrade from URL. """ - result = script.pip('install', 'INITools==0.2') - result.did_create(script.site_packages / 'initools') + result = script.pip("install", "INITools==0.2") + result.did_create(script.site_packages / "initools") result2 = script.pip( - 'install', - 'https://files.pythonhosted.org/packages/source/I/INITools/INITools-' - '0.3.tar.gz', + "install", + "https://files.pythonhosted.org/packages/source/I/INITools/INITools-" + "0.3.tar.gz", ) - assert result2.files_created, 'upgrade to INITools 0.3 failed' - result3 = script.pip('uninstall', 'initools', '-y') - assert_all_changes(result, result3, [script.venv / 'build', 'cache']) + assert result2.files_created, "upgrade to INITools 0.3 failed" + result3 = script.pip("uninstall", "initools", "-y") + assert_all_changes(result, result3, [script.venv / "build", "cache"]) @pytest.mark.network -def test_upgrade_to_same_version_from_url(script): +def test_upgrade_to_same_version_from_url(script: PipTestEnvironment) -> None: """ When installing from a URL the same version that is already installed, no need to uninstall and reinstall if --upgrade is not specified. """ - result = script.pip('install', 'INITools==0.3') - result.did_create(script.site_packages / 'initools') + result = script.pip("install", "INITools==0.3") + result.did_create(script.site_packages / "initools") result2 = script.pip( - 'install', - 'https://files.pythonhosted.org/packages/source/I/INITools/INITools-' - '0.3.tar.gz', - ) - assert script.site_packages / 'initools' not in result2.files_updated, ( - 'INITools 0.3 reinstalled same version' + "install", + "https://files.pythonhosted.org/packages/source/I/INITools/INITools-" + "0.3.tar.gz", ) - result3 = script.pip('uninstall', 'initools', '-y') - assert_all_changes(result, result3, [script.venv / 'build', 'cache']) + assert ( + script.site_packages / "initools" not in result2.files_updated + ), "INITools 0.3 reinstalled same version" + result3 = script.pip("uninstall", "initools", "-y") + assert_all_changes(result, result3, [script.venv / "build", "cache"]) @pytest.mark.network -def test_upgrade_from_reqs_file(script): +def test_upgrade_from_reqs_file(script: PipTestEnvironment) -> None: """ Upgrade from a requirements file. """ - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent("""\ + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """\ PyLogo<0.4 # and something else to test out: INITools==0.3 - """)) - install_result = script.pip( - 'install', '-r', script.scratch_path / 'test-req.txt' + """ + ) ) - script.scratch_path.joinpath("test-req.txt").write_text(textwrap.dedent("""\ + install_result = script.pip("install", "-r", script.scratch_path / "test-req.txt") + script.scratch_path.joinpath("test-req.txt").write_text( + textwrap.dedent( + """\ PyLogo # and something else to test out: INITools - """)) - script.pip( - 'install', '--upgrade', '-r', script.scratch_path / 'test-req.txt' + """ + ) ) + script.pip("install", "--upgrade", "-r", script.scratch_path / "test-req.txt") uninstall_result = script.pip( - 'uninstall', '-r', script.scratch_path / 'test-req.txt', '-y' + "uninstall", "-r", script.scratch_path / "test-req.txt", "-y" ) assert_all_changes( install_result, uninstall_result, - [script.venv / 'build', 'cache', script.scratch / 'test-req.txt'], + [script.venv / "build", "cache", script.scratch / "test-req.txt"], ) -def test_uninstall_rollback(script, data): +def test_uninstall_rollback(script: PipTestEnvironment, data: TestData) -> None: """ Test uninstall-rollback (using test package with a setup.py crafted to fail on install). """ - result = script.pip( - 'install', '-f', data.find_links, '--no-index', 'broken==0.1' - ) - result.did_create(script.site_packages / 'broken.py') + result = script.pip("install", "-f", data.find_links, "--no-index", "broken==0.1") + result.did_create(script.site_packages / "broken.py") result2 = script.pip( - 'install', '-f', data.find_links, '--no-index', 'broken===0.2broken', + "install", + "-f", + data.find_links, + "--no-index", + "broken===0.2broken", expect_error=True, ) assert result2.returncode == 1, str(result2) - assert script.run( - 'python', '-c', "import broken; print(broken.VERSION)" - ).stdout == '0.1\n' + assert ( + script.run("python", "-c", "import broken; print(broken.VERSION)").stdout + == "0.1\n" + ) assert_all_changes( result.files_after, result2, - [script.venv / 'build'], + [script.venv / "build"], ) @pytest.mark.network -def test_should_not_install_always_from_cache(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_should_not_install_always_from_cache(script: PipTestEnvironment) -> None: """ If there is an old cached package, pip should download the newer version Related to issue #175 """ - script.pip('install', 'INITools==0.2') - script.pip('uninstall', '-y', 'INITools') - result = script.pip('install', 'INITools==0.1') - result.did_not_create( - script.site_packages / - 'INITools-0.2.dist-info' - ) - result.did_create( - script.site_packages / - 'INITools-0.1.dist-info' - ) + script.pip("install", "INITools==0.2") + script.pip("uninstall", "-y", "INITools") + result = script.pip("install", "INITools==0.1") + result.did_not_create(script.site_packages / "INITools-0.2.dist-info") + result.did_create(script.site_packages / "INITools-0.1.dist-info") @pytest.mark.network -def test_install_with_ignoreinstalled_requested(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_with_ignoreinstalled_requested(script: PipTestEnvironment) -> None: """ Test old conflicting package is completely ignored """ - script.pip('install', 'INITools==0.1') - result = script.pip('install', '-I', 'INITools==0.3') - assert result.files_created, 'pip install -I did not install' + script.pip("install", "INITools==0.1") + result = script.pip("install", "-I", "INITools==0.3") + assert result.files_created, "pip install -I did not install" # both the old and new metadata should be present. - assert os.path.exists( - script.site_packages_path / - 'INITools-0.1.dist-info' - ) - assert os.path.exists( - script.site_packages_path / - 'INITools-0.3.dist-info' - ) + assert os.path.exists(script.site_packages_path / "INITools-0.1.dist-info") + assert os.path.exists(script.site_packages_path / "INITools-0.3.dist-info") @pytest.mark.network -def test_upgrade_vcs_req_with_no_dists_found(script, tmpdir): +def test_upgrade_vcs_req_with_no_dists_found( + script: PipTestEnvironment, tmpdir: Path +) -> None: """It can upgrade a VCS requirement that has no distributions otherwise.""" req = "{checkout}#egg=pip-test-package".format( checkout=local_checkout( - "git+https://github.com/pypa/pip-test-package.git", tmpdir, + "git+https://github.com/pypa/pip-test-package.git", + tmpdir, ) ) script.pip("install", req) @@ -379,74 +356,33 @@ def test_upgrade_vcs_req_with_no_dists_found(script, tmpdir): @pytest.mark.network -def test_upgrade_vcs_req_with_dist_found(script): +def test_upgrade_vcs_req_with_dist_found(script: PipTestEnvironment) -> None: """It can upgrade a VCS requirement that has distributions on the index.""" # TODO(pnasrat) Using local_checkout fails on windows - oddness with the # test path urls/git. - req = ( - "{url}#egg=pretend".format( - url=( - "git+git://github.com/alex/pretend@e7f26ad7dbcb4a02a4995aade4" - "743aad47656b27" - ), - ) + req = "{url}#egg=pretend".format( + url=( + "git+https://github.com/alex/pretend@e7f26ad7dbcb4a02a4995aade4" + "743aad47656b27" + ), ) script.pip("install", req, expect_stderr=True) result = script.pip("install", "-U", req, expect_stderr=True) assert "pypi.org" not in result.stdout, result.stdout -class TestUpgradeDistributeToSetuptools: - """ - From pip1.4 to pip6, pip supported a set of "hacks" (see Issue #1122) to - allow distribute to conflict with setuptools, so that the following would - work to upgrade distribute: - - ``pip install -U setuptools`` - - In pip7, the hacks were removed. This test remains to at least confirm pip - can upgrade distribute to setuptools using: - - ``pip install -U distribute`` - - The reason this works is that a final version of distribute (v0.7.3) was - released that is simple wrapper with: - - install_requires=['setuptools>=0.7'] - - The test use a fixed set of packages from our test packages dir. Note that - virtualenv-1.9.1 contains distribute-0.6.34 and virtualenv-1.10 contains - setuptools-0.9.7 - """ - - def prep_ve(self, script, version, pip_src, distribute=False): - self.script = script - self.script.pip_install_local( - 'virtualenv=={version}'.format(**locals())) - args = ['virtualenv', self.script.scratch_path / 'VE'] - if distribute: - args.insert(1, '--distribute') - if version == "1.9.1" and not distribute: - # setuptools 0.6 didn't support PYTHONDONTWRITEBYTECODE - del self.script.environ["PYTHONDONTWRITEBYTECODE"] - self.script.run(*args) - if sys.platform == 'win32': - bindir = "Scripts" - else: - bindir = "bin" - self.ve_bin = self.script.scratch_path / 'VE' / bindir - self.script.run(self.ve_bin / 'pip', 'uninstall', '-y', 'pip') - self.script.run( - self.ve_bin / 'python', 'setup.py', 'install', - cwd=pip_src, - expect_stderr=True, +@pytest.mark.parametrize( + "req1, req2", + list( + itertools.product( + ["foo.bar", "foo_bar", "foo-bar"], + ["foo.bar", "foo_bar", "foo-bar"], ) - - -@pytest.mark.parametrize("req1, req2", list(itertools.product( - ["foo.bar", "foo_bar", "foo-bar"], ["foo.bar", "foo_bar", "foo-bar"], -))) -def test_install_find_existing_package_canonicalize(script, req1, req2): + ), +) +def test_install_find_existing_package_canonicalize( + script: PipTestEnvironment, req1: str, req2: str +) -> None: """Ensure an already-installed dist is found no matter how the dist name was normalized on installation. (pypa/pip#8645) """ @@ -468,7 +404,22 @@ def test_install_find_existing_package_canonicalize(script, req1, req2): # Ensure the previously installed package can be correctly used to match # the dependency. result = script.pip( - "install", "--no-index", "--find-links", pkg_container, "pkg", + "install", + "--no-index", + "--find-links", + pkg_container, + "pkg", ) satisfied_message = f"Requirement already satisfied: {req2}" assert satisfied_message in result.stdout, str(result) + + +@pytest.mark.network +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test") +def test_modifying_pip_presents_error(script: PipTestEnvironment) -> None: + result = script.pip( + "install", "pip", "--force-reinstall", use_module=False, expect_error=True + ) + + assert "python.exe" in result.stderr or "python.EXE" in result.stderr, str(result) + assert " -m " in result.stderr, str(result) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index c5d7acced80..8657bc3ef2f 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -7,85 +7,96 @@ import pytest from tests.lib import pyversion # noqa: F401 -from tests.lib import need_svn +from tests.lib import PipTestEnvironment, TestData, need_svn from tests.lib.local_repos import local_checkout +from tests.lib.path import Path +from tests.lib.venv import VirtualEnvironment -def _patch_dist_in_site_packages(virtualenv): +def _patch_dist_in_site_packages(virtualenv: VirtualEnvironment) -> None: # Since the tests are run from a virtualenv, and to avoid the "Will not # install to the usersite because it will lack sys.path precedence..." - # error: Monkey patch `pip._internal.req.req_install.dist_in_site_packages` - # and `pip._internal.resolution.resolvelib.factory.dist_in_site_packages` - # so it's possible to install a conflicting distribution in the user site. - virtualenv.sitecustomize = textwrap.dedent(""" + # error: Monkey patch the Distribution class so it's possible to install a + # conflicting distribution in the user site. + virtualenv.sitecustomize = textwrap.dedent( + """ def dist_in_site_packages(dist): return False - from pip._internal.req import req_install - from pip._internal.resolution.resolvelib import factory - req_install.dist_in_site_packages = dist_in_site_packages - factory.dist_in_site_packages = dist_in_site_packages - """) + from pip._internal.metadata.base import BaseDistribution + BaseDistribution.in_site_packages = property(dist_in_site_packages) + """ + ) class Tests_UserSite: - @pytest.mark.network @pytest.mark.incompatible_with_test_venv - def test_reset_env_system_site_packages_usersite(self, script): + def test_reset_env_system_site_packages_usersite( + self, script: PipTestEnvironment + ) -> None: """ Check user site works as expected. """ - script.pip('install', '--user', 'INITools==0.2') + script.pip("install", "--user", "INITools==0.2") result = script.run( - 'python', '-c', + "python", + "-c", "import pkg_resources; print(pkg_resources.get_distribution" "('initools').project_name)", ) project_name = result.stdout.strip() - assert 'INITools' == project_name, project_name + assert "INITools" == project_name, project_name @pytest.mark.xfail @pytest.mark.network @need_svn @pytest.mark.incompatible_with_test_venv def test_install_subversion_usersite_editable_with_distribute( - self, script, tmpdir): + self, script: PipTestEnvironment, tmpdir: Path + ) -> None: """ Test installing current directory ('.') into usersite after installing distribute """ result = script.pip( - 'install', '--user', '-e', - '{checkout}#egg=initools'.format( + "install", + "--user", + "-e", + "{checkout}#egg=initools".format( checkout=local_checkout( - 'svn+http://svn.colorstudy.com/INITools', tmpdir) - ) + "svn+http://svn.colorstudy.com/INITools", tmpdir + ) + ), ) - result.assert_installed('INITools', use_user_site=True) + result.assert_installed("INITools", use_user_site=True) @pytest.mark.incompatible_with_test_venv + @pytest.mark.usefixtures("with_wheel") def test_install_from_current_directory_into_usersite( - self, script, data, with_wheel): + self, script: PipTestEnvironment, data: TestData + ) -> None: """ Test installing current directory ('.') into usersite """ run_from = data.packages.joinpath("FSPkg") result = script.pip( - 'install', '-vvv', '--user', curdir, + "install", + "-vvv", + "--user", + curdir, cwd=run_from, ) - fspkg_folder = script.user_site / 'fspkg' + fspkg_folder = script.user_site / "fspkg" result.did_create(fspkg_folder) - dist_info_folder = ( - script.user_site / 'FSPkg-0.1.dev0.dist-info' - ) + dist_info_folder = script.user_site / "FSPkg-0.1.dev0.dist-info" result.did_create(dist_info_folder) - def test_install_user_venv_nositepkgs_fails(self, virtualenv, - script, data): + def test_install_user_venv_nositepkgs_fails( + self, virtualenv: VirtualEnvironment, script: PipTestEnvironment, data: TestData + ) -> None: """ user install in virtualenv (with no system packages) fails with message """ @@ -94,7 +105,9 @@ def test_install_user_venv_nositepkgs_fails(self, virtualenv, virtualenv.user_site_packages = False run_from = data.packages.joinpath("FSPkg") result = script.pip( - 'install', '--user', curdir, + "install", + "--user", + curdir, cwd=run_from, expect_error=True, ) @@ -105,151 +118,155 @@ def test_install_user_venv_nositepkgs_fails(self, virtualenv, @pytest.mark.network @pytest.mark.incompatible_with_test_venv - def test_install_user_conflict_in_usersite(self, script): + def test_install_user_conflict_in_usersite( + self, script: PipTestEnvironment + ) -> None: """ Test user install with conflict in usersite updates usersite. """ - script.pip('install', '--user', 'INITools==0.3', '--no-binary=:all:') + script.pip("install", "--user", "INITools==0.3", "--no-binary=:all:") - result2 = script.pip( - 'install', '--user', 'INITools==0.1', '--no-binary=:all:') + result2 = script.pip("install", "--user", "INITools==0.1", "--no-binary=:all:") # usersite has 0.1 # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = ( - script.user_site / - 'INITools-0.1-py{pyversion}.egg-info'.format(**globals()) - ) + egg_info_folder = script.user_site / f"INITools-0.1-py{pyversion}.egg-info" initools_v3_file = ( # file only in 0.3 - script.base_path / script.user_site / 'initools' / - 'configparser.py' + script.base_path + / script.user_site + / "initools" + / "configparser.py" ) result2.did_create(egg_info_folder) assert not isfile(initools_v3_file), initools_v3_file @pytest.mark.network @pytest.mark.incompatible_with_test_venv - def test_install_user_conflict_in_globalsite(self, virtualenv, script): + def test_install_user_conflict_in_globalsite( + self, virtualenv: VirtualEnvironment, script: PipTestEnvironment + ) -> None: """ Test user install with conflict in global site ignores site and installs to usersite """ _patch_dist_in_site_packages(virtualenv) - script.pip('install', 'INITools==0.2', '--no-binary=:all:') + script.pip("install", "INITools==0.2", "--no-binary=:all:") - result2 = script.pip( - 'install', '--user', 'INITools==0.1', '--no-binary=:all:') + result2 = script.pip("install", "--user", "INITools==0.1", "--no-binary=:all:") # usersite has 0.1 # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = ( - script.user_site / - 'INITools-0.1-py{pyversion}.egg-info'.format(**globals()) - ) - initools_folder = script.user_site / 'initools' + egg_info_folder = script.user_site / f"INITools-0.1-py{pyversion}.egg-info" + initools_folder = script.user_site / "initools" result2.did_create(egg_info_folder) result2.did_create(initools_folder) # site still has 0.2 (can't look in result1; have to check) egg_info_folder = ( - script.base_path / script.site_packages / - 'INITools-0.2-py{pyversion}.egg-info'.format(**globals()) + script.base_path + / script.site_packages + / f"INITools-0.2-py{pyversion}.egg-info" ) - initools_folder = script.base_path / script.site_packages / 'initools' + initools_folder = script.base_path / script.site_packages / "initools" assert isdir(egg_info_folder) assert isdir(initools_folder) @pytest.mark.network @pytest.mark.incompatible_with_test_venv - def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script): + def test_upgrade_user_conflict_in_globalsite( + self, virtualenv: VirtualEnvironment, script: PipTestEnvironment + ) -> None: """ Test user install/upgrade with conflict in global site ignores site and installs to usersite """ _patch_dist_in_site_packages(virtualenv) - script.pip('install', 'INITools==0.2', '--no-binary=:all:') + script.pip("install", "INITools==0.2", "--no-binary=:all:") result2 = script.pip( - 'install', '--user', '--upgrade', 'INITools', '--no-binary=:all:') + "install", "--user", "--upgrade", "INITools", "--no-binary=:all:" + ) # usersite has 0.3.1 # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = ( - script.user_site / - 'INITools-0.3.1-py{pyversion}.egg-info'.format(**globals()) - ) - initools_folder = script.user_site / 'initools' + egg_info_folder = script.user_site / f"INITools-0.3.1-py{pyversion}.egg-info" + initools_folder = script.user_site / "initools" result2.did_create(egg_info_folder) result2.did_create(initools_folder) # site still has 0.2 (can't look in result1; have to check) egg_info_folder = ( - script.base_path / script.site_packages / - 'INITools-0.2-py{pyversion}.egg-info'.format(**globals()) + script.base_path + / script.site_packages + / f"INITools-0.2-py{pyversion}.egg-info" ) - initools_folder = script.base_path / script.site_packages / 'initools' + initools_folder = script.base_path / script.site_packages / "initools" assert isdir(egg_info_folder), result2.stdout assert isdir(initools_folder) @pytest.mark.network @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite_and_usersite( - self, virtualenv, script): + self, virtualenv: VirtualEnvironment, script: PipTestEnvironment + ) -> None: """ Test user install with conflict in globalsite and usersite ignores global site and updates usersite. """ _patch_dist_in_site_packages(virtualenv) - script.pip('install', 'INITools==0.2', '--no-binary=:all:') - script.pip('install', '--user', 'INITools==0.3', '--no-binary=:all:') + script.pip("install", "INITools==0.2", "--no-binary=:all:") + script.pip("install", "--user", "INITools==0.3", "--no-binary=:all:") - result3 = script.pip( - 'install', '--user', 'INITools==0.1', '--no-binary=:all:') + result3 = script.pip("install", "--user", "INITools==0.1", "--no-binary=:all:") # usersite has 0.1 # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = ( - script.user_site / - 'INITools-0.1-py{pyversion}.egg-info'.format(**globals()) - ) + egg_info_folder = script.user_site / f"INITools-0.1-py{pyversion}.egg-info" initools_v3_file = ( # file only in 0.3 - script.base_path / script.user_site / 'initools' / - 'configparser.py' + script.base_path + / script.user_site + / "initools" + / "configparser.py" ) result3.did_create(egg_info_folder) assert not isfile(initools_v3_file), initools_v3_file # site still has 0.2 (can't just look in result1; have to check) egg_info_folder = ( - script.base_path / script.site_packages / - 'INITools-0.2-py{pyversion}.egg-info'.format(**globals()) + script.base_path + / script.site_packages + / f"INITools-0.2-py{pyversion}.egg-info" ) - initools_folder = script.base_path / script.site_packages / 'initools' + initools_folder = script.base_path / script.site_packages / "initools" assert isdir(egg_info_folder) assert isdir(initools_folder) @pytest.mark.network @pytest.mark.incompatible_with_test_venv def test_install_user_in_global_virtualenv_with_conflict_fails( - self, script): + self, script: PipTestEnvironment + ) -> None: """ Test user install in --system-site-packages virtualenv with conflict in site fails. """ - script.pip('install', 'INITools==0.2') + script.pip("install", "INITools==0.2") result2 = script.pip( - 'install', '--user', 'INITools==0.1', + "install", + "--user", + "INITools==0.1", expect_error=True, ) resultp = script.run( - 'python', '-c', + "python", + "-c", "import pkg_resources; print(pkg_resources.get_distribution" "('initools').location)", ) @@ -257,7 +274,7 @@ def test_install_user_in_global_virtualenv_with_conflict_fails( assert ( "Will not install to the user site because it will lack sys.path " "precedence to {name} in {location}".format( - name='INITools', + name="INITools", location=dist_location, ) in result2.stderr diff --git a/tests/functional/test_install_vcs_git.py b/tests/functional/test_install_vcs_git.py index 4c26d8e88ac..04413c86d4b 100644 --- a/tests/functional/test_install_vcs_git.py +++ b/tests/functional/test_install_vcs_git.py @@ -1,7 +1,11 @@ +from typing import Optional + import pytest +from pip._internal.utils.urls import path_to_url from tests.lib import pyversion # noqa: F401 from tests.lib import ( + PipTestEnvironment, _change_test_package_version, _create_test_package, _test_path_to_file_url, @@ -12,38 +16,41 @@ _pull_in_submodule_changes_to_module, ) from tests.lib.local_repos import local_checkout +from tests.lib.path import Path -def _get_editable_repo_dir(script, package_name): +def _get_editable_repo_dir(script: PipTestEnvironment, package_name: str) -> Path: """ Return the repository directory for an editable install. """ - return script.venv_path / 'src' / package_name + return script.venv_path / "src" / package_name -def _get_editable_branch(script, package_name): +def _get_editable_branch(script: PipTestEnvironment, package_name: str) -> str: """ Return the current branch of an editable install. """ repo_dir = _get_editable_repo_dir(script, package_name) - result = script.run( - 'git', 'rev-parse', '--abbrev-ref', 'HEAD', cwd=repo_dir - ) + result = script.run("git", "rev-parse", "--abbrev-ref", "HEAD", cwd=repo_dir) return result.stdout.strip() -def _get_branch_remote(script, package_name, branch): - """ - - """ +def _get_branch_remote( + script: PipTestEnvironment, package_name: str, branch: str +) -> str: + """ """ repo_dir = _get_editable_repo_dir(script, package_name) - result = script.run( - 'git', 'config', f'branch.{branch}.remote', cwd=repo_dir - ) + result = script.run("git", "config", f"branch.{branch}.remote", cwd=repo_dir) return result.stdout.strip() -def _github_checkout(url_path, temp_dir, rev=None, egg=None, scheme=None): +def _github_checkout( + url_path: str, + temp_dir: Path, + rev: Optional[str] = None, + egg: Optional[str] = None, + scheme: Optional[str] = None, +) -> str: """ Call local_checkout() with a GitHub URL, and return the resulting URL. @@ -56,18 +63,20 @@ def _github_checkout(url_path, temp_dir, rev=None, egg=None, scheme=None): scheme: the scheme without the "git+" prefix. Defaults to "https". """ if scheme is None: - scheme = 'https' - url = f'git+{scheme}://github.com/{url_path}' + scheme = "https" + url = f"git+{scheme}://github.com/{url_path}" local_url = local_checkout(url, temp_dir) if rev is not None: - local_url += f'@{rev}' + local_url += f"@{rev}" if egg is not None: - local_url += f'#egg={egg}' + local_url += f"#egg={egg}" return local_url -def _make_version_pkg_url(path, rev=None, name="version_pkg"): +def _make_version_pkg_url( + path: Path, rev: Optional[str] = None, name: str = "version_pkg" +) -> str: """ Return a "git+file://" URL to the version_pkg test package. @@ -77,13 +86,18 @@ def _make_version_pkg_url(path, rev=None, name="version_pkg"): rev: an optional revision to install like a branch name, tag, or SHA. """ file_url = _test_path_to_file_url(path) - url_rev = '' if rev is None else f'@{rev}' - url = f'git+{file_url}{url_rev}#egg={name}' + url_rev = "" if rev is None else f"@{rev}" + url = f"git+{file_url}{url_rev}#egg={name}" return url -def _install_version_pkg_only(script, path, rev=None, expect_stderr=False): +def _install_version_pkg_only( + script: PipTestEnvironment, + path: Path, + rev: Optional[str] = None, + expect_stderr: bool = False, +) -> None: """ Install the version_pkg package in editable mode (without returning the version). @@ -94,10 +108,15 @@ def _install_version_pkg_only(script, path, rev=None, expect_stderr=False): rev: an optional revision to install like a branch name or tag. """ version_pkg_url = _make_version_pkg_url(path, rev=rev) - script.pip('install', '-e', version_pkg_url, expect_stderr=expect_stderr) + script.pip("install", "-e", version_pkg_url, expect_stderr=expect_stderr) -def _install_version_pkg(script, path, rev=None, expect_stderr=False): +def _install_version_pkg( + script: PipTestEnvironment, + path: Path, + rev: Optional[str] = None, + expect_stderr: bool = False, +) -> str: """ Install the version_pkg package in editable mode, and return the version installed. @@ -108,15 +127,18 @@ def _install_version_pkg(script, path, rev=None, expect_stderr=False): rev: an optional revision to install like a branch name or tag. """ _install_version_pkg_only( - script, path, rev=rev, expect_stderr=expect_stderr, + script, + path, + rev=rev, + expect_stderr=expect_stderr, ) - result = script.run('version_pkg') + result = script.run("version_pkg") version = result.stdout.strip() return version -def test_git_install_again_after_changes(script): +def test_git_install_again_after_changes(script: PipTestEnvironment) -> None: """ Test installing a repository a second time without specifying a revision, and after updates to the remote repository. @@ -127,290 +149,355 @@ def test_git_install_again_after_changes(script): """ version_pkg_path = _create_test_package(script) version = _install_version_pkg(script, version_pkg_path) - assert version == '0.1' + assert version == "0.1" _change_test_package_version(script, version_pkg_path) version = _install_version_pkg(script, version_pkg_path) - assert version == 'some different version' + assert version == "some different version" -def test_git_install_branch_again_after_branch_changes(script): +def test_git_install_branch_again_after_branch_changes( + script: PipTestEnvironment, +) -> None: """ Test installing a branch again after the branch is updated in the remote repository. """ version_pkg_path = _create_test_package(script) - version = _install_version_pkg(script, version_pkg_path, rev='master') - assert version == '0.1' + version = _install_version_pkg(script, version_pkg_path, rev="master") + assert version == "0.1" _change_test_package_version(script, version_pkg_path) - version = _install_version_pkg(script, version_pkg_path, rev='master') - assert version == 'some different version' + version = _install_version_pkg(script, version_pkg_path, rev="master") + assert version == "some different version" @pytest.mark.network -def test_install_editable_from_git_with_https(script, tmpdir): +def test_install_editable_from_git_with_https( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test cloning from Git with https. """ - url_path = 'pypa/pip-test-package.git' - local_url = _github_checkout(url_path, tmpdir, egg='pip-test-package') - result = script.pip('install', '-e', local_url) - result.assert_installed('pip-test-package', with_files=['.git']) + url_path = "pypa/pip-test-package.git" + local_url = _github_checkout(url_path, tmpdir, egg="pip-test-package") + result = script.pip("install", "-e", local_url) + result.assert_installed("pip-test-package", with_files=[".git"]) @pytest.mark.network -def test_install_noneditable_git(script, tmpdir, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_noneditable_git(script: PipTestEnvironment) -> None: """ Test installing from a non-editable git URL with a given tag. """ result = script.pip( - 'install', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1.1#egg=pip-test-package' - ) - dist_info_folder = ( - script.site_packages / - 'pip_test_package-0.1.1.dist-info' + "install", + "git+https://github.com/pypa/pip-test-package.git" + "@0.1.1#egg=pip-test-package", ) - result.assert_installed('piptestpackage', - without_egg_link=True, - editable=False) + dist_info_folder = script.site_packages / "pip_test_package-0.1.1.dist-info" + result.assert_installed("piptestpackage", without_egg_link=True, editable=False) result.did_create(dist_info_folder) -def test_git_with_sha1_revisions(script): +def test_git_with_sha1_revisions(script: PipTestEnvironment) -> None: """ Git backend should be able to install from SHA1 revisions """ version_pkg_path = _create_test_package(script) _change_test_package_version(script, version_pkg_path) sha1 = script.run( - 'git', 'rev-parse', 'HEAD~1', + "git", + "rev-parse", + "HEAD~1", cwd=version_pkg_path, ).stdout.strip() version = _install_version_pkg(script, version_pkg_path, rev=sha1) - assert '0.1' == version + assert "0.1" == version -def test_git_with_short_sha1_revisions(script): +def test_git_with_short_sha1_revisions(script: PipTestEnvironment) -> None: """ Git backend should be able to install from SHA1 revisions """ version_pkg_path = _create_test_package(script) _change_test_package_version(script, version_pkg_path) sha1 = script.run( - 'git', 'rev-parse', 'HEAD~1', + "git", + "rev-parse", + "HEAD~1", cwd=version_pkg_path, ).stdout.strip()[:7] version = _install_version_pkg(script, version_pkg_path, rev=sha1) - assert '0.1' == version + assert "0.1" == version -def test_git_with_branch_name_as_revision(script): +def test_git_with_branch_name_as_revision(script: PipTestEnvironment) -> None: """ Git backend should be able to install from branch names """ version_pkg_path = _create_test_package(script) - branch = 'test_branch' - script.run('git', 'checkout', '-b', branch, cwd=version_pkg_path) + branch = "test_branch" + script.run("git", "checkout", "-b", branch, cwd=version_pkg_path) _change_test_package_version(script, version_pkg_path) version = _install_version_pkg(script, version_pkg_path, rev=branch) - assert 'some different version' == version + assert "some different version" == version -def test_git_with_tag_name_as_revision(script): +def test_git_with_tag_name_as_revision(script: PipTestEnvironment) -> None: """ Git backend should be able to install from tag names """ version_pkg_path = _create_test_package(script) - script.run('git', 'tag', 'test_tag', cwd=version_pkg_path) + script.run("git", "tag", "test_tag", cwd=version_pkg_path) _change_test_package_version(script, version_pkg_path) - version = _install_version_pkg(script, version_pkg_path, rev='test_tag') - assert '0.1' == version + version = _install_version_pkg(script, version_pkg_path, rev="test_tag") + assert "0.1" == version -def _add_ref(script, path, ref): +def _add_ref(script: PipTestEnvironment, path: Path, ref: str) -> None: """ Add a new ref to a repository at the given path. """ - script.run('git', 'update-ref', ref, 'HEAD', cwd=path) + script.run("git", "update-ref", ref, "HEAD", cwd=path) -def test_git_install_ref(script): +def test_git_install_ref(script: PipTestEnvironment) -> None: """ The Git backend should be able to install a ref with the first install. """ version_pkg_path = _create_test_package(script) - _add_ref(script, version_pkg_path, 'refs/foo/bar') + _add_ref(script, version_pkg_path, "refs/foo/bar") _change_test_package_version(script, version_pkg_path) version = _install_version_pkg( - script, version_pkg_path, rev='refs/foo/bar', + script, + version_pkg_path, + rev="refs/foo/bar", ) - assert '0.1' == version + assert "0.1" == version -def test_git_install_then_install_ref(script): +def test_git_install_then_install_ref(script: PipTestEnvironment) -> None: """ The Git backend should be able to install a ref after a package has already been installed. """ version_pkg_path = _create_test_package(script) - _add_ref(script, version_pkg_path, 'refs/foo/bar') + _add_ref(script, version_pkg_path, "refs/foo/bar") _change_test_package_version(script, version_pkg_path) version = _install_version_pkg(script, version_pkg_path) - assert 'some different version' == version + assert "some different version" == version # Now install the ref. version = _install_version_pkg( - script, version_pkg_path, rev='refs/foo/bar', + script, + version_pkg_path, + rev="refs/foo/bar", ) - assert '0.1' == version + assert "0.1" == version + + +@pytest.mark.network +@pytest.mark.parametrize( + "rev, expected_sha", + [ + # Clone the default branch + ("", "5547fa909e83df8bd743d3978d6667497983a4b7"), + # Clone a specific tag + ("@0.1.1", "7d654e66c8fa7149c165ddeffa5b56bc06619458"), + # Clone a specific commit + ( + "@65cf0a5bdd906ecf48a0ac241c17d656d2071d56", + "65cf0a5bdd906ecf48a0ac241c17d656d2071d56", + ), + ], +) +def test_install_git_logs_commit_sha( + script: PipTestEnvironment, rev: str, expected_sha: str, tmpdir: Path +) -> None: + """ + Test installing from a git repository logs a commit SHA. + """ + url_path = "pypa/pip-test-package.git" + base_local_url = _github_checkout(url_path, tmpdir) + local_url = f"{base_local_url}{rev}#egg=pip-test-package" + result = script.pip("install", local_url) + # `[4:]` removes a 'git+' prefix + assert f"Resolved {base_local_url[4:]} to commit {expected_sha}" in result.stdout @pytest.mark.network -def test_git_with_tag_name_and_update(script, tmpdir): +def test_git_with_tag_name_and_update(script: PipTestEnvironment, tmpdir: Path) -> None: """ Test cloning a git repository and updating to a different version. """ - url_path = 'pypa/pip-test-package.git' + url_path = "pypa/pip-test-package.git" base_local_url = _github_checkout(url_path, tmpdir) - local_url = f'{base_local_url}#egg=pip-test-package' - result = script.pip('install', '-e', local_url) - result.assert_installed('pip-test-package', with_files=['.git']) + local_url = f"{base_local_url}#egg=pip-test-package" + result = script.pip("install", "-e", local_url) + result.assert_installed("pip-test-package", with_files=[".git"]) - new_local_url = f'{base_local_url}@0.1.2#egg=pip-test-package' + new_local_url = f"{base_local_url}@0.1.2#egg=pip-test-package" result = script.pip( - 'install', '--global-option=--version', '-e', new_local_url, + "install", + "--global-option=--version", + "-e", + new_local_url, + allow_stderr_warning=True, ) - assert '0.1.2' in result.stdout + assert "0.1.2" in result.stdout @pytest.mark.network -def test_git_branch_should_not_be_changed(script, tmpdir): +def test_git_branch_should_not_be_changed( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Editable installations should not change branch related to issue #32 and #161 """ - url_path = 'pypa/pip-test-package.git' - local_url = _github_checkout(url_path, tmpdir, egg='pip-test-package') - script.pip('install', '-e', local_url) - branch = _get_editable_branch(script, 'pip-test-package') - assert 'master' == branch + url_path = "pypa/pip-test-package.git" + local_url = _github_checkout(url_path, tmpdir, egg="pip-test-package") + script.pip("install", "-e", local_url) + branch = _get_editable_branch(script, "pip-test-package") + assert "master" == branch @pytest.mark.network -def test_git_with_non_editable_unpacking(script, tmpdir): +def test_git_with_non_editable_unpacking( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test cloning a git repository from a non-editable URL with a given tag. """ - url_path = 'pypa/pip-test-package.git' + url_path = "pypa/pip-test-package.git" local_url = _github_checkout( - url_path, tmpdir, rev='0.1.2', egg='pip-test-package', + url_path, + tmpdir, + rev="0.1.2", + egg="pip-test-package", + ) + result = script.pip( + "install", + "--global-option=--version", + local_url, + allow_stderr_warning=True, ) - result = script.pip('install', '--global-option=--version', local_url) - assert '0.1.2' in result.stdout + assert "0.1.2" in result.stdout @pytest.mark.network -def test_git_with_editable_where_egg_contains_dev_string(script, tmpdir): +def test_git_with_editable_where_egg_contains_dev_string( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test cloning a git repository from an editable url which contains "dev" string """ - url_path = 'dcramer/django-devserver.git' + url_path = "dcramer/django-devserver.git" local_url = _github_checkout( - url_path, tmpdir, egg='django-devserver', scheme='git', + url_path, + tmpdir, + egg="django-devserver", + scheme="https", ) - result = script.pip('install', '-e', local_url) - result.assert_installed('django-devserver', with_files=['.git']) + result = script.pip("install", "-e", local_url) + result.assert_installed("django-devserver", with_files=[".git"]) @pytest.mark.network -def test_git_with_non_editable_where_egg_contains_dev_string(script, tmpdir): +def test_git_with_non_editable_where_egg_contains_dev_string( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test cloning a git repository from a non-editable url which contains "dev" string """ - url_path = 'dcramer/django-devserver.git' + url_path = "dcramer/django-devserver.git" local_url = _github_checkout( - url_path, tmpdir, egg='django-devserver', scheme='git', + url_path, + tmpdir, + egg="django-devserver", + scheme="https", ) - result = script.pip('install', local_url) - devserver_folder = script.site_packages / 'devserver' + result = script.pip("install", local_url) + devserver_folder = script.site_packages / "devserver" result.did_create(devserver_folder) -def test_git_with_ambiguous_revs(script): +def test_git_with_ambiguous_revs(script: PipTestEnvironment) -> None: """ Test git with two "names" (tag/branch) pointing to the same commit """ version_pkg_path = _create_test_package(script) - version_pkg_url = _make_version_pkg_url(version_pkg_path, rev='0.1') - script.run('git', 'tag', '0.1', cwd=version_pkg_path) - result = script.pip('install', '-e', version_pkg_url) - assert 'Could not find a tag or branch' not in result.stdout + version_pkg_url = _make_version_pkg_url(version_pkg_path, rev="0.1") + script.run("git", "tag", "0.1", cwd=version_pkg_path) + result = script.pip("install", "-e", version_pkg_url) + assert "Could not find a tag or branch" not in result.stdout # it is 'version-pkg' instead of 'version_pkg' because # egg-link name is version-pkg.egg-link because it is a single .py module - result.assert_installed('version-pkg', with_files=['.git']) + result.assert_installed("version-pkg", with_files=[".git"]) -def test_editable__no_revision(script): +def test_editable__no_revision(script: PipTestEnvironment) -> None: """ Test a basic install in editable mode specifying no revision. """ version_pkg_path = _create_test_package(script) _install_version_pkg_only(script, version_pkg_path) - branch = _get_editable_branch(script, 'version-pkg') - assert branch == 'master' + branch = _get_editable_branch(script, "version-pkg") + assert branch == "master" - remote = _get_branch_remote(script, 'version-pkg', 'master') - assert remote == 'origin' + remote = _get_branch_remote(script, "version-pkg", "master") + assert remote == "origin" -def test_editable__branch_with_sha_same_as_default(script): +def test_editable__branch_with_sha_same_as_default(script: PipTestEnvironment) -> None: """ Test installing in editable mode a branch whose sha matches the sha of the default branch, but is different from the default branch. """ version_pkg_path = _create_test_package(script) # Create a second branch with the same SHA. - script.run('git', 'branch', 'develop', cwd=version_pkg_path) - _install_version_pkg_only(script, version_pkg_path, rev='develop') + script.run("git", "branch", "develop", cwd=version_pkg_path) + _install_version_pkg_only(script, version_pkg_path, rev="develop") - branch = _get_editable_branch(script, 'version-pkg') - assert branch == 'develop' + branch = _get_editable_branch(script, "version-pkg") + assert branch == "develop" - remote = _get_branch_remote(script, 'version-pkg', 'develop') - assert remote == 'origin' + remote = _get_branch_remote(script, "version-pkg", "develop") + assert remote == "origin" -def test_editable__branch_with_sha_different_from_default(script): +def test_editable__branch_with_sha_different_from_default( + script: PipTestEnvironment, +) -> None: """ Test installing in editable mode a branch whose sha is different from the sha of the default branch. """ version_pkg_path = _create_test_package(script) # Create a second branch. - script.run('git', 'branch', 'develop', cwd=version_pkg_path) + script.run("git", "branch", "develop", cwd=version_pkg_path) # Add another commit to the master branch to give it a different sha. _change_test_package_version(script, version_pkg_path) - version = _install_version_pkg(script, version_pkg_path, rev='develop') - assert version == '0.1' + version = _install_version_pkg(script, version_pkg_path, rev="develop") + assert version == "0.1" - branch = _get_editable_branch(script, 'version-pkg') - assert branch == 'develop' + branch = _get_editable_branch(script, "version-pkg") + assert branch == "develop" - remote = _get_branch_remote(script, 'version-pkg', 'develop') - assert remote == 'origin' + remote = _get_branch_remote(script, "version-pkg", "develop") + assert remote == "origin" -def test_editable__non_master_default_branch(script): +def test_editable__non_master_default_branch(script: PipTestEnvironment) -> None: """ Test the branch you get after an editable install from a remote repo with a non-master default branch. @@ -418,14 +505,16 @@ def test_editable__non_master_default_branch(script): version_pkg_path = _create_test_package(script) # Change the default branch of the remote repo to a name that is # alphabetically after "master". - script.run('git', 'checkout', '-b', 'release', cwd=version_pkg_path) + script.run("git", "checkout", "-b", "release", cwd=version_pkg_path) _install_version_pkg_only(script, version_pkg_path) - branch = _get_editable_branch(script, 'version-pkg') - assert branch == 'release' + branch = _get_editable_branch(script, "version-pkg") + assert branch == "release" -def test_reinstalling_works_with_editable_non_master_branch(script): +def test_reinstalling_works_with_editable_non_master_branch( + script: PipTestEnvironment, +) -> None: """ Reinstalling an editable installation should not assume that the "master" branch exists. See https://github.com/pypa/pip/issues/4448. @@ -433,50 +522,51 @@ def test_reinstalling_works_with_editable_non_master_branch(script): version_pkg_path = _create_test_package(script) # Switch the default branch to something other than 'master' - script.run('git', 'branch', '-m', 'foobar', cwd=version_pkg_path) + script.run("git", "branch", "-m", "foobar", cwd=version_pkg_path) version = _install_version_pkg(script, version_pkg_path) - assert '0.1' == version + assert "0.1" == version _change_test_package_version(script, version_pkg_path) version = _install_version_pkg(script, version_pkg_path) - assert 'some different version' == version + assert "some different version" == version # TODO(pnasrat) fix all helpers to do right things with paths on windows. @pytest.mark.skipif("sys.platform == 'win32'") -def test_check_submodule_addition(script): +def test_check_submodule_addition(script: PipTestEnvironment) -> None: """ Submodules are pulled in on install and updated on upgrade. """ - module_path, submodule_path = ( - _create_test_package_with_submodule(script, rel_path='testpkg/static') + module_path, submodule_path = _create_test_package_with_submodule( + script, rel_path="testpkg/static" ) install_result = script.pip( - 'install', '-e', 'git+' + module_path + '#egg=version_pkg' - ) - install_result.did_create( - script.venv / 'src/version-pkg/testpkg/static/testfile' + "install", "-e", "git+" + path_to_url(module_path) + "#egg=version_pkg" ) + install_result.did_create(script.venv / "src/version-pkg/testpkg/static/testfile") _change_test_package_submodule(script, submodule_path) _pull_in_submodule_changes_to_module( - script, module_path, rel_path='testpkg/static', + script, + module_path, + rel_path="testpkg/static", ) # expect error because git may write to stderr update_result = script.pip( - 'install', '-e', 'git+' + module_path + '#egg=version_pkg', - '--upgrade', + "install", + "-e", + "git+" + path_to_url(module_path) + "#egg=version_pkg", + "--upgrade", ) - update_result.did_create( - script.venv / 'src/version-pkg/testpkg/static/testfile2' - ) + update_result.did_create(script.venv / "src/version-pkg/testpkg/static/testfile2") -def test_install_git_branch_not_cached(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_git_branch_not_cached(script: PipTestEnvironment) -> None: """ Installing git urls with a branch revision does not cause wheel caching. """ @@ -488,26 +578,21 @@ def test_install_git_branch_not_cached(script, with_wheel): script.pip("uninstall", "-y", PKG) # build occurs on the second install too because it is not cached result = script.pip("install", url) - assert ( - f"Successfully built {PKG}" in result.stdout - ), result.stdout + assert f"Successfully built {PKG}" in result.stdout, result.stdout -def test_install_git_sha_cached(script, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_git_sha_cached(script: PipTestEnvironment) -> None: """ Installing git urls with a sha revision does cause wheel caching. """ PKG = "gitshacached" repo_dir = _create_test_package(script, name=PKG) - commit = script.run( - 'git', 'rev-parse', 'HEAD', cwd=repo_dir - ).stdout.strip() + commit = script.run("git", "rev-parse", "HEAD", cwd=repo_dir).stdout.strip() url = _make_version_pkg_url(repo_dir, rev=commit, name=PKG) result = script.pip("install", url) assert f"Successfully built {PKG}" in result.stdout, result.stdout script.pip("uninstall", "-y", PKG) # build does not occur on the second install because it is cached result = script.pip("install", url) - assert ( - f"Successfully built {PKG}" not in result.stdout - ), result.stdout + assert f"Successfully built {PKG}" not in result.stdout, result.stdout diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index 8df208bb7da..86c5e5fbeb2 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -3,195 +3,222 @@ import glob import os import shutil +from typing import Any import pytest -from tests.lib import create_basic_wheel_for_package +from tests.lib import PipTestEnvironment, TestData, create_basic_wheel_for_package from tests.lib.path import Path -from tests.lib.wheel import make_wheel +from tests.lib.wheel import WheelBuilder, make_wheel # assert_installed expects a package subdirectory, so give it to them -def make_wheel_with_file(name, version, **kwargs): +def make_wheel_with_file(name: str, version: str, **kwargs: Any) -> WheelBuilder: extra_files = kwargs.setdefault("extra_files", {}) extra_files[f"{name}/__init__.py"] = "# example" return make_wheel(name=name, version=version, **kwargs) -def test_install_from_future_wheel_version(script, tmpdir): +def test_install_from_future_wheel_version( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test installing a wheel with a WHEEL metadata version that is: - a major version ahead of what we expect (not ok), and - a minor version ahead of what we expect (ok) """ from tests.lib import TestFailure + package = make_wheel_with_file( name="futurewheel", version="3.0", wheel_metadata_updates={"Wheel-Version": "3.0"}, ).save_to_dir(tmpdir) - result = script.pip('install', package, '--no-index', expect_error=True) + result = script.pip("install", package, "--no-index", expect_error=True) with pytest.raises(TestFailure): - result.assert_installed('futurewheel', without_egg_link=True, - editable=False) + result.assert_installed("futurewheel", without_egg_link=True, editable=False) package = make_wheel_with_file( name="futurewheel", version="1.9", wheel_metadata_updates={"Wheel-Version": "1.9"}, ).save_to_dir(tmpdir) - result = script.pip( - 'install', package, '--no-index', expect_stderr=True - ) - result.assert_installed('futurewheel', without_egg_link=True, - editable=False) + result = script.pip("install", package, "--no-index", expect_stderr=True) + result.assert_installed("futurewheel", without_egg_link=True, editable=False) -def test_install_from_broken_wheel(script, data): +@pytest.mark.parametrize( + "wheel_name", + [ + "brokenwheel-1.0-py2.py3-none-any.whl", + "corruptwheel-1.0-py2.py3-none-any.whl", + ], +) +def test_install_from_broken_wheel( + script: PipTestEnvironment, data: TestData, wheel_name: str +) -> None: """ Test that installing a broken wheel fails properly """ from tests.lib import TestFailure - package = data.packages.joinpath("brokenwheel-1.0-py2.py3-none-any.whl") - result = script.pip('install', package, '--no-index', expect_error=True) + + package = data.packages.joinpath(wheel_name) + result = script.pip("install", package, "--no-index", expect_error=True) with pytest.raises(TestFailure): - result.assert_installed('futurewheel', without_egg_link=True, - editable=False) + result.assert_installed("futurewheel", without_egg_link=True, editable=False) -def test_basic_install_from_wheel(script, shared_data, tmpdir): +def test_basic_install_from_wheel( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing from a wheel (that has a script) """ - shutil.copy( - shared_data.packages / "has.script-1.0-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "has.script-1.0-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'has.script==1.0', '--no-index', - '--find-links', tmpdir, + "install", + "has.script==1.0", + "--no-index", + "--find-links", + tmpdir, ) - dist_info_folder = script.site_packages / 'has.script-1.0.dist-info' + dist_info_folder = script.site_packages / "has.script-1.0.dist-info" result.did_create(dist_info_folder) - script_file = script.bin / 'script.py' + script_file = script.bin / "script.py" result.did_create(script_file) -def test_basic_install_from_wheel_with_extras(script, shared_data, tmpdir): +def test_basic_install_from_wheel_with_extras( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing from a wheel with extras. """ - shutil.copy( - shared_data.packages / "complex_dist-0.1-py2.py3-none-any.whl", tmpdir - ) - shutil.copy( - shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "complex_dist-0.1-py2.py3-none-any.whl", tmpdir) + shutil.copy(shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'complex-dist[simple]', '--no-index', - '--find-links', tmpdir, + "install", + "complex-dist[simple]", + "--no-index", + "--find-links", + tmpdir, ) - dist_info_folder = script.site_packages / 'complex_dist-0.1.dist-info' + dist_info_folder = script.site_packages / "complex_dist-0.1.dist-info" result.did_create(dist_info_folder) - dist_info_folder = script.site_packages / 'simple.dist-0.1.dist-info' + dist_info_folder = script.site_packages / "simple.dist-0.1.dist-info" result.did_create(dist_info_folder) -def test_basic_install_from_wheel_file(script, data): +def test_basic_install_from_wheel_file( + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing directly from a wheel file. """ package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") - result = script.pip('install', package, '--no-index') - dist_info_folder = script.site_packages / 'simple.dist-0.1.dist-info' + result = script.pip("install", package, "--no-index") + dist_info_folder = script.site_packages / "simple.dist-0.1.dist-info" result.did_create(dist_info_folder) - installer = dist_info_folder / 'INSTALLER' + installer = dist_info_folder / "INSTALLER" result.did_create(installer) - with open(script.base_path / installer, 'rb') as installer_file: + with open(script.base_path / installer, "rb") as installer_file: installer_details = installer_file.read() - assert installer_details == b'pip\n' - installer_temp = dist_info_folder / 'INSTALLER.pip' + assert installer_details == b"pip\n" + installer_temp = dist_info_folder / "INSTALLER.pip" result.did_not_create(installer_temp) # Installation seems to work, but scripttest fails to check. # I really don't care now since we're desupporting it soon anyway. -def test_basic_install_from_unicode_wheel(script, data): +def test_basic_install_from_unicode_wheel( + script: PipTestEnvironment, data: TestData +) -> None: """ Test installing from a wheel (that has a script) """ make_wheel( - 'unicode_package', - '1.0', + "unicode_package", + "1.0", extra_files={ - 'வணக்கம்/__init__.py': b'', - 'வணக்கம்/નમસ્તે.py': b'', + "வணக்கம்/__init__.py": b"", + "வணக்கம்/નમસ્તે.py": b"", }, ).save_to_dir(script.scratch_path) result = script.pip( - 'install', 'unicode_package==1.0', '--no-index', - '--find-links', script.scratch_path, + "install", + "unicode_package==1.0", + "--no-index", + "--find-links", + script.scratch_path, ) - dist_info_folder = script.site_packages / 'unicode_package-1.0.dist-info' + dist_info_folder = script.site_packages / "unicode_package-1.0.dist-info" result.did_create(dist_info_folder) - file1 = script.site_packages.joinpath('வணக்கம்', '__init__.py') + file1 = script.site_packages.joinpath("வணக்கம்", "__init__.py") result.did_create(file1) - file2 = script.site_packages.joinpath('வணக்கம்', 'નમસ્તે.py') + file2 = script.site_packages.joinpath("வணக்கம்", "નમસ્તે.py") result.did_create(file2) -def get_header_scheme_path_for_script(script, dist_name): +def get_header_scheme_path_for_script( + script: PipTestEnvironment, dist_name: str +) -> Path: command = ( "from pip._internal.locations import get_scheme;" "scheme = get_scheme({!r});" "print(scheme.headers);" ).format(dist_name) - result = script.run('python', '-c', command).stdout + result = script.run("python", "-c", command).stdout return Path(result.strip()) -def test_install_from_wheel_with_headers(script): +def test_install_from_wheel_with_headers(script: PipTestEnvironment) -> None: """ Test installing from a wheel file with headers """ - header_text = '/* hello world */\n' + header_text = "/* hello world */\n" package = make_wheel( - 'headers.dist', - '0.1', - extra_data_files={ - 'headers/header.h': header_text - }, + "headers.dist", + "0.1", + extra_data_files={"headers/header.h": header_text}, ).save_to_dir(script.scratch_path) - result = script.pip('install', package, '--no-index') - dist_info_folder = script.site_packages / 'headers.dist-0.1.dist-info' + result = script.pip("install", package, "--no-index") + dist_info_folder = script.site_packages / "headers.dist-0.1.dist-info" result.did_create(dist_info_folder) - header_scheme_path = get_header_scheme_path_for_script( - script, 'headers.dist' - ) - header_path = header_scheme_path / 'header.h' + header_scheme_path = get_header_scheme_path_for_script(script, "headers.dist") + header_path = header_scheme_path / "header.h" assert header_path.read_text() == header_text -def test_install_wheel_with_target(script, shared_data, with_wheel, tmpdir): +@pytest.mark.usefixtures("with_wheel") +def test_install_wheel_with_target( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing a wheel using pip install --target """ - shutil.copy( - shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir - ) - target_dir = script.scratch_path / 'target' + shutil.copy(shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir) + target_dir = script.scratch_path / "target" result = script.pip( - 'install', 'simple.dist==0.1', '-t', target_dir, - '--no-index', '--find-links', tmpdir, + "install", + "simple.dist==0.1", + "-t", + target_dir, + "--no-index", + "--find-links", + tmpdir, ) - result.did_create(Path('scratch') / 'target' / 'simpledist') + result.did_create(Path("scratch") / "target" / "simpledist") -def test_install_wheel_with_target_and_data_files(script, data, with_wheel): +@pytest.mark.usefixtures("with_wheel") +def test_install_wheel_with_target_and_data_files( + script: PipTestEnvironment, data: TestData +) -> None: """ Test for issue #4092. It will be checked that a data_files specification in setup.py is handled correctly when a wheel is installed with the --target @@ -210,100 +237,116 @@ def test_install_wheel_with_target_and_data_files(script, data, with_wheel): ] ) """ - target_dir = script.scratch_path / 'prjwithdatafile' - package = data.packages.joinpath( - "prjwithdatafile-1.0-py2.py3-none-any.whl" - ) - result = script.pip('install', package, - '-t', target_dir, - '--no-index') - project_path = Path('scratch') / 'prjwithdatafile' - result.did_create(project_path / 'packages1' / 'README.txt') - result.did_create(project_path / 'packages2' / 'README.txt') - result.did_not_create(project_path / 'lib' / 'python') + target_dir = script.scratch_path / "prjwithdatafile" + package = data.packages.joinpath("prjwithdatafile-1.0-py2.py3-none-any.whl") + result = script.pip("install", package, "-t", target_dir, "--no-index") + project_path = Path("scratch") / "prjwithdatafile" + result.did_create(project_path / "packages1" / "README.txt") + result.did_create(project_path / "packages2" / "README.txt") + result.did_not_create(project_path / "lib" / "python") -def test_install_wheel_with_root(script, shared_data, tmpdir): +def test_install_wheel_with_root( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing a wheel using pip install --root """ - root_dir = script.scratch_path / 'root' - shutil.copy( - shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir - ) + root_dir = script.scratch_path / "root" + shutil.copy(shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'simple.dist==0.1', '--root', root_dir, - '--no-index', '--find-links', tmpdir, + "install", + "simple.dist==0.1", + "--root", + root_dir, + "--no-index", + "--find-links", + tmpdir, ) - result.did_create(Path('scratch') / 'root') + result.did_create(Path("scratch") / "root") -def test_install_wheel_with_prefix(script, shared_data, tmpdir): +def test_install_wheel_with_prefix( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing a wheel using pip install --prefix """ - prefix_dir = script.scratch_path / 'prefix' - shutil.copy( - shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir - ) + prefix_dir = script.scratch_path / "prefix" + shutil.copy(shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'simple.dist==0.1', '--prefix', prefix_dir, - '--no-index', '--find-links', tmpdir, + "install", + "simple.dist==0.1", + "--prefix", + prefix_dir, + "--no-index", + "--find-links", + tmpdir, ) - lib = distutils.sysconfig.get_python_lib(prefix=Path('scratch') / 'prefix') + lib = distutils.sysconfig.get_python_lib(prefix=Path("scratch") / "prefix") result.did_create(lib) -def test_install_from_wheel_installs_deps(script, data, tmpdir): +def test_install_from_wheel_installs_deps( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test can install dependencies of wheels """ # 'requires_source' depends on the 'source' project - package = data.packages.joinpath( - "requires_source-1.0-py2.py3-none-any.whl" - ) + package = data.packages.joinpath("requires_source-1.0-py2.py3-none-any.whl") shutil.copy(data.packages / "source-1.0.tar.gz", tmpdir) result = script.pip( - 'install', '--no-index', '--find-links', tmpdir, package, + "install", + "--no-index", + "--find-links", + tmpdir, + package, ) - result.assert_installed('source', editable=False) + result.assert_installed("source", editable=False) -def test_install_from_wheel_no_deps(script, data, tmpdir): +def test_install_from_wheel_no_deps( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test --no-deps works with wheel installs """ # 'requires_source' depends on the 'source' project - package = data.packages.joinpath( - "requires_source-1.0-py2.py3-none-any.whl" - ) + package = data.packages.joinpath("requires_source-1.0-py2.py3-none-any.whl") shutil.copy(data.packages / "source-1.0.tar.gz", tmpdir) result = script.pip( - 'install', '--no-index', '--find-links', tmpdir, '--no-deps', + "install", + "--no-index", + "--find-links", + tmpdir, + "--no-deps", package, ) - pkg_folder = script.site_packages / 'source' + pkg_folder = script.site_packages / "source" result.did_not_create(pkg_folder) -def test_wheel_record_lines_in_deterministic_order(script, data): +def test_wheel_record_lines_in_deterministic_order( + script: PipTestEnvironment, data: TestData +) -> None: to_install = data.packages.joinpath("simplewheel-1.0-py2.py3-none-any.whl") - result = script.pip('install', to_install) + result = script.pip("install", to_install) - dist_info_folder = script.site_packages / 'simplewheel-1.0.dist-info' - record_path = dist_info_folder / 'RECORD' + dist_info_folder = script.site_packages / "simplewheel-1.0.dist-info" + record_path = dist_info_folder / "RECORD" result.did_create(dist_info_folder) result.did_create(record_path) record_path = result.files_created[record_path].full - record_lines = [ - p for p in Path(record_path).read_text().split('\n') if p - ] + record_lines = [p for p in Path(record_path).read_text().split("\n") if p] assert record_lines == sorted(record_lines) -def test_wheel_record_lines_have_hash_for_data_files(script): +def test_wheel_record_lines_have_hash_for_data_files( + script: PipTestEnvironment, +) -> None: package = make_wheel( "simple", "0.1.0", @@ -312,38 +355,42 @@ def test_wheel_record_lines_have_hash_for_data_files(script): }, ).save_to_dir(script.scratch_path) script.pip("install", package) - record_file = ( - script.site_packages_path / "simple-0.1.0.dist-info" / "RECORD" - ) + record_file = script.site_packages_path / "simple-0.1.0.dist-info" / "RECORD" record_text = record_file.read_text() record_rows = list(csv.reader(record_text.splitlines())) - records = { - r[0]: r[1:] for r in record_rows - } + records = {r[0]: r[1:] for r in record_rows} assert records["info.txt"] == [ - "sha256=Ln0sA6lQeuJl7PW1NWiFpTOTogKdJBOUmXJloaJa78Y", "1" + "sha256=Ln0sA6lQeuJl7PW1NWiFpTOTogKdJBOUmXJloaJa78Y", + "1", ] @pytest.mark.incompatible_with_test_venv -def test_install_user_wheel(script, shared_data, with_wheel, tmpdir): +@pytest.mark.usefixtures("with_wheel") +def test_install_user_wheel( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test user install from wheel (that has a script) """ - shutil.copy( - shared_data.packages / "has.script-1.0-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "has.script-1.0-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'has.script==1.0', '--user', '--no-index', - '--find-links', tmpdir, + "install", + "has.script==1.0", + "--user", + "--no-index", + "--find-links", + tmpdir, ) - dist_info_folder = script.user_site / 'has.script-1.0.dist-info' + dist_info_folder = script.user_site / "has.script-1.0.dist-info" result.did_create(dist_info_folder) - script_file = script.user_bin / 'script.py' + script_file = script.user_bin / "script.py" result.did_create(script_file) -def test_install_from_wheel_gen_entrypoint(script, shared_data, tmpdir): +def test_install_from_wheel_gen_entrypoint( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing scripts (entry points are generated) """ @@ -352,13 +399,16 @@ def test_install_from_wheel_gen_entrypoint(script, shared_data, tmpdir): tmpdir, ) result = script.pip( - 'install', 'script.wheel1a==0.1', '--no-index', - '--find-links', tmpdir, + "install", + "script.wheel1a==0.1", + "--no-index", + "--find-links", + tmpdir, ) - if os.name == 'nt': - wrapper_file = script.bin / 't1.exe' + if os.name == "nt": + wrapper_file = script.bin / "t1.exe" else: - wrapper_file = script.bin / 't1' + wrapper_file = script.bin / "t1" result.did_create(wrapper_file) if os.name != "nt": @@ -366,32 +416,34 @@ def test_install_from_wheel_gen_entrypoint(script, shared_data, tmpdir): def test_install_from_wheel_gen_uppercase_entrypoint( - script, shared_data, tmpdir -): + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing scripts with uppercase letters in entry point names """ shutil.copy( - shared_data.packages / - "console_scripts_uppercase-1.0-py2.py3-none-any.whl", + shared_data.packages / "console_scripts_uppercase-1.0-py2.py3-none-any.whl", tmpdir, ) result = script.pip( - 'install', 'console-scripts-uppercase==1.0', '--no-index', - '--find-links', tmpdir, + "install", + "console-scripts-uppercase==1.0", + "--no-index", + "--find-links", + tmpdir, ) - if os.name == 'nt': + if os.name == "nt": # Case probably doesn't make any difference on NT - wrapper_file = script.bin / 'cmdName.exe' + wrapper_file = script.bin / "cmdName.exe" else: - wrapper_file = script.bin / 'cmdName' + wrapper_file = script.bin / "cmdName" result.did_create(wrapper_file) if os.name != "nt": assert bool(os.access(script.base_path / wrapper_file, os.X_OK)) -def test_install_from_wheel_gen_unicode_entrypoint(script): +def test_install_from_wheel_gen_unicode_entrypoint(script: PipTestEnvironment) -> None: make_wheel( "script_wheel_unicode", "1.0", @@ -411,7 +463,9 @@ def test_install_from_wheel_gen_unicode_entrypoint(script): result.did_create(script.bin.joinpath("進入點")) -def test_install_from_wheel_with_legacy(script, shared_data, tmpdir): +def test_install_from_wheel_with_legacy( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing scripts (legacy scripts are preserved) """ @@ -420,36 +474,40 @@ def test_install_from_wheel_with_legacy(script, shared_data, tmpdir): tmpdir, ) result = script.pip( - 'install', 'script.wheel2a==0.1', '--no-index', - '--find-links', tmpdir, + "install", + "script.wheel2a==0.1", + "--no-index", + "--find-links", + tmpdir, ) - legacy_file1 = script.bin / 'testscript1.bat' - legacy_file2 = script.bin / 'testscript2' + legacy_file1 = script.bin / "testscript1.bat" + legacy_file2 = script.bin / "testscript2" result.did_create(legacy_file1) result.did_create(legacy_file2) def test_install_from_wheel_no_setuptools_entrypoint( - script, shared_data, tmpdir -): + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test that when we generate scripts, any existing setuptools wrappers in the wheel are skipped. """ - shutil.copy( - shared_data.packages / "script.wheel1-0.1-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "script.wheel1-0.1-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'script.wheel1==0.1', '--no-index', - '--find-links', tmpdir, + "install", + "script.wheel1==0.1", + "--no-index", + "--find-links", + tmpdir, ) - if os.name == 'nt': - wrapper_file = script.bin / 't1.exe' + if os.name == "nt": + wrapper_file = script.bin / "t1.exe" else: - wrapper_file = script.bin / 't1' - wrapper_helper = script.bin / 't1-script.py' + wrapper_file = script.bin / "t1" + wrapper_helper = script.bin / "t1-script.py" # The wheel has t1.exe and t1-script.py. We will be generating t1 or # t1.exe depending on the platform. So we check that the correct wrapper @@ -460,131 +518,139 @@ def test_install_from_wheel_no_setuptools_entrypoint( result.did_not_create(wrapper_helper) -def test_skipping_setuptools_doesnt_skip_legacy(script, shared_data, tmpdir): +def test_skipping_setuptools_doesnt_skip_legacy( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing scripts (legacy scripts are preserved even when we skip setuptools wrappers) """ - shutil.copy( - shared_data.packages / "script.wheel2-0.1-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "script.wheel2-0.1-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'script.wheel2==0.1', '--no-index', - '--find-links', tmpdir, + "install", + "script.wheel2==0.1", + "--no-index", + "--find-links", + tmpdir, ) - legacy_file1 = script.bin / 'testscript1.bat' - legacy_file2 = script.bin / 'testscript2' - wrapper_helper = script.bin / 't1-script.py' + legacy_file1 = script.bin / "testscript1.bat" + legacy_file2 = script.bin / "testscript2" + wrapper_helper = script.bin / "t1-script.py" result.did_create(legacy_file1) result.did_create(legacy_file2) result.did_not_create(wrapper_helper) -def test_install_from_wheel_gui_entrypoint(script, shared_data, tmpdir): +def test_install_from_wheel_gui_entrypoint( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing scripts (gui entry points are generated) """ - shutil.copy( - shared_data.packages / "script.wheel3-0.1-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "script.wheel3-0.1-py2.py3-none-any.whl", tmpdir) result = script.pip( - 'install', 'script.wheel3==0.1', '--no-index', - '--find-links', tmpdir, + "install", + "script.wheel3==0.1", + "--no-index", + "--find-links", + tmpdir, ) - if os.name == 'nt': - wrapper_file = script.bin / 't1.exe' + if os.name == "nt": + wrapper_file = script.bin / "t1.exe" else: - wrapper_file = script.bin / 't1' + wrapper_file = script.bin / "t1" result.did_create(wrapper_file) -def test_wheel_compiles_pyc(script, shared_data, tmpdir): +def test_wheel_compiles_pyc( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing from wheel with --compile on """ - shutil.copy( - shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir) script.pip( - "install", "--compile", "simple.dist==0.1", "--no-index", - "--find-links", tmpdir, + "install", + "--compile", + "simple.dist==0.1", + "--no-index", + "--find-links", + tmpdir, ) # There are many locations for the __init__.pyc file so attempt to find # any of them exists = [ os.path.exists(script.site_packages_path / "simpledist/__init__.pyc"), + *glob.glob(script.site_packages_path / "simpledist/__pycache__/__init__*.pyc"), ] - - exists += glob.glob( - script.site_packages_path / "simpledist/__pycache__/__init__*.pyc" - ) - assert any(exists) -def test_wheel_no_compiles_pyc(script, shared_data, tmpdir): +def test_wheel_no_compiles_pyc( + script: PipTestEnvironment, shared_data: TestData, tmpdir: Path +) -> None: """ Test installing from wheel with --compile on """ - shutil.copy( - shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir - ) + shutil.copy(shared_data.packages / "simple.dist-0.1-py2.py3-none-any.whl", tmpdir) script.pip( - "install", "--no-compile", "simple.dist==0.1", "--no-index", - "--find-links", tmpdir, + "install", + "--no-compile", + "simple.dist==0.1", + "--no-index", + "--find-links", + tmpdir, ) # There are many locations for the __init__.pyc file so attempt to find # any of them exists = [ os.path.exists(script.site_packages_path / "simpledist/__init__.pyc"), + *glob.glob(script.site_packages_path / "simpledist/__pycache__/__init__*.pyc"), ] - exists += glob.glob( - script.site_packages_path / "simpledist/__pycache__/__init__*.pyc" - ) - assert not any(exists) -def test_install_from_wheel_uninstalls_old_version(script, data): +def test_install_from_wheel_uninstalls_old_version( + script: PipTestEnvironment, data: TestData +) -> None: # regression test for https://github.com/pypa/pip/issues/1825 package = data.packages.joinpath("simplewheel-1.0-py2.py3-none-any.whl") - result = script.pip('install', package, '--no-index') + result = script.pip("install", package, "--no-index") package = data.packages.joinpath("simplewheel-2.0-py2.py3-none-any.whl") - result = script.pip('install', package, '--no-index') - dist_info_folder = script.site_packages / 'simplewheel-2.0.dist-info' + result = script.pip("install", package, "--no-index") + dist_info_folder = script.site_packages / "simplewheel-2.0.dist-info" result.did_create(dist_info_folder) - dist_info_folder = script.site_packages / 'simplewheel-1.0.dist-info' + dist_info_folder = script.site_packages / "simplewheel-1.0.dist-info" result.did_not_create(dist_info_folder) -def test_wheel_compile_syntax_error(script, data): +def test_wheel_compile_syntax_error(script: PipTestEnvironment, data: TestData) -> None: package = data.packages.joinpath("compilewheel-1.0-py2.py3-none-any.whl") - result = script.pip('install', '--compile', package, '--no-index') - assert 'yield from' not in result.stdout - assert 'SyntaxError: ' not in result.stdout + result = script.pip("install", "--compile", package, "--no-index") + assert "yield from" not in result.stdout + assert "SyntaxError: " not in result.stdout -def test_wheel_install_with_no_cache_dir(script, tmpdir, data): - """Check wheel installations work, even with no cache. - """ +def test_wheel_install_with_no_cache_dir( + script: PipTestEnvironment, data: TestData +) -> None: + """Check wheel installations work, even with no cache.""" package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") - result = script.pip('install', '--no-cache-dir', '--no-index', package) - result.assert_installed('simpledist', editable=False) + result = script.pip("install", "--no-cache-dir", "--no-index", package) + result.assert_installed("simpledist", editable=False) -def test_wheel_install_fails_with_extra_dist_info(script): +def test_wheel_install_fails_with_extra_dist_info(script: PipTestEnvironment) -> None: package = create_basic_wheel_for_package( script, "simple", "0.1.0", extra_files={ "unrelated-2.0.0.dist-info/WHEEL": "Wheel-Version: 1.0", - "unrelated-2.0.0.dist-info/METADATA": ( - "Name: unrelated\nVersion: 2.0.0\n" - ), + "unrelated-2.0.0.dist-info/METADATA": ("Name: unrelated\nVersion: 2.0.0\n"), }, ) result = script.pip( @@ -593,7 +659,9 @@ def test_wheel_install_fails_with_extra_dist_info(script): assert "multiple .dist-info directories" in result.stderr -def test_wheel_install_fails_with_unrelated_dist_info(script): +def test_wheel_install_fails_with_unrelated_dist_info( + script: PipTestEnvironment, +) -> None: package = create_basic_wheel_for_package(script, "simple", "0.1.0") new_name = "unrelated-2.0.0-py2.py3-none-any.whl" new_package = os.path.join(os.path.dirname(package), new_name) @@ -607,13 +675,10 @@ def test_wheel_install_fails_with_unrelated_dist_info(script): expect_error=True, ) - assert ( - "'simple-0.1.0.dist-info' does not start with 'unrelated'" - in result.stderr - ) + assert "'simple-0.1.0.dist-info' does not start with 'unrelated'" in result.stderr -def test_wheel_installs_ok_with_nested_dist_info(script): +def test_wheel_installs_ok_with_nested_dist_info(script: PipTestEnvironment) -> None: package = create_basic_wheel_for_package( script, "simple", @@ -625,35 +690,29 @@ def test_wheel_installs_ok_with_nested_dist_info(script): ), }, ) - script.pip( - "install", "--no-cache-dir", "--no-index", package - ) + script.pip("install", "--no-cache-dir", "--no-index", package) def test_wheel_installs_ok_with_badly_encoded_irrelevant_dist_info_file( - script -): + script: PipTestEnvironment, +) -> None: package = create_basic_wheel_for_package( script, "simple", "0.1.0", - extra_files={ - "simple-0.1.0.dist-info/AUTHORS.txt": b"\xff" - }, - ) - script.pip( - "install", "--no-cache-dir", "--no-index", package + extra_files={"simple-0.1.0.dist-info/AUTHORS.txt": b"\xff"}, ) + script.pip("install", "--no-cache-dir", "--no-index", package) -def test_wheel_install_fails_with_badly_encoded_metadata(script): +def test_wheel_install_fails_with_badly_encoded_metadata( + script: PipTestEnvironment, +) -> None: package = create_basic_wheel_for_package( script, "simple", "0.1.0", - extra_files={ - "simple-0.1.0.dist-info/METADATA": b"\xff" - }, + extra_files={"simple-0.1.0.dist-info/METADATA": b"\xff"}, ) result = script.pip( "install", "--no-cache-dir", "--no-index", package, expect_error=True @@ -664,22 +723,24 @@ def test_wheel_install_fails_with_badly_encoded_metadata(script): @pytest.mark.parametrize( - 'package_name', - ['simple-package', 'simple_package'], + "package_name", + ["simple-package", "simple_package"], ) -def test_correct_package_name_while_creating_wheel_bug(script, package_name): +def test_correct_package_name_while_creating_wheel_bug( + script: PipTestEnvironment, package_name: str +) -> None: """Check that the package name is correctly named while creating a .whl file with a given format """ - package = create_basic_wheel_for_package(script, package_name, '1.0') + package = create_basic_wheel_for_package(script, package_name, "1.0") wheel_name = os.path.basename(package) - assert wheel_name == 'simple_package-1.0-py2.py3-none-any.whl' + assert wheel_name == "simple_package-1.0-py2.py3-none-any.whl" @pytest.mark.parametrize("name", ["purelib", "abc"]) def test_wheel_with_file_in_data_dir_has_reasonable_error( - script, tmpdir, name -): + script: PipTestEnvironment, tmpdir: Path, name: str +) -> None: """Normally we expect entities in the .data directory to be in a subdirectory, but if they are not then we should show a reasonable error message that includes the path. @@ -688,22 +749,16 @@ def test_wheel_with_file_in_data_dir_has_reasonable_error( "simple", "0.1.0", extra_data_files={name: "hello world"} ).save_to_dir(tmpdir) - result = script.pip( - "install", "--no-index", str(wheel_path), expect_error=True - ) + result = script.pip("install", "--no-index", str(wheel_path), expect_error=True) assert f"simple-0.1.0.data/{name}" in result.stderr def test_wheel_with_unknown_subdir_in_data_dir_has_reasonable_error( - script, tmpdir -): + script: PipTestEnvironment, tmpdir: Path +) -> None: wheel_path = make_wheel( - "simple", - "0.1.0", - extra_data_files={"unknown/hello.txt": "hello world"} + "simple", "0.1.0", extra_data_files={"unknown/hello.txt": "hello world"} ).save_to_dir(tmpdir) - result = script.pip( - "install", "--no-index", str(wheel_path), expect_error=True - ) + result = script.pip("install", "--no-index", str(wheel_path), expect_error=True) assert "simple-0.1.0.data/unknown/hello.txt" in result.stderr diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index 40dfbdea30d..b9d0f0fa340 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -3,540 +3,678 @@ import pytest -from tests.lib import create_test_package_with_setup, wheel +from pip._internal.models.direct_url import DirectUrl, DirInfo +from tests.conftest import ScriptFactory +from tests.lib import ( + PipTestEnvironment, + TestData, + _create_test_package, + create_test_package_with_setup, + wheel, +) +from tests.lib.direct_url import get_created_direct_url_path from tests.lib.path import Path @pytest.fixture(scope="session") -def simple_script(tmpdir_factory, script_factory, shared_data): +def simple_script( + tmpdir_factory: pytest.TempdirFactory, + script_factory: ScriptFactory, + shared_data: TestData, +) -> PipTestEnvironment: tmpdir = Path(str(tmpdir_factory.mktemp("pip_test_package"))) script = script_factory(tmpdir.joinpath("workspace")) script.pip( - 'install', '-f', shared_data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', + "install", + "-f", + shared_data.find_links, + "--no-index", + "simple==1.0", + "simple2==3.0", ) return script -def test_basic_list(simple_script): +def test_basic_list(simple_script: PipTestEnvironment) -> None: """ Test default behavior of list command without format specifier. """ - result = simple_script.pip('list') - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + result = simple_script.pip("list") + assert "simple 1.0" in result.stdout, str(result) + assert "simple2 3.0" in result.stdout, str(result) -def test_verbose_flag(simple_script): +def test_verbose_flag(simple_script: PipTestEnvironment) -> None: """ Test the list command with the '-v' option """ - result = simple_script.pip('list', '-v', '--format=columns') - assert 'Package' in result.stdout, str(result) - assert 'Version' in result.stdout, str(result) - assert 'Location' in result.stdout, str(result) - assert 'Installer' in result.stdout, str(result) - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + result = simple_script.pip("list", "-v", "--format=columns") + assert "Package" in result.stdout, str(result) + assert "Version" in result.stdout, str(result) + assert "Location" in result.stdout, str(result) + assert "Installer" in result.stdout, str(result) + assert "simple 1.0" in result.stdout, str(result) + assert "simple2 3.0" in result.stdout, str(result) -def test_columns_flag(simple_script): +def test_columns_flag(simple_script: PipTestEnvironment) -> None: """ Test the list command with the '--format=columns' option """ - result = simple_script.pip('list', '--format=columns') - assert 'Package' in result.stdout, str(result) - assert 'Version' in result.stdout, str(result) - assert 'simple (1.0)' not in result.stdout, str(result) - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + result = simple_script.pip("list", "--format=columns") + assert "Package" in result.stdout, str(result) + assert "Version" in result.stdout, str(result) + assert "simple (1.0)" not in result.stdout, str(result) + assert "simple 1.0" in result.stdout, str(result) + assert "simple2 3.0" in result.stdout, str(result) -def test_format_priority(simple_script): +def test_format_priority(simple_script: PipTestEnvironment) -> None: """ Test that latest format has priority over previous ones. """ - result = simple_script.pip('list', '--format=columns', '--format=freeze', - expect_stderr=True) - assert 'simple==1.0' in result.stdout, str(result) - assert 'simple2==3.0' in result.stdout, str(result) - assert 'simple 1.0' not in result.stdout, str(result) - assert 'simple2 3.0' not in result.stdout, str(result) + result = simple_script.pip( + "list", "--format=columns", "--format=freeze", expect_stderr=True + ) + assert "simple==1.0" in result.stdout, str(result) + assert "simple2==3.0" in result.stdout, str(result) + assert "simple 1.0" not in result.stdout, str(result) + assert "simple2 3.0" not in result.stdout, str(result) - result = simple_script.pip('list', '--format=freeze', '--format=columns') - assert 'Package' in result.stdout, str(result) - assert 'Version' in result.stdout, str(result) - assert 'simple==1.0' not in result.stdout, str(result) - assert 'simple2==3.0' not in result.stdout, str(result) - assert 'simple 1.0' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + result = simple_script.pip("list", "--format=freeze", "--format=columns") + assert "Package" in result.stdout, str(result) + assert "Version" in result.stdout, str(result) + assert "simple==1.0" not in result.stdout, str(result) + assert "simple2==3.0" not in result.stdout, str(result) + assert "simple 1.0" in result.stdout, str(result) + assert "simple2 3.0" in result.stdout, str(result) -def test_local_flag(simple_script): +def test_local_flag(simple_script: PipTestEnvironment) -> None: """ Test the behavior of --local flag in the list command """ - result = simple_script.pip('list', '--local', '--format=json') + result = simple_script.pip("list", "--local", "--format=json") assert {"name": "simple", "version": "1.0"} in json.loads(result.stdout) -def test_local_columns_flag(simple_script): +def test_local_columns_flag(simple_script: PipTestEnvironment) -> None: """ Test the behavior of --local --format=columns flags in the list command """ - result = simple_script.pip('list', '--local', '--format=columns') - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'simple (1.0)' not in result.stdout - assert 'simple 1.0' in result.stdout, str(result) + result = simple_script.pip("list", "--local", "--format=columns") + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "simple (1.0)" not in result.stdout + assert "simple 1.0" in result.stdout, str(result) -def test_multiple_exclude_and_normalization(script, tmpdir): - req_path = wheel.make_wheel( - name="Normalizable_Name", version="1.0").save_to_dir(tmpdir) +def test_multiple_exclude_and_normalization( + script: PipTestEnvironment, tmpdir: Path +) -> None: + req_path = wheel.make_wheel(name="Normalizable_Name", version="1.0").save_to_dir( + tmpdir + ) script.pip("install", "--no-index", req_path) result = script.pip("list") print(result.stdout) - assert "Normalizable-Name" in result.stdout + assert "Normalizable_Name" in result.stdout assert "pip" in result.stdout result = script.pip("list", "--exclude", "normalizablE-namE", "--exclude", "pIp") - assert "Normalizable-Name" not in result.stdout + assert "Normalizable_Name" not in result.stdout assert "pip" not in result.stdout @pytest.mark.network @pytest.mark.incompatible_with_test_venv -def test_user_flag(script, data): +def test_user_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --user flag in the list command """ - script.pip('download', 'setuptools', 'wheel', '-d', data.packages) - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip('install', '-f', data.find_links, '--no-index', - '--user', 'simple2==2.0') - result = script.pip('list', '--user', '--format=json') - assert {"name": "simple", "version": "1.0"} \ - not in json.loads(result.stdout) + script.pip("download", "setuptools", "wheel", "-d", data.packages) + script.pip("install", "-f", data.find_links, "--no-index", "simple==1.0") + script.pip("install", "-f", data.find_links, "--no-index", "--user", "simple2==2.0") + result = script.pip("list", "--user", "--format=json") + assert {"name": "simple", "version": "1.0"} not in json.loads(result.stdout) assert {"name": "simple2", "version": "2.0"} in json.loads(result.stdout) @pytest.mark.network @pytest.mark.incompatible_with_test_venv -def test_user_columns_flag(script, data): +def test_user_columns_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --user --format=columns flags in the list command """ - script.pip('download', 'setuptools', 'wheel', '-d', data.packages) - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip('install', '-f', data.find_links, '--no-index', - '--user', 'simple2==2.0') - result = script.pip('list', '--user', '--format=columns') - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'simple2 (2.0)' not in result.stdout - assert 'simple2 2.0' in result.stdout, str(result) + script.pip("download", "setuptools", "wheel", "-d", data.packages) + script.pip("install", "-f", data.find_links, "--no-index", "simple==1.0") + script.pip("install", "-f", data.find_links, "--no-index", "--user", "simple2==2.0") + result = script.pip("list", "--user", "--format=columns") + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "simple2 (2.0)" not in result.stdout + assert "simple2 2.0" in result.stdout, str(result) @pytest.mark.network -def test_uptodate_flag(script, data): +def test_uptodate_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --uptodate flag in the list command """ script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', + "install", + "-f", + data.find_links, + "--no-index", + "simple==1.0", + "simple2==3.0", ) script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--uptodate', - '--format=json', - ) - assert {"name": "simple", "version": "1.0"} \ - not in json.loads(result.stdout) # 3.0 is latest - assert {"name": "pip-test-package", "version": "0.1.1"} \ - in json.loads(result.stdout) # editables included - assert {"name": "simple2", "version": "3.0"} in json.loads(result.stdout) + "list", + "-f", + data.find_links, + "--no-index", + "--uptodate", + "--format=json", + ) + json_output = json.loads(result.stdout) + for item in json_output: + if "editable_project_location" in item: + item["editable_project_location"] = "" + assert {"name": "simple", "version": "1.0"} not in json_output # 3.0 is latest + assert { + "name": "pip-test-package", + "version": "0.1.1", + "editable_project_location": "", + } in json_output # editables included + assert {"name": "simple2", "version": "3.0"} in json_output @pytest.mark.network -def test_uptodate_columns_flag(script, data): +def test_uptodate_columns_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --uptodate --format=columns flag in the list command """ script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', + "install", + "-f", + data.find_links, + "--no-index", + "simple==1.0", + "simple2==3.0", ) script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--uptodate', - '--format=columns', + "list", + "-f", + data.find_links, + "--no-index", + "--uptodate", + "--format=columns", ) - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'Location' in result.stdout # editables included - assert 'pip-test-package (0.1.1,' not in result.stdout - assert 'pip-test-package 0.1.1' in result.stdout, str(result) - assert 'simple2 3.0' in result.stdout, str(result) + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "Editable project location" in result.stdout # editables included + assert "pip-test-package (0.1.1," not in result.stdout + assert "pip-test-package 0.1.1" in result.stdout, str(result) + assert "simple2 3.0" in result.stdout, str(result) @pytest.mark.network -def test_outdated_flag(script, data): +def test_outdated_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --outdated flag in the list command """ script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', 'simplewheel==1.0', + "install", + "-f", + data.find_links, + "--no-index", + "simple==1.0", + "simple2==3.0", + "simplewheel==1.0", ) script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git@0.1#egg=pip-test-package", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--outdated', - '--format=json', - ) - assert {"name": "simple", "version": "1.0", - "latest_version": "3.0", "latest_filetype": "sdist"} \ - in json.loads(result.stdout) - assert dict(name="simplewheel", version="1.0", - latest_version="2.0", latest_filetype="wheel") \ - in json.loads(result.stdout) - assert dict(name="pip-test-package", version="0.1", - latest_version="0.1.1", latest_filetype="sdist") \ - in json.loads(result.stdout) - assert "simple2" not in {p["name"] for p in json.loads(result.stdout)} + "list", + "-f", + data.find_links, + "--no-index", + "--outdated", + "--format=json", + ) + json_output = json.loads(result.stdout) + for item in json_output: + if "editable_project_location" in item: + item["editable_project_location"] = "" + assert { + "name": "simple", + "version": "1.0", + "latest_version": "3.0", + "latest_filetype": "sdist", + } in json_output + assert ( + dict( + name="simplewheel", + version="1.0", + latest_version="2.0", + latest_filetype="wheel", + ) + in json_output + ) + assert ( + dict( + name="pip-test-package", + version="0.1", + latest_version="0.1.1", + latest_filetype="sdist", + editable_project_location="", + ) + in json_output + ) + assert "simple2" not in {p["name"] for p in json_output} @pytest.mark.network -def test_outdated_columns_flag(script, data): +def test_outdated_columns_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --outdated --format=columns flag in the list command """ script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', 'simplewheel==1.0', + "install", + "-f", + data.find_links, + "--no-index", + "simple==1.0", + "simple2==3.0", + "simplewheel==1.0", ) script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git@0.1#egg=pip-test-package", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--outdated', - '--format=columns', - ) - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'Latest' in result.stdout - assert 'Type' in result.stdout - assert 'simple (1.0) - Latest: 3.0 [sdist]' not in result.stdout - assert 'simplewheel (1.0) - Latest: 2.0 [wheel]' not in result.stdout - assert 'simple 1.0 3.0 sdist' in result.stdout, ( - str(result) - ) - assert 'simplewheel 1.0 2.0 wheel' in result.stdout, ( - str(result) - ) - assert 'simple2' not in result.stdout, str(result) # 3.0 is latest + "list", + "-f", + data.find_links, + "--no-index", + "--outdated", + "--format=columns", + ) + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "Latest" in result.stdout + assert "Type" in result.stdout + assert "simple (1.0) - Latest: 3.0 [sdist]" not in result.stdout + assert "simplewheel (1.0) - Latest: 2.0 [wheel]" not in result.stdout + assert "simple 1.0 3.0 sdist" in result.stdout, str(result) + assert "simplewheel 1.0 2.0 wheel" in result.stdout, str(result) + assert "simple2" not in result.stdout, str(result) # 3.0 is latest @pytest.fixture(scope="session") -def pip_test_package_script(tmpdir_factory, script_factory, shared_data): +def pip_test_package_script( + tmpdir_factory: pytest.TempdirFactory, + script_factory: ScriptFactory, + shared_data: TestData, +) -> PipTestEnvironment: tmpdir = Path(str(tmpdir_factory.mktemp("pip_test_package"))) script = script_factory(tmpdir.joinpath("workspace")) + script.pip("install", "-f", shared_data.find_links, "--no-index", "simple==1.0") script.pip( - 'install', '-f', shared_data.find_links, '--no-index', 'simple==1.0' - ) - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package", ) return script @pytest.mark.network -def test_editables_flag(pip_test_package_script): +def test_editables_flag(pip_test_package_script: PipTestEnvironment) -> None: """ Test the behavior of --editables flag in the list command """ - result = pip_test_package_script.pip('list', '--editable', '--format=json') - result2 = pip_test_package_script.pip('list', '--editable') - assert {"name": "simple", "version": "1.0"} \ - not in json.loads(result.stdout) - assert os.path.join('src', 'pip-test-package') in result2.stdout + result = pip_test_package_script.pip("list", "--editable", "--format=json") + result2 = pip_test_package_script.pip("list", "--editable") + assert {"name": "simple", "version": "1.0"} not in json.loads(result.stdout) + assert os.path.join("src", "pip-test-package") in result2.stdout @pytest.mark.network -def test_exclude_editable_flag(pip_test_package_script): +def test_exclude_editable_flag(pip_test_package_script: PipTestEnvironment) -> None: """ Test the behavior of --editables flag in the list command """ - result = pip_test_package_script.pip( - 'list', '--exclude-editable', '--format=json' - ) + result = pip_test_package_script.pip("list", "--exclude-editable", "--format=json") assert {"name": "simple", "version": "1.0"} in json.loads(result.stdout) - assert "pip-test-package" \ - not in {p["name"] for p in json.loads(result.stdout)} + assert "pip-test-package" not in {p["name"] for p in json.loads(result.stdout)} @pytest.mark.network -def test_editables_columns_flag(pip_test_package_script): +def test_editables_columns_flag(pip_test_package_script: PipTestEnvironment) -> None: """ Test the behavior of --editables flag in the list command """ - result = pip_test_package_script.pip( - 'list', '--editable', '--format=columns' - ) - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'Location' in result.stdout - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) + result = pip_test_package_script.pip("list", "--editable", "--format=columns") + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "Editable project location" in result.stdout + assert os.path.join("src", "pip-test-package") in result.stdout, str(result) @pytest.mark.network -def test_uptodate_editables_flag(pip_test_package_script, data): +def test_uptodate_editables_flag( + pip_test_package_script: PipTestEnvironment, data: TestData +) -> None: """ test the behavior of --editable --uptodate flag in the list command """ result = pip_test_package_script.pip( - 'list', '-f', data.find_links, '--no-index', - '--editable', '--uptodate', - ) - assert 'simple' not in result.stdout - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) + "list", + "-f", + data.find_links, + "--no-index", + "--editable", + "--uptodate", ) + assert "simple" not in result.stdout + assert os.path.join("src", "pip-test-package") in result.stdout, str(result) @pytest.mark.network -def test_uptodate_editables_columns_flag(pip_test_package_script, data): +def test_uptodate_editables_columns_flag( + pip_test_package_script: PipTestEnvironment, data: TestData +) -> None: """ test the behavior of --editable --uptodate --format=columns flag in the list command """ result = pip_test_package_script.pip( - 'list', '-f', data.find_links, '--no-index', - '--editable', '--uptodate', '--format=columns', - ) - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'Location' in result.stdout - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) + "list", + "-f", + data.find_links, + "--no-index", + "--editable", + "--uptodate", + "--format=columns", ) + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "Editable project location" in result.stdout + assert os.path.join("src", "pip-test-package") in result.stdout, str(result) @pytest.mark.network -def test_outdated_editables_flag(script, data): +def test_outdated_editables_flag(script: PipTestEnvironment, data: TestData) -> None: """ test the behavior of --editable --outdated flag in the list command """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') + script.pip("install", "-f", data.find_links, "--no-index", "simple==1.0") result = script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git@0.1#egg=pip-test-package", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', - '--editable', '--outdated', + "list", + "-f", + data.find_links, + "--no-index", + "--editable", + "--outdated", ) - assert 'simple' not in result.stdout - assert os.path.join('src', 'pip-test-package') in result.stdout + assert "simple" not in result.stdout + assert os.path.join("src", "pip-test-package") in result.stdout @pytest.mark.network -def test_outdated_editables_columns_flag(script, data): +def test_outdated_editables_columns_flag( + script: PipTestEnvironment, data: TestData +) -> None: """ test the behavior of --editable --outdated flag in the list command """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') + script.pip("install", "-f", data.find_links, "--no-index", "simple==1.0") result = script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' + "install", + "-e", + "git+https://github.com/pypa/pip-test-package.git@0.1#egg=pip-test-package", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', - '--editable', '--outdated', '--format=columns', - ) - assert 'Package' in result.stdout - assert 'Version' in result.stdout - assert 'Location' in result.stdout - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) + "list", + "-f", + data.find_links, + "--no-index", + "--editable", + "--outdated", + "--format=columns", ) + assert "Package" in result.stdout + assert "Version" in result.stdout + assert "Editable project location" in result.stdout + assert os.path.join("src", "pip-test-package") in result.stdout, str(result) -def test_outdated_not_required_flag(script, data): +def test_outdated_not_required_flag(script: PipTestEnvironment, data: TestData) -> None: """ test the behavior of --outdated --not-required flag in the list command """ script.pip( - 'install', '-f', data.find_links, '--no-index', - 'simple==2.0', 'require_simple==1.0' + "install", + "-f", + data.find_links, + "--no-index", + "simple==2.0", + "require_simple==1.0", ) result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--outdated', - '--not-required', '--format=json', + "list", + "-f", + data.find_links, + "--no-index", + "--outdated", + "--not-required", + "--format=json", ) assert [] == json.loads(result.stdout) -def test_outdated_pre(script, data): - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') +def test_outdated_pre(script: PipTestEnvironment, data: TestData) -> None: + script.pip("install", "-f", data.find_links, "--no-index", "simple==1.0") # Let's build a fake wheelhouse script.scratch_path.joinpath("wheelhouse").mkdir() - wheelhouse_path = script.scratch_path / 'wheelhouse' - wheelhouse_path.joinpath('simple-1.1-py2.py3-none-any.whl').write_text('') - wheelhouse_path.joinpath( - 'simple-2.0.dev0-py2.py3-none-any.whl' - ).write_text('') + wheelhouse_path = script.scratch_path / "wheelhouse" + wheelhouse_path.joinpath("simple-1.1-py2.py3-none-any.whl").write_text("") + wheelhouse_path.joinpath("simple-2.0.dev0-py2.py3-none-any.whl").write_text("") result = script.pip( - 'list', '--no-index', '--find-links', wheelhouse_path, - '--format=json', + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--format=json", ) assert {"name": "simple", "version": "1.0"} in json.loads(result.stdout) result = script.pip( - 'list', '--no-index', '--find-links', wheelhouse_path, '--outdated', - '--format=json', - ) - assert {"name": "simple", "version": "1.0", - "latest_version": "1.1", "latest_filetype": "wheel"} \ - in json.loads(result.stdout) - result_pre = script.pip('list', '--no-index', - '--find-links', wheelhouse_path, - '--outdated', '--pre', '--format=json') - assert {"name": "simple", "version": "1.0", - "latest_version": "2.0.dev0", "latest_filetype": "wheel"} \ - in json.loads(result_pre.stdout) - - -def test_outdated_formats(script, data): - """ Test of different outdated formats """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--outdated", + "--format=json", + ) + assert { + "name": "simple", + "version": "1.0", + "latest_version": "1.1", + "latest_filetype": "wheel", + } in json.loads(result.stdout) + result_pre = script.pip( + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--outdated", + "--pre", + "--format=json", + ) + assert { + "name": "simple", + "version": "1.0", + "latest_version": "2.0.dev0", + "latest_filetype": "wheel", + } in json.loads(result_pre.stdout) + + +def test_outdated_formats(script: PipTestEnvironment, data: TestData) -> None: + """Test of different outdated formats""" + script.pip("install", "-f", data.find_links, "--no-index", "simple==1.0") # Let's build a fake wheelhouse script.scratch_path.joinpath("wheelhouse").mkdir() - wheelhouse_path = script.scratch_path / 'wheelhouse' - wheelhouse_path.joinpath('simple-1.1-py2.py3-none-any.whl').write_text('') + wheelhouse_path = script.scratch_path / "wheelhouse" + wheelhouse_path.joinpath("simple-1.1-py2.py3-none-any.whl").write_text("") result = script.pip( - 'list', '--no-index', '--find-links', wheelhouse_path, - '--format=freeze', + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--format=freeze", ) - assert 'simple==1.0' in result.stdout + assert "simple==1.0" in result.stdout # Check columns result = script.pip( - 'list', '--no-index', '--find-links', wheelhouse_path, - '--outdated', '--format=columns', + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--outdated", + "--format=columns", ) - assert 'Package Version Latest Type' in result.stdout - assert 'simple 1.0 1.1 wheel' in result.stdout + assert "Package Version Latest Type" in result.stdout + assert "simple 1.0 1.1 wheel" in result.stdout # Check freeze result = script.pip( - 'list', '--no-index', '--find-links', wheelhouse_path, - '--outdated', '--format=freeze', + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--outdated", + "--format=freeze", ) - assert 'simple==1.0' in result.stdout + assert "simple==1.0" in result.stdout # Check json result = script.pip( - 'list', '--no-index', '--find-links', wheelhouse_path, - '--outdated', '--format=json', + "list", + "--no-index", + "--find-links", + wheelhouse_path, + "--outdated", + "--format=json", ) data = json.loads(result.stdout) - assert data == [{'name': 'simple', 'version': '1.0', - 'latest_version': '1.1', 'latest_filetype': 'wheel'}] + assert data == [ + { + "name": "simple", + "version": "1.0", + "latest_version": "1.1", + "latest_filetype": "wheel", + } + ] -def test_not_required_flag(script, data): - script.pip( - 'install', '-f', data.find_links, '--no-index', 'TopoRequires4' - ) - result = script.pip('list', '--not-required', expect_stderr=True) - assert 'TopoRequires4 ' in result.stdout, str(result) - assert 'TopoRequires ' not in result.stdout - assert 'TopoRequires2 ' not in result.stdout - assert 'TopoRequires3 ' not in result.stdout +def test_not_required_flag(script: PipTestEnvironment, data: TestData) -> None: + script.pip("install", "-f", data.find_links, "--no-index", "TopoRequires4") + result = script.pip("list", "--not-required", expect_stderr=True) + assert "TopoRequires4 " in result.stdout, str(result) + assert "TopoRequires " not in result.stdout + assert "TopoRequires2 " not in result.stdout + assert "TopoRequires3 " not in result.stdout -def test_list_freeze(simple_script): +def test_list_freeze(simple_script: PipTestEnvironment) -> None: """ Test freeze formatting of list command """ - result = simple_script.pip('list', '--format=freeze') - assert 'simple==1.0' in result.stdout, str(result) - assert 'simple2==3.0' in result.stdout, str(result) + result = simple_script.pip("list", "--format=freeze") + assert "simple==1.0" in result.stdout, str(result) + assert "simple2==3.0" in result.stdout, str(result) -def test_list_json(simple_script): +def test_list_json(simple_script: PipTestEnvironment) -> None: """ Test json formatting of list command """ - result = simple_script.pip('list', '--format=json') + result = simple_script.pip("list", "--format=json") data = json.loads(result.stdout) - assert {'name': 'simple', 'version': '1.0'} in data - assert {'name': 'simple2', 'version': '3.0'} in data + assert {"name": "simple", "version": "1.0"} in data + assert {"name": "simple2", "version": "3.0"} in data -def test_list_path(tmpdir, script, data): +def test_list_path(tmpdir: Path, script: PipTestEnvironment, data: TestData) -> None: """ Test list with --path. """ - result = script.pip('list', '--path', tmpdir, '--format=json') + result = script.pip("list", "--path", tmpdir, "--format=json") json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '2.0'} not in json_result + assert {"name": "simple", "version": "2.0"} not in json_result - script.pip_install_local('--target', tmpdir, 'simple==2.0') - result = script.pip('list', '--path', tmpdir, '--format=json') + script.pip_install_local("--target", tmpdir, "simple==2.0") + result = script.pip("list", "--path", tmpdir, "--format=json") json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '2.0'} in json_result + assert {"name": "simple", "version": "2.0"} in json_result @pytest.mark.incompatible_with_test_venv -def test_list_path_exclude_user(tmpdir, script, data): +def test_list_path_exclude_user( + tmpdir: Path, script: PipTestEnvironment, data: TestData +) -> None: """ Test list with --path and make sure packages from --user are not picked up. """ - script.pip_install_local('--user', 'simple2') - script.pip_install_local('--target', tmpdir, 'simple==1.0') + script.pip_install_local("--user", "simple2") + script.pip_install_local("--target", tmpdir, "simple==1.0") - result = script.pip('list', '--user', '--format=json') + result = script.pip("list", "--user", "--format=json") json_result = json.loads(result.stdout) - assert {'name': 'simple2', 'version': '3.0'} in json_result + assert {"name": "simple2", "version": "3.0"} in json_result - result = script.pip('list', '--path', tmpdir, '--format=json') + result = script.pip("list", "--path", tmpdir, "--format=json") json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '1.0'} in json_result + assert {"name": "simple", "version": "1.0"} in json_result -def test_list_path_multiple(tmpdir, script, data): +def test_list_path_multiple( + tmpdir: Path, script: PipTestEnvironment, data: TestData +) -> None: """ Test list with multiple --path arguments. """ @@ -545,53 +683,74 @@ def test_list_path_multiple(tmpdir, script, data): path2 = tmpdir / "path2" os.mkdir(path2) - script.pip_install_local('--target', path1, 'simple==2.0') - script.pip_install_local('--target', path2, 'simple2==3.0') + script.pip_install_local("--target", path1, "simple==2.0") + script.pip_install_local("--target", path2, "simple2==3.0") - result = script.pip('list', '--path', path1, '--format=json') + result = script.pip("list", "--path", path1, "--format=json") json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '2.0'} in json_result + assert {"name": "simple", "version": "2.0"} in json_result - result = script.pip('list', '--path', path1, '--path', path2, - '--format=json') + result = script.pip("list", "--path", path1, "--path", path2, "--format=json") json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '2.0'} in json_result - assert {'name': 'simple2', 'version': '3.0'} in json_result + assert {"name": "simple", "version": "2.0"} in json_result + assert {"name": "simple2", "version": "3.0"} in json_result -def test_list_skip_work_dir_pkg(script): +def test_list_skip_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that list should not include package in working directory """ # Create a test package and create .egg-info dir - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) # List should not include package simple when run from package directory - result = script.pip('list', '--format=json', cwd=pkg_path) + result = script.pip("list", "--format=json", cwd=pkg_path) json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '1.0'} not in json_result + assert {"name": "simple", "version": "1.0"} not in json_result -def test_list_include_work_dir_pkg(script): +def test_list_include_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that list should include package in working directory if working directory is added in PYTHONPATH """ # Create a test package and create .egg-info dir - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) - script.environ.update({'PYTHONPATH': pkg_path}) + script.environ.update({"PYTHONPATH": pkg_path}) # List should include package simple when run from package directory # when the package directory is in PYTHONPATH - result = script.pip('list', '--format=json', cwd=pkg_path) + result = script.pip("list", "--format=json", cwd=pkg_path) json_result = json.loads(result.stdout) - assert {'name': 'simple', 'version': '1.0'} in json_result + assert {"name": "simple", "version": "1.0"} in json_result + + +@pytest.mark.usefixtures("with_wheel") +def test_list_pep610_editable(script: PipTestEnvironment) -> None: + """ + Test that a package installed with a direct_url.json with editable=true + is correctly listed as editable. + """ + pkg_path = _create_test_package(script, name="testpkg") + result = script.pip("install", pkg_path) + direct_url_path = get_created_direct_url_path(result, "testpkg") + assert direct_url_path + # patch direct_url.json to simulate an editable install + with open(direct_url_path) as f: + direct_url = DirectUrl.from_json(f.read()) + assert isinstance(direct_url.info, DirInfo) + direct_url.info.editable = True + with open(direct_url_path, "w") as f: + f.write(direct_url.to_json()) + result = script.pip("list", "--format=json") + for item in json.loads(result.stdout): + if item["name"] == "testpkg": + assert item["editable_project_location"] + break + else: + assert False, "package 'testpkg' not found in pip list result" diff --git a/tests/functional/test_new_resolver.py b/tests/functional/test_new_resolver.py index f3850c208c6..21b9e0b7fa9 100644 --- a/tests/functional/test_new_resolver.py +++ b/tests/functional/test_new_resolver.py @@ -1,51 +1,54 @@ -import json import os +import pathlib import sys import textwrap +from typing import TYPE_CHECKING, Callable, Dict, List, Tuple import pytest -from pip._vendor.packaging.utils import canonicalize_name from tests.lib import ( + PipTestEnvironment, create_basic_sdist_for_package, create_basic_wheel_for_package, create_test_package_with_setup, + path_to_url, ) +from tests.lib.direct_url import get_created_direct_url +from tests.lib.path import Path from tests.lib.wheel import make_wheel +if TYPE_CHECKING: + from typing import Protocol -def assert_installed(script, **kwargs): - ret = script.pip('list', '--format=json') - installed = set( - (canonicalize_name(val['name']), val['version']) - for val in json.loads(ret.stdout) - ) - expected = set((canonicalize_name(k), v) for k, v in kwargs.items()) - assert expected <= installed, f"{expected!r} not all in {installed!r}" - - -def assert_not_installed(script, *args): - ret = script.pip("list", "--format=json") - installed = set( - canonicalize_name(val["name"]) - for val in json.loads(ret.stdout) - ) - # None of the given names should be listed as installed, i.e. their - # intersection should be empty. - expected = set(canonicalize_name(k) for k in args) - assert not (expected & installed), f"{expected!r} contained in {installed!r}" - -def assert_editable(script, *args): +def assert_editable(script: PipTestEnvironment, *args: str) -> None: # This simply checks whether all of the listed packages have a # corresponding .egg-link file installed. # TODO: Implement a more rigorous way to test for editable installations. - egg_links = set(f"{arg}.egg-link" for arg in args) - assert egg_links <= set(os.listdir(script.site_packages_path)), \ - f"{args!r} not all found in {script.site_packages_path!r}" + egg_links = {f"{arg}.egg-link" for arg in args} + assert egg_links <= set( + os.listdir(script.site_packages_path) + ), f"{args!r} not all found in {script.site_packages_path!r}" + + +@pytest.fixture() +def make_fake_wheel(script: PipTestEnvironment) -> Callable[[str, str, str], Path]: + def _make_fake_wheel(name: str, version: str, wheel_tag: str) -> Path: + wheel_house = script.scratch_path.joinpath("wheelhouse") + wheel_house.mkdir() + wheel_builder = make_wheel( + name=name, + version=version, + wheel_metadata_updates={"Tag": []}, + ) + wheel_path = wheel_house.joinpath(f"{name}-{version}-{wheel_tag}.whl") + wheel_builder.save_to(wheel_path) + return wheel_path + return _make_fake_wheel -def test_new_resolver_can_install(script): + +def test_new_resolver_can_install(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "simple", @@ -53,14 +56,16 @@ def test_new_resolver_can_install(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple", ) - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") -def test_new_resolver_can_install_with_version(script): +def test_new_resolver_can_install_with_version(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "simple", @@ -68,14 +73,16 @@ def test_new_resolver_can_install_with_version(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple==0.1.0" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple==0.1.0", ) - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") -def test_new_resolver_picks_latest_version(script): +def test_new_resolver_picks_latest_version(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "simple", @@ -88,14 +95,16 @@ def test_new_resolver_picks_latest_version(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple", ) - assert_installed(script, simple="0.2.0") + script.assert_installed(simple="0.2.0") -def test_new_resolver_picks_installed_version(script): +def test_new_resolver_picks_installed_version(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "simple", @@ -108,23 +117,29 @@ def test_new_resolver_picks_installed_version(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple==0.1.0" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple==0.1.0", ) - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple", ) assert "Collecting" not in result.stdout, "Should not fetch new version" - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") -def test_new_resolver_picks_installed_version_if_no_match_found(script): +def test_new_resolver_picks_installed_version_if_no_match_found( + script: PipTestEnvironment, +) -> None: create_basic_wheel_for_package( script, "simple", @@ -137,22 +152,20 @@ def test_new_resolver_picks_installed_version_if_no_match_found(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple==0.1.0" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple==0.1.0", ) - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") - result = script.pip( - "install", - "--no-cache-dir", "--no-index", - "simple" - ) + result = script.pip("install", "--no-cache-dir", "--no-index", "simple") assert "Collecting" not in result.stdout, "Should not fetch new version" - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") -def test_new_resolver_installs_dependencies(script): +def test_new_resolver_installs_dependencies(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -166,14 +179,16 @@ def test_new_resolver_installs_dependencies(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "base" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "base", ) - assert_installed(script, base="0.1.0", dep="0.1.0") + script.assert_installed(base="0.1.0", dep="0.1.0") -def test_new_resolver_ignore_dependencies(script): +def test_new_resolver_ignore_dependencies(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -187,12 +202,15 @@ def test_new_resolver_ignore_dependencies(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", "--no-deps", - "--find-links", script.scratch_path, - "base" + "--no-cache-dir", + "--no-index", + "--no-deps", + "--find-links", + script.scratch_path, + "base", ) - assert_installed(script, base="0.1.0") - assert_not_installed(script, "dep") + script.assert_installed(base="0.1.0") + script.assert_not_installed("dep") @pytest.mark.parametrize( @@ -202,7 +220,9 @@ def test_new_resolver_ignore_dependencies(script): "base[add] >= 0.1.0", ], ) -def test_new_resolver_installs_extras(tmpdir, script, root_dep): +def test_new_resolver_installs_extras( + tmpdir: Path, script: PipTestEnvironment, root_dep: str +) -> None: req_file = tmpdir.joinpath("requirements.txt") req_file.write_text(root_dep) @@ -219,40 +239,17 @@ def test_new_resolver_installs_extras(tmpdir, script, root_dep): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-r", req_file, - ) - assert_installed(script, base="0.1.0", dep="0.1.0") - - -def test_new_resolver_installs_extras_deprecated(tmpdir, script): - req_file = tmpdir.joinpath("requirements.txt") - req_file.write_text("base >= 0.1.0[add]") - - create_basic_wheel_for_package( - script, - "base", - "0.1.0", - extras={"add": ["dep"]}, - ) - create_basic_wheel_for_package( - script, - "dep", - "0.1.0", - ) - result = script.pip( - "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-r", req_file, - expect_stderr=True + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-r", + req_file, ) - assert "DEPRECATION: Extras after version" in result.stderr - assert_installed(script, base="0.1.0", dep="0.1.0") + script.assert_installed(base="0.1.0", dep="0.1.0") -def test_new_resolver_installs_extras_warn_missing(script): +def test_new_resolver_installs_extras_warn_missing(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -266,34 +263,40 @@ def test_new_resolver_installs_extras_warn_missing(script): ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base[add,missing]", expect_stderr=True, ) assert "does not provide the extra" in result.stderr, str(result) assert "missing" in result.stderr, str(result) - assert_installed(script, base="0.1.0", dep="0.1.0") + script.assert_installed(base="0.1.0", dep="0.1.0") -def test_new_resolver_installed_message(script): +def test_new_resolver_installed_message(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "A", "1.0") result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "A", expect_stderr=False, ) assert "Successfully installed A-1.0" in result.stdout, str(result) -def test_new_resolver_no_dist_message(script): +def test_new_resolver_no_dist_message(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "A", "1.0") result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "B", expect_error=True, expect_stderr=True, @@ -304,12 +307,13 @@ def test_new_resolver_no_dist_message(script): # requirement xxx (from versions: none) # ERROR: No matching distribution found for xxx - assert "Could not find a version that satisfies the requirement B" \ - in result.stderr, str(result) + assert ( + "Could not find a version that satisfies the requirement B" in result.stderr + ), str(result) assert "No matching distribution found for B" in result.stderr, str(result) -def test_new_resolver_installs_editable(script): +def test_new_resolver_installs_editable(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -323,12 +327,15 @@ def test_new_resolver_installs_editable(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base", - "--editable", source_dir, + "--editable", + source_dir, ) - assert_installed(script, base="0.1.0", dep="0.1.0") + script.assert_installed(base="0.1.0", dep="0.1.0") assert_editable(script, "dep") @@ -338,18 +345,17 @@ def test_new_resolver_installs_editable(script): # Something impossible to satisfy. ("<2", False, "0.1.0"), ("<2", True, "0.2.0"), - # Something guaranteed to satisfy. (">=2", False, "0.2.0"), (">=2", True, "0.2.0"), ], ) def test_new_resolver_requires_python( - script, - requires_python, - ignore_requires_python, - dep_version, -): + script: PipTestEnvironment, + requires_python: str, + ignore_requires_python: bool, + dep_version: str, +) -> None: create_basic_wheel_for_package( script, "base", @@ -372,7 +378,8 @@ def test_new_resolver_requires_python( "install", "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--find-links", + script.scratch_path, ] if ignore_requires_python: args.append("--ignore-requires-python") @@ -380,10 +387,10 @@ def test_new_resolver_requires_python( script.pip(*args) - assert_installed(script, base="0.1.0", dep=dep_version) + script.assert_installed(base="0.1.0", dep=dep_version) -def test_new_resolver_requires_python_error(script): +def test_new_resolver_requires_python_error(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -392,8 +399,10 @@ def test_new_resolver_requires_python_error(script): ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base", expect_error=True, ) @@ -405,7 +414,7 @@ def test_new_resolver_requires_python_error(script): assert message in result.stderr, str(result) -def test_new_resolver_installed(script): +def test_new_resolver_installed(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -420,27 +429,29 @@ def test_new_resolver_installed(script): result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base", ) assert "Requirement already satisfied" not in result.stdout, str(result) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base~=0.1.0", ) - assert "Requirement already satisfied: base~=0.1.0" in result.stdout, \ - str(result) + assert "Requirement already satisfied: base~=0.1.0" in result.stdout, str(result) result.did_not_update( - script.site_packages / "base", - message="base 0.1.0 reinstalled" + script.site_packages / "base", message="base 0.1.0 reinstalled" ) -def test_new_resolver_ignore_installed(script): +def test_new_resolver_ignore_installed(script: PipTestEnvironment) -> None: create_basic_wheel_for_package( script, "base", @@ -450,26 +461,32 @@ def test_new_resolver_ignore_installed(script): result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base", ) assert satisfied_output not in result.stdout, str(result) result = script.pip( "install", - "--no-cache-dir", "--no-index", "--ignore-installed", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--ignore-installed", + "--find-links", + script.scratch_path, "base", ) assert satisfied_output not in result.stdout, str(result) result.did_update( - script.site_packages / "base", - message="base 0.1.0 not reinstalled" + script.site_packages / "base", message="base 0.1.0 not reinstalled" ) -def test_new_resolver_only_builds_sdists_when_needed(script): +def test_new_resolver_only_builds_sdists_when_needed( + script: PipTestEnvironment, +) -> None: create_basic_wheel_for_package( script, "base", @@ -491,57 +508,65 @@ def test_new_resolver_only_builds_sdists_when_needed(script): # We only ever need to check dep 0.2.0 as it's the latest version script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "base" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "base", ) - assert_installed(script, base="0.1.0", dep="0.2.0") + script.assert_installed(base="0.1.0", dep="0.2.0") # We merge criteria here, as we have two "dep" requirements script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "base", "dep" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "base", + "dep", ) - assert_installed(script, base="0.1.0", dep="0.2.0") + script.assert_installed(base="0.1.0", dep="0.2.0") -def test_new_resolver_install_different_version(script): +def test_new_resolver_install_different_version(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "base", "0.1.0") create_basic_wheel_for_package(script, "base", "0.2.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==0.1.0", ) # This should trigger an uninstallation of base. result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==0.2.0", ) assert "Uninstalling base-0.1.0" in result.stdout, str(result) assert "Successfully uninstalled base-0.1.0" in result.stdout, str(result) - result.did_update( - script.site_packages / "base", - message="base not upgraded" - ) - assert_installed(script, base="0.2.0") + result.did_update(script.site_packages / "base", message="base not upgraded") + script.assert_installed(base="0.2.0") -def test_new_resolver_force_reinstall(script): +def test_new_resolver_force_reinstall(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "base", "0.1.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==0.1.0", ) @@ -549,19 +574,18 @@ def test_new_resolver_force_reinstall(script): # even though the installed version matches. result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--force-reinstall", "base==0.1.0", ) assert "Uninstalling base-0.1.0" in result.stdout, str(result) assert "Successfully uninstalled base-0.1.0" in result.stdout, str(result) - result.did_update( - script.site_packages / "base", - message="base not reinstalled" - ) - assert_installed(script, base="0.1.0") + result.did_update(script.site_packages / "base", message="base not reinstalled") + script.assert_installed(base="0.1.0") @pytest.mark.parametrize( @@ -579,20 +603,22 @@ def test_new_resolver_force_reinstall(script): ids=["default", "exact-pre", "explicit-pre", "no-stable"], ) def test_new_resolver_handles_prerelease( - script, - available_versions, - pip_args, - expected_version, -): + script: PipTestEnvironment, + available_versions: List[str], + pip_args: List[str], + expected_version: str, +) -> None: for version in available_versions: create_basic_wheel_for_package(script, "pkg", version) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - *pip_args + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + *pip_args, ) - assert_installed(script, pkg=expected_version) + script.assert_installed(pkg=expected_version) @pytest.mark.parametrize( @@ -602,20 +628,24 @@ def test_new_resolver_handles_prerelease( (["dep; os_name == 'nonexist_os'"], ["pkg"]), # This tests the marker is picked up from a root dependency. ([], ["pkg", "dep; os_name == 'nonexist_os'"]), - ] + ], ) -def test_new_reolver_skips_marker(script, pkg_deps, root_deps): +def test_new_resolver_skips_marker( + script: PipTestEnvironment, pkg_deps: List[str], root_deps: List[str] +) -> None: create_basic_wheel_for_package(script, "pkg", "1.0", depends=pkg_deps) create_basic_wheel_for_package(script, "dep", "1.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - *root_deps + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + *root_deps, ) - assert_installed(script, pkg="1.0") - assert_not_installed(script, "dep") + script.assert_installed(pkg="1.0") + script.assert_not_installed("dep") @pytest.mark.parametrize( @@ -625,9 +655,11 @@ def test_new_reolver_skips_marker(script, pkg_deps, root_deps): # This also tests the pkg constraint don't get merged with the # requirement prematurely. (pypa/pip#8134) ["pkg<2.0"], - ] + ], ) -def test_new_resolver_constraints(script, constraints): +def test_new_resolver_constraints( + script: PipTestEnvironment, constraints: List[str] +) -> None: create_basic_wheel_for_package(script, "pkg", "1.0") create_basic_wheel_for_package(script, "pkg", "2.0") create_basic_wheel_for_package(script, "pkg", "3.0") @@ -635,28 +667,34 @@ def test_new_resolver_constraints(script, constraints): constraints_file.write_text("\n".join(constraints)) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-c", constraints_file, - "pkg" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "pkg", ) - assert_installed(script, pkg="1.0") - assert_not_installed(script, "constraint_only") + script.assert_installed(pkg="1.0") + script.assert_not_installed("constraint_only") -def test_new_resolver_constraint_no_specifier(script): +def test_new_resolver_constraint_no_specifier(script: PipTestEnvironment) -> None: "It's allowed (but useless...) for a constraint to have no specifier" create_basic_wheel_for_package(script, "pkg", "1.0") constraints_file = script.scratch_path / "constraints.txt" constraints_file.write_text("pkg") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-c", constraints_file, - "pkg" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "pkg", ) - assert_installed(script, pkg="1.0") + script.assert_installed(pkg="1.0") @pytest.mark.parametrize( @@ -667,8 +705,8 @@ def test_new_resolver_constraint_no_specifier(script): "Unnamed requirements are not allowed as constraints", ), ( - "req @ https://example.com/dist.zip", - "Links are not allowed as constraints", + "-e git+https://example.com/dist.git#egg=req", + "Editable requirements are not allowed as constraints", ), ( "pkg[extra]", @@ -676,15 +714,20 @@ def test_new_resolver_constraint_no_specifier(script): ), ], ) -def test_new_resolver_constraint_reject_invalid(script, constraint, error): +def test_new_resolver_constraint_reject_invalid( + script: PipTestEnvironment, constraint: str, error: str +) -> None: create_basic_wheel_for_package(script, "pkg", "1.0") constraints_file = script.scratch_path / "constraints.txt" constraints_file.write_text(constraint) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-c", constraints_file, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, "pkg", expect_error=True, expect_stderr=True, @@ -692,7 +735,7 @@ def test_new_resolver_constraint_reject_invalid(script, constraint, error): assert error in result.stderr, str(result) -def test_new_resolver_constraint_on_dependency(script): +def test_new_resolver_constraint_on_dependency(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "base", "1.0", depends=["dep"]) create_basic_wheel_for_package(script, "dep", "1.0") create_basic_wheel_for_package(script, "dep", "2.0") @@ -701,30 +744,32 @@ def test_new_resolver_constraint_on_dependency(script): constraints_file.write_text("dep==2.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-c", constraints_file, - "base" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "base", ) - assert_installed(script, base="1.0") - assert_installed(script, dep="2.0") + script.assert_installed(base="1.0") + script.assert_installed(dep="2.0") @pytest.mark.parametrize( "constraint_version, expect_error, message", [ - ("1.0", True, "ERROR: No matching distribution found for foo 2.0"), + ("1.0", True, "Cannot install foo 2.0"), ("2.0", False, "Successfully installed foo-2.0"), ], ) def test_new_resolver_constraint_on_path_empty( - script, - constraint_version, - expect_error, - message, -): - """A path requirement can be filtered by a constraint. - """ + script: PipTestEnvironment, + constraint_version: str, + expect_error: bool, + message: str, +) -> None: + """A path requirement can be filtered by a constraint.""" setup_py = script.scratch_path / "setup.py" text = "from setuptools import setup\nsetup(name='foo', version='2.0')" setup_py.write_text(text) @@ -734,8 +779,10 @@ def test_new_resolver_constraint_on_path_empty( result = script.pip( "install", - "--no-cache-dir", "--no-index", - "-c", constraints_txt, + "--no-cache-dir", + "--no-index", + "-c", + constraints_txt, str(script.scratch_path), expect_error=expect_error, ) @@ -746,37 +793,42 @@ def test_new_resolver_constraint_on_path_empty( assert message in result.stdout, str(result) -def test_new_resolver_constraint_only_marker_match(script): +def test_new_resolver_constraint_only_marker_match(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "pkg", "1.0") create_basic_wheel_for_package(script, "pkg", "2.0") create_basic_wheel_for_package(script, "pkg", "3.0") - constrants_content = textwrap.dedent( + constraints_content = textwrap.dedent( """ pkg==1.0; python_version == "{ver[0]}.{ver[1]}" # Always satisfies. pkg==2.0; python_version < "0" # Never satisfies. """ ).format(ver=sys.version_info) constraints_txt = script.scratch_path / "constraints.txt" - constraints_txt.write_text(constrants_content) + constraints_txt.write_text(constraints_content) script.pip( "install", - "--no-cache-dir", "--no-index", - "-c", constraints_txt, - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "-c", + constraints_txt, + "--find-links", + script.scratch_path, "pkg", ) - assert_installed(script, pkg="1.0") + script.assert_installed(pkg="1.0") -def test_new_resolver_upgrade_needs_option(script): +def test_new_resolver_upgrade_needs_option(script: PipTestEnvironment) -> None: # Install pkg 1.0.0 create_basic_wheel_for_package(script, "pkg", "1.0.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "pkg", ) @@ -786,44 +838,47 @@ def test_new_resolver_upgrade_needs_option(script): # This should not upgrade because we don't specify --upgrade result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "pkg", ) assert "Requirement already satisfied" in result.stdout, str(result) - assert_installed(script, pkg="1.0.0") + script.assert_installed(pkg="1.0.0") # This should upgrade result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--upgrade", "PKG", # Deliberately uppercase to check canonicalization ) assert "Uninstalling pkg-1.0.0" in result.stdout, str(result) assert "Successfully uninstalled pkg-1.0.0" in result.stdout, str(result) - result.did_update( - script.site_packages / "pkg", - message="pkg not upgraded" - ) - assert_installed(script, pkg="2.0.0") + result.did_update(script.site_packages / "pkg", message="pkg not upgraded") + script.assert_installed(pkg="2.0.0") -def test_new_resolver_upgrade_strategy(script): +def test_new_resolver_upgrade_strategy(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "base", "1.0.0", depends=["dep"]) create_basic_wheel_for_package(script, "dep", "1.0.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base", ) - assert_installed(script, base="1.0.0") - assert_installed(script, dep="1.0.0") + script.assert_installed(base="1.0.0") + script.assert_installed(dep="1.0.0") # Now release new versions create_basic_wheel_for_package(script, "base", "2.0.0", depends=["dep"]) @@ -831,29 +886,100 @@ def test_new_resolver_upgrade_strategy(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--upgrade", "base", ) # With upgrade strategy "only-if-needed" (the default), dep should not # be upgraded. - assert_installed(script, base="2.0.0") - assert_installed(script, dep="1.0.0") + script.assert_installed(base="2.0.0") + script.assert_installed(dep="1.0.0") create_basic_wheel_for_package(script, "base", "3.0.0", depends=["dep"]) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "--upgrade", "--upgrade-strategy=eager", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "--upgrade", + "--upgrade-strategy=eager", "base", ) # With upgrade strategy "eager", dep should be upgraded. - assert_installed(script, base="3.0.0") - assert_installed(script, dep="2.0.0") + script.assert_installed(base="3.0.0") + script.assert_installed(dep="2.0.0") + + +if TYPE_CHECKING: + + class PackageBuilder(Protocol): + def __call__( + self, + script: PipTestEnvironment, + name: str, + version: str, + requires: List[str], + extras: Dict[str, List[str]], + ) -> str: + ... + + +def _local_with_setup( + script: PipTestEnvironment, + name: str, + version: str, + requires: List[str], + extras: Dict[str, List[str]], +) -> str: + """Create the package as a local source directory to install from path.""" + return create_test_package_with_setup( + script, + name=name, + version=version, + install_requires=requires, + extras_require=extras, + ) + + +def _direct_wheel( + script: PipTestEnvironment, + name: str, + version: str, + requires: List[str], + extras: Dict[str, List[str]], +) -> str: + """Create the package as a wheel to install from path directly.""" + return create_basic_wheel_for_package( + script, + name=name, + version=version, + depends=requires, + extras=extras, + ) + + +def _wheel_from_index( + script: PipTestEnvironment, + name: str, + version: str, + requires: List[str], + extras: Dict[str, List[str]], +) -> str: + """Create the package as a wheel to install from index.""" + create_basic_wheel_for_package( + script, + name=name, + version=version, + depends=requires, + extras=extras, + ) + return name class TestExtraMerge: @@ -862,40 +988,6 @@ class TestExtraMerge: extras, one listed as required and the other as in extra. """ - def _local_with_setup(script, name, version, requires, extras): - """Create the package as a local source directory to install from path. - """ - return create_test_package_with_setup( - script, - name=name, - version=version, - install_requires=requires, - extras_require=extras, - ) - - def _direct_wheel(script, name, version, requires, extras): - """Create the package as a wheel to install from path directly. - """ - return create_basic_wheel_for_package( - script, - name=name, - version=version, - depends=requires, - extras=extras, - ) - - def _wheel_from_index(script, name, version, requires, extras): - """Create the package as a wheel to install from index. - """ - create_basic_wheel_for_package( - script, - name=name, - version=version, - depends=requires, - extras=extras, - ) - return name - @pytest.mark.parametrize( "pkg_builder", [ @@ -905,8 +997,8 @@ def _wheel_from_index(script, name, version, requires, extras): ], ) def test_new_resolver_extra_merge_in_package( - self, monkeypatch, script, pkg_builder, - ): + self, script: PipTestEnvironment, pkg_builder: "PackageBuilder" + ) -> None: create_basic_wheel_for_package(script, "depdev", "1.0.0") create_basic_wheel_for_package( script, @@ -924,14 +1016,16 @@ def test_new_resolver_extra_merge_in_package( script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, requirement + "[dev]", ) - assert_installed(script, pkg="1.0.0", dep="1.0.0", depdev="1.0.0") + script.assert_installed(pkg="1.0.0", dep="1.0.0", depdev="1.0.0") -def test_new_resolver_build_directory_error_zazo_19(script): +def test_new_resolver_build_directory_error_zazo_19(script: PipTestEnvironment) -> None: """https://github.com/pradyunsg/zazo/issues/19#issuecomment-631615674 This will first resolve like this: @@ -953,7 +1047,10 @@ def test_new_resolver_build_directory_error_zazo_19(script): can delete this. Please delete it and try again. """ create_basic_wheel_for_package( - script, "pkg_a", "3.0.0", depends=["pkg-b<2"], + script, + "pkg_a", + "3.0.0", + depends=["pkg-b<2"], ) create_basic_wheel_for_package(script, "pkg_a", "2.0.0") create_basic_wheel_for_package(script, "pkg_a", "1.0.0") @@ -963,36 +1060,43 @@ def test_new_resolver_build_directory_error_zazo_19(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "pkg-a", "pkg-b", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "pkg-a", + "pkg-b", ) - assert_installed(script, pkg_a="3.0.0", pkg_b="1.0.0") + script.assert_installed(pkg_a="3.0.0", pkg_b="1.0.0") -def test_new_resolver_upgrade_same_version(script): +def test_new_resolver_upgrade_same_version(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "pkg", "2") create_basic_wheel_for_package(script, "pkg", "1") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "pkg", ) - assert_installed(script, pkg="2") + script.assert_installed(pkg="2") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--upgrade", "pkg", ) - assert_installed(script, pkg="2") + script.assert_installed(pkg="2") -def test_new_resolver_local_and_req(script): +def test_new_resolver_local_and_req(script: PipTestEnvironment) -> None: source_dir = create_test_package_with_setup( script, name="pkg", @@ -1000,13 +1104,17 @@ def test_new_resolver_local_and_req(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - source_dir, "pkg!=0.1.0", + "--no-cache-dir", + "--no-index", + source_dir, + "pkg!=0.1.0", expect_error=True, ) -def test_new_resolver_no_deps_checks_requires_python(script): +def test_new_resolver_no_deps_checks_requires_python( + script: PipTestEnvironment, +) -> None: create_basic_wheel_for_package( script, "base", @@ -1025,7 +1133,8 @@ def test_new_resolver_no_deps_checks_requires_python(script): "--no-cache-dir", "--no-index", "--no-deps", - "--find-links", script.scratch_path, + "--find-links", + script.scratch_path, "base", expect_error=True, ) @@ -1037,7 +1146,9 @@ def test_new_resolver_no_deps_checks_requires_python(script): assert message in result.stderr -def test_new_resolver_prefers_installed_in_upgrade_if_latest(script): +def test_new_resolver_prefers_installed_in_upgrade_if_latest( + script: PipTestEnvironment, +) -> None: create_basic_wheel_for_package(script, "pkg", "1") local_pkg = create_test_package_with_setup(script, name="pkg", version="2") @@ -1054,17 +1165,20 @@ def test_new_resolver_prefers_installed_in_upgrade_if_latest(script): "install", "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--find-links", + script.scratch_path, "--upgrade", "pkg", ) - assert_installed(script, pkg="2") + script.assert_installed(pkg="2") @pytest.mark.parametrize("N", [2, 10, 20]) -def test_new_resolver_presents_messages_when_backtracking_a_lot(script, N): +def test_new_resolver_presents_messages_when_backtracking_a_lot( + script: PipTestEnvironment, N: int +) -> None: # Generate a set of wheels that will definitely cause backtracking. - for index in range(1, N+1): + for index in range(1, N + 1): A_version = f"{index}.0.0" B_version = f"{index}.0.0" C_version = "{index_minus_one}.0.0".format(index_minus_one=index - 1) @@ -1076,7 +1190,7 @@ def test_new_resolver_presents_messages_when_backtracking_a_lot(script, N): print("A", A_version, "B", B_version, "C", C_version) create_basic_wheel_for_package(script, "A", A_version, depends=depends) - for index in range(1, N+1): + for index in range(1, N + 1): B_version = f"{index}.0.0" C_version = f"{index}.0.0" depends = ["C == " + C_version] @@ -1084,7 +1198,7 @@ def test_new_resolver_presents_messages_when_backtracking_a_lot(script, N): print("B", B_version, "C", C_version) create_basic_wheel_for_package(script, "B", B_version, depends=depends) - for index in range(1, N+1): + for index in range(1, N + 1): C_version = f"{index}.0.0" print("C", C_version) create_basic_wheel_for_package(script, "C", C_version) @@ -1094,11 +1208,12 @@ def test_new_resolver_presents_messages_when_backtracking_a_lot(script, N): "install", "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "A" + "--find-links", + script.scratch_path, + "A", ) - assert_installed(script, A="1.0.0", B="1.0.0", C="1.0.0") + script.assert_installed(A="1.0.0", B="1.0.0", C="1.0.0") # These numbers are hard-coded in the code. if N >= 1: assert "This could take a while." in result.stdout @@ -1113,7 +1228,6 @@ def test_new_resolver_presents_messages_when_backtracking_a_lot(script, N): [ "0.1.0+local.1", # Normalized form. "0.1.0+local_1", # Non-normalized form containing an underscore. - # Non-normalized form containing a dash. This is allowed, installation # works correctly, but assert_installed() fails because pkg_resources # cannot handle it correctly. Nobody is complaining about it right now, @@ -1132,10 +1246,10 @@ def test_new_resolver_presents_messages_when_backtracking_a_lot(script, N): ids=["file_dot", "file_underscore"], ) def test_new_resolver_check_wheel_version_normalized( - script, - metadata_version, - filename_version, -): + script: PipTestEnvironment, + metadata_version: str, + filename_version: str, +) -> None: filename = f"simple-{filename_version}-py2.py3-none-any.whl" wheel_builder = make_wheel(name="simple", version=metadata_version) @@ -1143,56 +1257,63 @@ def test_new_resolver_check_wheel_version_normalized( script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple", ) - assert_installed(script, simple="0.1.0+local.1") + script.assert_installed(simple="0.1.0+local.1") -def test_new_resolver_does_reinstall_local_sdists(script): +def test_new_resolver_does_reinstall_local_sdists(script: PipTestEnvironment) -> None: archive_path = create_basic_sdist_for_package( script, "pkg", "1.0", ) script.pip( - "install", "--no-cache-dir", "--no-index", + "install", + "--no-cache-dir", + "--no-index", archive_path, ) - assert_installed(script, pkg="1.0") + script.assert_installed(pkg="1.0") result = script.pip( - "install", "--no-cache-dir", "--no-index", + "install", + "--no-cache-dir", + "--no-index", archive_path, expect_stderr=True, ) assert "Installing collected packages: pkg" in result.stdout, str(result) - assert "DEPRECATION" in result.stderr, str(result) - assert_installed(script, pkg="1.0") + script.assert_installed(pkg="1.0") -def test_new_resolver_does_reinstall_local_paths(script): - pkg = create_test_package_with_setup( - script, - name="pkg", - version="1.0" - ) +def test_new_resolver_does_reinstall_local_paths(script: PipTestEnvironment) -> None: + pkg = create_test_package_with_setup(script, name="pkg", version="1.0") script.pip( - "install", "--no-cache-dir", "--no-index", + "install", + "--no-cache-dir", + "--no-index", pkg, ) - assert_installed(script, pkg="1.0") + script.assert_installed(pkg="1.0") result = script.pip( - "install", "--no-cache-dir", "--no-index", + "install", + "--no-cache-dir", + "--no-index", pkg, ) assert "Installing collected packages: pkg" in result.stdout, str(result) - assert_installed(script, pkg="1.0") + script.assert_installed(pkg="1.0") -def test_new_resolver_does_not_reinstall_when_from_a_local_index(script): +def test_new_resolver_does_not_reinstall_when_from_a_local_index( + script: PipTestEnvironment, +) -> None: create_basic_sdist_for_package( script, "simple", @@ -1200,25 +1321,29 @@ def test_new_resolver_does_not_reinstall_when_from_a_local_index(script): ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple", ) - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "simple" + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "simple", ) # Should not reinstall! assert "Installing collected packages: simple" not in result.stdout, str(result) assert "Requirement already satisfied: simple" in result.stdout, str(result) - assert_installed(script, simple="0.1.0") + script.assert_installed(simple="0.1.0") -def test_new_resolver_skip_inconsistent_metadata(script): +def test_new_resolver_skip_inconsistent_metadata(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "A", "1") a_2 = create_basic_wheel_for_package(script, "A", "2") @@ -1226,15 +1351,19 @@ def test_new_resolver_skip_inconsistent_metadata(script): result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--verbose", "A", allow_stderr_warning=True, ) - assert " different version in metadata: '2'" in result.stderr, str(result) - assert_installed(script, a="1") + assert ( + " inconsistent version: filename has '3', but metadata has '2'" + ) in result.stdout, str(result) + script.assert_installed(a="1") @pytest.mark.parametrize( @@ -1242,7 +1371,9 @@ def test_new_resolver_skip_inconsistent_metadata(script): [True, False], ids=["upgrade", "no-upgrade"], ) -def test_new_resolver_lazy_fetch_candidates(script, upgrade): +def test_new_resolver_lazy_fetch_candidates( + script: PipTestEnvironment, upgrade: bool +) -> None: create_basic_wheel_for_package(script, "myuberpkg", "1") create_basic_wheel_for_package(script, "myuberpkg", "2") create_basic_wheel_for_package(script, "myuberpkg", "3") @@ -1250,8 +1381,10 @@ def test_new_resolver_lazy_fetch_candidates(script, upgrade): # Install an old version first. script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "myuberpkg==1", ) @@ -1262,18 +1395,978 @@ def test_new_resolver_lazy_fetch_candidates(script, upgrade): pip_upgrade_args = [] result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "myuberpkg", - *pip_upgrade_args # Trailing comma fails on Python 2. + *pip_upgrade_args, # Trailing comma fails on Python 2. ) # pip should install the version preferred by the strategy... if upgrade: - assert_installed(script, myuberpkg="3") + script.assert_installed(myuberpkg="3") else: - assert_installed(script, myuberpkg="1") + script.assert_installed(myuberpkg="1") # But should reach there in the best route possible, without trying # candidates it does not need to. assert "myuberpkg-2" not in result.stdout, str(result) + + +def test_new_resolver_no_fetch_no_satisfying(script: PipTestEnvironment) -> None: + create_basic_wheel_for_package(script, "myuberpkg", "1") + + # Install the package. This should emit a "Processing" message for + # fetching the distribution from the --find-links page. + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "myuberpkg", + ) + assert "Processing " in result.stdout, str(result) + + # Try to upgrade the package. This should NOT emit the "Processing" + # message because the currently installed version is latest. + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "--upgrade", + "myuberpkg", + ) + assert "Processing " not in result.stdout, str(result) + + +def test_new_resolver_does_not_install_unneeded_packages_with_url_constraint( + script: PipTestEnvironment, +) -> None: + archive_path = create_basic_wheel_for_package( + script, + "installed", + "0.1.0", + ) + not_installed_path = create_basic_wheel_for_package( + script, + "not_installed", + "0.1.0", + ) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("not_installed @ " + path_to_url(not_installed_path)) + + (script.scratch_path / "index").mkdir() + archive_path.rename(script.scratch_path / "index" / archive_path.name) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path / "index", + "-c", + constraints_file, + "installed", + ) + + script.assert_installed(installed="0.1.0") + script.assert_not_installed("not_installed") + + +def test_new_resolver_installs_packages_with_url_constraint( + script: PipTestEnvironment, +) -> None: + installed_path = create_basic_wheel_for_package( + script, + "installed", + "0.1.0", + ) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("installed @ " + path_to_url(installed_path)) + + script.pip( + "install", "--no-cache-dir", "--no-index", "-c", constraints_file, "installed" + ) + + script.assert_installed(installed="0.1.0") + + +def test_new_resolver_reinstall_link_requirement_with_constraint( + script: PipTestEnvironment, +) -> None: + installed_path = create_basic_wheel_for_package( + script, + "installed", + "0.1.0", + ) + + cr_file = script.scratch_path / "constraints.txt" + cr_file.write_text("installed @ " + path_to_url(installed_path)) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "-r", + cr_file, + ) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "-c", + cr_file, + "-r", + cr_file, + ) + # TODO: strengthen assertion to "second invocation does no work" + # I don't think this is true yet, but it should be in the future. + + script.assert_installed(installed="0.1.0") + + +def test_new_resolver_prefers_url_constraint(script: PipTestEnvironment) -> None: + installed_path = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + not_installed_path = create_basic_wheel_for_package( + script, + "test_pkg", + "0.2.0", + ) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("test_pkg @ " + path_to_url(installed_path)) + + (script.scratch_path / "index").mkdir() + not_installed_path.rename(script.scratch_path / "index" / not_installed_path.name) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path / "index", + "-c", + constraints_file, + "test_pkg", + ) + + script.assert_installed(test_pkg="0.1.0") + + +def test_new_resolver_prefers_url_constraint_on_update( + script: PipTestEnvironment, +) -> None: + installed_path = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + not_installed_path = create_basic_wheel_for_package( + script, + "test_pkg", + "0.2.0", + ) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("test_pkg @ " + path_to_url(installed_path)) + + (script.scratch_path / "index").mkdir() + not_installed_path.rename(script.scratch_path / "index" / not_installed_path.name) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path / "index", + "test_pkg", + ) + + script.assert_installed(test_pkg="0.2.0") + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path / "index", + "-c", + constraints_file, + "test_pkg", + ) + + script.assert_installed(test_pkg="0.1.0") + + +@pytest.mark.parametrize("version_option", ["--constraint", "--requirement"]) +def test_new_resolver_fails_with_url_constraint_and_incompatible_version( + script: PipTestEnvironment, + version_option: str, +) -> None: + not_installed_path = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + not_installed_path = create_basic_wheel_for_package( + script, + "test_pkg", + "0.2.0", + ) + + url_constraint = script.scratch_path / "constraints.txt" + url_constraint.write_text("test_pkg @ " + path_to_url(not_installed_path)) + + version_req = script.scratch_path / "requirements.txt" + version_req.write_text("test_pkg<0.2.0") + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "--constraint", + url_constraint, + version_option, + version_req, + "test_pkg", + expect_error=True, + ) + + assert "Cannot install test_pkg" in result.stderr, str(result) + assert ( + "because these package versions have conflicting dependencies." + ) in result.stderr, str(result) + + script.assert_not_installed("test_pkg") + + # Assert that pip works properly in the absence of the constraints file. + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + version_option, + version_req, + "test_pkg", + ) + + +def test_new_resolver_ignores_unneeded_conflicting_constraints( + script: PipTestEnvironment, +) -> None: + version_1 = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + version_2 = create_basic_wheel_for_package( + script, + "test_pkg", + "0.2.0", + ) + create_basic_wheel_for_package( + script, + "installed", + "0.1.0", + ) + + constraints = [ + "test_pkg @ " + path_to_url(version_1), + "test_pkg @ " + path_to_url(version_2), + ] + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("\n".join(constraints)) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "installed", + ) + + script.assert_not_installed("test_pkg") + script.assert_installed(installed="0.1.0") + + +def test_new_resolver_fails_on_needed_conflicting_constraints( + script: PipTestEnvironment, +) -> None: + version_1 = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + version_2 = create_basic_wheel_for_package( + script, + "test_pkg", + "0.2.0", + ) + + constraints = [ + "test_pkg @ " + path_to_url(version_1), + "test_pkg @ " + path_to_url(version_2), + ] + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("\n".join(constraints)) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "test_pkg", + expect_error=True, + ) + + assert ( + "Cannot install test_pkg because these package versions have conflicting " + "dependencies." + ) in result.stderr, str(result) + + script.assert_not_installed("test_pkg") + + # Assert that pip works properly in the absence of the constraints file. + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "test_pkg", + ) + + +def test_new_resolver_fails_on_conflicting_constraint_and_requirement( + script: PipTestEnvironment, +) -> None: + version_1 = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + version_2 = create_basic_wheel_for_package( + script, + "test_pkg", + "0.2.0", + ) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("test_pkg @ " + path_to_url(version_1)) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "test_pkg @ " + path_to_url(version_2), + expect_error=True, + ) + + assert "Cannot install test-pkg 0.2.0" in result.stderr, str(result) + assert ( + "because these package versions have conflicting dependencies." + ) in result.stderr, str(result) + + script.assert_not_installed("test_pkg") + + # Assert that pip works properly in the absence of the constraints file. + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "test_pkg @ " + path_to_url(version_2), + ) + + +@pytest.mark.parametrize("editable", [False, True]) +def test_new_resolver_succeeds_on_matching_constraint_and_requirement( + script: PipTestEnvironment, editable: bool +) -> None: + if editable: + source_dir = create_test_package_with_setup( + script, name="test_pkg", version="0.1.0" + ) + else: + source_dir = create_basic_wheel_for_package( + script, + "test_pkg", + "0.1.0", + ) + + req_line = "test_pkg @ " + path_to_url(source_dir) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text(req_line) + + last_args: Tuple[str, ...] + if editable: + last_args = ("-e", source_dir) + else: + last_args = (req_line,) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "-c", + constraints_file, + *last_args, + ) + + script.assert_installed(test_pkg="0.1.0") + if editable: + assert_editable(script, "test-pkg") + + +def test_new_resolver_applies_url_constraint_to_dep(script: PipTestEnvironment) -> None: + version_1 = create_basic_wheel_for_package( + script, + "dep", + "0.1.0", + ) + version_2 = create_basic_wheel_for_package( + script, + "dep", + "0.2.0", + ) + + base = create_basic_wheel_for_package(script, "base", "0.1.0", depends=["dep"]) + + (script.scratch_path / "index").mkdir() + base.rename(script.scratch_path / "index" / base.name) + version_2.rename(script.scratch_path / "index" / version_2.name) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("dep @ " + path_to_url(version_1)) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "-c", + constraints_file, + "--find-links", + script.scratch_path / "index", + "base", + ) + + script.assert_installed(dep="0.1.0") + + +def test_new_resolver_handles_compatible_wheel_tags_in_constraint_url( + script: PipTestEnvironment, make_fake_wheel: Callable[[str, str, str], Path] +) -> None: + initial_path = make_fake_wheel("base", "0.1.0", "fakepy1-fakeabi-fakeplat") + + constrained = script.scratch_path / "constrained" + constrained.mkdir() + + final_path = constrained / initial_path.name + + initial_path.rename(final_path) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("base @ " + path_to_url(final_path)) + + result = script.pip( + "install", + "--implementation", + "fakepy", + "--only-binary=:all:", + "--python-version", + "1", + "--abi", + "fakeabi", + "--platform", + "fakeplat", + "--target", + script.scratch_path / "target", + "--no-cache-dir", + "--no-index", + "-c", + constraints_file, + "base", + ) + + dist_info = Path("scratch", "target", "base-0.1.0.dist-info") + result.did_create(dist_info) + + +def test_new_resolver_handles_incompatible_wheel_tags_in_constraint_url( + script: PipTestEnvironment, make_fake_wheel: Callable[[str, str, str], Path] +) -> None: + initial_path = make_fake_wheel("base", "0.1.0", "fakepy1-fakeabi-fakeplat") + + constrained = script.scratch_path / "constrained" + constrained.mkdir() + + final_path = constrained / initial_path.name + + initial_path.rename(final_path) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("base @ " + path_to_url(final_path)) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "-c", + constraints_file, + "base", + expect_error=True, + ) + + assert ( + "Cannot install base because these package versions have conflicting " + "dependencies." + ) in result.stderr, str(result) + + script.assert_not_installed("base") + + +def test_new_resolver_avoids_incompatible_wheel_tags_in_constraint_url( + script: PipTestEnvironment, make_fake_wheel: Callable[[str, str, str], Path] +) -> None: + initial_path = make_fake_wheel("dep", "0.1.0", "fakepy1-fakeabi-fakeplat") + + constrained = script.scratch_path / "constrained" + constrained.mkdir() + + final_path = constrained / initial_path.name + + initial_path.rename(final_path) + + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("dep @ " + path_to_url(final_path)) + + index = script.scratch_path / "index" + index.mkdir() + + index_dep = create_basic_wheel_for_package(script, "dep", "0.2.0") + + base = create_basic_wheel_for_package(script, "base", "0.1.0") + base_2 = create_basic_wheel_for_package(script, "base", "0.2.0", depends=["dep"]) + + index_dep.rename(index / index_dep.name) + base.rename(index / base.name) + base_2.rename(index / base_2.name) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "-c", + constraints_file, + "--find-links", + script.scratch_path / "index", + "base", + ) + + script.assert_installed(base="0.1.0") + script.assert_not_installed("dep") + + +@pytest.mark.parametrize( + "suffixes_equivalent, depend_suffix, request_suffix", + [ + pytest.param( + True, + "#egg=foo", + "", + id="drop-depend-egg", + ), + pytest.param( + True, + "", + "#egg=foo", + id="drop-request-egg", + ), + pytest.param( + True, + "#subdirectory=bar&egg=foo", + "#subdirectory=bar&egg=bar", + id="drop-egg-only", + ), + pytest.param( + True, + "#subdirectory=bar&egg=foo", + "#egg=foo&subdirectory=bar", + id="fragment-ordering", + ), + pytest.param( + True, + "?a=1&b=2", + "?b=2&a=1", + id="query-opordering", + ), + pytest.param( + False, + "#sha512=1234567890abcdef", + "#sha512=abcdef1234567890", + id="different-keys", + ), + pytest.param( + False, + "#sha512=1234567890abcdef", + "#md5=1234567890abcdef", + id="different-values", + ), + pytest.param( + False, + "#subdirectory=bar&egg=foo", + "#subdirectory=rex", + id="drop-egg-still-different", + ), + ], +) +def test_new_resolver_direct_url_equivalent( + tmp_path: pathlib.Path, + script: PipTestEnvironment, + suffixes_equivalent: bool, + depend_suffix: str, + request_suffix: str, +) -> None: + pkga = create_basic_wheel_for_package(script, name="pkga", version="1") + pkgb = create_basic_wheel_for_package( + script, + name="pkgb", + version="1", + depends=[f"pkga@{path_to_url(pkga)}{depend_suffix}"], + ) + + # Make pkgb visible via --find-links, but not pkga. + find_links = tmp_path.joinpath("find_links") + find_links.mkdir() + with open(pkgb, "rb") as f: + find_links.joinpath(pkgb.name).write_bytes(f.read()) + + # Install pkgb from --find-links, and pkga directly but from a different + # URL suffix as specified in pkgb. This should work! + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + str(find_links), + f"{path_to_url(pkga)}{request_suffix}", + "pkgb", + expect_error=(not suffixes_equivalent), + ) + + if suffixes_equivalent: + script.assert_installed(pkga="1", pkgb="1") + else: + script.assert_not_installed("pkga", "pkgb") + + +def test_new_resolver_direct_url_with_extras( + tmp_path: pathlib.Path, script: PipTestEnvironment +) -> None: + pkg1 = create_basic_wheel_for_package(script, name="pkg1", version="1") + pkg2 = create_basic_wheel_for_package( + script, + name="pkg2", + version="1", + extras={"ext": ["pkg1"]}, + ) + pkg3 = create_basic_wheel_for_package( + script, + name="pkg3", + version="1", + depends=["pkg2[ext]"], + ) + + # Make pkg1 and pkg3 visible via --find-links, but not pkg2. + find_links = tmp_path.joinpath("find_links") + find_links.mkdir() + with open(pkg1, "rb") as f: + find_links.joinpath(pkg1.name).write_bytes(f.read()) + with open(pkg3, "rb") as f: + find_links.joinpath(pkg3.name).write_bytes(f.read()) + + # Install with pkg2 only available with direct URL. The extra-ed direct + # URL pkg2 should be able to provide pkg2[ext] required by pkg3. + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + str(find_links), + pkg2, + "pkg3", + ) + + script.assert_installed(pkg1="1", pkg2="1", pkg3="1") + assert not get_created_direct_url(result, "pkg1") + assert get_created_direct_url(result, "pkg2") + assert not get_created_direct_url(result, "pkg3") + + +def test_new_resolver_modifies_installed_incompatible( + script: PipTestEnvironment, +) -> None: + create_basic_wheel_for_package(script, name="a", version="1") + create_basic_wheel_for_package(script, name="a", version="2") + create_basic_wheel_for_package(script, name="a", version="3") + create_basic_wheel_for_package(script, name="b", version="1", depends=["a==1"]) + create_basic_wheel_for_package(script, name="b", version="2", depends=["a==2"]) + create_basic_wheel_for_package(script, name="c", version="1", depends=["a!=1"]) + create_basic_wheel_for_package(script, name="c", version="2", depends=["a!=1"]) + create_basic_wheel_for_package(script, name="d", version="1", depends=["b", "c"]) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "b==1", + ) + + # d-1 depends on b and c. b-1 is already installed and therefore first + # pinned, but later found to be incompatible since the "a==1" dependency + # makes all c versions impossible to satisfy. The resolver should be able to + # discard b-1 and backtrack, so b-2 is selected instead. + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "d==1", + ) + script.assert_installed(d="1", c="2", b="2", a="2") + + +def test_new_resolver_transitively_depends_on_unnamed_local( + script: PipTestEnvironment, +) -> None: + create_basic_wheel_for_package(script, name="certbot-docs", version="1") + certbot = create_test_package_with_setup( + script, + name="certbot", + version="99.99.0.dev0", + extras_require={"docs": ["certbot-docs"]}, + ) + certbot_apache = create_test_package_with_setup( + script, + name="certbot-apache", + version="99.99.0.dev0", + install_requires=["certbot>=99.99.0.dev0"], + ) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + f"{certbot}[docs]", + certbot_apache, + ) + script.assert_installed( + certbot="99.99.0.dev0", + certbot_apache="99.99.0.dev0", + certbot_docs="1", + ) + + +def _to_uri(path: str) -> str: + # Something like file:///path/to/package + return pathlib.Path(path).as_uri() + + +def _to_localhost_uri(path: str) -> str: + # Something like file://localhost/path/to/package + return pathlib.Path(path).as_uri().replace("///", "//localhost/") + + +@pytest.mark.parametrize( + "format_dep", + [ + pytest.param(_to_uri, id="emptyhost"), + pytest.param(_to_localhost_uri, id="localhost"), + ], +) +@pytest.mark.parametrize( + "format_input", + [ + pytest.param(lambda path: path, id="path"), + pytest.param(_to_uri, id="emptyhost"), + pytest.param(_to_localhost_uri, id="localhost"), + ], +) +def test_new_resolver_file_url_normalize( + script: PipTestEnvironment, + format_dep: Callable[[str], str], + format_input: Callable[[str], str], +) -> None: + lib_a = create_test_package_with_setup( + script, + name="lib_a", + version="1", + ) + lib_b = create_test_package_with_setup( + script, + name="lib_b", + version="1", + install_requires=[f"lib_a @ {format_dep(lib_a)}"], + ) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + format_input(lib_a), + lib_b, + ) + script.assert_installed(lib_a="1", lib_b="1") + + +def test_new_resolver_dont_backtrack_on_extra_if_base_constrained( + script: PipTestEnvironment, +) -> None: + create_basic_wheel_for_package(script, "dep", "1.0") + create_basic_wheel_for_package(script, "pkg", "1.0", extras={"ext": ["dep"]}) + create_basic_wheel_for_package(script, "pkg", "2.0", extras={"ext": ["dep"]}) + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("pkg==1.0") + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "--constraint", + constraints_file, + "pkg[ext]", + ) + assert "pkg-2.0" not in result.stdout, "Should not try 2.0 due to constraint" + script.assert_installed(pkg="1.0", dep="1.0") + + +def test_new_resolver_respect_user_requested_if_extra_is_installed( + script: PipTestEnvironment, +) -> None: + create_basic_wheel_for_package(script, "pkg1", "1.0") + create_basic_wheel_for_package(script, "pkg2", "1.0", extras={"ext": ["pkg1"]}) + create_basic_wheel_for_package(script, "pkg2", "2.0", extras={"ext": ["pkg1"]}) + create_basic_wheel_for_package(script, "pkg3", "1.0", depends=["pkg2[ext]"]) + + # Install pkg3 with an older pkg2. + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "pkg3", + "pkg2==1.0", + ) + script.assert_installed(pkg3="1.0", pkg2="1.0", pkg1="1.0") + + # Now upgrade both pkg3 and pkg2. pkg2 should be upgraded although pkg2[ext] + # is not requested by the user. + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "--upgrade", + "pkg3", + "pkg2", + ) + script.assert_installed(pkg3="1.0", pkg2="2.0", pkg1="1.0") + + +def test_new_resolver_do_not_backtrack_on_build_failure( + script: PipTestEnvironment, +) -> None: + create_basic_sdist_for_package(script, "pkg1", "2.0", fails_egg_info=True) + create_basic_wheel_for_package(script, "pkg1", "1.0") + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "pkg1", + expect_error=True, + ) + + assert "egg_info" in result.stderr + + +def test_new_resolver_flag_permits_backtracking_on_build_failure( + script: PipTestEnvironment, +) -> None: + create_basic_sdist_for_package(script, "pkg1", "2.0", fails_egg_info=True) + create_basic_wheel_for_package(script, "pkg1", "1.0") + + script.pip( + "install", + "--use-deprecated=backtrack-on-build-failures", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "pkg1", + allow_stderr_warning=True, + ) + + script.assert_installed(pkg1="1.0") + + +def test_new_resolver_works_when_failing_package_builds_are_disallowed( + script: PipTestEnvironment, +) -> None: + create_basic_wheel_for_package(script, "pkg2", "1.0", depends=["pkg1"]) + create_basic_sdist_for_package(script, "pkg1", "2.0", fails_egg_info=True) + create_basic_wheel_for_package(script, "pkg1", "1.0") + constraints_file = script.scratch_path / "constraints.txt" + constraints_file.write_text("pkg1 != 2.0") + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "pkg2", + ) + + script.assert_installed(pkg2="1.0", pkg1="1.0") diff --git a/tests/functional/test_new_resolver_errors.py b/tests/functional/test_new_resolver_errors.py index 830acc764e9..1278bc3edde 100644 --- a/tests/functional/test_new_resolver_errors.py +++ b/tests/functional/test_new_resolver_errors.py @@ -1,14 +1,30 @@ -from tests.lib import create_basic_wheel_for_package +import pathlib +import sys +from tests.lib import ( + PipTestEnvironment, + create_basic_wheel_for_package, + create_test_package_with_setup, +) +from tests.lib.path import Path -def test_new_resolver_conflict_requirements_file(tmpdir, script): + +def test_new_resolver_conflict_requirements_file( + tmpdir: Path, script: PipTestEnvironment +) -> None: create_basic_wheel_for_package(script, "base", "1.0") create_basic_wheel_for_package(script, "base", "2.0") create_basic_wheel_for_package( - script, "pkga", "1.0", depends=["base==1.0"], + script, + "pkga", + "1.0", + depends=["base==1.0"], ) create_basic_wheel_for_package( - script, "pkgb", "1.0", depends=["base==2.0"], + script, + "pkgb", + "1.0", + depends=["base==2.0"], ) req_file = tmpdir.joinpath("requirements.txt") @@ -16,11 +32,102 @@ def test_new_resolver_conflict_requirements_file(tmpdir, script): result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, - "-r", req_file, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-r", + req_file, expect_error=True, ) message = "package versions have conflicting dependencies" assert message in result.stderr, str(result) + + +def test_new_resolver_conflict_constraints_file( + tmpdir: Path, script: PipTestEnvironment +) -> None: + create_basic_wheel_for_package(script, "pkg", "1.0") + + constraints_file = tmpdir.joinpath("constraints.txt") + constraints_file.write_text("pkg!=1.0") + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "-c", + constraints_file, + "pkg==1.0", + expect_error=True, + ) + + assert "ResolutionImpossible" in result.stderr, str(result) + + message = "The user requested (constraint) pkg!=1.0" + assert message in result.stdout, str(result) + + +def test_new_resolver_requires_python_error(script: PipTestEnvironment) -> None: + compatible_python = ">={0.major}.{0.minor}".format(sys.version_info) + incompatible_python = "<{0.major}.{0.minor}".format(sys.version_info) + + pkga = create_test_package_with_setup( + script, + name="pkga", + version="1.0", + python_requires=compatible_python, + ) + pkgb = create_test_package_with_setup( + script, + name="pkgb", + version="1.0", + python_requires=incompatible_python, + ) + + # This always fails because pkgb can never be satisfied. + result = script.pip("install", "--no-index", pkga, pkgb, expect_error=True) + + # The error message should mention the Requires-Python: value causing the + # conflict, not the compatible one. + assert incompatible_python in result.stderr, str(result) + assert compatible_python not in result.stderr, str(result) + + +def test_new_resolver_checks_requires_python_before_dependencies( + script: PipTestEnvironment, +) -> None: + incompatible_python = "<{0.major}.{0.minor}".format(sys.version_info) + + pkg_dep = create_basic_wheel_for_package( + script, + name="pkg-dep", + version="1", + ) + create_basic_wheel_for_package( + script, + name="pkg-root", + version="1", + # Refer the dependency by URL to prioritise it as much as possible, + # to test that Requires-Python is *still* inspected first. + depends=[f"pkg-dep@{pathlib.Path(pkg_dep).as_uri()}"], + requires_python=incompatible_python, + ) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "pkg-root", + expect_error=True, + ) + + # Resolution should fail because of pkg-a's Requires-Python. + # This check should be done before pkg-b, so pkg-b should never be pulled. + assert incompatible_python in result.stderr, str(result) + assert "pkg-b" not in result.stderr, str(result) diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 854b66418ae..008a4284c1d 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -4,14 +4,19 @@ import pytest from pip._internal.utils.urls import path_to_url -from tests.lib import create_basic_sdist_for_package, create_basic_wheel_for_package +from tests.lib import ( + PipTestEnvironment, + create_basic_sdist_for_package, + create_basic_wheel_for_package, +) _FindLinks = collections.namedtuple( - "_FindLinks", "index_html sdist_hash wheel_hash", + "_FindLinks", + "index_html sdist_hash wheel_hash", ) -def _create_find_links(script): +def _create_find_links(script: PipTestEnvironment) -> _FindLinks: sdist_path = create_basic_sdist_for_package(script, "base", "0.1.0") wheel_path = create_basic_wheel_for_package(script, "base", "0.1.0") @@ -21,6 +26,7 @@ def _create_find_links(script): index_html = script.scratch_path / "index.html" index_html.write_text( """ + {sdist_path.stem} {wheel_path.stem} """.format( @@ -30,7 +36,7 @@ def _create_find_links(script): wheel_url=path_to_url(wheel_path), wheel_hash=wheel_hash, wheel_path=wheel_path, - ) + ).strip() ) return _FindLinks(index_html, sdist_hash, wheel_hash) @@ -59,7 +65,9 @@ def _create_find_links(script): ], ids=["identical", "intersect"], ) -def test_new_resolver_hash_intersect(script, requirements_template, message): +def test_new_resolver_hash_intersect( + script: PipTestEnvironment, requirements_template: str, message: str +) -> None: find_links = _create_find_links(script) requirements_txt = script.scratch_path / "requirements.txt" @@ -75,15 +83,19 @@ def test_new_resolver_hash_intersect(script, requirements_template, message): "--no-cache-dir", "--no-deps", "--no-index", - "--find-links", find_links.index_html, - "--verbose", - "--requirement", requirements_txt, + "--find-links", + find_links.index_html, + "-vv", + "--requirement", + requirements_txt, ) assert message.format(name="base") in result.stdout, str(result) -def test_new_resolver_hash_intersect_from_constraint(script): +def test_new_resolver_hash_intersect_from_constraint( + script: PipTestEnvironment, +) -> None: find_links = _create_find_links(script) constraints_txt = script.scratch_path / "constraints.txt" @@ -107,10 +119,13 @@ def test_new_resolver_hash_intersect_from_constraint(script): "--no-cache-dir", "--no-deps", "--no-index", - "--find-links", find_links.index_html, - "--verbose", - "--constraint", constraints_txt, - "--requirement", requirements_txt, + "--find-links", + find_links.index_html, + "-vv", + "--constraint", + constraints_txt, + "--requirement", + requirements_txt, ) message = ( @@ -138,8 +153,10 @@ def test_new_resolver_hash_intersect_from_constraint(script): ids=["both-requirements", "one-each"], ) def test_new_resolver_hash_intersect_empty( - script, requirements_template, constraints_template, -): + script: PipTestEnvironment, + requirements_template: str, + constraints_template: str, +) -> None: find_links = _create_find_links(script) constraints_txt = script.scratch_path / "constraints.txt" @@ -163,9 +180,12 @@ def test_new_resolver_hash_intersect_empty( "--no-cache-dir", "--no-deps", "--no-index", - "--find-links", find_links.index_html, - "--constraint", constraints_txt, - "--requirement", requirements_txt, + "--find-links", + find_links.index_html, + "--constraint", + constraints_txt, + "--requirement", + requirements_txt, expect_error=True, ) @@ -174,7 +194,9 @@ def test_new_resolver_hash_intersect_empty( ) in result.stderr, str(result) -def test_new_resolver_hash_intersect_empty_from_constraint(script): +def test_new_resolver_hash_intersect_empty_from_constraint( + script: PipTestEnvironment, +) -> None: find_links = _create_find_links(script) constraints_txt = script.scratch_path / "constraints.txt" @@ -193,8 +215,10 @@ def test_new_resolver_hash_intersect_empty_from_constraint(script): "--no-cache-dir", "--no-deps", "--no-index", - "--find-links", find_links.index_html, - "--constraint", constraints_txt, + "--find-links", + find_links.index_html, + "--constraint", + constraints_txt, "base==0.1.0", expect_error=True, ) @@ -204,3 +228,179 @@ def test_new_resolver_hash_intersect_empty_from_constraint(script): "from some requirements." ) assert message in result.stderr, str(result) + + +@pytest.mark.parametrize("constrain_by_hash", [False, True]) +def test_new_resolver_hash_requirement_and_url_constraint_can_succeed( + script: PipTestEnvironment, + constrain_by_hash: bool, +) -> None: + wheel_path = create_basic_wheel_for_package(script, "base", "0.1.0") + + wheel_hash = hashlib.sha256(wheel_path.read_bytes()).hexdigest() + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + """ + base==0.1.0 --hash=sha256:{wheel_hash} + """.format( + wheel_hash=wheel_hash, + ), + ) + + constraints_txt = script.scratch_path / "constraints.txt" + constraint_text = "base @ {wheel_url}\n".format(wheel_url=path_to_url(wheel_path)) + if constrain_by_hash: + constraint_text += "base==0.1.0 --hash=sha256:{wheel_hash}\n".format( + wheel_hash=wheel_hash, + ) + constraints_txt.write_text(constraint_text) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--constraint", + constraints_txt, + "--requirement", + requirements_txt, + ) + + script.assert_installed(base="0.1.0") + + +@pytest.mark.parametrize("constrain_by_hash", [False, True]) +def test_new_resolver_hash_requirement_and_url_constraint_can_fail( + script: PipTestEnvironment, + constrain_by_hash: bool, +) -> None: + wheel_path = create_basic_wheel_for_package(script, "base", "0.1.0") + other_path = create_basic_wheel_for_package(script, "other", "0.1.0") + + other_hash = hashlib.sha256(other_path.read_bytes()).hexdigest() + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + """ + base==0.1.0 --hash=sha256:{other_hash} + """.format( + other_hash=other_hash, + ), + ) + + constraints_txt = script.scratch_path / "constraints.txt" + constraint_text = "base @ {wheel_url}\n".format(wheel_url=path_to_url(wheel_path)) + if constrain_by_hash: + constraint_text += "base==0.1.0 --hash=sha256:{other_hash}\n".format( + other_hash=other_hash, + ) + constraints_txt.write_text(constraint_text) + + result = script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--constraint", + constraints_txt, + "--requirement", + requirements_txt, + expect_error=True, + ) + + assert ( + "THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE." + ) in result.stderr, str(result) + + script.assert_not_installed("base", "other") + + +def test_new_resolver_hash_with_extras(script: PipTestEnvironment) -> None: + parent_with_extra_path = create_basic_wheel_for_package( + script, "parent_with_extra", "0.1.0", depends=["child[extra]"] + ) + parent_with_extra_hash = hashlib.sha256( + parent_with_extra_path.read_bytes() + ).hexdigest() + + parent_without_extra_path = create_basic_wheel_for_package( + script, "parent_without_extra", "0.1.0", depends=["child"] + ) + parent_without_extra_hash = hashlib.sha256( + parent_without_extra_path.read_bytes() + ).hexdigest() + + child_path = create_basic_wheel_for_package( + script, "child", "0.1.0", extras={"extra": ["extra"]} + ) + child_hash = hashlib.sha256(child_path.read_bytes()).hexdigest() + + # Newer release + create_basic_wheel_for_package( + script, "child", "0.2.0", extras={"extra": ["extra"]} + ) + + extra_path = create_basic_wheel_for_package(script, "extra", "0.1.0") + extra_hash = hashlib.sha256(extra_path.read_bytes()).hexdigest() + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text( + """ + child[extra]==0.1.0 --hash=sha256:{child_hash} + parent_with_extra==0.1.0 --hash=sha256:{parent_with_extra_hash} + parent_without_extra==0.1.0 --hash=sha256:{parent_without_extra_hash} + extra==0.1.0 --hash=sha256:{extra_hash} + """.format( + child_hash=child_hash, + parent_with_extra_hash=parent_with_extra_hash, + parent_without_extra_hash=parent_without_extra_hash, + extra_hash=extra_hash, + ), + ) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, + "--requirement", + requirements_txt, + ) + + script.assert_installed( + parent_with_extra="0.1.0", + parent_without_extra="0.1.0", + child="0.1.0", + extra="0.1.0", + ) + + +def test_new_resolver_hash_with_pin(script: PipTestEnvironment) -> None: + find_links = _create_find_links(script) + + requirements_txt = script.scratch_path / "requirements.txt" + requirements_txt.write_text("base") + + constraints_txt = script.scratch_path / "constraints.txt" + constraints_txt.write_text( + """ + base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash} + """.format( + sdist_hash=find_links.sdist_hash, + wheel_hash=find_links.wheel_hash, + ) + ) + + script.pip( + "install", + "--no-cache-dir", + "--no-index", + "--find-links", + find_links.index_html, + "--requirement", + requirements_txt, + "--constraint", + constraints_txt, + ) + + script.assert_installed(base="0.1.0") diff --git a/tests/functional/test_new_resolver_target.py b/tests/functional/test_new_resolver_target.py index f5ec6ac7a09..398695b5a83 100644 --- a/tests/functional/test_new_resolver_target.py +++ b/tests/functional/test_new_resolver_target.py @@ -1,14 +1,18 @@ +from typing import Callable, Optional + import pytest from pip._internal.cli.status_codes import ERROR, SUCCESS +from tests.lib import PipTestEnvironment from tests.lib.path import Path from tests.lib.wheel import make_wheel +MakeFakeWheel = Callable[[str], Path] -@pytest.fixture() -def make_fake_wheel(script): - def _make_fake_wheel(wheel_tag): +@pytest.fixture() +def make_fake_wheel(script: PipTestEnvironment) -> MakeFakeWheel: + def _make_fake_wheel(wheel_tag: str) -> Path: wheel_house = script.scratch_path.joinpath("wheelhouse") wheel_house.mkdir() wheel_builder = make_wheel( @@ -28,19 +32,21 @@ def _make_fake_wheel(wheel_tag): @pytest.mark.parametrize("abi", [None, "fakeabi"]) @pytest.mark.parametrize("platform", [None, "fakeplat"]) def test_new_resolver_target_checks_compatibility_failure( - script, - make_fake_wheel, - implementation, - python_version, - abi, - platform, -): + script: PipTestEnvironment, + make_fake_wheel: MakeFakeWheel, + implementation: Optional[str], + python_version: Optional[str], + abi: Optional[str], + platform: Optional[str], +) -> None: fake_wheel_tag = "fakepy1-fakeabi-fakeplat" args = [ "install", "--only-binary=:all:", - "--no-cache-dir", "--no-index", - "--target", str(script.scratch_path.joinpath("target")), + "--no-cache-dir", + "--no-index", + "--target", + str(script.scratch_path.joinpath("target")), make_fake_wheel(fake_wheel_tag), ] if implementation: @@ -58,7 +64,7 @@ def test_new_resolver_target_checks_compatibility_failure( abi, platform, ) - wheel_tag_matches = (args_tag == fake_wheel_tag) + wheel_tag_matches = args_tag == fake_wheel_tag result = script.pip(*args, expect_error=(not wheel_tag_matches)) diff --git a/tests/functional/test_new_resolver_user.py b/tests/functional/test_new_resolver_user.py index dd617318cef..acdae71c9cf 100644 --- a/tests/functional/test_new_resolver_user.py +++ b/tests/functional/test_new_resolver_user.py @@ -3,16 +3,19 @@ import pytest -from tests.lib import create_basic_wheel_for_package +from tests.lib import PipTestEnvironment, create_basic_wheel_for_package +from tests.lib.venv import VirtualEnvironment @pytest.mark.incompatible_with_test_venv -def test_new_resolver_install_user(script): +def test_new_resolver_install_user(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "base", "0.1.0") result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base", ) @@ -20,7 +23,9 @@ def test_new_resolver_install_user(script): @pytest.mark.incompatible_with_test_venv -def test_new_resolver_install_user_satisfied_by_global_site(script): +def test_new_resolver_install_user_satisfied_by_global_site( + script: PipTestEnvironment, +) -> None: """ An install a matching version to user site should re-use a global site installation if it satisfies. @@ -29,14 +34,18 @@ def test_new_resolver_install_user_satisfied_by_global_site(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==1.0.0", ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base==1.0.0", ) @@ -45,7 +54,9 @@ def test_new_resolver_install_user_satisfied_by_global_site(script): @pytest.mark.incompatible_with_test_venv -def test_new_resolver_install_user_conflict_in_user_site(script): +def test_new_resolver_install_user_conflict_in_user_site( + script: PipTestEnvironment, +) -> None: """ Installing a different version in user site should uninstall an existing different version in user site. @@ -55,16 +66,20 @@ def test_new_resolver_install_user_conflict_in_user_site(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base==2.0.0", ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base==1.0.0", ) @@ -77,20 +92,26 @@ def test_new_resolver_install_user_conflict_in_user_site(script): @pytest.mark.incompatible_with_test_venv -def test_new_resolver_install_user_in_virtualenv_with_conflict_fails(script): +def test_new_resolver_install_user_in_virtualenv_with_conflict_fails( + script: PipTestEnvironment, +) -> None: create_basic_wheel_for_package(script, "base", "1.0.0") create_basic_wheel_for_package(script, "base", "2.0.0") script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==2.0.0", ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base==1.0.0", expect_error=True, @@ -104,23 +125,27 @@ def test_new_resolver_install_user_in_virtualenv_with_conflict_fails(script): @pytest.fixture() -def patch_dist_in_site_packages(virtualenv): +def patch_dist_in_site_packages(virtualenv: VirtualEnvironment) -> None: # Since the tests are run from a virtualenv, and to avoid the "Will not # install to the usersite because it will lack sys.path precedence..." - # error: Monkey patch `dist_in_site_packages` in the resolver module so - # it's possible to install a conflicting distribution in the user site. - virtualenv.sitecustomize = textwrap.dedent(""" + # error: Monkey patch `pip._internal.utils.misc.dist_in_site_packages` + # so it's possible to install a conflicting distribution in the user site. + virtualenv.sitecustomize = textwrap.dedent( + """ def dist_in_site_packages(dist): return False - from pip._internal.resolution.resolvelib import factory - factory.dist_in_site_packages = dist_in_site_packages - """) + from pip._internal.metadata.base import BaseDistribution + BaseDistribution.in_site_packages = property(dist_in_site_packages) + """ + ) @pytest.mark.incompatible_with_test_venv @pytest.mark.usefixtures("patch_dist_in_site_packages") -def test_new_resolver_install_user_reinstall_global_site(script): +def test_new_resolver_install_user_reinstall_global_site( + script: PipTestEnvironment, +) -> None: """ Specifying --force-reinstall makes a different version in user site, ignoring the matching installation in global site. @@ -129,14 +154,18 @@ def test_new_resolver_install_user_reinstall_global_site(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==1.0.0", ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "--force-reinstall", "base==1.0.0", @@ -150,7 +179,9 @@ def test_new_resolver_install_user_reinstall_global_site(script): @pytest.mark.incompatible_with_test_venv @pytest.mark.usefixtures("patch_dist_in_site_packages") -def test_new_resolver_install_user_conflict_in_global_site(script): +def test_new_resolver_install_user_conflict_in_global_site( + script: PipTestEnvironment, +) -> None: """ Installing a different version in user site should ignore an existing different version in global site, and simply add to the user site. @@ -160,15 +191,19 @@ def test_new_resolver_install_user_conflict_in_global_site(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==1.0.0", ) result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base==2.0.0", ) @@ -182,7 +217,9 @@ def test_new_resolver_install_user_conflict_in_global_site(script): @pytest.mark.incompatible_with_test_venv @pytest.mark.usefixtures("patch_dist_in_site_packages") -def test_new_resolver_install_user_conflict_in_global_and_user_sites(script): +def test_new_resolver_install_user_conflict_in_global_and_user_sites( + script: PipTestEnvironment, +) -> None: """ Installing a different version in user site should ignore an existing different version in global site, but still upgrade the user site. @@ -192,14 +229,18 @@ def test_new_resolver_install_user_conflict_in_global_and_user_sites(script): script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "base==2.0.0", ) script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "--force-reinstall", "base==2.0.0", @@ -207,8 +248,10 @@ def test_new_resolver_install_user_conflict_in_global_and_user_sites(script): result = script.pip( "install", - "--no-cache-dir", "--no-index", - "--find-links", script.scratch_path, + "--no-cache-dir", + "--no-index", + "--find-links", + script.scratch_path, "--user", "base==1.0.0", ) diff --git a/tests/functional/test_no_color.py b/tests/functional/test_no_color.py index 48ed3ff7848..4094bdd167a 100644 --- a/tests/functional/test_no_color.py +++ b/tests/functional/test_no_color.py @@ -2,14 +2,18 @@ Test specific for the --no-color option """ import os +import shutil import subprocess +import sys import pytest +from tests.lib import PipTestEnvironment -def test_no_color(script): - """Ensure colour output disabled when --no-color is passed. - """ + +@pytest.mark.skipif(shutil.which("script") is None, reason="no 'script' executable") +def test_no_color(script: PipTestEnvironment) -> None: + """Ensure colour output disabled when --no-color is passed.""" # Using 'script' in this test allows for transparently testing pip's output # since pip is smart enough to disable colour output when piped, which is # not the behaviour we want to be testing here. @@ -18,27 +22,28 @@ def test_no_color(script): # 'script' and well as the mere use of the same. # # This test will stay until someone has the time to rewrite it. - command = ( - 'script --flush --quiet --return /tmp/pip-test-no-color.txt ' - '--command "pip uninstall {} noSuchPackage"' - ) + pip_command = "pip uninstall {} noSuchPackage" + if sys.platform == "darwin": + command = f"script -q /tmp/pip-test-no-color.txt {pip_command}" + else: + command = f'script -q /tmp/pip-test-no-color.txt --command "{pip_command}"' - def get_run_output(option): + def get_run_output(option: str = "") -> str: cmd = command.format(option) proc = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) proc.communicate() - if proc.returncode: - pytest.skip("Unable to capture output using script: " + cmd) try: - with open("/tmp/pip-test-no-color.txt", "r") as output_file: + with open("/tmp/pip-test-no-color.txt") as output_file: retval = output_file.read() return retval finally: os.unlink("/tmp/pip-test-no-color.txt") - assert "\x1b" in get_run_output(option=""), "Expected color in output" - assert "\x1b" not in get_run_output(option="--no-color"), \ - "Expected no color in output" + assert "\x1b[3" in get_run_output(""), "Expected color in output" + assert "\x1b[3" not in get_run_output("--no-color"), "Expected no color in output" diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index bcad4793672..cae3040ab0b 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -1,26 +1,34 @@ +from typing import Any, Dict, List, Optional, Tuple + import pytest -from pip._vendor import toml +import tomli_w from pip._internal.build_env import BuildEnvironment from pip._internal.req import InstallRequirement -from tests.lib import make_test_finder, path_to_url, windows_workaround_7667 +from tests.lib import PipTestEnvironment, TestData, make_test_finder, path_to_url +from tests.lib.path import Path -def make_project(tmpdir, requires=None, backend=None, backend_path=None): +def make_project( + tmpdir: Path, + requires: Optional[List[str]] = None, + backend: Optional[str] = None, + backend_path: Optional[List[str]] = None, +) -> Path: requires = requires or [] - project_dir = tmpdir / 'project' + project_dir = tmpdir / "project" project_dir.mkdir() - buildsys = {'requires': requires} + buildsys: Dict[str, Any] = {"requires": requires} if backend: - buildsys['build-backend'] = backend + buildsys["build-backend"] = backend if backend_path: - buildsys['backend-path'] = backend_path - data = toml.dumps({'build-system': buildsys}) - project_dir.joinpath('pyproject.toml').write_text(data) + buildsys["backend-path"] = backend_path + data = tomli_w.dumps({"build-system": buildsys}) + project_dir.joinpath("pyproject.toml").write_text(data) return project_dir -def test_backend(tmpdir, data): +def test_backend(tmpdir: Path, data: TestData) -> None: """Check we can call a requirement's backend successfully""" project_dir = make_project(tmpdir, backend="dummy_backend") req = InstallRequirement(None, None) @@ -28,11 +36,12 @@ def test_backend(tmpdir, data): req.load_pyproject_toml() env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], 'normal', "Installing") + env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") conflicting, missing = env.check_requirements(["dummy_backend"]) assert not conflicting and not missing - assert hasattr(req.pep517_backend, 'build_wheel') + assert hasattr(req.pep517_backend, "build_wheel") with env: + assert req.pep517_backend is not None assert req.pep517_backend.build_wheel("dir") == "Backend called" @@ -46,28 +55,27 @@ def build_wheel( """ -def test_backend_path(tmpdir, data): +def test_backend_path(tmpdir: Path, data: TestData) -> None: """Check we can call a backend inside the project""" - project_dir = make_project( - tmpdir, backend="dummy_backend", backend_path=['.'] - ) - (project_dir / 'dummy_backend.py').write_text(dummy_backend_code) + project_dir = make_project(tmpdir, backend="dummy_backend", backend_path=["."]) + (project_dir / "dummy_backend.py").write_text(dummy_backend_code) req = InstallRequirement(None, None) req.source_dir = project_dir # make req believe it has been unpacked req.load_pyproject_toml() env = BuildEnvironment() - assert hasattr(req.pep517_backend, 'build_wheel') + assert hasattr(req.pep517_backend, "build_wheel") with env: + assert req.pep517_backend is not None assert req.pep517_backend.build_wheel("dir") == "Backend called" -def test_backend_path_and_dep(tmpdir, data): +def test_backend_path_and_dep(tmpdir: Path, data: TestData) -> None: """Check we can call a requirement's backend successfully""" project_dir = make_project( - tmpdir, backend="dummy_internal_backend", backend_path=['.'] + tmpdir, backend="dummy_internal_backend", backend_path=["."] ) - (project_dir / 'dummy_internal_backend.py').write_text( + (project_dir / "dummy_internal_backend.py").write_text( "from dummy_backend import build_wheel" ) req = InstallRequirement(None, None) @@ -75,196 +83,295 @@ def test_backend_path_and_dep(tmpdir, data): req.load_pyproject_toml() env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], 'normal', "Installing") + env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") - assert hasattr(req.pep517_backend, 'build_wheel') + assert hasattr(req.pep517_backend, "build_wheel") with env: + assert req.pep517_backend is not None assert req.pep517_backend.build_wheel("dir") == "Backend called" -def test_pep517_install(script, tmpdir, data): +def test_pep517_install( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: """Check we can build with a custom backend""" project_dir = make_project( - tmpdir, requires=['test_backend'], - backend="test_backend" - ) - result = script.pip( - 'install', '--no-index', '-f', data.backends, project_dir + tmpdir, requires=["test_backend"], backend="test_backend" ) - result.assert_installed('project', editable=False) + result = script.pip("install", "--no-index", "-f", data.backends, project_dir) + result.assert_installed("project", editable=False) -def test_pep517_install_with_reqs(script, tmpdir, data): +def test_pep517_install_with_reqs( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: """Backend generated requirements are installed in the build env""" project_dir = make_project( - tmpdir, requires=['test_backend'], - backend="test_backend" + tmpdir, requires=["test_backend"], backend="test_backend" ) project_dir.joinpath("backend_reqs.txt").write_text("simplewheel") result = script.pip( - 'install', '--no-index', - '-f', data.backends, - '-f', data.packages, - project_dir + "install", "--no-index", "-f", data.backends, "-f", data.packages, project_dir ) - result.assert_installed('project', editable=False) + result.assert_installed("project", editable=False) -def test_no_use_pep517_without_setup_py(script, tmpdir, data): +def test_no_use_pep517_without_setup_py( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: """Using --no-use-pep517 requires setup.py""" project_dir = make_project( - tmpdir, requires=['test_backend'], - backend="test_backend" + tmpdir, requires=["test_backend"], backend="test_backend" ) result = script.pip( - 'install', '--no-index', '--no-use-pep517', - '-f', data.backends, + "install", + "--no-index", + "--no-use-pep517", + "-f", + data.backends, project_dir, - expect_error=True + expect_error=True, ) - assert 'project does not have a setup.py' in result.stderr + assert "project does not have a setup.py" in result.stderr -def test_conflicting_pep517_backend_requirements(script, tmpdir, data): +def test_conflicting_pep517_backend_requirements( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: project_dir = make_project( - tmpdir, requires=['test_backend', 'simplewheel==1.0'], - backend="test_backend" + tmpdir, requires=["test_backend", "simplewheel==1.0"], backend="test_backend" ) project_dir.joinpath("backend_reqs.txt").write_text("simplewheel==2.0") result = script.pip( - 'install', '--no-index', - '-f', data.backends, - '-f', data.packages, + "install", + "--no-index", + "-f", + data.backends, + "-f", + data.packages, + project_dir, + expect_error=True, + ) + msg = ( + "Some build dependencies for {url} conflict with the backend " + "dependencies: simplewheel==1.0 is incompatible with " + "simplewheel==2.0.".format(url=path_to_url(project_dir)) + ) + assert result.returncode != 0 and msg in result.stderr, str(result) + + +def test_validate_missing_pep517_backend_requirements( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: + project_dir = make_project( + tmpdir, requires=["test_backend", "simplewheel==1.0"], backend="test_backend" + ) + result = script.pip( + "install", + "--no-index", + "-f", + data.backends, + "-f", + data.packages, + "--no-build-isolation", + project_dir, + expect_error=True, + ) + msg = ( + "Some build dependencies for {url} are missing: " + "'simplewheel==1.0', 'test_backend'.".format(url=path_to_url(project_dir)) + ) + assert result.returncode != 0 and msg in result.stderr, str(result) + + +def test_validate_conflicting_pep517_backend_requirements( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: + project_dir = make_project( + tmpdir, requires=["simplewheel==1.0"], backend="test_backend" + ) + script.pip("install", "simplewheel==2.0", "--no-index", "-f", data.packages) + result = script.pip( + "install", + "--no-index", + "-f", + data.backends, + "-f", + data.packages, + "--no-build-isolation", project_dir, - expect_error=True + expect_error=True, ) msg = ( - 'Some build dependencies for {url} conflict with the backend ' - 'dependencies: simplewheel==1.0 is incompatible with ' - 'simplewheel==2.0.'.format(url=path_to_url(project_dir))) - assert ( - result.returncode != 0 and - msg in result.stderr - ), str(result) + "Some build dependencies for {url} conflict with the backend " + "dependencies: simplewheel==2.0 is incompatible with " + "simplewheel==1.0.".format(url=path_to_url(project_dir)) + ) + assert result.returncode != 0 and msg in result.stderr, str(result) -def test_pep517_backend_requirements_already_satisfied(script, tmpdir, data): +def test_pep517_backend_requirements_already_satisfied( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: project_dir = make_project( - tmpdir, requires=['test_backend', 'simplewheel==1.0'], - backend="test_backend" + tmpdir, requires=["test_backend", "simplewheel==1.0"], backend="test_backend" ) project_dir.joinpath("backend_reqs.txt").write_text("simplewheel") result = script.pip( - 'install', '--no-index', - '-f', data.backends, - '-f', data.packages, + "install", + "--no-index", + "-f", + data.backends, + "-f", + data.packages, project_dir, ) - assert 'Installing backend dependencies:' not in result.stdout + assert "Installing backend dependencies:" not in result.stdout -def test_pep517_install_with_no_cache_dir(script, tmpdir, data): - """Check builds with a custom backends work, even with no cache. - """ +def test_pep517_install_with_no_cache_dir( + script: PipTestEnvironment, tmpdir: Path, data: TestData +) -> None: + """Check builds with a custom backends work, even with no cache.""" project_dir = make_project( - tmpdir, requires=['test_backend'], - backend="test_backend" + tmpdir, requires=["test_backend"], backend="test_backend" ) result = script.pip( - 'install', '--no-cache-dir', '--no-index', '-f', data.backends, + "install", + "--no-cache-dir", + "--no-index", + "-f", + data.backends, project_dir, ) - result.assert_installed('project', editable=False) + result.assert_installed("project", editable=False) -def make_pyproject_with_setup(tmpdir, build_system=True, set_backend=True): - project_dir = tmpdir / 'project' +def make_pyproject_with_setup( + tmpdir: Path, build_system: bool = True, set_backend: bool = True +) -> Tuple[Path, str]: + project_dir = tmpdir / "project" project_dir.mkdir() - setup_script = ( - 'from setuptools import setup\n' - ) + setup_script = "from setuptools import setup\n" expect_script_dir_on_path = True if build_system: - buildsys = { - 'requires': ['setuptools', 'wheel'], + buildsys: Dict[str, Any] = { + "requires": ["setuptools", "wheel"], } if set_backend: - buildsys['build-backend'] = 'setuptools.build_meta' + buildsys["build-backend"] = "setuptools.build_meta" expect_script_dir_on_path = False - project_data = toml.dumps({'build-system': buildsys}) + project_data = tomli_w.dumps({"build-system": buildsys}) else: - project_data = '' + project_data = "" if expect_script_dir_on_path: - setup_script += ( - 'from pep517_test import __version__\n' - ) + setup_script += "from pep517_test import __version__\n" else: setup_script += ( - 'try:\n' - ' import pep517_test\n' - 'except ImportError:\n' - ' pass\n' - 'else:\n' + "try:\n" + " import pep517_test\n" + "except ImportError:\n" + " pass\n" + "else:\n" ' raise RuntimeError("Source dir incorrectly on sys.path")\n' ) - setup_script += ( - 'setup(name="pep517_test", version="0.1", packages=["pep517_test"])' - ) + setup_script += 'setup(name="pep517_test", version="0.1", packages=["pep517_test"])' - project_dir.joinpath('pyproject.toml').write_text(project_data) - project_dir.joinpath('setup.py').write_text(setup_script) + project_dir.joinpath("pyproject.toml").write_text(project_data) + project_dir.joinpath("setup.py").write_text(setup_script) package_dir = project_dir / "pep517_test" package_dir.mkdir() - package_dir.joinpath('__init__.py').write_text('__version__ = "0.1"') + package_dir.joinpath("__init__.py").write_text('__version__ = "0.1"') return project_dir, "pep517_test" -def test_no_build_system_section(script, tmpdir, data, common_wheels): - """Check builds with setup.py, pyproject.toml, but no build-system section. - """ +def test_no_build_system_section( + script: PipTestEnvironment, tmpdir: Path, data: TestData, common_wheels: Path +) -> None: + """Check builds with setup.py, pyproject.toml, but no build-system section.""" project_dir, name = make_pyproject_with_setup(tmpdir, build_system=False) result = script.pip( - 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + "install", + "--no-cache-dir", + "--no-index", + "-f", + common_wheels, project_dir, ) result.assert_installed(name, editable=False) -def test_no_build_backend_entry(script, tmpdir, data, common_wheels): - """Check builds with setup.py, pyproject.toml, but no build-backend entry. - """ +def test_no_build_backend_entry( + script: PipTestEnvironment, tmpdir: Path, data: TestData, common_wheels: Path +) -> None: + """Check builds with setup.py, pyproject.toml, but no build-backend entry.""" project_dir, name = make_pyproject_with_setup(tmpdir, set_backend=False) result = script.pip( - 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + "install", + "--no-cache-dir", + "--no-index", + "-f", + common_wheels, project_dir, ) result.assert_installed(name, editable=False) -def test_explicit_setuptools_backend(script, tmpdir, data, common_wheels): - """Check builds with setup.py, pyproject.toml, and a build-backend entry. - """ +def test_explicit_setuptools_backend( + script: PipTestEnvironment, tmpdir: Path, data: TestData, common_wheels: Path +) -> None: + """Check builds with setup.py, pyproject.toml, and a build-backend entry.""" project_dir, name = make_pyproject_with_setup(tmpdir) result = script.pip( - 'install', '--no-cache-dir', '--no-index', '-f', common_wheels, + "install", + "--no-cache-dir", + "--no-index", + "-f", + common_wheels, project_dir, ) result.assert_installed(name, editable=False) @pytest.mark.network -@windows_workaround_7667 -def test_pep517_and_build_options(script, tmpdir, data, common_wheels): +def test_pep517_and_build_options( + script: PipTestEnvironment, tmpdir: Path, data: TestData, common_wheels: Path +) -> None: + """Backend generated requirements are installed in the build env""" + project_dir, name = make_pyproject_with_setup(tmpdir) + result = script.pip( + "wheel", + "--wheel-dir", + tmpdir, + "--build-option", + "foo", + "-f", + common_wheels, + project_dir, + allow_stderr_warning=True, + ) + assert "Ignoring --build-option when building" in result.stderr + assert "using PEP 517" in result.stderr + + +@pytest.mark.network +def test_pep517_and_global_options( + script: PipTestEnvironment, tmpdir: Path, data: TestData, common_wheels: Path +) -> None: """Backend generated requirements are installed in the build env""" project_dir, name = make_pyproject_with_setup(tmpdir) result = script.pip( - 'wheel', '--wheel-dir', tmpdir, - '--build-option', 'foo', - '-f', common_wheels, + "wheel", + "--wheel-dir", + tmpdir, + "--global-option", + "foo", + "-f", + common_wheels, project_dir, - expect_error=True + allow_stderr_warning=True, ) - assert 'Cannot build wheel' in result.stderr - assert 'when --build-option is present' in result.stderr + assert "Ignoring --global-option when building" in result.stderr + assert "using PEP 517" in result.stderr diff --git a/tests/functional/test_pep660.py b/tests/functional/test_pep660.py new file mode 100644 index 00000000000..c55fb80d37b --- /dev/null +++ b/tests/functional/test_pep660.py @@ -0,0 +1,233 @@ +import os +from typing import Any, Dict + +import pytest +import tomli_w + +from pip._internal.utils.urls import path_to_url +from tests.lib import PipTestEnvironment +from tests.lib.path import Path + +SETUP_PY = """ +from setuptools import setup + +setup() +""" + +SETUP_CFG = """ +[metadata] +name = project +version = 1.0.0 +""" + +BACKEND_WITHOUT_PEP660 = """ +from setuptools.build_meta import ( + build_wheel as _build_wheel, + prepare_metadata_for_build_wheel as _prepare_metadata_for_build_wheel, + get_requires_for_build_wheel as _get_requires_for_build_wheel, +) + +def get_requires_for_build_wheel(config_settings=None): + with open("log.txt", "a") as f: + print(":get_requires_for_build_wheel called", file=f) + return _get_requires_for_build_wheel(config_settings) + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): + with open("log.txt", "a") as f: + print(":prepare_metadata_for_build_wheel called", file=f) + return _prepare_metadata_for_build_wheel(metadata_directory, config_settings) + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + with open("log.txt", "a") as f: + print(":build_wheel called", file=f) + return _build_wheel(wheel_directory, config_settings, metadata_directory) +""" + +# fmt: off +BACKEND_WITH_PEP660 = BACKEND_WITHOUT_PEP660 + """ +def get_requires_for_build_editable(config_settings=None): + with open("log.txt", "a") as f: + print(":get_requires_for_build_editable called", file=f) + return _get_requires_for_build_wheel(config_settings) + +def prepare_metadata_for_build_editable(metadata_directory, config_settings=None): + with open("log.txt", "a") as f: + print(":prepare_metadata_for_build_editable called", file=f) + return _prepare_metadata_for_build_wheel(metadata_directory, config_settings) + +def build_editable(wheel_directory, config_settings=None, metadata_directory=None): + with open("log.txt", "a") as f: + print(":build_editable called", file=f) + return _build_wheel(wheel_directory, config_settings, metadata_directory) +""" +# fmt: on + + +def _make_project( + tmpdir: Path, backend_code: str, with_setup_py: bool, with_pyproject: bool = True +) -> Path: + project_dir = tmpdir / "project" + project_dir.mkdir() + project_dir.joinpath("setup.cfg").write_text(SETUP_CFG) + if with_setup_py: + project_dir.joinpath("setup.py").write_text(SETUP_PY) + if backend_code: + assert with_pyproject + buildsys: Dict[str, Any] = {"requires": ["setuptools", "wheel"]} + buildsys["build-backend"] = "test_backend" + buildsys["backend-path"] = ["."] + data = tomli_w.dumps({"build-system": buildsys}) + project_dir.joinpath("pyproject.toml").write_text(data) + project_dir.joinpath("test_backend.py").write_text(backend_code) + elif with_pyproject: + project_dir.joinpath("pyproject.toml").touch() + project_dir.joinpath("log.txt").touch() + return project_dir + + +def _assert_hook_called(project_dir: Path, hook: str) -> None: + log = project_dir.joinpath("log.txt").read_text() + assert f":{hook} called" in log, f"{hook} has not been called" + + +def _assert_hook_not_called(project_dir: Path, hook: str) -> None: + log = project_dir.joinpath("log.txt").read_text() + assert f":{hook} called" not in log, f"{hook} should not have been called" + + +@pytest.mark.usefixtures("with_wheel") +def test_install_pep517_basic(tmpdir: Path, script: PipTestEnvironment) -> None: + """ + Check that the test harness we have in this file is sane. + """ + project_dir = _make_project(tmpdir, BACKEND_WITHOUT_PEP660, with_setup_py=False) + script.pip( + "install", + "--no-index", + "--no-build-isolation", + project_dir, + ) + _assert_hook_called(project_dir, "prepare_metadata_for_build_wheel") + _assert_hook_called(project_dir, "build_wheel") + + +@pytest.mark.usefixtures("with_wheel") +def test_install_pep660_basic(tmpdir: Path, script: PipTestEnvironment) -> None: + """ + Test with backend that supports build_editable. + """ + project_dir = _make_project(tmpdir, BACKEND_WITH_PEP660, with_setup_py=False) + result = script.pip( + "install", + "--no-index", + "--no-build-isolation", + "--editable", + project_dir, + ) + _assert_hook_called(project_dir, "prepare_metadata_for_build_editable") + _assert_hook_called(project_dir, "build_editable") + assert ( + result.test_env.site_packages.joinpath("project.egg-link") + not in result.files_created + ), "a .egg-link file should not have been created" + + +@pytest.mark.usefixtures("with_wheel") +def test_install_no_pep660_setup_py_fallback( + tmpdir: Path, script: PipTestEnvironment +) -> None: + """ + Test that we fall back to setuptools develop when using a backend that + does not support build_editable. Since there is a pyproject.toml, + the prepare_metadata_for_build_wheel hook is called. + """ + project_dir = _make_project(tmpdir, BACKEND_WITHOUT_PEP660, with_setup_py=True) + result = script.pip( + "install", + "--no-index", + "--no-build-isolation", + "--editable", + project_dir, + allow_stderr_warning=False, + ) + _assert_hook_called(project_dir, "prepare_metadata_for_build_wheel") + assert ( + result.test_env.site_packages.joinpath("project.egg-link") + in result.files_created + ), "a .egg-link file should have been created" + + +@pytest.mark.usefixtures("with_wheel") +def test_install_no_pep660_setup_cfg_fallback( + tmpdir: Path, script: PipTestEnvironment +) -> None: + """ + Test that we fall back to setuptools develop when using a backend that + does not support build_editable. Since there is a pyproject.toml, + the prepare_metadata_for_build_wheel hook is called. + """ + project_dir = _make_project(tmpdir, BACKEND_WITHOUT_PEP660, with_setup_py=False) + result = script.pip( + "install", + "--no-index", + "--no-build-isolation", + "--editable", + project_dir, + allow_stderr_warning=False, + ) + print(result.stdout, result.stderr) + _assert_hook_called(project_dir, "prepare_metadata_for_build_wheel") + assert ( + result.test_env.site_packages.joinpath("project.egg-link") + in result.files_created + ), ".egg-link file should have been created" + + +@pytest.mark.usefixtures("with_wheel") +def test_wheel_editable_pep660_basic(tmpdir: Path, script: PipTestEnvironment) -> None: + """ + Test 'pip wheel' of an editable pep 660 project. + It must *not* call prepare_metadata_for_build_editable. + """ + project_dir = _make_project(tmpdir, BACKEND_WITH_PEP660, with_setup_py=False) + wheel_dir = tmpdir / "dist" + script.pip( + "wheel", + "--no-index", + "--no-build-isolation", + "--editable", + project_dir, + "-w", + wheel_dir, + ) + _assert_hook_not_called(project_dir, "prepare_metadata_for_build_editable") + _assert_hook_not_called(project_dir, "build_editable") + _assert_hook_called(project_dir, "prepare_metadata_for_build_wheel") + _assert_hook_called(project_dir, "build_wheel") + assert len(os.listdir(str(wheel_dir))) == 1, "a wheel should have been created" + + +@pytest.mark.usefixtures("with_wheel") +def test_download_editable_pep660_basic( + tmpdir: Path, script: PipTestEnvironment +) -> None: + """ + Test 'pip download' of an editable pep 660 project. + It must *not* call prepare_metadata_for_build_editable. + """ + project_dir = _make_project(tmpdir, BACKEND_WITH_PEP660, with_setup_py=False) + reqs_file = tmpdir / "requirements.txt" + reqs_file.write_text(f"-e {path_to_url(project_dir)}\n") + download_dir = tmpdir / "download" + script.pip( + "download", + "--no-index", + "--no-build-isolation", + "-r", + reqs_file, + "-d", + download_dir, + ) + _assert_hook_not_called(project_dir, "prepare_metadata_for_build_editable") + _assert_hook_called(project_dir, "prepare_metadata_for_build_wheel") + assert len(os.listdir(str(download_dir))) == 1, "a zip should have been created" diff --git a/tests/functional/test_requests.py b/tests/functional/test_requests.py index f8eef787c95..0fbc4ae0e36 100644 --- a/tests/functional/test_requests.py +++ b/tests/functional/test_requests.py @@ -1,17 +1,20 @@ import pytest +from tests.lib import PipTestEnvironment -@pytest.mark.skipif -def test_timeout(script): + +@pytest.mark.network +def test_timeout(script: PipTestEnvironment) -> None: result = script.pip( - "--timeout", "0.01", "install", "-vvv", "INITools", + "--timeout", + "0.0001", + "install", + "-vvv", + "INITools", expect_error=True, ) assert ( - "Could not fetch URL https://pypi.org/simple/INITools/: " - "timed out" in result.stdout - ) - assert ( - "Could not fetch URL https://pypi.org/simple/: " - "timed out" in result.stdout - ) + "Could not fetch URL https://pypi.org/simple/initools/: " + "connection error: HTTPSConnectionPool(host='pypi.org', port=443): " + "Max retries exceeded with url: /simple/initools/ " + ) in result.stdout diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py index b8bc6d51e2f..3f784e5dd1c 100644 --- a/tests/functional/test_search.py +++ b/tests/functional/test_search.py @@ -1,55 +1,60 @@ import logging +from typing import TYPE_CHECKING, Dict, List +from unittest import mock -import pretend import pytest from pip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pip._internal.commands import create_command from pip._internal.commands.search import highest_version, print_results, transform_hits +from tests.lib import PipTestEnvironment +if TYPE_CHECKING: + from pip._internal.commands.search import TransformedHit -def test_version_compare(): + +def test_version_compare() -> None: """ Test version comparison. """ - assert highest_version(['1.0', '2.0', '0.1']) == '2.0' - assert highest_version(['1.0a1', '1.0']) == '1.0' + assert highest_version(["1.0", "2.0", "0.1"]) == "2.0" + assert highest_version(["1.0a1", "1.0"]) == "1.0" -def test_pypi_xml_transformation(): +def test_pypi_xml_transformation() -> None: """ Test transformation of data structures (PyPI xmlrpc to custom list). """ - pypi_hits = [ + pypi_hits: List[Dict[str, str]] = [ { - 'name': 'foo', - 'summary': 'foo summary', - 'version': '1.0', + "name": "foo", + "summary": "foo summary", + "version": "1.0", }, { - 'name': 'foo', - 'summary': 'foo summary v2', - 'version': '2.0', + "name": "foo", + "summary": "foo summary v2", + "version": "2.0", }, { - '_pypi_ordering': 50, - 'name': 'bar', - 'summary': 'bar summary', - 'version': '1.0', + "_pypi_ordering": 50, # type: ignore[dict-item] + "name": "bar", + "summary": "bar summary", + "version": "1.0", }, ] - expected = [ + expected: List["TransformedHit"] = [ { - 'versions': ['1.0', '2.0'], - 'name': 'foo', - 'summary': 'foo summary v2', + "versions": ["1.0", "2.0"], + "name": "foo", + "summary": "foo summary v2", }, { - 'versions': ['1.0'], - 'name': 'bar', - 'summary': 'bar summary', + "versions": ["1.0"], + "name": "bar", + "summary": "bar summary", }, ] assert transform_hits(pypi_hits) == expected @@ -57,55 +62,51 @@ def test_pypi_xml_transformation(): @pytest.mark.network @pytest.mark.search -def test_basic_search(script): +def test_basic_search(script: PipTestEnvironment) -> None: """ End to end test of search command. """ - output = script.pip('search', 'pip') - assert ( - 'The PyPA recommended tool for installing ' - 'Python packages.' in output.stdout - ) + output = script.pip("search", "pip") + assert "The PyPA recommended tool for installing Python packages." in output.stdout @pytest.mark.network @pytest.mark.skip( - reason=("Warehouse search behavior is different and no longer returns " - "multiple results. See " - "https://github.com/pypa/warehouse/issues/3717 for more " - "information."), + reason=( + "Warehouse search behavior is different and no longer returns " + "multiple results. See " + "https://github.com/pypa/warehouse/issues/3717 for more " + "information." + ), ) @pytest.mark.search -def test_multiple_search(script): +def test_multiple_search(script: PipTestEnvironment) -> None: """ Test searching for multiple packages at once. """ - output = script.pip('search', 'pip', 'INITools') - assert ( - 'The PyPA recommended tool for installing ' - 'Python packages.' in output.stdout - ) - assert 'Tools for parsing and using INI-style files' in output.stdout + output = script.pip("search", "pip", "INITools") + assert "The PyPA recommended tool for installing Python packages." in output.stdout + assert "Tools for parsing and using INI-style files" in output.stdout @pytest.mark.search -def test_search_missing_argument(script): +def test_search_missing_argument(script: PipTestEnvironment) -> None: """ Test missing required argument for search """ - result = script.pip('search', expect_error=True) - assert 'ERROR: Missing required argument (search query).' in result.stderr + result = script.pip("search", expect_error=True) + assert "ERROR: Missing required argument (search query)." in result.stderr @pytest.mark.network @pytest.mark.search -def test_run_method_should_return_success_when_find_packages(): +def test_run_method_should_return_success_when_find_packages() -> None: """ Test SearchCommand.run for found package """ - command = create_command('search') + command = create_command("search") cmdline = "--index=https://pypi.org/pypi pip" with command.main_context(): options, args = command.parse_args(cmdline.split()) @@ -115,11 +116,11 @@ def test_run_method_should_return_success_when_find_packages(): @pytest.mark.network @pytest.mark.search -def test_run_method_should_return_no_matches_found_when_does_not_find_pkgs(): +def test_run_method_should_return_no_matches_found_when_does_not_find_pkgs() -> None: """ Test SearchCommand.run for no matches """ - command = create_command('search') + command = create_command("search") cmdline = "--index=https://pypi.org/pypi nonexistentpackage" with command.main_context(): options, args = command.parse_args(cmdline.split()) @@ -129,74 +130,80 @@ def test_run_method_should_return_no_matches_found_when_does_not_find_pkgs(): @pytest.mark.network @pytest.mark.search -def test_search_should_exit_status_code_zero_when_find_packages(script): +def test_search_should_exit_status_code_zero_when_find_packages( + script: PipTestEnvironment, +) -> None: """ Test search exit status code for package found """ - result = script.pip('search', 'pip') + result = script.pip("search", "pip") assert result.returncode == SUCCESS @pytest.mark.network @pytest.mark.search -def test_search_exit_status_code_when_finds_no_package(script): +def test_search_exit_status_code_when_finds_no_package( + script: PipTestEnvironment, +) -> None: """ Test search exit status code for no matches """ - result = script.pip('search', 'nonexistentpackage', expect_error=True) + result = script.pip("search", "nonexistentpackage", expect_error=True) assert result.returncode == NO_MATCHES_FOUND, result.returncode @pytest.mark.search -def test_latest_prerelease_install_message(caplog, monkeypatch): +def test_latest_prerelease_install_message( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: """ Test documentation for installing pre-release packages is displayed """ - hits = [ + hits: List["TransformedHit"] = [ { - 'name': 'ni', - 'summary': 'For knights who say Ni!', - 'versions': ['1.0.0', '1.0.1a'] + "name": "ni", + "summary": "For knights who say Ni!", + "versions": ["1.0.0", "1.0.1a"], } ] - installed_package = pretend.stub(project_name="ni") - monkeypatch.setattr("pip._vendor.pkg_resources.working_set", - [installed_package]) + installed_package = mock.Mock(project_name="ni") + monkeypatch.setattr("pip._vendor.pkg_resources.working_set", [installed_package]) - dist = pretend.stub(version="1.0.0") - get_dist = pretend.call_recorder(lambda x: dist) - monkeypatch.setattr("pip._internal.commands.search.get_distribution", - get_dist) + get_dist = mock.Mock() + get_dist.return_value = mock.Mock(version="1.0.0") + monkeypatch.setattr("pip._internal.commands.search.get_distribution", get_dist) with caplog.at_level(logging.INFO): print_results(hits) message = caplog.records[-1].getMessage() assert 'pre-release; install with "pip install --pre"' in message - assert get_dist.calls == [pretend.call('ni')] + assert get_dist.call_args_list == [mock.call("ni")] @pytest.mark.search -def test_search_print_results_should_contain_latest_versions(caplog): +def test_search_print_results_should_contain_latest_versions( + caplog: pytest.LogCaptureFixture, +) -> None: """ Test that printed search results contain the latest package versions """ - hits = [ + hits: List["TransformedHit"] = [ { - 'name': 'testlib1', - 'summary': 'Test library 1.', - 'versions': ['1.0.5', '1.0.3'] + "name": "testlib1", + "summary": "Test library 1.", + "versions": ["1.0.5", "1.0.3"], }, { - 'name': 'testlib2', - 'summary': 'Test library 1.', - 'versions': ['2.0.1', '2.0.3'] - } + "name": "testlib2", + "summary": "Test library 1.", + "versions": ["2.0.1", "2.0.3"], + }, ] with caplog.at_level(logging.INFO): print_results(hits) log_messages = sorted([r.getMessage() for r in caplog.records]) - assert log_messages[0].startswith('testlib1 (1.0.5)') - assert log_messages[1].startswith('testlib2 (2.0.3)') + assert log_messages[0].startswith("testlib1 (1.0.5)") + assert log_messages[1].startswith("testlib2 (2.0.3)") diff --git a/tests/functional/test_show.py b/tests/functional/test_show.py index 7047aa63aa8..a704911e643 100644 --- a/tests/functional/test_show.py +++ b/tests/functional/test_show.py @@ -1,86 +1,112 @@ import os +import pathlib import re -import pytest - from pip import __version__ from pip._internal.commands.show import search_packages_info -from tests.lib import create_test_package_with_setup +from pip._internal.operations.install.legacy import ( + write_installed_files_from_setuptools_record, +) +from pip._internal.utils.unpacking import untar_file +from tests.lib import PipTestEnvironment, TestData, create_test_package_with_setup -def test_basic_show(script): +def test_basic_show(script: PipTestEnvironment) -> None: """ Test end to end test for show command. """ - result = script.pip('show', 'pip') + result = script.pip("show", "pip") lines = result.stdout.splitlines() assert len(lines) == 10 - assert 'Name: pip' in lines - assert f'Version: {__version__}' in lines - assert any(line.startswith('Location: ') for line in lines) - assert 'Requires: ' in lines + assert "Name: pip" in lines + assert f"Version: {__version__}" in lines + assert any(line.startswith("Location: ") for line in lines) + assert "Requires: " in lines -def test_show_with_files_not_found(script, data): +def test_show_with_files_not_found(script: PipTestEnvironment, data: TestData) -> None: """ Test for show command with installed files listing enabled and installed-files.txt not found. """ - editable = data.packages.joinpath('SetupPyUTF8') - script.pip('install', '-e', editable) - result = script.pip('show', '-f', 'SetupPyUTF8') + editable = data.packages.joinpath("SetupPyUTF8") + script.pip("install", "-e", editable) + result = script.pip("show", "-f", "SetupPyUTF8") lines = result.stdout.splitlines() assert len(lines) == 12 - assert 'Name: SetupPyUTF8' in lines - assert 'Version: 0.0.0' in lines - assert any(line.startswith('Location: ') for line in lines) - assert 'Requires: ' in lines - assert 'Files:' in lines - assert 'Cannot locate installed-files.txt' in lines + assert "Name: SetupPyUTF8" in lines + assert "Version: 0.0.0" in lines + assert any(line.startswith("Location: ") for line in lines) + assert "Requires: " in lines + assert "Files:" in lines + assert "Cannot locate RECORD or installed-files.txt" in lines -def test_show_with_files_from_wheel(script, data): +def test_show_with_files_from_wheel(script: PipTestEnvironment, data: TestData) -> None: """ - Test that a wheel's files can be listed + Test that a wheel's files can be listed. """ - wheel_file = data.packages.joinpath('simple.dist-0.1-py2.py3-none-any.whl') - script.pip('install', '--no-index', wheel_file) - result = script.pip('show', '-f', 'simple.dist') + wheel_file = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") + script.pip("install", "--no-index", wheel_file) + result = script.pip("show", "-f", "simple.dist") lines = result.stdout.splitlines() - assert 'Name: simple.dist' in lines - assert 'Cannot locate installed-files.txt' not in lines[6], lines[6] + assert "Name: simple.dist" in lines + assert "Cannot locate RECORD or installed-files.txt" not in lines[6], lines[6] assert re.search(r"Files:\n( .+\n)+", result.stdout) + assert f" simpledist{os.sep}__init__.py" in lines[6:] + + +def test_show_with_files_from_legacy( + tmp_path: pathlib.Path, script: PipTestEnvironment, data: TestData +) -> None: + """ + Test listing files in the show command (legacy installed-files.txt). + """ + # Since 'pip install' now always tries to build a wheel from sdist, it + # cannot properly generate a setup. The legacy code path is basically + # 'setup.py install' plus installed-files.txt, which we manually generate. + source_dir = tmp_path.joinpath("unpacked-sdist") + setuptools_record = tmp_path.joinpath("installed-record.txt") + untar_file(data.packages.joinpath("simple-1.0.tar.gz"), str(source_dir)) + script.run( + "python", + "setup.py", + "install", + "--single-version-externally-managed", + "--record", + str(setuptools_record), + cwd=source_dir, + ) + write_installed_files_from_setuptools_record( + setuptools_record.read_text().splitlines(), + root=None, + req_description="simple==1.0", + ) - -@pytest.mark.network -def test_show_with_all_files(script): - """ - Test listing all files in the show command. - """ - script.pip('install', 'initools==0.2') - result = script.pip('show', '--files', 'initools') + result = script.pip("show", "--files", "simple") lines = result.stdout.splitlines() - assert 'Cannot locate installed-files.txt' not in lines[6], lines[6] + assert "Cannot locate RECORD or installed-files.txt" not in lines[6], lines[6] assert re.search(r"Files:\n( .+\n)+", result.stdout) + assert f" simple{os.sep}__init__.py" in lines[6:] -def test_missing_argument(script): +def test_missing_argument(script: PipTestEnvironment) -> None: """ Test show command with no arguments. """ - result = script.pip('show', expect_error=True) - assert 'ERROR: Please provide a package name or names.' in result.stderr + result = script.pip("show", expect_error=True) + assert "ERROR: Please provide a package name or names." in result.stderr -def test_find_package_not_found(): +def test_find_package_not_found() -> None: """ Test trying to get info about a nonexistent package. """ - result = search_packages_info(['abcd3']) + result = search_packages_info(["abcd3"]) assert len(list(result)) == 0 -def test_report_single_not_found(script): +def test_report_single_not_found(script: PipTestEnvironment) -> None: """ Test passing one name and that isn't found. """ @@ -89,212 +115,213 @@ def test_report_single_not_found(script): # Also, the following should report an error as there are no results # to print. Consequently, there is no need to pass # allow_stderr_warning=True since this is implied by expect_error=True. - result = script.pip('show', 'Abcd-3', expect_error=True) - assert 'WARNING: Package(s) not found: Abcd-3' in result.stderr + result = script.pip("show", "Abcd-3", expect_error=True) + assert "WARNING: Package(s) not found: Abcd-3" in result.stderr assert not result.stdout.splitlines() -def test_report_mixed_not_found(script): +def test_report_mixed_not_found(script: PipTestEnvironment) -> None: """ Test passing a mixture of found and not-found names. """ # We test passing non-canonicalized names. - result = script.pip( - 'show', 'Abcd3', 'A-B-C', 'pip', allow_stderr_warning=True - ) - assert 'WARNING: Package(s) not found: A-B-C, Abcd3' in result.stderr + result = script.pip("show", "Abcd3", "A-B-C", "pip", allow_stderr_warning=True) + assert "WARNING: Package(s) not found: A-B-C, Abcd3" in result.stderr lines = result.stdout.splitlines() assert len(lines) == 10 - assert 'Name: pip' in lines + assert "Name: pip" in lines -def test_search_any_case(): +def test_search_any_case() -> None: """ Search for a package in any case. """ - result = list(search_packages_info(['PIP'])) + result = list(search_packages_info(["PIP"])) assert len(result) == 1 - assert result[0]['name'] == 'pip' + assert result[0].name == "pip" -def test_more_than_one_package(): +def test_more_than_one_package() -> None: """ Search for more than one package. """ - result = list(search_packages_info(['pIp', 'pytest', 'Virtualenv'])) + result = list(search_packages_info(["pIp", "pytest", "Virtualenv"])) assert len(result) == 3 -def test_show_verbose_with_classifiers(script): +def test_show_verbose_with_classifiers(script: PipTestEnvironment) -> None: """ Test that classifiers can be listed """ - result = script.pip('show', 'pip', '--verbose') + result = script.pip("show", "pip", "--verbose") lines = result.stdout.splitlines() - assert 'Name: pip' in lines + assert "Name: pip" in lines assert re.search(r"Classifiers:\n( .+\n)+", result.stdout) assert "Intended Audience :: Developers" in result.stdout -def test_show_verbose_installer(script, data): +def test_show_verbose_installer(script: PipTestEnvironment, data: TestData) -> None: """ Test that the installer is shown (this currently needs a wheel install) """ - wheel_file = data.packages.joinpath('simple.dist-0.1-py2.py3-none-any.whl') - script.pip('install', '--no-index', wheel_file) - result = script.pip('show', '--verbose', 'simple.dist') + wheel_file = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") + script.pip("install", "--no-index", wheel_file) + result = script.pip("show", "--verbose", "simple.dist") lines = result.stdout.splitlines() - assert 'Name: simple.dist' in lines - assert 'Installer: pip' in lines + assert "Name: simple.dist" in lines + assert "Installer: pip" in lines -def test_show_verbose(script): +def test_show_verbose(script: PipTestEnvironment) -> None: """ Test end to end test for verbose show command. """ - result = script.pip('show', '--verbose', 'pip') + result = script.pip("show", "--verbose", "pip") lines = result.stdout.splitlines() - assert any(line.startswith('Metadata-Version: ') for line in lines) - assert any(line.startswith('Installer: ') for line in lines) - assert 'Entry-points:' in lines - assert 'Classifiers:' in lines + assert any(line.startswith("Metadata-Version: ") for line in lines) + assert any(line.startswith("Installer: ") for line in lines) + assert "Entry-points:" in lines + assert "Classifiers:" in lines -def test_all_fields(script): +def test_all_fields(script: PipTestEnvironment) -> None: """ Test that all the fields are present """ - result = script.pip('show', 'pip') + result = script.pip("show", "pip") lines = result.stdout.splitlines() - expected = {'Name', 'Version', 'Summary', 'Home-page', 'Author', - 'Author-email', 'License', 'Location', 'Requires', - 'Required-by'} - actual = {re.sub(':.*$', '', line) for line in lines} + expected = { + "Name", + "Version", + "Summary", + "Home-page", + "Author", + "Author-email", + "License", + "Location", + "Requires", + "Required-by", + } + actual = {re.sub(":.*$", "", line) for line in lines} assert actual == expected -def test_pip_show_is_short(script): +def test_pip_show_is_short(script: PipTestEnvironment) -> None: """ Test that pip show stays short """ - result = script.pip('show', 'pip') + result = script.pip("show", "pip") lines = result.stdout.splitlines() assert len(lines) <= 10 -def test_pip_show_divider(script, data): +def test_pip_show_divider(script: PipTestEnvironment, data: TestData) -> None: """ Expect a divider between packages """ - script.pip('install', 'pip-test-package', '--no-index', - '-f', data.packages) - result = script.pip('show', 'pip', 'pip-test-package') + script.pip("install", "pip-test-package", "--no-index", "-f", data.packages) + result = script.pip("show", "pip", "pip-test-package") lines = result.stdout.splitlines() assert "---" in lines -def test_package_name_is_canonicalized(script, data): - script.pip('install', 'pip-test-package', '--no-index', '-f', - data.packages) +def test_package_name_is_canonicalized( + script: PipTestEnvironment, data: TestData +) -> None: + script.pip("install", "pip-test-package", "--no-index", "-f", data.packages) - dash_show_result = script.pip('show', 'pip-test-package') - underscore_upper_show_result = script.pip('show', 'pip-test_Package') + dash_show_result = script.pip("show", "pip-test-package") + underscore_upper_show_result = script.pip("show", "pip-test_Package") assert underscore_upper_show_result.returncode == 0 assert underscore_upper_show_result.stdout == dash_show_result.stdout -def test_show_required_by_packages_basic(script, data): +def test_show_required_by_packages_basic( + script: PipTestEnvironment, data: TestData +) -> None: """ Test that installed packages that depend on this package are shown """ - editable_path = os.path.join(data.src, 'requires_simple') - script.pip( - 'install', '--no-index', '-f', data.find_links, editable_path - ) + editable_path = os.path.join(data.src, "requires_simple") + script.pip("install", "--no-index", "-f", data.find_links, editable_path) - result = script.pip('show', 'simple') + result = script.pip("show", "simple") lines = result.stdout.splitlines() - assert 'Name: simple' in lines - assert 'Required-by: requires-simple' in lines + assert "Name: simple" in lines + assert "Required-by: requires-simple" in lines -def test_show_required_by_packages_capitalized(script, data): +def test_show_required_by_packages_capitalized( + script: PipTestEnvironment, data: TestData +) -> None: """ Test that the installed packages which depend on a package are shown where the package has a capital letter """ - editable_path = os.path.join(data.src, 'requires_capitalized') - script.pip( - 'install', '--no-index', '-f', data.find_links, editable_path - ) + editable_path = os.path.join(data.src, "requires_capitalized") + script.pip("install", "--no-index", "-f", data.find_links, editable_path) - result = script.pip('show', 'simple') + result = script.pip("show", "simple") lines = result.stdout.splitlines() - assert 'Name: simple' in lines - assert 'Required-by: Requires-Capitalized' in lines + assert "Name: simple" in lines + assert "Required-by: Requires-Capitalized" in lines -def test_show_required_by_packages_requiring_capitalized(script, data): +def test_show_required_by_packages_requiring_capitalized( + script: PipTestEnvironment, data: TestData +) -> None: """ Test that the installed packages which depend on a package are shown where the package has a name with a mix of lower and upper case letters """ - required_package_path = os.path.join(data.src, 'requires_capitalized') - script.pip( - 'install', '--no-index', '-f', data.find_links, required_package_path - ) - editable_path = os.path.join(data.src, 'requires_requires_capitalized') - script.pip( - 'install', '--no-index', '-f', data.find_links, editable_path - ) + required_package_path = os.path.join(data.src, "requires_capitalized") + script.pip("install", "--no-index", "-f", data.find_links, required_package_path) + editable_path = os.path.join(data.src, "requires_requires_capitalized") + script.pip("install", "--no-index", "-f", data.find_links, editable_path) - result = script.pip('show', 'Requires_Capitalized') + result = script.pip("show", "Requires_Capitalized") lines = result.stdout.splitlines() print(lines) - assert 'Name: Requires-Capitalized' in lines - assert 'Required-by: requires-requires-capitalized' in lines + assert "Name: Requires-Capitalized" in lines + assert "Required-by: requires-requires-capitalized" in lines -def test_show_skip_work_dir_pkg(script): +def test_show_skip_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that show should not include package present in working directory """ # Create a test package and create .egg-info dir - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) # Show should not include package simple when run from package directory - result = script.pip('show', 'simple', expect_error=True, cwd=pkg_path) - assert 'WARNING: Package(s) not found: simple' in result.stderr + result = script.pip("show", "simple", expect_error=True, cwd=pkg_path) + assert "WARNING: Package(s) not found: simple" in result.stderr -def test_show_include_work_dir_pkg(script): +def test_show_include_work_dir_pkg(script: PipTestEnvironment) -> None: """ Test that show should include package in working directory if working directory is added in PYTHONPATH """ # Create a test package and create .egg-info dir - pkg_path = create_test_package_with_setup( - script, name='simple', version='1.0') - script.run('python', 'setup.py', 'egg_info', - expect_stderr=True, cwd=pkg_path) + pkg_path = create_test_package_with_setup(script, name="simple", version="1.0") + script.run("python", "setup.py", "egg_info", expect_stderr=True, cwd=pkg_path) - script.environ.update({'PYTHONPATH': pkg_path}) + script.environ.update({"PYTHONPATH": pkg_path}) # Show should include package simple when run from package directory, # when package directory is in PYTHONPATH - result = script.pip('show', 'simple', cwd=pkg_path) + result = script.pip("show", "simple", cwd=pkg_path) lines = result.stdout.splitlines() - assert 'Name: simple' in lines + assert "Name: simple" in lines diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index d3c6a392ec4..a9d6a828bd7 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -1,55 +1,66 @@ -import json import logging import os import sys import textwrap from os.path import join, normpath from tempfile import mkdtemp +from typing import Any +from unittest.mock import Mock -import pretend import pytest from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import rmtree -from tests.lib import assert_all_changes, create_test_package_with_setup, need_svn +from tests.lib import ( + PipTestEnvironment, + TestData, + assert_all_changes, + create_test_package_with_setup, + need_svn, +) from tests.lib.local_repos import local_checkout, local_repo +from tests.lib.path import Path @pytest.mark.network -def test_basic_uninstall(script): +def test_basic_uninstall(script: PipTestEnvironment) -> None: """ Test basic install and uninstall. """ - result = script.pip('install', 'INITools==0.2') - result.did_create(join(script.site_packages, 'initools')) + result = script.pip("install", "INITools==0.2") + result.did_create(join(script.site_packages, "initools")) # the import forces the generation of __pycache__ if the version of python # supports it - script.run('python', '-c', "import initools") - result2 = script.pip('uninstall', 'INITools', '-y') - assert_all_changes(result, result2, [script.venv / 'build', 'cache']) + script.run("python", "-c", "import initools") + result2 = script.pip("uninstall", "INITools", "-y") + assert_all_changes(result, result2, [script.venv / "build", "cache"]) -def test_basic_uninstall_distutils(script): +def test_basic_uninstall_distutils(script: PipTestEnvironment) -> None: """ Test basic install and uninstall. """ script.scratch_path.joinpath("distutils_install").mkdir() - pkg_path = script.scratch_path / 'distutils_install' - pkg_path.joinpath("setup.py").write_text(textwrap.dedent(""" + pkg_path = script.scratch_path / "distutils_install" + pkg_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ from distutils.core import setup setup( name='distutils-install', version='0.1', ) - """)) - result = script.run('python', pkg_path / 'setup.py', 'install') - result = script.pip('list', '--format=json') - assert {"name": "distutils-install", "version": "0.1"} \ - in json.loads(result.stdout) - result = script.pip('uninstall', 'distutils_install', '-y', - expect_stderr=True, expect_error=True) + """ + ) + ) + result = script.run("python", pkg_path / "setup.py", "install") + result = script.pip("list", "--format=json") + script.assert_installed(distutils_install="0.1") + result = script.pip( + "uninstall", "distutils_install", "-y", expect_stderr=True, expect_error=True + ) assert ( "Cannot uninstall 'distutils-install'. It is a distutils installed " "project and thus we cannot accurately determine which files belong " @@ -58,72 +69,92 @@ def test_basic_uninstall_distutils(script): @pytest.mark.network -def test_basic_uninstall_with_scripts(script): +def test_basic_uninstall_with_scripts(script: PipTestEnvironment) -> None: """ Uninstall an easy_installed package with scripts. """ - result = script.easy_install('PyLogo', expect_stderr=True) - easy_install_pth = script.site_packages / 'easy-install.pth' - pylogo = sys.platform == 'win32' and 'pylogo' or 'PyLogo' - assert(pylogo in result.files_updated[easy_install_pth].bytes) - result2 = script.pip('uninstall', 'pylogo', '-y') + # setuptools 52 removed easy_install. + script.pip("install", "setuptools==51.3.3", use_module=True) + + result = script.easy_install("PyLogo", expect_stderr=True) + easy_install_pth = script.site_packages / "easy-install.pth" + pylogo = sys.platform == "win32" and "pylogo" or "PyLogo" + assert pylogo in result.files_updated[easy_install_pth].bytes + result2 = script.pip("uninstall", "pylogo", "-y") assert_all_changes( result, result2, - [script.venv / 'build', 'cache', easy_install_pth], + [script.venv / "build", "cache", easy_install_pth], + ) + + +@pytest.mark.parametrize("name", ["GTrolls.tar.gz", "https://guyto.com/archives/"]) +def test_uninstall_invalid_parameter( + script: PipTestEnvironment, caplog: pytest.LogCaptureFixture, name: str +) -> None: + result = script.pip("uninstall", name, "-y", expect_error=True) + expected_message = ( + f"Invalid requirement: '{name}' ignored -" + f" the uninstall command expects named requirements." ) + assert expected_message in result.stderr @pytest.mark.network -def test_uninstall_easy_install_after_import(script): +def test_uninstall_easy_install_after_import(script: PipTestEnvironment) -> None: """ Uninstall an easy_installed package after it's been imported """ - result = script.easy_install('INITools==0.2', expect_stderr=True) + # setuptools 52 removed easy_install. + script.pip("install", "setuptools==51.3.3", use_module=True) + + result = script.easy_install("INITools==0.2", expect_stderr=True) # the import forces the generation of __pycache__ if the version of python # supports it - script.run('python', '-c', "import initools") - result2 = script.pip('uninstall', 'INITools', '-y') + script.run("python", "-c", "import initools") + result2 = script.pip("uninstall", "INITools", "-y") assert_all_changes( result, result2, [ - script.venv / 'build', - 'cache', - script.site_packages / 'easy-install.pth', - ] + script.venv / "build", + "cache", + script.site_packages / "easy-install.pth", + ], ) @pytest.mark.network -def test_uninstall_trailing_newline(script): +def test_uninstall_trailing_newline(script: PipTestEnvironment) -> None: """ Uninstall behaves appropriately if easy-install.pth lacks a trailing newline """ - script.easy_install('INITools==0.2', expect_stderr=True) - script.easy_install('PyLogo', expect_stderr=True) - easy_install_pth = script.site_packages_path / 'easy-install.pth' + # setuptools 52 removed easy_install. + script.pip("install", "setuptools==51.3.3", use_module=True) + + script.easy_install("INITools==0.2", expect_stderr=True) + script.easy_install("PyLogo", expect_stderr=True) + easy_install_pth = script.site_packages_path / "easy-install.pth" # trim trailing newline from easy-install.pth with open(easy_install_pth) as f: pth_before = f.read() - with open(easy_install_pth, 'w') as f: + with open(easy_install_pth, "w") as f: f.write(pth_before.rstrip()) # uninstall initools - script.pip('uninstall', 'INITools', '-y') + script.pip("uninstall", "INITools", "-y") with open(easy_install_pth) as f: pth_after = f.read() # verify that only initools is removed before_without_initools = [ - line for line in pth_before.splitlines() - if 'initools' not in line.lower() + line for line in pth_before.splitlines() if "initools" not in line.lower() ] lines_after = pth_after.splitlines() @@ -131,24 +162,26 @@ def test_uninstall_trailing_newline(script): @pytest.mark.network -def test_basic_uninstall_namespace_package(script): +def test_basic_uninstall_namespace_package(script: PipTestEnvironment) -> None: """ Uninstall a distribution with a namespace package without clobbering the namespace and everything in it. """ - result = script.pip('install', 'pd.requires==0.0.3') - result.did_create(join(script.site_packages, 'pd')) - result2 = script.pip('uninstall', 'pd.find', '-y') - assert join(script.site_packages, 'pd') not in result2.files_deleted, ( - sorted(result2.files_deleted.keys()) + result = script.pip("install", "pd.requires==0.0.3") + result.did_create(join(script.site_packages, "pd")) + result2 = script.pip("uninstall", "pd.find", "-y") + assert join(script.site_packages, "pd") not in result2.files_deleted, sorted( + result2.files_deleted.keys() ) - assert join(script.site_packages, 'pd', 'find') in result2.files_deleted, ( - sorted(result2.files_deleted.keys()) + assert join(script.site_packages, "pd", "find") in result2.files_deleted, sorted( + result2.files_deleted.keys() ) -def test_uninstall_overlapping_package(script, data): +def test_uninstall_overlapping_package( + script: PipTestEnvironment, data: TestData +) -> None: """ Uninstalling a distribution that adds modules to a pre-existing package should only remove those added modules, not the rest of the existing @@ -159,68 +192,70 @@ def test_uninstall_overlapping_package(script, data): parent_pkg = data.packages.joinpath("parent-0.1.tar.gz") child_pkg = data.packages.joinpath("child-0.1.tar.gz") - result1 = script.pip('install', parent_pkg) - result1.did_create(join(script.site_packages, 'parent')) - result2 = script.pip('install', child_pkg) - result2.did_create(join(script.site_packages, 'child')) - result2.did_create(normpath( - join(script.site_packages, 'parent/plugins/child_plugin.py') - )) + result1 = script.pip("install", parent_pkg) + result1.did_create(join(script.site_packages, "parent")) + result2 = script.pip("install", child_pkg) + result2.did_create(join(script.site_packages, "child")) + result2.did_create( + normpath(join(script.site_packages, "parent/plugins/child_plugin.py")) + ) # The import forces the generation of __pycache__ if the version of python # supports it - script.run('python', '-c', "import parent.plugins.child_plugin, child") - result3 = script.pip('uninstall', '-y', 'child') - assert join(script.site_packages, 'child') in result3.files_deleted, ( - sorted(result3.files_created.keys()) + script.run("python", "-c", "import parent.plugins.child_plugin, child") + result3 = script.pip("uninstall", "-y", "child") + assert join(script.site_packages, "child") in result3.files_deleted, sorted( + result3.files_created.keys() ) - assert normpath( - join(script.site_packages, 'parent/plugins/child_plugin.py') - ) in result3.files_deleted, sorted(result3.files_deleted.keys()) - assert join(script.site_packages, 'parent') not in result3.files_deleted, ( - sorted(result3.files_deleted.keys()) + assert ( + normpath(join(script.site_packages, "parent/plugins/child_plugin.py")) + in result3.files_deleted + ), sorted(result3.files_deleted.keys()) + assert join(script.site_packages, "parent") not in result3.files_deleted, sorted( + result3.files_deleted.keys() ) # Additional check: uninstalling 'child' should return things to the # previous state, without unintended side effects. assert_all_changes(result2, result3, []) -@pytest.mark.parametrize("console_scripts", - ["test_ = distutils_install", - "test_:test_ = distutils_install"]) -def test_uninstall_entry_point_colon_in_name(script, console_scripts): +@pytest.mark.parametrize( + "console_scripts", ["test_ = distutils_install", "test_:test_ = distutils_install"] +) +def test_uninstall_entry_point_colon_in_name( + script: PipTestEnvironment, console_scripts: str +) -> None: """ Test uninstall package with two or more entry points in the same section, whose name contain a colon. """ - pkg_name = 'ep_install' + pkg_name = "ep_install" pkg_path = create_test_package_with_setup( script, name=pkg_name, - version='0.1', - entry_points={"console_scripts": [console_scripts, ], - "pip_test.ep": - ["ep:name1 = distutils_install", - "ep:name2 = distutils_install"] - } - ) - script_name = script.bin_path.joinpath( - console_scripts.split('=')[0].strip() + version="0.1", + entry_points={ + "console_scripts": [ + console_scripts, + ], + "pip_test.ep": [ + "ep:name1 = distutils_install", + "ep:name2 = distutils_install", + ], + }, ) - if sys.platform == 'win32': - script_name += '.exe' - result = script.pip('install', pkg_path) + script_name = script.bin_path.joinpath(console_scripts.split("=")[0].strip()) + if sys.platform == "win32": + script_name += ".exe" + script.pip("install", pkg_path) assert script_name.exists() - result = script.pip('list', '--format=json') - assert {"name": "ep-install", "version": "0.1"} \ - in json.loads(result.stdout) - script.pip('uninstall', 'ep_install', '-y') + script.assert_installed(ep_install="0.1") + + script.pip("uninstall", "ep_install", "-y") assert not script_name.exists() - result2 = script.pip('list', '--format=json') - assert {"name": "ep-install", "version": "0.1"} \ - not in json.loads(result2.stdout) + script.assert_not_installed("ep-install") -def test_uninstall_gui_scripts(script): +def test_uninstall_gui_scripts(script: PipTestEnvironment) -> None: """ Make sure that uninstall removes gui scripts """ @@ -228,115 +263,131 @@ def test_uninstall_gui_scripts(script): pkg_path = create_test_package_with_setup( script, name=pkg_name, - version='0.1', - entry_points={"gui_scripts": ["test_ = distutils_install", ], } + version="0.1", + entry_points={ + "gui_scripts": [ + "test_ = distutils_install", + ], + }, ) - script_name = script.bin_path.joinpath('test_') - if sys.platform == 'win32': - script_name += '.exe' - script.pip('install', pkg_path) + script_name = script.bin_path.joinpath("test_") + if sys.platform == "win32": + script_name += ".exe" + script.pip("install", pkg_path) assert script_name.exists() - script.pip('uninstall', pkg_name, '-y') + script.pip("uninstall", pkg_name, "-y") assert not script_name.exists() -def test_uninstall_console_scripts(script): +def test_uninstall_console_scripts(script: PipTestEnvironment) -> None: """ Test uninstalling a package with more files (console_script entry points, extra directories). """ pkg_path = create_test_package_with_setup( script, - name='discover', - version='0.1', - entry_points={'console_scripts': ['discover = discover:main']}, + name="discover", + version="0.1", + entry_points={"console_scripts": ["discover = discover:main"]}, + ) + result = script.pip("install", pkg_path) + result.did_create(script.bin / "discover" + script.exe) + result2 = script.pip("uninstall", "discover", "-y") + assert_all_changes( + result, + result2, + [ + script.venv / "build", + "cache", + Path("scratch") / "discover" / "discover.egg-info", + ], ) - result = script.pip('install', pkg_path) - result.did_create(script.bin / 'discover' + script.exe) - result2 = script.pip('uninstall', 'discover', '-y') - assert_all_changes(result, result2, [script.venv / 'build', 'cache']) -def test_uninstall_console_scripts_uppercase_name(script): +def test_uninstall_console_scripts_uppercase_name(script: PipTestEnvironment) -> None: """ Test uninstalling console script with uppercase character. """ pkg_path = create_test_package_with_setup( script, - name='ep_install', - version='0.1', + name="ep_install", + version="0.1", entry_points={ "console_scripts": [ "Test = distutils_install", ], }, ) - script_name = script.bin_path.joinpath('Test' + script.exe) + script_name = script.bin_path.joinpath("Test" + script.exe) - script.pip('install', pkg_path) + script.pip("install", pkg_path) assert script_name.exists() - script.pip('uninstall', 'ep_install', '-y') + script.pip("uninstall", "ep_install", "-y") assert not script_name.exists() @pytest.mark.network -def test_uninstall_easy_installed_console_scripts(script): +def test_uninstall_easy_installed_console_scripts(script: PipTestEnvironment) -> None: """ Test uninstalling package with console_scripts that is easy_installed. """ - # setuptools >= 42.0.0 deprecates easy_install and prints a warning when - # used - result = script.easy_install('discover', allow_stderr_warning=True) - result.did_create(script.bin / 'discover' + script.exe) - result2 = script.pip('uninstall', 'discover', '-y') + # setuptools 52 removed easy_install and prints a warning after 42 when + # the command is used. + script.pip("install", "setuptools==51.3.3", use_module=True) + + result = script.easy_install("discover", allow_stderr_warning=True) + result.did_create(script.bin / "discover" + script.exe) + result2 = script.pip("uninstall", "discover", "-y") assert_all_changes( result, result2, [ - script.venv / 'build', - 'cache', - script.site_packages / 'easy-install.pth', - ] + script.venv / "build", + "cache", + script.site_packages / "easy-install.pth", + ], ) @pytest.mark.xfail @pytest.mark.network @need_svn -def test_uninstall_editable_from_svn(script, tmpdir): +def test_uninstall_editable_from_svn(script: PipTestEnvironment, tmpdir: Path) -> None: """ Test uninstalling an editable installation from svn. """ result = script.pip( - 'install', '-e', - '{checkout}#egg=initools'.format( - checkout=local_checkout( - 'svn+http://svn.colorstudy.com/INITools', tmpdir) + "install", + "-e", + "{checkout}#egg=initools".format( + checkout=local_checkout("svn+http://svn.colorstudy.com/INITools", tmpdir) ), ) - result.assert_installed('INITools') - result2 = script.pip('uninstall', '-y', 'initools') - assert (script.venv / 'src' / 'initools' in result2.files_after) + result.assert_installed("INITools") + result2 = script.pip("uninstall", "-y", "initools") + assert script.venv / "src" / "initools" in result2.files_after assert_all_changes( result, result2, [ - script.venv / 'src', - script.venv / 'build', - script.site_packages / 'easy-install.pth' + script.venv / "src", + script.venv / "build", + script.site_packages / "easy-install.pth", ], ) @pytest.mark.network -def test_uninstall_editable_with_source_outside_venv(script, tmpdir): +def test_uninstall_editable_with_source_outside_venv( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Test uninstalling editable install from existing source outside the venv. """ try: temp = mkdtemp() - temp_pkg_dir = join(temp, 'pip-test-package') + temp_pkg_dir = join(temp, "pip-test-package") _test_uninstall_editable_with_source_outside_venv( script, tmpdir, @@ -347,47 +398,52 @@ def test_uninstall_editable_with_source_outside_venv(script, tmpdir): def _test_uninstall_editable_with_source_outside_venv( - script, tmpdir, temp_pkg_dir, -): + script: PipTestEnvironment, + tmpdir: Path, + temp_pkg_dir: str, +) -> None: result = script.run( - 'git', 'clone', - local_repo('git+git://github.com/pypa/pip-test-package', tmpdir), + "git", + "clone", + local_repo("git+https://github.com/pypa/pip-test-package", tmpdir), temp_pkg_dir, expect_stderr=True, ) - result2 = script.pip('install', '-e', temp_pkg_dir) - result2.did_create(join( - script.site_packages, 'pip-test-package.egg-link' - )) - result3 = script.pip('uninstall', '-y', 'pip-test-package') + result2 = script.pip("install", "-e", temp_pkg_dir) + result2.did_create(join(script.site_packages, "pip-test-package.egg-link")) + result3 = script.pip("uninstall", "-y", "pip-test-package") assert_all_changes( result, result3, - [script.venv / 'build', script.site_packages / 'easy-install.pth'], + [script.venv / "build", script.site_packages / "easy-install.pth"], ) @pytest.mark.xfail @pytest.mark.network @need_svn -def test_uninstall_from_reqs_file(script, tmpdir): +def test_uninstall_from_reqs_file(script: PipTestEnvironment, tmpdir: Path) -> None: """ Test uninstall from a requirements file. """ local_svn_url = local_checkout( - 'svn+http://svn.colorstudy.com/INITools', tmpdir, + "svn+http://svn.colorstudy.com/INITools", + tmpdir, ) script.scratch_path.joinpath("test-req.txt").write_text( - textwrap.dedent(""" + textwrap.dedent( + """ -e {url}#egg=initools # and something else to test out: PyLogo<0.4 - """).format(url=local_svn_url) + """ + ).format(url=local_svn_url) ) - result = script.pip('install', '-r', 'test-req.txt') + result = script.pip("install", "-r", "test-req.txt") script.scratch_path.joinpath("test-req.txt").write_text( - textwrap.dedent(""" + textwrap.dedent( + """ # -f, -i, and --extra-index-url should all be ignored by uninstall -f http://www.example.com -i http://www.example.com @@ -396,55 +452,55 @@ def test_uninstall_from_reqs_file(script, tmpdir): -e {url}#egg=initools # and something else to test out: PyLogo<0.4 - """).format(url=local_svn_url) + """ + ).format(url=local_svn_url) ) - result2 = script.pip('uninstall', '-r', 'test-req.txt', '-y') + result2 = script.pip("uninstall", "-r", "test-req.txt", "-y") assert_all_changes( result, result2, [ - script.venv / 'build', - script.venv / 'src', - script.scratch / 'test-req.txt', - script.site_packages / 'easy-install.pth', + script.venv / "build", + script.venv / "src", + script.scratch / "test-req.txt", + script.site_packages / "easy-install.pth", ], ) -def test_uninstallpathset_no_paths(caplog): +def test_uninstallpathset_no_paths(caplog: pytest.LogCaptureFixture) -> None: """ Test UninstallPathSet logs notification when there are no paths to uninstall """ - from pkg_resources import get_distribution - + from pip._internal.metadata import get_default_environment from pip._internal.req.req_uninstall import UninstallPathSet caplog.set_level(logging.INFO) - test_dist = get_distribution('pip') + test_dist = get_default_environment().get_distribution("pip") + assert test_dist is not None, "pip not installed" + uninstall_set = UninstallPathSet(test_dist) uninstall_set.remove() # with no files added to set - assert ( - "Can't uninstall 'pip'. No files were found to uninstall." - in caplog.text - ) + assert "Can't uninstall 'pip'. No files were found to uninstall." in caplog.text -def test_uninstall_non_local_distutils(caplog, monkeypatch, tmpdir): +def test_uninstall_non_local_distutils( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, tmpdir: Path +) -> None: einfo = tmpdir.joinpath("thing-1.0.egg-info") with open(einfo, "wb"): pass - dist = pretend.stub( + get_dist = Mock() + get_dist.return_value = Mock( key="thing", project_name="thing", egg_info=einfo, location=einfo, - _provider=pretend.stub(), ) - get_dist = pretend.call_recorder(lambda x: dist) monkeypatch.setattr("pip._vendor.pkg_resources.get_distribution", get_dist) req = install_req_from_line("thing") @@ -453,152 +509,215 @@ def test_uninstall_non_local_distutils(caplog, monkeypatch, tmpdir): assert os.path.exists(einfo) -def test_uninstall_wheel(script, data): +def test_uninstall_wheel(script: PipTestEnvironment, data: TestData) -> None: """ Test uninstalling a wheel """ package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") - result = script.pip('install', package, '--no-index') - dist_info_folder = script.site_packages / 'simple.dist-0.1.dist-info' + result = script.pip("install", package, "--no-index") + dist_info_folder = script.site_packages / "simple.dist-0.1.dist-info" result.did_create(dist_info_folder) - result2 = script.pip('uninstall', 'simple.dist', '-y') + result2 = script.pip("uninstall", "simple.dist", "-y") assert_all_changes(result, result2, []) +@pytest.mark.parametrize( + "installer", + [ + FileNotFoundError, + IsADirectoryError, + "", + os.linesep, + b"\xc0\xff\xee", + "pip", + "MegaCorp Cloud Install-O-Matic", + ], +) +def test_uninstall_without_record_fails( + script: PipTestEnvironment, data: TestData, installer: Any +) -> None: + """ + Test uninstalling a package installed without RECORD + """ + package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") + result = script.pip("install", package, "--no-index") + dist_info_folder = script.site_packages / "simple.dist-0.1.dist-info" + result.did_create(dist_info_folder) + + # Remove RECORD + record_path = dist_info_folder / "RECORD" + (script.base_path / record_path).unlink() + ignore_changes = [record_path] + + # Populate, remove or otherwise break INSTALLER + installer_path = dist_info_folder / "INSTALLER" + ignore_changes += [installer_path] + installer_path = script.base_path / installer_path + if installer in (FileNotFoundError, IsADirectoryError): + installer_path.unlink() + if installer is IsADirectoryError: + installer_path.mkdir() + else: + if isinstance(installer, bytes): + installer_path.write_bytes(installer) + else: + installer_path.write_text(installer + os.linesep) + + result2 = script.pip("uninstall", "simple.dist", "-y", expect_error=True) + expected_error_message = ( + "ERROR: Cannot uninstall simple.dist 0.1, RECORD file not found." + ) + if not isinstance(installer, str) or not installer.strip() or installer == "pip": + expected_error_message += ( + " You might be able to recover from this via: " + "'pip install --force-reinstall --no-deps " + "simple.dist==0.1'." + ) + elif installer: + expected_error_message += " Hint: The package was installed by {}.".format( + installer + ) + assert result2.stderr.rstrip() == expected_error_message + assert_all_changes(result.files_after, result2, ignore_changes) + + @pytest.mark.skipif("sys.platform == 'win32'") -def test_uninstall_with_symlink(script, data, tmpdir): +def test_uninstall_with_symlink( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test uninstalling a wheel, with an additional symlink https://github.com/pypa/pip/issues/6892 """ package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") - script.pip('install', package, '--no-index') + script.pip("install", package, "--no-index") symlink_target = tmpdir / "target" symlink_target.mkdir() symlink_source = script.site_packages / "symlink" (script.base_path / symlink_source).symlink_to(symlink_target) st_mode = symlink_target.stat().st_mode - distinfo_path = script.site_packages_path / 'simple.dist-0.1.dist-info' - record_path = distinfo_path / 'RECORD' + distinfo_path = script.site_packages_path / "simple.dist-0.1.dist-info" + record_path = distinfo_path / "RECORD" with open(record_path, "a") as f: f.write("symlink,,\n") - uninstall_result = script.pip('uninstall', 'simple.dist', '-y') + uninstall_result = script.pip("uninstall", "simple.dist", "-y") assert symlink_source in uninstall_result.files_deleted assert symlink_target.stat().st_mode == st_mode -def test_uninstall_setuptools_develop_install(script, data): +def test_uninstall_setuptools_develop_install( + script: PipTestEnvironment, data: TestData +) -> None: """Try uninstall after setup.py develop followed of setup.py install""" pkg_path = data.packages.joinpath("FSPkg") - script.run('python', 'setup.py', 'develop', - expect_stderr=True, cwd=pkg_path) - script.run('python', 'setup.py', 'install', - expect_stderr=True, cwd=pkg_path) - list_result = script.pip('list', '--format=json') - assert {"name": os.path.normcase("FSPkg"), "version": "0.1.dev0"} \ - in json.loads(list_result.stdout), str(list_result) + script.run("python", "setup.py", "develop", expect_stderr=True, cwd=pkg_path) + script.run("python", "setup.py", "install", expect_stderr=True, cwd=pkg_path) + script.assert_installed(FSPkg="0.1.dev0") # Uninstall both develop and install - uninstall = script.pip('uninstall', 'FSPkg', '-y') - assert any(filename.endswith('.egg') - for filename in uninstall.files_deleted.keys()) - uninstall2 = script.pip('uninstall', 'FSPkg', '-y') - assert join( - script.site_packages, 'FSPkg.egg-link' - ) in uninstall2.files_deleted, list(uninstall2.files_deleted.keys()) - list_result2 = script.pip('list', '--format=json') - assert "FSPkg" not in {p["name"] for p in json.loads(list_result2.stdout)} - - -def test_uninstall_editable_and_pip_install(script, data): + uninstall = script.pip("uninstall", "FSPkg", "-y") + assert any(filename.endswith(".egg") for filename in uninstall.files_deleted.keys()) + uninstall2 = script.pip("uninstall", "FSPkg", "-y") + assert ( + join(script.site_packages, "FSPkg.egg-link") in uninstall2.files_deleted + ), list(uninstall2.files_deleted.keys()) + script.assert_not_installed("FSPkg") + + +def test_uninstall_editable_and_pip_install( + script: PipTestEnvironment, data: TestData +) -> None: """Try uninstall after pip install -e after pip install""" # SETUPTOOLS_SYS_PATH_TECHNIQUE=raw removes the assumption that `-e` # installs are always higher priority than regular installs. # This becomes the default behavior in setuptools 25. - script.environ['SETUPTOOLS_SYS_PATH_TECHNIQUE'] = 'raw' + script.environ["SETUPTOOLS_SYS_PATH_TECHNIQUE"] = "raw" pkg_path = data.packages.joinpath("FSPkg") - script.pip('install', '-e', '.', - expect_stderr=True, cwd=pkg_path) + script.pip("install", "-e", ".", expect_stderr=True, cwd=pkg_path) # ensure both are installed with --ignore-installed: - script.pip('install', '--ignore-installed', '.', - expect_stderr=True, cwd=pkg_path) - list_result = script.pip('list', '--format=json') - assert {"name": "FSPkg", "version": "0.1.dev0"} \ - in json.loads(list_result.stdout) + script.pip("install", "--ignore-installed", ".", expect_stderr=True, cwd=pkg_path) + script.assert_installed(FSPkg="0.1.dev0") # Uninstall both develop and install - uninstall = script.pip('uninstall', 'FSPkg', '-y') - assert not any(filename.endswith('.egg-link') - for filename in uninstall.files_deleted.keys()) - uninstall2 = script.pip('uninstall', 'FSPkg', '-y') - assert join( - script.site_packages, 'FSPkg.egg-link' - ) in uninstall2.files_deleted, list(uninstall2.files_deleted.keys()) - list_result2 = script.pip('list', '--format=json') - assert "FSPkg" not in {p["name"] for p in json.loads(list_result2.stdout)} - - -def test_uninstall_editable_and_pip_install_easy_install_remove(script, data): + uninstall = script.pip("uninstall", "FSPkg", "-y") + assert not any( + filename.endswith(".egg-link") for filename in uninstall.files_deleted.keys() + ) + uninstall2 = script.pip("uninstall", "FSPkg", "-y") + assert ( + join(script.site_packages, "FSPkg.egg-link") in uninstall2.files_deleted + ), list(uninstall2.files_deleted.keys()) + script.assert_not_installed("FSPkg") + + +def test_uninstall_editable_and_pip_install_easy_install_remove( + script: PipTestEnvironment, data: TestData +) -> None: """Try uninstall after pip install -e after pip install and removing easy-install.pth""" # SETUPTOOLS_SYS_PATH_TECHNIQUE=raw removes the assumption that `-e` # installs are always higher priority than regular installs. # This becomes the default behavior in setuptools 25. - script.environ['SETUPTOOLS_SYS_PATH_TECHNIQUE'] = 'raw' + script.environ["SETUPTOOLS_SYS_PATH_TECHNIQUE"] = "raw" # Rename easy-install.pth to pip-test.pth - easy_install_pth = join(script.site_packages_path, 'easy-install.pth') - pip_test_pth = join(script.site_packages_path, 'pip-test.pth') + easy_install_pth = join(script.site_packages_path, "easy-install.pth") + pip_test_pth = join(script.site_packages_path, "pip-test.pth") os.rename(easy_install_pth, pip_test_pth) # Install FSPkg pkg_path = data.packages.joinpath("FSPkg") - script.pip('install', '-e', '.', - expect_stderr=True, cwd=pkg_path) + script.pip("install", "-e", ".", expect_stderr=True, cwd=pkg_path) # Rename easy-install.pth to pip-test-fspkg.pth - pip_test_fspkg_pth = join(script.site_packages_path, 'pip-test-fspkg.pth') + pip_test_fspkg_pth = join(script.site_packages_path, "pip-test-fspkg.pth") os.rename(easy_install_pth, pip_test_fspkg_pth) # Confirm that FSPkg is installed - list_result = script.pip('list', '--format=json') - assert {"name": "FSPkg", "version": "0.1.dev0"} \ - in json.loads(list_result.stdout) + script.assert_installed(FSPkg="0.1.dev0") # Remove pip-test-fspkg.pth os.remove(pip_test_fspkg_pth) # Uninstall will fail with given warning - uninstall = script.pip('uninstall', 'FSPkg', '-y') + uninstall = script.pip("uninstall", "FSPkg", "-y") assert "Cannot remove entries from nonexistent file" in uninstall.stderr - assert join( - script.site_packages, 'FSPkg.egg-link' - ) in uninstall.files_deleted, list(uninstall.files_deleted.keys()) + assert ( + join(script.site_packages, "FSPkg.egg-link") in uninstall.files_deleted + ), list(uninstall.files_deleted.keys()) # Confirm that FSPkg is uninstalled - list_result = script.pip('list', '--format=json') - assert {"name": "FSPkg", "version": "0.1.dev0"} \ - not in json.loads(list_result.stdout) + script.assert_not_installed("FSPkg") # Rename pip-test.pth back to easy-install.pth os.rename(pip_test_pth, easy_install_pth) -def test_uninstall_ignores_missing_packages(script, data): - """Uninstall of a non existent package prints a warning and exits cleanly - """ +def test_uninstall_ignores_missing_packages( + script: PipTestEnvironment, data: TestData +) -> None: + """Uninstall of a non existent package prints a warning and exits cleanly""" result = script.pip( - 'uninstall', '-y', 'non-existent-pkg', expect_stderr=True, + "uninstall", + "-y", + "non-existent-pkg", + expect_stderr=True, ) assert "Skipping non-existent-pkg as it is not installed." in result.stderr assert result.returncode == 0, "Expected clean exit" -def test_uninstall_ignores_missing_packages_and_uninstalls_rest(script, data): - script.pip_install_local('simple') +def test_uninstall_ignores_missing_packages_and_uninstalls_rest( + script: PipTestEnvironment, data: TestData +) -> None: + script.pip_install_local("simple") result = script.pip( - 'uninstall', '-y', 'non-existent-pkg', 'simple', expect_stderr=True, + "uninstall", + "-y", + "non-existent-pkg", + "simple", + expect_stderr=True, ) assert "Skipping non-existent-pkg as it is not installed." in result.stderr diff --git a/tests/functional/test_uninstall_user.py b/tests/functional/test_uninstall_user.py index 2dbf032ac38..6d48fe1627a 100644 --- a/tests/functional/test_uninstall_user.py +++ b/tests/functional/test_uninstall_user.py @@ -7,50 +7,55 @@ from tests.functional.test_install_user import _patch_dist_in_site_packages from tests.lib import pyversion # noqa: F401 -from tests.lib import assert_all_changes +from tests.lib import PipTestEnvironment, TestData, assert_all_changes +from tests.lib.venv import VirtualEnvironment @pytest.mark.incompatible_with_test_venv class Tests_UninstallUserSite: - @pytest.mark.network - def test_uninstall_from_usersite(self, script): + def test_uninstall_from_usersite(self, script: PipTestEnvironment) -> None: """ Test uninstall from usersite """ - result1 = script.pip('install', '--user', 'INITools==0.3') - result2 = script.pip('uninstall', '-y', 'INITools') - assert_all_changes(result1, result2, [script.venv / 'build', 'cache']) + result1 = script.pip("install", "--user", "INITools==0.3") + result2 = script.pip("uninstall", "-y", "INITools") + assert_all_changes(result1, result2, [script.venv / "build", "cache"]) def test_uninstall_from_usersite_with_dist_in_global_site( - self, virtualenv, script): + self, virtualenv: VirtualEnvironment, script: PipTestEnvironment + ) -> None: """ Test uninstall from usersite (with same dist in global site) """ _patch_dist_in_site_packages(virtualenv) - script.pip_install_local('pip-test-package==0.1', '--no-binary=:all:') + script.pip_install_local("pip-test-package==0.1", "--no-binary=:all:") result2 = script.pip_install_local( - '--user', 'pip-test-package==0.1.1', '--no-binary=:all:') - result3 = script.pip('uninstall', '-vy', 'pip-test-package') + "--user", "pip-test-package==0.1.1", "--no-binary=:all:" + ) + result3 = script.pip("uninstall", "-vy", "pip-test-package") # uninstall console is mentioning user scripts, but not global scripts assert normcase(script.user_bin_path) in result3.stdout, str(result3) assert normcase(script.bin_path) not in result3.stdout, str(result3) # uninstall worked - assert_all_changes(result2, result3, [script.venv / 'build', 'cache']) + assert_all_changes(result2, result3, [script.venv / "build", "cache"]) # site still has 0.2 (can't look in result1; have to check) # keep checking for egg-info because no-binary implies setup.py install egg_info_folder = ( - script.base_path / script.site_packages / - 'pip_test_package-0.1-py{pyversion}.egg-info'.format(**globals()) + script.base_path + / script.site_packages + / f"pip_test_package-0.1-py{pyversion}.egg-info" ) assert isdir(egg_info_folder) - def test_uninstall_editable_from_usersite(self, script, data): + def test_uninstall_editable_from_usersite( + self, script: PipTestEnvironment, data: TestData + ) -> None: """ Test uninstall editable local user install """ @@ -58,22 +63,20 @@ def test_uninstall_editable_from_usersite(self, script, data): # install to_install = data.packages.joinpath("FSPkg") - result1 = script.pip( - 'install', '--user', '-e', to_install - ) - egg_link = script.user_site / 'FSPkg.egg-link' + result1 = script.pip("install", "--user", "-e", to_install) + egg_link = script.user_site / "FSPkg.egg-link" result1.did_create(egg_link) # uninstall - result2 = script.pip('uninstall', '-y', 'FSPkg') + result2 = script.pip("uninstall", "-y", "FSPkg") assert not isfile(script.base_path / egg_link) assert_all_changes( result1, result2, [ - script.venv / 'build', - 'cache', - script.user_site / 'easy-install.pth', - ] + script.venv / "build", + "cache", + script.user_site / "easy-install.pth", + ], ) diff --git a/tests/functional/test_vcs_bazaar.py b/tests/functional/test_vcs_bazaar.py index ad24d73d5ba..06f9cbc8107 100644 --- a/tests/functional/test_vcs_bazaar.py +++ b/tests/functional/test_vcs_bazaar.py @@ -3,78 +3,31 @@ """ import os +import sys import pytest -from pip._internal.utils.misc import hide_url from pip._internal.vcs.bazaar import Bazaar from pip._internal.vcs.versioncontrol import RemoteNotFoundError -from tests.lib import ( - _test_path_to_file_url, - _vcs_add, - create_file, - is_bzr_installed, - need_bzr, -) +from tests.lib import PipTestEnvironment, is_bzr_installed, need_bzr +from tests.lib.path import Path @pytest.mark.skipif( - 'TRAVIS' not in os.environ, - reason='Bazaar is only required under Travis') -def test_ensure_bzr_available(): - """Make sure that bzr is available when running in Travis.""" + sys.platform == "win32" or "CI" not in os.environ, + reason="Bazaar is only required under CI", +) +def test_ensure_bzr_available() -> None: + """Make sure that bzr is available when running in CI.""" assert is_bzr_installed() @need_bzr -def test_export(script, tmpdir): - """Test that a Bazaar branch can be exported.""" - source_dir = tmpdir / 'test-source' - source_dir.mkdir() - - create_file(source_dir / 'test_file', 'something') - - _vcs_add(script, str(source_dir), vcs='bazaar') - - export_dir = str(tmpdir / 'export') - url = hide_url('bzr+' + _test_path_to_file_url(source_dir)) - Bazaar().export(export_dir, url=url) - - assert os.listdir(export_dir) == ['test_file'] - - -@need_bzr -def test_export_rev(script, tmpdir): - """Test that a Bazaar branch can be exported, specifying a rev.""" - source_dir = tmpdir / 'test-source' - source_dir.mkdir() - - # Create a single file that is changed by two revisions. - create_file(source_dir / 'test_file', 'something initial') - _vcs_add(script, str(source_dir), vcs='bazaar') - - create_file(source_dir / 'test_file', 'something new') - script.run( - 'bzr', 'commit', '-q', - '--author', 'pip ', - '-m', 'change test file', cwd=source_dir, - ) - - export_dir = tmpdir / 'export' - url = hide_url('bzr+' + _test_path_to_file_url(source_dir) + '@1') - Bazaar().export(str(export_dir), url=url) - - with open(export_dir / 'test_file', 'r') as f: - assert f.read() == 'something initial' - - -@need_bzr -def test_get_remote_url__no_remote(script, tmpdir): - repo_dir = tmpdir / 'temp-repo' +def test_get_remote_url__no_remote(script: PipTestEnvironment, tmpdir: Path) -> None: + repo_dir = tmpdir / "temp-repo" repo_dir.mkdir() - repo_dir = str(repo_dir) - script.run('bzr', 'init', repo_dir) + script.run("bzr", "init", repo_dir) with pytest.raises(RemoteNotFoundError): Bazaar().get_remote_url(repo_dir) diff --git a/tests/functional/test_vcs_git.py b/tests/functional/test_vcs_git.py index d5de1a2fd77..b535d2eab87 100644 --- a/tests/functional/test_vcs_git.py +++ b/tests/functional/test_vcs_git.py @@ -1,44 +1,58 @@ """ Contains functional tests of the Git class. """ - +import logging import os +import pathlib +from typing import List, Optional, Tuple +from unittest.mock import Mock, patch import pytest +from pip._internal.utils.misc import HiddenText from pip._internal.vcs import vcs from pip._internal.vcs.git import Git, RemoteNotFoundError -from tests.lib import _create_test_package, _git_commit, _test_path_to_file_url +from tests.lib import ( + PipTestEnvironment, + _create_test_package, + _git_commit, + _test_path_to_file_url, +) +from tests.lib.path import Path -def test_get_backend_for_scheme(): +def test_get_backend_for_scheme() -> None: assert vcs.get_backend_for_scheme("git+https") is vcs.get_backend("Git") -def get_head_sha(script, dest): +def get_head_sha(script: PipTestEnvironment, dest: str) -> str: """Return the HEAD sha.""" - result = script.run('git', 'rev-parse', 'HEAD', cwd=dest) + result = script.run("git", "rev-parse", "HEAD", cwd=dest) sha = result.stdout.strip() return sha -def checkout_ref(script, repo_dir, ref): - script.run('git', 'checkout', ref, cwd=repo_dir) +def checkout_ref(script: PipTestEnvironment, repo_dir: str, ref: str) -> None: + script.run("git", "checkout", ref, cwd=repo_dir) -def checkout_new_branch(script, repo_dir, branch): +def checkout_new_branch(script: PipTestEnvironment, repo_dir: str, branch: str) -> None: script.run( - 'git', 'checkout', '-b', branch, cwd=repo_dir, + "git", + "checkout", + "-b", + branch, + cwd=repo_dir, ) -def do_commit(script, dest): - _git_commit(script, dest, message='test commit', allow_empty=True) +def do_commit(script: PipTestEnvironment, dest: str) -> str: + _git_commit(script, dest, message="test commit", allow_empty=True) return get_head_sha(script, dest) -def add_commits(script, dest, count): +def add_commits(script: PipTestEnvironment, dest: str, count: int) -> List[str]: """Return a list of the commit hashes from oldest to newest.""" shas = [] for _ in range(count): @@ -48,111 +62,113 @@ def add_commits(script, dest, count): return shas -def check_rev(repo_dir, rev, expected): +def check_rev(repo_dir: str, rev: str, expected: Tuple[Optional[str], bool]) -> None: assert Git.get_revision_sha(repo_dir, rev) == expected -def test_git_dir_ignored(tmpdir): +def test_git_dir_ignored(tmpdir: Path) -> None: """ Test that a GIT_DIR environment variable is ignored. """ - repo_path = tmpdir / 'test-repo' + repo_path = tmpdir / "test-repo" repo_path.mkdir() repo_dir = str(repo_path) - env = {'GIT_DIR': 'foo'} + env = {"GIT_DIR": "foo"} # If GIT_DIR is not ignored, then os.listdir() will return ['foo']. - Git.run_command(['init', repo_dir], cwd=repo_dir, extra_environ=env) - assert os.listdir(repo_dir) == ['.git'] + Git.run_command(["init", repo_dir], cwd=repo_dir, extra_environ=env) + assert os.listdir(repo_dir) == [".git"] -def test_git_work_tree_ignored(tmpdir): +def test_git_work_tree_ignored(tmpdir: Path) -> None: """ Test that a GIT_WORK_TREE environment variable is ignored. """ - repo_path = tmpdir / 'test-repo' + repo_path = tmpdir / "test-repo" repo_path.mkdir() repo_dir = str(repo_path) - Git.run_command(['init', repo_dir], cwd=repo_dir) + Git.run_command(["init", repo_dir], cwd=repo_dir) # Choose a directory relative to the cwd that does not exist. # If GIT_WORK_TREE is not ignored, then the command will error out # with: "fatal: This operation must be run in a work tree". - env = {'GIT_WORK_TREE': 'foo'} - Git.run_command(['status', repo_dir], extra_environ=env, cwd=repo_dir) + env = {"GIT_WORK_TREE": "foo"} + Git.run_command(["status", repo_dir], extra_environ=env, cwd=repo_dir) -def test_get_remote_url(script, tmpdir): - source_dir = tmpdir / 'source' - source_dir.mkdir() - source_url = _test_path_to_file_url(source_dir) +def test_get_remote_url(script: PipTestEnvironment, tmpdir: Path) -> None: + source_path = tmpdir / "source" + source_path.mkdir() + source_url = _test_path_to_file_url(source_path) - source_dir = str(source_dir) - script.run('git', 'init', cwd=source_dir) + source_dir = str(source_path) + script.run("git", "init", cwd=source_dir) do_commit(script, source_dir) - repo_dir = str(tmpdir / 'repo') - script.run('git', 'clone', source_url, repo_dir) + repo_dir = str(tmpdir / "repo") + script.run("git", "clone", source_url, repo_dir) remote_url = Git.get_remote_url(repo_dir) assert remote_url == source_url -def test_get_remote_url__no_remote(script, tmpdir): +def test_get_remote_url__no_remote(script: PipTestEnvironment, tmpdir: Path) -> None: """ Test a repo with no remote. """ - repo_dir = tmpdir / 'temp-repo' - repo_dir.mkdir() - repo_dir = str(repo_dir) + repo_path = tmpdir / "temp-repo" + repo_path.mkdir() + repo_dir = str(repo_path) - script.run('git', 'init', cwd=repo_dir) + script.run("git", "init", cwd=repo_dir) with pytest.raises(RemoteNotFoundError): Git.get_remote_url(repo_dir) -def test_get_current_branch(script): +def test_get_current_branch(script: PipTestEnvironment) -> None: repo_dir = str(script.scratch_path) - script.run('git', 'init', cwd=repo_dir) + script.run("git", "init", cwd=repo_dir) sha = do_commit(script, repo_dir) - assert Git.get_current_branch(repo_dir) == 'master' + assert Git.get_current_branch(repo_dir) == "master" # Switch to a branch with the same SHA as "master" but whose name # is alphabetically after. - checkout_new_branch(script, repo_dir, 'release') - assert Git.get_current_branch(repo_dir) == 'release' + checkout_new_branch(script, repo_dir, "release") + assert Git.get_current_branch(repo_dir) == "release" # Also test the detached HEAD case. checkout_ref(script, repo_dir, sha) assert Git.get_current_branch(repo_dir) is None -def test_get_current_branch__branch_and_tag_same_name(script, tmpdir): +def test_get_current_branch__branch_and_tag_same_name( + script: PipTestEnvironment, tmpdir: Path +) -> None: """ Check calling get_current_branch() from a branch or tag when the branch and tag have the same name. """ repo_dir = str(tmpdir) - script.run('git', 'init', cwd=repo_dir) + script.run("git", "init", cwd=repo_dir) do_commit(script, repo_dir) - checkout_new_branch(script, repo_dir, 'dev') + checkout_new_branch(script, repo_dir, "dev") # Create a tag with the same name as the branch. - script.run('git', 'tag', 'dev', cwd=repo_dir) + script.run("git", "tag", "dev", cwd=repo_dir) - assert Git.get_current_branch(repo_dir) == 'dev' + assert Git.get_current_branch(repo_dir) == "dev" # Now try with the tag checked out. - checkout_ref(script, repo_dir, 'refs/tags/dev') + checkout_ref(script, repo_dir, "refs/tags/dev") assert Git.get_current_branch(repo_dir) is None -def test_get_revision_sha(script): +def test_get_revision_sha(script: PipTestEnvironment) -> None: repo_dir = str(script.scratch_path) - script.run('git', 'init', cwd=repo_dir) + script.run("git", "init", cwd=repo_dir) shas = add_commits(script, repo_dir, count=3) tag_sha = shas[0] @@ -160,99 +176,96 @@ def test_get_revision_sha(script): head_sha = shas[2] assert head_sha == shas[-1] - origin_ref = 'refs/remotes/origin/origin-branch' - generic_ref = 'refs/generic-ref' + origin_ref = "refs/remotes/origin/origin-branch" + generic_ref = "refs/generic-ref" + script.run("git", "branch", "local-branch", head_sha, cwd=repo_dir) + script.run("git", "tag", "v1.0", tag_sha, cwd=repo_dir) + script.run("git", "update-ref", origin_ref, origin_sha, cwd=repo_dir) script.run( - 'git', 'branch', 'local-branch', head_sha, cwd=repo_dir + "git", + "update-ref", + "refs/remotes/upstream/upstream-branch", + head_sha, + cwd=repo_dir, ) - script.run('git', 'tag', 'v1.0', tag_sha, cwd=repo_dir) - script.run('git', 'update-ref', origin_ref, origin_sha, cwd=repo_dir) - script.run( - 'git', 'update-ref', 'refs/remotes/upstream/upstream-branch', - head_sha, cwd=repo_dir - ) - script.run('git', 'update-ref', generic_ref, head_sha, cwd=repo_dir) + script.run("git", "update-ref", generic_ref, head_sha, cwd=repo_dir) # Test two tags pointing to the same sha. - script.run('git', 'tag', 'v2.0', tag_sha, cwd=repo_dir) + script.run("git", "tag", "v2.0", tag_sha, cwd=repo_dir) # Test tags sharing the same suffix as another tag, both before and # after the suffix alphabetically. - script.run('git', 'tag', 'aaa/v1.0', head_sha, cwd=repo_dir) - script.run('git', 'tag', 'zzz/v1.0', head_sha, cwd=repo_dir) + script.run("git", "tag", "aaa/v1.0", head_sha, cwd=repo_dir) + script.run("git", "tag", "zzz/v1.0", head_sha, cwd=repo_dir) - check_rev(repo_dir, 'v1.0', (tag_sha, False)) - check_rev(repo_dir, 'v2.0', (tag_sha, False)) - check_rev(repo_dir, 'origin-branch', (origin_sha, True)) + check_rev(repo_dir, "v1.0", (tag_sha, False)) + check_rev(repo_dir, "v2.0", (tag_sha, False)) + check_rev(repo_dir, "origin-branch", (origin_sha, True)) ignored_names = [ # Local branches should be ignored. - 'local-branch', + "local-branch", # Non-origin remote branches should be ignored. - 'upstream-branch', + "upstream-branch", # Generic refs should be ignored. - 'generic-ref', + "generic-ref", # Fully spelled-out refs should be ignored. origin_ref, generic_ref, # Test passing a valid commit hash. tag_sha, # Test passing a non-existent name. - 'does-not-exist', + "does-not-exist", ] for name in ignored_names: check_rev(repo_dir, name, (None, False)) -def test_is_commit_id_equal(script): +def test_is_commit_id_equal(script: PipTestEnvironment) -> None: """ Test Git.is_commit_id_equal(). """ version_pkg_path = _create_test_package(script) - script.run('git', 'branch', 'branch0.1', cwd=version_pkg_path) - commit = script.run( - 'git', 'rev-parse', 'HEAD', - cwd=version_pkg_path - ).stdout.strip() + script.run("git", "branch", "branch0.1", cwd=version_pkg_path) + commit = script.run("git", "rev-parse", "HEAD", cwd=version_pkg_path).stdout.strip() assert Git.is_commit_id_equal(version_pkg_path, commit) assert not Git.is_commit_id_equal(version_pkg_path, commit[:7]) - assert not Git.is_commit_id_equal(version_pkg_path, 'branch0.1') - assert not Git.is_commit_id_equal(version_pkg_path, 'abc123') + assert not Git.is_commit_id_equal(version_pkg_path, "branch0.1") + assert not Git.is_commit_id_equal(version_pkg_path, "abc123") # Also check passing a None value. assert not Git.is_commit_id_equal(version_pkg_path, None) -def test_is_immutable_rev_checkout(script): +def test_is_immutable_rev_checkout(script: PipTestEnvironment) -> None: version_pkg_path = _create_test_package(script) - commit = script.run( - 'git', 'rev-parse', 'HEAD', - cwd=version_pkg_path - ).stdout.strip() + commit = script.run("git", "rev-parse", "HEAD", cwd=version_pkg_path).stdout.strip() assert Git().is_immutable_rev_checkout( "git+https://g.c/o/r@" + commit, version_pkg_path ) - assert not Git().is_immutable_rev_checkout( - "git+https://g.c/o/r", version_pkg_path - ) + assert not Git().is_immutable_rev_checkout("git+https://g.c/o/r", version_pkg_path) assert not Git().is_immutable_rev_checkout( "git+https://g.c/o/r@master", version_pkg_path ) -def test_get_repository_root(script): +def test_get_repository_root(script: PipTestEnvironment) -> None: version_pkg_path = _create_test_package(script) tests_path = version_pkg_path.joinpath("tests") tests_path.mkdir() root1 = Git.get_repository_root(version_pkg_path) + assert root1 is not None assert os.path.normcase(root1) == os.path.normcase(version_pkg_path) root2 = Git.get_repository_root(version_pkg_path.joinpath("tests")) + assert root2 is not None assert os.path.normcase(root2) == os.path.normcase(version_pkg_path) -def test_resolve_commit_not_on_branch(script, tmp_path): +def test_resolve_commit_not_on_branch( + script: PipTestEnvironment, tmp_path: pathlib.Path +) -> None: repo_path = tmp_path / "repo" repo_file = repo_path / "file.txt" clone_path = repo_path / "clone" @@ -267,9 +280,7 @@ def test_resolve_commit_not_on_branch(script, tmp_path): # create a commit repo_file.write_text("..") script.run("git", "commit", "-a", "-m", "commit 1", cwd=str(repo_path)) - commit = script.run( - "git", "rev-parse", "HEAD", cwd=str(repo_path) - ).stdout.strip() + commit = script.run("git", "rev-parse", "HEAD", cwd=str(repo_path)).stdout.strip() # make sure our commit is not on a branch script.run("git", "checkout", "master", cwd=str(repo_path)) @@ -281,4 +292,172 @@ def test_resolve_commit_not_on_branch(script, tmp_path): # check we can fetch our commit rev_options = Git.make_rev_options(commit) - Git().fetch_new(str(clone_path), repo_path.as_uri(), rev_options) + Git().fetch_new( + str(clone_path), + HiddenText(repo_path.as_uri(), redacted="*"), + rev_options, + verbosity=0, + ) + + +def _initialize_clonetest_server( + repo_path: pathlib.Path, script: PipTestEnvironment, enable_partial_clone: bool +) -> pathlib.Path: + repo_path.mkdir() + script.run("git", "init", cwd=str(repo_path)) + repo_file = repo_path / "file.txt" + repo_file.write_text(".") + script.run("git", "add", "file.txt", cwd=str(repo_path)) + script.run("git", "commit", "-m", "initial commit", cwd=str(repo_path)) + + # Enable filtering support on server + if enable_partial_clone: + script.run("git", "config", "uploadpack.allowFilter", "true", cwd=repo_path) + script.run( + "git", "config", "uploadpack.allowanysha1inwant", "true", cwd=repo_path + ) + + return repo_file + + +@pytest.mark.parametrize( + "version_out, expected_message", + ( + ("git version -2.25.1", "Can't parse git version: git version -2.25.1"), + ("git version 2.a.1", "Can't parse git version: git version 2.a.1"), + ("git ver. 2.25.1", "Can't parse git version: git ver. 2.25.1"), + ), +) +@patch("pip._internal.vcs.versioncontrol.VersionControl.run_command") +def test_git_parse_fail_warning( + mock_run_command: Mock, + caplog: pytest.LogCaptureFixture, + version_out: str, + expected_message: str, +) -> None: + """Test invalid git version logs adds an explicit warning log.""" + mock_run_command.return_value = version_out + + caplog.set_level(logging.WARNING) + + git_tuple = Git().get_git_version() + # Returns an empty tuple if it is an invalid git version + assert git_tuple == () + + # Check for warning log + assert expected_message in caplog.text.strip() + + +@pytest.mark.skipif(Git().get_git_version() < (2, 17), reason="git too old") +def test_partial_clone(script: PipTestEnvironment, tmp_path: pathlib.Path) -> None: + """Test partial clone w/ a git-server that supports it""" + repo_path = tmp_path / "repo" + repo_file = _initialize_clonetest_server( + repo_path, script, enable_partial_clone=True + ) + clone_path1 = repo_path / "clone1" + clone_path2 = repo_path / "clone2" + + commit = script.run("git", "rev-parse", "HEAD", cwd=str(repo_path)).stdout.strip() + + # Check that we can clone at HEAD + Git().fetch_new( + str(clone_path1), + HiddenText(repo_path.as_uri(), redacted="*"), + Git.make_rev_options(), + verbosity=0, + ) + # Check that we can clone to commit + Git().fetch_new( + str(clone_path2), + HiddenText(repo_path.as_uri(), redacted="*"), + Git.make_rev_options(commit), + verbosity=0, + ) + + # Write some additional stuff to git pull + repo_file.write_text("..") + script.run("git", "commit", "-am", "second commit", cwd=str(repo_path)) + + # Make sure git pull works - with server supporting filtering + assert ( + "warning: filtering not recognized by server, ignoring" + not in script.run("git", "pull", cwd=clone_path1).stderr + ) + assert ( + "warning: filtering not recognized by server, ignoring" + not in script.run("git", "pull", cwd=clone_path2).stderr + ) + + +@pytest.mark.skipif(Git().get_git_version() < (2, 17), reason="git too old") +def test_partial_clone_without_server_support( + script: PipTestEnvironment, tmp_path: pathlib.Path +) -> None: + """Test partial clone w/ a git-server that does not support it""" + repo_path = tmp_path / "repo" + repo_file = _initialize_clonetest_server( + repo_path, script, enable_partial_clone=False + ) + clone_path1 = repo_path / "clone1" + clone_path2 = repo_path / "clone2" + + commit = script.run("git", "rev-parse", "HEAD", cwd=str(repo_path)).stdout.strip() + + # Check that we can clone at HEAD + Git().fetch_new( + str(clone_path1), + HiddenText(repo_path.as_uri(), redacted="*"), + Git.make_rev_options(), + verbosity=0, + ) + # Check that we can clone to commit + Git().fetch_new( + str(clone_path2), + HiddenText(repo_path.as_uri(), redacted="*"), + Git.make_rev_options(commit), + verbosity=0, + ) + + # Write some additional stuff to git pull + repo_file.write_text("..") + script.run("git", "commit", "-am", "second commit", cwd=str(repo_path)) + + # Make sure git pull works - even though server doesn't support filtering + assert ( + "warning: filtering not recognized by server, ignoring" + in script.run("git", "pull", cwd=clone_path1).stderr + ) + assert ( + "warning: filtering not recognized by server, ignoring" + in script.run("git", "pull", cwd=clone_path2).stderr + ) + + +def test_clone_without_partial_clone_support( + script: PipTestEnvironment, tmp_path: pathlib.Path +) -> None: + """Older git clients don't support partial clone. Test the fallback path""" + repo_path = tmp_path / "repo" + repo_file = _initialize_clonetest_server( + repo_path, script, enable_partial_clone=True + ) + clone_path = repo_path / "clone1" + + # Check that we can clone w/ old version of git w/o --filter + with patch("pip._internal.vcs.git.Git.get_git_version", return_value=(2, 16)): + Git().fetch_new( + str(clone_path), + HiddenText(repo_path.as_uri(), redacted="*"), + Git.make_rev_options(), + verbosity=0, + ) + + repo_file.write_text("...") + script.run("git", "commit", "-am", "third commit", cwd=str(repo_path)) + + # Should work fine w/o attempting to use `--filter` args + assert ( + "warning: filtering not recognized by server, ignoring" + not in script.run("git", "pull", cwd=clone_path).stderr + ) diff --git a/tests/functional/test_vcs_mercurial.py b/tests/functional/test_vcs_mercurial.py index 841c4d8218e..3ac7ac3d1fc 100644 --- a/tests/functional/test_vcs_mercurial.py +++ b/tests/functional/test_vcs_mercurial.py @@ -1,17 +1,19 @@ import os from pip._internal.vcs.mercurial import Mercurial -from tests.lib import _create_test_package, need_mercurial +from tests.lib import PipTestEnvironment, _create_test_package, need_mercurial @need_mercurial -def test_get_repository_root(script): +def test_get_repository_root(script: PipTestEnvironment) -> None: version_pkg_path = _create_test_package(script, vcs="hg") tests_path = version_pkg_path.joinpath("tests") tests_path.mkdir() root1 = Mercurial.get_repository_root(version_pkg_path) + assert root1 is not None assert os.path.normcase(root1) == os.path.normcase(version_pkg_path) root2 = Mercurial.get_repository_root(version_pkg_path.joinpath("tests")) + assert root2 is not None assert os.path.normcase(root2) == os.path.normcase(version_pkg_path) diff --git a/tests/functional/test_vcs_subversion.py b/tests/functional/test_vcs_subversion.py index 194019da955..91627af8688 100644 --- a/tests/functional/test_vcs_subversion.py +++ b/tests/functional/test_vcs_subversion.py @@ -2,14 +2,15 @@ from pip._internal.vcs.subversion import Subversion from pip._internal.vcs.versioncontrol import RemoteNotFoundError -from tests.lib import _create_svn_repo, need_svn +from tests.lib import PipTestEnvironment, _create_svn_repo, need_svn +from tests.lib.path import Path @need_svn -def test_get_remote_url__no_remote(script, tmpdir): - repo_dir = tmpdir / 'temp-repo' - repo_dir.mkdir() - repo_dir = str(repo_dir) +def test_get_remote_url__no_remote(script: PipTestEnvironment, tmpdir: Path) -> None: + repo_path = tmpdir / "temp-repo" + repo_path.mkdir() + repo_dir = str(repo_path) _create_svn_repo(script, repo_dir) @@ -18,12 +19,14 @@ def test_get_remote_url__no_remote(script, tmpdir): @need_svn -def test_get_remote_url__no_remote_with_setup(script, tmpdir): - repo_dir = tmpdir / 'temp-repo' - repo_dir.mkdir() - setup = repo_dir / "setup.py" +def test_get_remote_url__no_remote_with_setup( + script: PipTestEnvironment, tmpdir: Path +) -> None: + repo_path = tmpdir / "temp-repo" + repo_path.mkdir() + setup = repo_path / "setup.py" setup.touch() - repo_dir = str(repo_dir) + repo_dir = str(repo_path) _create_svn_repo(script, repo_dir) diff --git a/tests/functional/test_warning.py b/tests/functional/test_warning.py index 3558704bcef..a7018944873 100644 --- a/tests/functional/test_warning.py +++ b/tests/functional/test_warning.py @@ -1,44 +1,68 @@ +import sys import textwrap import pytest +from tests.lib import PipTestEnvironment +from tests.lib.path import Path + @pytest.fixture -def warnings_demo(tmpdir): - demo = tmpdir.joinpath('warnings_demo.py') - demo.write_text(textwrap.dedent(''' +def warnings_demo(tmpdir: Path) -> Path: + demo = tmpdir.joinpath("warnings_demo.py") + demo.write_text( + textwrap.dedent( + """ from logging import basicConfig from pip._internal.utils import deprecation deprecation.install_warning_logger() basicConfig() - deprecation.deprecated("deprecated!", replacement=None, gone_in=None) - ''')) + deprecation.deprecated(reason="deprecated!", replacement=None, gone_in=None) + """ + ) + ) return demo -def test_deprecation_warnings_are_correct(script, warnings_demo): - result = script.run('python', warnings_demo, expect_stderr=True) - expected = 'WARNING:pip._internal.deprecations:DEPRECATION: deprecated!\n' +def test_deprecation_warnings_are_correct( + script: PipTestEnvironment, warnings_demo: Path +) -> None: + result = script.run("python", warnings_demo, expect_stderr=True) + expected = "WARNING:pip._internal.deprecations:DEPRECATION: deprecated!\n" assert result.stderr == expected -def test_deprecation_warnings_can_be_silenced(script, warnings_demo): - script.environ['PYTHONWARNINGS'] = 'ignore' - result = script.run('python', warnings_demo) - assert result.stderr == '' +def test_deprecation_warnings_can_be_silenced( + script: PipTestEnvironment, warnings_demo: Path +) -> None: + script.environ["PYTHONWARNINGS"] = "ignore" + result = script.run("python", warnings_demo) + assert result.stderr == "" DEPRECATION_TEXT = "drop support for Python 2.7" CPYTHON_DEPRECATION_TEXT = "January 1st, 2020" -def test_version_warning_is_not_shown_if_python_version_is_not_2(script): +def test_version_warning_is_not_shown_if_python_version_is_not_2( + script: PipTestEnvironment, +) -> None: result = script.pip("debug", allow_stderr_warning=True) assert DEPRECATION_TEXT not in result.stderr, str(result) assert CPYTHON_DEPRECATION_TEXT not in result.stderr, str(result) -def test_flag_does_nothing_if_python_version_is_not_2(script): +def test_flag_does_nothing_if_python_version_is_not_2( + script: PipTestEnvironment, +) -> None: script.pip("list", "--no-python-version-warning") + + +@pytest.mark.skipif( + sys.version_info >= (3, 10), reason="distutils is deprecated in 3.10+" +) +def test_pip_works_with_warnings_as_errors(script: PipTestEnvironment) -> None: + script.environ["PYTHONWARNINGS"] = "error" + script.pip("--version") diff --git a/tests/functional/test_wheel.py b/tests/functional/test_wheel.py index 3c1a3299c36..dedf22ba8cc 100644 --- a/tests/functional/test_wheel.py +++ b/tests/functional/test_wheel.py @@ -2,76 +2,84 @@ import os import re import sys -from os.path import exists import pytest from pip._internal.cli.status_codes import ERROR from tests.lib import pyversion # noqa: F401 +from tests.lib import PipTestEnvironment, TestData +from tests.lib.path import Path +pytestmark = pytest.mark.usefixtures("with_wheel") -@pytest.fixture(autouse=True) -def auto_with_wheel(with_wheel): - pass - -def add_files_to_dist_directory(folder): - (folder / 'dist').mkdir(parents=True) - (folder / 'dist' / 'a_name-0.0.1.tar.gz').write_text("hello") +def add_files_to_dist_directory(folder: Path) -> None: + (folder / "dist").mkdir(parents=True) + (folder / "dist" / "a_name-0.0.1.tar.gz").write_text("hello") # Not adding a wheel file since that confuses setuptools' backend. # (folder / 'dist' / 'a_name-0.0.1-py2.py3-none-any.whl').write_text( # "hello" # ) -def test_wheel_exit_status_code_when_no_requirements(script): +def test_wheel_exit_status_code_when_no_requirements( + script: PipTestEnvironment, +) -> None: """ Test wheel exit status code when no requirements specified """ - result = script.pip('wheel', expect_error=True) + result = script.pip("wheel", expect_error=True) assert "You must give at least one requirement to wheel" in result.stderr assert result.returncode == ERROR -def test_wheel_exit_status_code_when_blank_requirements_file(script): +def test_wheel_exit_status_code_when_blank_requirements_file( + script: PipTestEnvironment, +) -> None: """ Test wheel exit status code when blank requirements file specified """ script.scratch_path.joinpath("blank.txt").write_text("\n") - script.pip('wheel', '-r', 'blank.txt') + script.pip("wheel", "-r", "blank.txt") -def test_pip_wheel_success(script, data): +def test_pip_wheel_success(script: PipTestEnvironment, data: TestData) -> None: """ Test 'pip wheel' success. """ result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - 'simple==3.0', + "wheel", + "--no-index", + "-f", + data.find_links, + "simple==3.0", ) - wheel_file_name = 'simple-3.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"simple-3.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name assert re.search( r"Created wheel for simple: " - r"filename={filename} size=\d+ sha256=[A-Fa-f0-9]{{64}}" - .format(filename=re.escape(wheel_file_name)), result.stdout) - assert re.search( - r"^\s+Stored in directory: ", result.stdout, re.M) + r"filename={filename} size=\d+ sha256=[A-Fa-f0-9]{{64}}".format( + filename=re.escape(wheel_file_name) + ), + result.stdout, + ) + assert re.search(r"^\s+Stored in directory: ", result.stdout, re.M) result.did_create(wheel_file_path) assert "Successfully built simple" in result.stdout, result.stdout -def test_pip_wheel_build_cache(script, data): +def test_pip_wheel_build_cache(script: PipTestEnvironment, data: TestData) -> None: """ Test 'pip wheel' builds and caches. """ result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - 'simple==3.0', + "wheel", + "--no-index", + "-f", + data.find_links, + "simple==3.0", ) - wheel_file_name = 'simple-3.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"simple-3.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) assert "Successfully built simple" in result.stdout, result.stdout @@ -80,122 +88,170 @@ def test_pip_wheel_build_cache(script, data): # pip wheel again and test that no build occurs since # we get the wheel from cache result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - 'simple==3.0', + "wheel", + "--no-index", + "-f", + data.find_links, + "simple==3.0", ) result.did_create(wheel_file_path) assert "Successfully built simple" not in result.stdout, result.stdout -def test_basic_pip_wheel_downloads_wheels(script, data): +def test_basic_pip_wheel_downloads_wheels( + script: PipTestEnvironment, data: TestData +) -> None: """ Test 'pip wheel' downloads wheels """ result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, 'simple.dist', + "wheel", + "--no-index", + "-f", + data.find_links, + "simple.dist", ) - wheel_file_name = 'simple.dist-0.1-py2.py3-none-any.whl' + wheel_file_name = "simple.dist-0.1-py2.py3-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) assert "Saved" in result.stdout, result.stdout -def test_pip_wheel_build_relative_cachedir(script, data): +def test_pip_wheel_build_relative_cachedir( + script: PipTestEnvironment, data: TestData +) -> None: """ Test 'pip wheel' builds and caches with a non-absolute cache directory. """ result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - '--cache-dir', './cache', - 'simple==3.0', + "wheel", + "--no-index", + "-f", + data.find_links, + "--cache-dir", + "./cache", + "simple==3.0", ) assert result.returncode == 0 -def test_pip_wheel_builds_when_no_binary_set(script, data): - data.packages.joinpath('simple-3.0-py2.py3-none-any.whl').touch() +def test_pip_wheel_builds_when_no_binary_set( + script: PipTestEnvironment, data: TestData +) -> None: + data.packages.joinpath("simple-3.0-py2.py3-none-any.whl").touch() # Check that the wheel package is ignored res = script.pip( - 'wheel', '--no-index', '--no-binary', ':all:', - '-f', data.find_links, - 'simple==3.0') + "wheel", + "--no-index", + "--no-binary", + ":all:", + "-f", + data.find_links, + "simple==3.0", + ) assert "Building wheel for simple" in str(res), str(res) @pytest.mark.skipif("sys.platform == 'win32'") -def test_pip_wheel_readonly_cache(script, data, tmpdir): +def test_pip_wheel_readonly_cache( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: cache_dir = tmpdir / "cache" cache_dir.mkdir() os.chmod(cache_dir, 0o400) # read-only cache # Check that the wheel package is ignored res = script.pip( - 'wheel', '--no-index', - '-f', data.find_links, - '--cache-dir', cache_dir, - 'simple==3.0', + "wheel", + "--no-index", + "-f", + data.find_links, + "--cache-dir", + cache_dir, + "simple==3.0", allow_stderr_warning=True, ) assert res.returncode == 0 assert "The cache has been disabled." in str(res), str(res) -def test_pip_wheel_builds_editable_deps(script, data): +def test_pip_wheel_builds_editable_deps( + script: PipTestEnvironment, data: TestData +) -> None: """ Test 'pip wheel' finds and builds dependencies of editables """ - editable_path = os.path.join(data.src, 'requires_simple') + editable_path = os.path.join(data.src, "requires_simple") result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - '-e', editable_path + "wheel", "--no-index", "-f", data.find_links, "-e", editable_path ) - wheel_file_name = 'simple-1.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"simple-1.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) -def test_pip_wheel_builds_editable(script, data): +def test_pip_wheel_builds_editable(script: PipTestEnvironment, data: TestData) -> None: """ Test 'pip wheel' builds an editable package """ - editable_path = os.path.join(data.src, 'simplewheel-1.0') + editable_path = os.path.join(data.src, "simplewheel-1.0") result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - '-e', editable_path + "wheel", "--no-index", "-f", data.find_links, "-e", editable_path ) - wheel_file_name = 'simplewheel-1.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"simplewheel-1.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) -def test_pip_wheel_builds_editable_does_not_create_zip(script, data, tmpdir): +@pytest.mark.network +def test_pip_wheel_git_editable_keeps_clone( + script: PipTestEnvironment, tmpdir: Path +) -> None: + """ + Test that `pip wheel -e giturl` preserves a git clone in src. + """ + script.pip( + "wheel", + "--no-deps", + "-e", + "git+https://github.com/pypa/pip-test-package#egg=pip-test-package", + "--src", + tmpdir / "src", + "--wheel-dir", + tmpdir, + ) + assert (tmpdir / "src" / "pip-test-package").exists() + assert (tmpdir / "src" / "pip-test-package" / ".git").exists() + + +def test_pip_wheel_builds_editable_does_not_create_zip( + script: PipTestEnvironment, data: TestData, tmpdir: Path +) -> None: """ Test 'pip wheel' of editables does not create zip files (regression test for issue #9122) """ wheel_dir = tmpdir / "wheel_dir" wheel_dir.mkdir() - editable_path = os.path.join(data.src, 'simplewheel-1.0') - script.pip( - 'wheel', '--no-deps', '-e', editable_path, '-w', wheel_dir - ) + editable_path = os.path.join(data.src, "simplewheel-1.0") + script.pip("wheel", "--no-deps", "-e", editable_path, "-w", wheel_dir) wheels = os.listdir(wheel_dir) assert len(wheels) == 1 assert wheels[0].endswith(".whl") -def test_pip_wheel_fail(script, data): +def test_pip_wheel_fail(script: PipTestEnvironment, data: TestData) -> None: """ Test 'pip wheel' failure. """ result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - 'wheelbroken==0.1', + "wheel", + "--no-index", + "-f", + data.find_links, + "wheelbroken==0.1", expect_error=True, ) - wheel_file_name = 'wheelbroken-0.1-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"wheelbroken-0.1-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_not_create(wheel_file_path) assert "FakeError" in result.stderr, result.stderr @@ -203,139 +259,132 @@ def test_pip_wheel_fail(script, data): assert result.returncode != 0 -@pytest.mark.xfail( - reason="The --build option was removed" -) -def test_no_clean_option_blocks_cleaning_after_wheel( - script, - data, - resolver_variant, -): - """ - Test --no-clean option blocks cleaning after wheel build - """ - build = script.venv_path / 'build' - result = script.pip( - 'wheel', '--no-clean', '--no-index', '--build', build, - '--find-links={data.find_links}'.format(**locals()), - 'simple', - expect_temp=True, - # TODO: allow_stderr_warning is used for the --build deprecation, - # remove it when removing support for --build - allow_stderr_warning=True, - ) - - if resolver_variant == "legacy": - build = build / 'simple' - message = f"build/simple should still exist {result}" - assert exists(build), message - - -def test_pip_wheel_source_deps(script, data): +def test_pip_wheel_source_deps(script: PipTestEnvironment, data: TestData) -> None: """ Test 'pip wheel' finds and builds source archive dependencies of wheels """ # 'requires_source' is a wheel that depends on the 'source' project result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - 'requires_source', + "wheel", + "--no-index", + "-f", + data.find_links, + "requires_source", ) - wheel_file_name = 'source-1.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"source-1.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) assert "Successfully built source" in result.stdout, result.stdout -def test_wheel_package_with_latin1_setup(script, data): +def test_wheel_package_with_latin1_setup( + script: PipTestEnvironment, data: TestData +) -> None: """Create a wheel from a package with latin-1 encoded setup.py.""" pkg_to_wheel = data.packages.joinpath("SetupPyLatin1") - result = script.pip('wheel', pkg_to_wheel) - assert 'Successfully built SetupPyUTF8' in result.stdout + result = script.pip("wheel", pkg_to_wheel) + assert "Successfully built SetupPyUTF8" in result.stdout -def test_pip_wheel_with_pep518_build_reqs(script, data, common_wheels): - result = script.pip('wheel', '--no-index', '-f', data.find_links, - '-f', common_wheels, 'pep518==3.0',) - wheel_file_name = 'pep518-3.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) +def test_pip_wheel_with_pep518_build_reqs( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: + result = script.pip( + "wheel", + "--no-index", + "-f", + data.find_links, + "-f", + common_wheels, + "pep518==3.0", + ) + wheel_file_name = f"pep518-3.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) assert "Successfully built pep518" in result.stdout, result.stdout assert "Installing build dependencies" in result.stdout, result.stdout -def test_pip_wheel_with_pep518_build_reqs_no_isolation(script, data): - script.pip_install_local('simplewheel==2.0') +def test_pip_wheel_with_pep518_build_reqs_no_isolation( + script: PipTestEnvironment, data: TestData +) -> None: + script.pip_install_local("simplewheel==2.0") result = script.pip( - 'wheel', '--no-index', '-f', data.find_links, - '--no-build-isolation', 'pep518==3.0', + "wheel", + "--no-index", + "-f", + data.find_links, + "--no-build-isolation", + "pep518==3.0", ) - wheel_file_name = 'pep518-3.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"pep518-3.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) assert "Successfully built pep518" in result.stdout, result.stdout assert "Installing build dependencies" not in result.stdout, result.stdout -def test_pip_wheel_with_user_set_in_config(script, data, common_wheels): - config_file = script.scratch_path / 'pip.conf' - script.environ['PIP_CONFIG_FILE'] = str(config_file) +def test_pip_wheel_with_user_set_in_config( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: + config_file = script.scratch_path / "pip.conf" + script.environ["PIP_CONFIG_FILE"] = str(config_file) config_file.write_text("[install]\nuser = true") result = script.pip( - 'wheel', data.src / 'withpyproject', - '--no-index', '-f', common_wheels + "wheel", data.src / "withpyproject", "--no-index", "-f", common_wheels ) assert "Successfully built withpyproject" in result.stdout, result.stdout -@pytest.mark.skipif(sys.platform.startswith('win'), - reason='The empty extension module does not work on Win') -def test_pip_wheel_ext_module_with_tmpdir_inside(script, data, common_wheels): - tmpdir = data.src / 'extension/tmp' +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason="The empty extension module does not work on Win", +) +def test_pip_wheel_ext_module_with_tmpdir_inside( + script: PipTestEnvironment, data: TestData, common_wheels: Path +) -> None: + tmpdir = data.src / "extension/tmp" tmpdir.mkdir() - script.environ['TMPDIR'] = str(tmpdir) + script.environ["TMPDIR"] = str(tmpdir) # To avoid a test dependency on a C compiler, we set the env vars to "noop" # The .c source is empty anyway - script.environ['CC'] = script.environ['LDSHARED'] = 'true' + script.environ["CC"] = script.environ["LDSHARED"] = "true" result = script.pip( - 'wheel', data.src / 'extension', - '--no-index', '-f', common_wheels + "wheel", data.src / "extension", "--no-index", "-f", common_wheels ) assert "Successfully built extension" in result.stdout, result.stdout @pytest.mark.network -def test_pep517_wheels_are_not_confused_with_other_files(script, tmpdir, data): - """Check correct wheels are copied. (#6196) - """ - pkg_to_wheel = data.src / 'withpyproject' +def test_pep517_wheels_are_not_confused_with_other_files( + script: PipTestEnvironment, data: TestData +) -> None: + """Check correct wheels are copied. (#6196)""" + pkg_to_wheel = data.src / "withpyproject" add_files_to_dist_directory(pkg_to_wheel) - result = script.pip('wheel', pkg_to_wheel, '-w', script.scratch_path) + result = script.pip("wheel", pkg_to_wheel, "-w", script.scratch_path) assert "Installing build dependencies" in result.stdout, result.stdout - wheel_file_name = 'withpyproject-0.0.1-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"withpyproject-0.0.1-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) -def test_legacy_wheels_are_not_confused_with_other_files(script, tmpdir, data): - """Check correct wheels are copied. (#6196) - """ - pkg_to_wheel = data.src / 'simplewheel-1.0' +def test_legacy_wheels_are_not_confused_with_other_files( + script: PipTestEnvironment, data: TestData +) -> None: + """Check correct wheels are copied. (#6196)""" + pkg_to_wheel = data.src / "simplewheel-1.0" add_files_to_dist_directory(pkg_to_wheel) - result = script.pip('wheel', pkg_to_wheel, '-w', script.scratch_path) + result = script.pip("wheel", pkg_to_wheel, "-w", script.scratch_path) assert "Installing build dependencies" not in result.stdout, result.stdout - wheel_file_name = 'simplewheel-1.0-py{pyversion[0]}-none-any.whl' \ - .format(**globals()) + wheel_file_name = f"simplewheel-1.0-py{pyversion[0]}-none-any.whl" wheel_file_path = script.scratch / wheel_file_name result.did_create(wheel_file_path) diff --git a/tests/functional/test_yaml.py b/tests/functional/test_yaml.py deleted file mode 100644 index ba7b17531ef..00000000000 --- a/tests/functional/test_yaml.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Tests for the resolver -""" - -import os -import re -import sys - -import pytest -import yaml - -from tests.lib import DATA_DIR, create_basic_wheel_for_package, path_to_url - - -def generate_yaml_tests(directory): - """ - Generate yaml test cases from the yaml files in the given directory - """ - for yml_file in directory.glob("*.yml"): - data = yaml.safe_load(yml_file.read_text()) - assert "cases" in data, "A fixture needs cases to be used in testing" - - # Strip the parts of the directory to only get a name without - # extension and resolver directory - base_name = str(yml_file)[len(str(directory)) + 1:-4] - - base = data.get("base", {}) - cases = data["cases"] - - for resolver in 'legacy', '2020-resolver': - for i, case_template in enumerate(cases): - case = base.copy() - case.update(case_template) - - case[":name:"] = base_name - if len(cases) > 1: - case[":name:"] += "-" + str(i) - case[":name:"] += "*" + resolver - case[":resolver:"] = resolver - - skip = case.pop("skip", False) - assert skip in [False, True, 'legacy', '2020-resolver'] - if skip is True or skip == resolver: - case = pytest.param(case, marks=pytest.mark.xfail) - - yield case - - -def id_func(param): - """ - Give a nice parameter name to the generated function parameters - """ - if isinstance(param, dict) and ":name:" in param: - return param[":name:"] - - retval = str(param) - if len(retval) > 25: - retval = retval[:20] + "..." + retval[-2:] - return retval - - -def convert_to_dict(string): - - def stripping_split(my_str, splitwith, count=None): - if count is None: - return [x.strip() for x in my_str.strip().split(splitwith)] - else: - return [x.strip() for x in my_str.strip().split(splitwith, count)] - - parts = stripping_split(string, ";") - - retval = {} - retval["depends"] = [] - retval["extras"] = {} - - retval["name"], retval["version"] = stripping_split(parts[0], " ") - - for part in parts[1:]: - verb, args_str = stripping_split(part, " ", 1) - assert verb in ["depends"], f"Unknown verb {verb!r}" - - retval[verb] = stripping_split(args_str, ",") - - return retval - - -def handle_request(script, action, requirement, options, resolver_variant): - if action == 'install': - args = ['install'] - if resolver_variant == "legacy": - args.append("--use-deprecated=legacy-resolver") - args.extend(["--no-index", "--find-links", - path_to_url(script.scratch_path)]) - elif action == 'uninstall': - args = ['uninstall', '--yes'] - else: - raise f"Did not excpet action: {action!r}" - - if isinstance(requirement, str): - args.append(requirement) - elif isinstance(requirement, list): - args.extend(requirement) - else: - raise f"requirement neither str nor list {requirement!r}" - - args.extend(options) - args.append("--verbose") - - result = script.pip(*args, - allow_stderr_error=True, - allow_stderr_warning=True, - allow_error=True) - - # Check which packages got installed - state = [] - for path in os.listdir(script.site_packages_path): - if path.endswith(".dist-info"): - name, version = ( - os.path.basename(path)[:-len(".dist-info")] - ).rsplit("-", 1) - # TODO: information about extras. - state.append(" ".join((name, version))) - - return {"result": result, "state": sorted(state)} - - -def check_error(error, result): - return_code = error.get('code') - if return_code: - assert result.returncode == return_code - - stderr = error.get('stderr') - if not stderr: - return - - if isinstance(stderr, str): - patters = [stderr] - elif isinstance(stderr, list): - patters = stderr - else: - raise "string or list expected, found %r" % stderr - - for patter in patters: - match = re.search(patter, result.stderr, re.I) - assert match, 'regex %r not found in stderr: %r' % ( - stderr, result.stderr) - - -@pytest.mark.yaml -@pytest.mark.parametrize( - "case", generate_yaml_tests(DATA_DIR.parent / "yaml"), ids=id_func -) -def test_yaml_based(script, case): - available = case.get("available", []) - requests = case.get("request", []) - responses = case.get("response", []) - - assert len(requests) == len(responses), ( - "Expected requests and responses counts to be same" - ) - - # Create a custom index of all the packages that are supposed to be - # available - # XXX: This doesn't work because this isn't making an index of files. - for package in available: - if isinstance(package, str): - package = convert_to_dict(package) - - assert isinstance(package, dict), "Needs to be a dictionary" - - create_basic_wheel_for_package(script, **package) - - # use scratch path for index - for request, response in zip(requests, responses): - - for action in 'install', 'uninstall': - if action in request: - break - else: - raise f"Unsupported request {request!r}" - - # Perform the requested action - effect = handle_request(script, action, - request[action], - request.get('options', '').split(), - resolver_variant=case[':resolver:']) - result = effect['result'] - - if 0: # for analyzing output easier - with open(DATA_DIR.parent / "yaml" / - case[':name:'].replace('*', '-'), 'w') as fo: - fo.write("=== RETURNCODE = %d\n" % result.returncode) - fo.write("=== STDERR ===:\n%s\n" % result.stderr) - - if 'state' in response: - assert effect['state'] == (response['state'] or []), str(result) - - error = response.get('error') - if error and case[":resolver:"] == 'new' and sys.platform != 'win32': - # Note: we currently skip running these tests on Windows, as they - # were failing due to different error codes. There should not - # be a reason for not running these this check on Windows. - check_error(error, result) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index 6a98d4acf78..c8d68fea9b0 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -1,4 +1,6 @@ +import json import os +import pathlib import re import shutil import site @@ -10,43 +12,60 @@ from hashlib import sha256 from io import BytesIO from textwrap import dedent +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterator, + List, + Optional, + Tuple, + Union, + cast, +) from zipfile import ZipFile import pytest -from pip._vendor.six import ensure_binary -from scripttest import FoundDir, TestFileEnvironment +from pip._vendor.packaging.utils import canonicalize_name +from scripttest import FoundDir, FoundFile, ProcResult, TestFileEnvironment from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.locations import get_major_minor_version from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.models.target_python import TargetPython from pip._internal.network.session import PipSession from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX -from pip._internal.utils.typing import MYPY_CHECK_RUNNING from tests.lib.path import Path, curdir +from tests.lib.venv import VirtualEnvironment from tests.lib.wheel import make_wheel -if MYPY_CHECK_RUNNING: - from typing import List, Optional - - from pip._internal.models.target_python import TargetPython +if TYPE_CHECKING: + # Literal was introduced in Python 3.8. + from typing import Literal + ResolverVariant = Literal["resolvelib", "legacy"] +else: + ResolverVariant = str DATA_DIR = Path(__file__).parent.parent.joinpath("data").resolve() SRC_DIR = Path(__file__).resolve().parent.parent.parent pyversion = get_major_minor_version() -pyversion_tuple = sys.version_info CURRENT_PY_VERSION_INFO = sys.version_info[:3] +_Test = Callable[..., None] +_FilesState = Dict[str, Union[FoundDir, FoundFile]] + -def assert_paths_equal(actual, expected): +def assert_paths_equal(actual: str, expected: str) -> None: assert os.path.normpath(actual) == os.path.normpath(expected) -def path_to_url(path): +def path_to_url(path: str) -> str: """ Convert a path to URI. The path will be made absolute and will not have quoted path parts. @@ -55,27 +74,26 @@ def path_to_url(path): path = os.path.normpath(os.path.abspath(path)) drive, path = os.path.splitdrive(path) filepath = path.split(os.path.sep) - url = '/'.join(filepath) + url = "/".join(filepath) if drive: # Note: match urllib.request.pathname2url's # behavior: uppercase the drive letter. - return 'file:///' + drive.upper() + url - return 'file://' + url + return "file:///" + drive.upper() + url + return "file://" + url -def _test_path_to_file_url(path): +def _test_path_to_file_url(path: Path) -> str: """ Convert a test Path to a "file://" URL. Args: path: a tests.lib.path.Path object. """ - return 'file://' + path.resolve().replace('\\', '/') + return "file://" + path.resolve().replace("\\", "/") -def create_file(path, contents=None): - """Create a file on the path, with the given contents - """ +def create_file(path: str, contents: Optional[str] = None) -> None: + """Create a file on the path, with the given contents""" from pip._internal.utils.misc import ensure_dir ensure_dir(os.path.dirname(path)) @@ -87,9 +105,9 @@ def create_file(path, contents=None): def make_test_search_scope( - find_links=None, # type: Optional[List[str]] - index_urls=None, # type: Optional[List[str]] -): + find_links: Optional[List[str]] = None, + index_urls: Optional[List[str]] = None, +) -> SearchScope: if find_links is None: find_links = [] if index_urls is None: @@ -99,11 +117,10 @@ def make_test_search_scope( def make_test_link_collector( - find_links=None, # type: Optional[List[str]] - index_urls=None, # type: Optional[List[str]] - session=None, # type: Optional[PipSession] -): - # type: (...) -> LinkCollector + find_links: Optional[List[str]] = None, + index_urls: Optional[List[str]] = None, + session: Optional[PipSession] = None, +) -> LinkCollector: """ Create a LinkCollector object for testing purposes. """ @@ -119,13 +136,13 @@ def make_test_link_collector( def make_test_finder( - find_links=None, # type: Optional[List[str]] - index_urls=None, # type: Optional[List[str]] - allow_all_prereleases=False, # type: bool - session=None, # type: Optional[PipSession] - target_python=None, # type: Optional[TargetPython] -): - # type: (...) -> PackageFinder + find_links: Optional[List[str]] = None, + index_urls: Optional[List[str]] = None, + allow_all_prereleases: bool = False, + session: Optional[PipSession] = None, + target_python: Optional[TargetPython] = None, + use_deprecated_html5lib: bool = False, +) -> PackageFinder: """ Create a PackageFinder for testing purposes. """ @@ -143,6 +160,7 @@ def make_test_finder( link_collector=link_collector, selection_prefs=selection_prefs, target_python=target_python, + use_deprecated_html5lib=use_deprecated_html5lib, ) @@ -158,17 +176,19 @@ class TestData: data into a directory and operating on the copied data. """ - def __init__(self, root, source=None): + __test__ = False + + def __init__(self, root: str, source: Optional[Path] = None) -> None: self.source = source or DATA_DIR self.root = Path(root).resolve() @classmethod - def copy(cls, root): + def copy(cls, root: str) -> "TestData": obj = cls(root) obj.reset() return obj - def reset(self): + def reset(self) -> None: # Check explicitly for the target directory to avoid overly-broad # try/except. if self.root.exists(): @@ -176,50 +196,50 @@ def reset(self): shutil.copytree(self.source, self.root, symlinks=True) @property - def packages(self): + def packages(self) -> Path: return self.root.joinpath("packages") @property - def packages2(self): + def packages2(self) -> Path: return self.root.joinpath("packages2") @property - def packages3(self): + def packages3(self) -> Path: return self.root.joinpath("packages3") @property - def src(self): + def src(self) -> Path: return self.root.joinpath("src") @property - def indexes(self): + def indexes(self) -> Path: return self.root.joinpath("indexes") @property - def reqfiles(self): + def reqfiles(self) -> Path: return self.root.joinpath("reqfiles") @property - def completion_paths(self): + def completion_paths(self) -> Path: return self.root.joinpath("completion_paths") @property - def find_links(self): + def find_links(self) -> str: return path_to_url(self.packages) @property - def find_links2(self): + def find_links2(self) -> str: return path_to_url(self.packages2) @property - def find_links3(self): + def find_links3(self) -> str: return path_to_url(self.packages3) @property - def backends(self): + def backends(self) -> str: return path_to_url(self.root.joinpath("backends")) - def index_url(self, index="simple"): + def index_url(self, index: str = "simple") -> str: return path_to_url(self.root.joinpath("indexes", index)) @@ -227,51 +247,61 @@ class TestFailure(AssertionError): """ An "assertion" failed during testing. """ + pass class TestPipResult: + __test__ = False - def __init__(self, impl, verbose=False): + def __init__(self, impl: ProcResult, verbose: bool = False) -> None: self._impl = impl if verbose: print(self.stdout) if self.stderr: - print('======= stderr ========') + print("======= stderr ========") print(self.stderr) - print('=======================') + print("=======================") - def __getattr__(self, attr): + def __getattr__(self, attr: str) -> Any: return getattr(self._impl, attr) - if sys.platform == 'win32': + if sys.platform == "win32": @property - def stdout(self): - return self._impl.stdout.replace('\r\n', '\n') + def stdout(self) -> str: + return self._impl.stdout.replace("\r\n", "\n") @property - def stderr(self): - return self._impl.stderr.replace('\r\n', '\n') + def stderr(self) -> str: + return self._impl.stderr.replace("\r\n", "\n") + + def __str__(self) -> str: + return str(self._impl).replace("\r\n", "\n") - def __str__(self): - return str(self._impl).replace('\r\n', '\n') else: # Python doesn't automatically forward __str__ through __getattr__ - def __str__(self): + def __str__(self) -> str: return str(self._impl) - def assert_installed(self, pkg_name, editable=True, with_files=None, - without_files=None, without_egg_link=False, - use_user_site=False, sub_dir=False): + def assert_installed( + self, + pkg_name: str, + editable: bool = True, + with_files: Optional[List[str]] = None, + without_files: Optional[List[str]] = None, + without_egg_link: bool = False, + use_user_site: bool = False, + sub_dir: Optional[str] = None, + ) -> None: with_files = with_files or [] without_files = without_files or [] e = self.test_env if editable: - pkg_dir = e.venv / 'src' / pkg_name.lower() + pkg_dir = e.venv / "src" / pkg_name.lower() # If package was installed in a sub directory if sub_dir: pkg_dir = pkg_dir / sub_dir @@ -280,116 +310,117 @@ def assert_installed(self, pkg_name, editable=True, with_files=None, pkg_dir = e.site_packages / pkg_name if use_user_site: - egg_link_path = e.user_site / pkg_name + '.egg-link' + egg_link_path = e.user_site / pkg_name + ".egg-link" else: - egg_link_path = e.site_packages / pkg_name + '.egg-link' + egg_link_path = e.site_packages / pkg_name + ".egg-link" if without_egg_link: if egg_link_path in self.files_created: raise TestFailure( - 'unexpected egg link file created: ' - '{egg_link_path!r}\n{self}' - .format(**locals()) + f"unexpected egg link file created: {egg_link_path!r}\n{self}" ) else: if egg_link_path not in self.files_created: raise TestFailure( - 'expected egg link file missing: ' - '{egg_link_path!r}\n{self}' - .format(**locals()) + f"expected egg link file missing: {egg_link_path!r}\n{self}" ) egg_link_file = self.files_created[egg_link_path] - egg_link_contents = egg_link_file.bytes.replace(os.linesep, '\n') + egg_link_contents = egg_link_file.bytes.replace(os.linesep, "\n") # FIXME: I don't understand why there's a trailing . here - if not (egg_link_contents.endswith('\n.') and - egg_link_contents[:-2].endswith(pkg_dir)): - raise TestFailure(textwrap.dedent( - '''\ - Incorrect egg_link file {egg_link_file!r} - Expected ending: {expected_ending!r} - ------- Actual contents ------- - {egg_link_contents!r} - -------------------------------'''.format( - expected_ending=pkg_dir + '\n.', - **locals()) - )) + if not ( + egg_link_contents.endswith("\n.") + and egg_link_contents[:-2].endswith(pkg_dir) + ): + expected_ending = pkg_dir + "\n." + raise TestFailure( + textwrap.dedent( + f""" + Incorrect egg_link file {egg_link_file!r} + Expected ending: {expected_ending!r} + ------- Actual contents ------- + {egg_link_contents!r} + ------------------------------- + """ + ).strip() + ) if use_user_site: - pth_file = e.user_site / 'easy-install.pth' + pth_file = e.user_site / "easy-install.pth" else: - pth_file = e.site_packages / 'easy-install.pth' + pth_file = e.site_packages / "easy-install.pth" if (pth_file in self.files_updated) == without_egg_link: - raise TestFailure( - '{pth_file} unexpectedly {maybe}updated by install'.format( - maybe=not without_egg_link and 'not ' or '', - **locals())) + maybe = "" if without_egg_link else "not " + raise TestFailure(f"{pth_file} unexpectedly {maybe}updated by install") if (pkg_dir in self.files_created) == (curdir in without_files): - raise TestFailure(textwrap.dedent('''\ - expected package directory {pkg_dir!r} {maybe}to be created - actually created: - {files} - ''').format( - pkg_dir=pkg_dir, - maybe=curdir in without_files and 'not ' or '', - files=sorted(self.files_created.keys()), - )) + maybe = "not " if curdir in without_files else "" + files = sorted(self.files_created) + raise TestFailure( + textwrap.dedent( + f""" + expected package directory {pkg_dir!r} {maybe}to be created + actually created: + {files} + """ + ) + ) for f in with_files: normalized_path = os.path.normpath(pkg_dir / f) if normalized_path not in self.files_created: raise TestFailure( - 'Package directory {pkg_dir!r} missing ' - 'expected content {f!r}'.format(**locals()) + f"Package directory {pkg_dir!r} missing expected content {f!r}" ) for f in without_files: normalized_path = os.path.normpath(pkg_dir / f) if normalized_path in self.files_created: raise TestFailure( - 'Package directory {pkg_dir!r} has unexpected content {f}' - .format(**locals()) + f"Package directory {pkg_dir!r} has unexpected content {f}" ) - def did_create(self, path, message=None): + def did_create(self, path: str, message: Optional[str] = None) -> None: assert str(path) in self.files_created, _one_or_both(message, self) - def did_not_create(self, path, message=None): + def did_not_create(self, path: str, message: Optional[str] = None) -> None: assert str(path) not in self.files_created, _one_or_both(message, self) - def did_update(self, path, message=None): + def did_update(self, path: str, message: Optional[str] = None) -> None: assert str(path) in self.files_updated, _one_or_both(message, self) - def did_not_update(self, path, message=None): + def did_not_update(self, path: str, message: Optional[str] = None) -> None: assert str(path) not in self.files_updated, _one_or_both(message, self) -def _one_or_both(a, b): - """Returns f"{a}\n{b}" if a is truthy, else returns str(b). - """ +def _one_or_both(a: Optional[str], b: Any) -> str: + """Returns f"{a}\n{b}" if a is truthy, else returns str(b).""" if not a: return str(b) return f"{a}\n{b}" -def make_check_stderr_message(stderr, line, reason): +def make_check_stderr_message(stderr: str, line: str, reason: str) -> str: """ Create an exception message to use inside check_stderr(). """ - return dedent("""\ + return dedent( + """\ {reason}: Caused by line: {line!r} Complete stderr: {stderr} - """).format(stderr=stderr, line=line, reason=reason) + """ + ).format(stderr=stderr, line=line, reason=reason) def _check_stderr( - stderr, allow_stderr_warning, allow_stderr_error, -): + stderr: str, + allow_stderr_warning: bool, + allow_stderr_error: bool, +) -> None: """ Check the given stderr for logged warnings and errors. @@ -409,29 +440,29 @@ def _check_stderr( # sent directly to stderr and so bypass any configured log formatter. # The "--- Logging error ---" string is used in Python 3.4+, and # "Logged from file " is used in Python 2. - if (line.startswith('--- Logging error ---') or - line.startswith('Logged from file ')): - reason = 'stderr has a logging error, which is never allowed' + if line.startswith("--- Logging error ---") or line.startswith( + "Logged from file " + ): + reason = "stderr has a logging error, which is never allowed" msg = make_check_stderr_message(stderr, line=line, reason=reason) raise RuntimeError(msg) if allow_stderr_error: continue - if line.startswith('ERROR: '): + if line.startswith("ERROR: "): reason = ( - 'stderr has an unexpected error ' - '(pass allow_stderr_error=True to permit this)' + "stderr has an unexpected error " + "(pass allow_stderr_error=True to permit this)" ) msg = make_check_stderr_message(stderr, line=line, reason=reason) raise RuntimeError(msg) if allow_stderr_warning: continue - if (line.startswith('WARNING: ') or - line.startswith(DEPRECATION_MSG_PREFIX)): + if line.startswith("WARNING: ") or line.startswith(DEPRECATION_MSG_PREFIX): reason = ( - 'stderr has an unexpected warning ' - '(pass allow_stderr_warning=True to permit this)' + "stderr has an unexpected warning " + "(pass allow_stderr_warning=True to permit this)" ) msg = make_check_stderr_message(stderr, line=line, reason=reason) raise RuntimeError(msg) @@ -451,33 +482,37 @@ class PipTestEnvironment(TestFileEnvironment): # a name of the form xxxx_path and relative paths have a name that # does not end in '_path'. - exe = sys.platform == 'win32' and '.exe' or '' + exe = sys.platform == "win32" and ".exe" or "" verbose = False - def __init__(self, base_path, *args, **kwargs): + def __init__( + self, + base_path: str, + *args: Any, + virtualenv: VirtualEnvironment, + pip_expect_warning: bool = False, + **kwargs: Any, + ) -> None: # Make our base_path a test.lib.path.Path object base_path = Path(base_path) # Store paths related to the virtual environment - venv = kwargs.pop("virtualenv") - self.venv_path = venv.location - self.lib_path = venv.lib - self.site_packages_path = venv.site - self.bin_path = venv.bin + self.venv_path = virtualenv.location + self.lib_path = virtualenv.lib + self.site_packages_path = virtualenv.site + self.bin_path = virtualenv.bin + + assert site.USER_BASE is not None + assert site.USER_SITE is not None self.user_base_path = self.venv_path.joinpath("user") self.user_site_path = self.venv_path.joinpath( "user", - site.USER_SITE[len(site.USER_BASE) + 1:], + site.USER_SITE[len(site.USER_BASE) + 1 :], ) - if sys.platform == 'win32': - if sys.version_info >= (3, 5): - scripts_base = Path( - os.path.normpath(self.user_site_path.joinpath('..')) - ) - else: - scripts_base = self.user_base_path - self.user_bin_path = scripts_base.joinpath('Scripts') + if sys.platform == "win32": + scripts_base = Path(os.path.normpath(self.user_site_path.joinpath(".."))) + self.user_bin_path = scripts_base.joinpath("Scripts") else: self.user_bin_path = self.user_base_path.joinpath( os.path.relpath(self.bin_path, self.venv_path) @@ -503,22 +538,31 @@ def __init__(self, base_path, *args, **kwargs): # Whether all pip invocations should expect stderr # (useful for Python version deprecation) - self.pip_expect_warning = kwargs.pop('pip_expect_warning', None) + self.pip_expect_warning = pip_expect_warning # Call the TestFileEnvironment __init__ super().__init__(base_path, *args, **kwargs) # Expand our absolute path directories into relative - for name in ["base", "venv", "bin", "lib", "site_packages", - "user_base", "user_site", "user_bin", "scratch"]: - real_name = "{name}_path".format(**locals()) - relative_path = Path(os.path.relpath( - getattr(self, real_name), self.base_path - )) + for name in [ + "base", + "venv", + "bin", + "lib", + "site_packages", + "user_base", + "user_site", + "user_bin", + "scratch", + ]: + real_name = f"{name}_path" + relative_path = Path( + os.path.relpath(getattr(self, real_name), self.base_path) + ) setattr(self, name, relative_path) # Make sure temp_path is a Path object - self.temp_path = Path(self.temp_path) + self.temp_path: Path = Path(self.temp_path) # Ensure the tmp dir exists, things break horribly if it doesn't self.temp_path.mkdir() @@ -527,24 +571,32 @@ def __init__(self, base_path, *args, **kwargs): self.user_site_path.mkdir(parents=True) self.user_site_path.joinpath("easy-install.pth").touch() - def _ignore_file(self, fn): - if fn.endswith('__pycache__') or fn.endswith(".pyc"): + def _ignore_file(self, fn: str) -> bool: + if fn.endswith("__pycache__") or fn.endswith(".pyc"): result = True else: result = super()._ignore_file(fn) return result - def _find_traverse(self, path, result): + def _find_traverse(self, path: str, result: Dict[str, FoundDir]) -> None: # Ignore symlinked directories to avoid duplicates in `run()` # results because of venv `lib64 -> lib/` symlink on Linux. full = os.path.join(self.base_path, path) if os.path.isdir(full) and os.path.islink(full): - if not self.temp_path or path != 'tmp': + if not self.temp_path or path != "tmp": result[path] = FoundDir(self.base_path, path) else: super()._find_traverse(path, result) - def run(self, *args, **kw): + def run( + self, + *args: str, + cwd: Union[None, str, pathlib.Path] = None, + allow_stderr_error: Optional[bool] = None, + allow_stderr_warning: Optional[bool] = None, + allow_error: bool = False, + **kw: Any, + ) -> TestPipResult: """ :param allow_stderr_error: whether a logged error is allowed in stderr. Passing True for this argument implies @@ -565,50 +617,39 @@ def run(self, *args, **kw): compatibility. """ if self.verbose: - print('>> running {args} {kw}'.format(**locals())) + print(f">> running {args} {kw}") - cwd = kw.pop('cwd', None) - run_from = kw.pop('run_from', None) - assert not cwd or not run_from, "Don't use run_from; it's going away" - cwd = cwd or run_from or self.cwd - if sys.platform == 'win32': + cwd = cwd or self.cwd + if sys.platform == "win32": # Partial fix for ScriptTest.run using `shell=True` on Windows. - args = [str(a).replace('^', '^^').replace('&', '^&') for a in args] - - # Remove `allow_stderr_error`, `allow_stderr_warning` and - # `allow_error` before calling run() because PipTestEnvironment - # doesn't support them. - allow_stderr_error = kw.pop('allow_stderr_error', None) - allow_stderr_warning = kw.pop('allow_stderr_warning', None) - allow_error = kw.pop('allow_error', None) + args = tuple(str(a).replace("^", "^^").replace("&", "^&") for a in args) + if allow_error: - kw['expect_error'] = True + kw["expect_error"] = True # Propagate default values. - expect_error = kw.get('expect_error') + expect_error = kw.get("expect_error") if expect_error: # Then default to allowing logged errors. if allow_stderr_error is not None and not allow_stderr_error: raise RuntimeError( - 'cannot pass allow_stderr_error=False with ' - 'expect_error=True' + "cannot pass allow_stderr_error=False with expect_error=True" ) allow_stderr_error = True - elif kw.get('expect_stderr'): + elif kw.get("expect_stderr"): # Then default to allowing logged warnings. if allow_stderr_warning is not None and not allow_stderr_warning: raise RuntimeError( - 'cannot pass allow_stderr_warning=False with ' - 'expect_stderr=True' + "cannot pass allow_stderr_warning=False with expect_stderr=True" ) allow_stderr_warning = True if allow_stderr_error: if allow_stderr_warning is not None and not allow_stderr_warning: raise RuntimeError( - 'cannot pass allow_stderr_warning=False with ' - 'allow_stderr_error=True' + "cannot pass allow_stderr_warning=False with " + "allow_stderr_error=True" ) # Default values if not set. @@ -619,48 +660,75 @@ def run(self, *args, **kw): # Pass expect_stderr=True to allow any stderr. We do this because # we do our checking of stderr further on in check_stderr(). - kw['expect_stderr'] = True + kw["expect_stderr"] = True result = super().run(cwd=cwd, *args, **kw) if expect_error and not allow_error: if result.returncode == 0: __tracebackhide__ = True - raise AssertionError("Script passed unexpectedly.") + raise AssertionError(f"Script passed unexpectedly:\n{result}") _check_stderr( - result.stderr, allow_stderr_error=allow_stderr_error, + result.stderr, + allow_stderr_error=allow_stderr_error, allow_stderr_warning=allow_stderr_warning, ) return TestPipResult(result, verbose=self.verbose) - def pip(self, *args, **kwargs): + def pip(self, *args: str, use_module: bool = True, **kwargs: Any) -> TestPipResult: __tracebackhide__ = True if self.pip_expect_warning: - kwargs['allow_stderr_warning'] = True - if kwargs.pop('use_module', True): - exe = 'python' - args = ('-m', 'pip') + args + kwargs["allow_stderr_warning"] = True + if use_module: + exe = "python" + args = ("-m", "pip") + args else: - exe = 'pip' + exe = "pip" return self.run(exe, *args, **kwargs) - def pip_install_local(self, *args, **kwargs): + def pip_install_local(self, *args: str, **kwargs: Any) -> TestPipResult: return self.pip( - "install", "--no-index", - "--find-links", path_to_url(os.path.join(DATA_DIR, "packages")), - *args, **kwargs + "install", + "--no-index", + "--find-links", + path_to_url(os.path.join(DATA_DIR, "packages")), + *args, + **kwargs, ) - def easy_install(self, *args, **kwargs): - args = ('-m', 'easy_install') + args - return self.run('python', *args, **kwargs) + def easy_install(self, *args: str, **kwargs: Any) -> TestPipResult: + args = ("-m", "easy_install") + args + return self.run("python", *args, **kwargs) + + def assert_installed(self, **kwargs: str) -> None: + ret = self.pip("list", "--format=json") + installed = set( + (canonicalize_name(val["name"]), val["version"]) + for val in json.loads(ret.stdout) + ) + expected = set((canonicalize_name(k), v) for k, v in kwargs.items()) + assert expected <= installed, "{!r} not all in {!r}".format(expected, installed) + + def assert_not_installed(self, *args: str) -> None: + ret = self.pip("list", "--format=json") + installed = set( + canonicalize_name(val["name"]) for val in json.loads(ret.stdout) + ) + # None of the given names should be listed as installed, i.e. their + # intersection should be empty. + expected = set(canonicalize_name(k) for k in args) + assert not (expected & installed), "{!r} contained in {!r}".format( + expected, installed + ) # FIXME ScriptTest does something similar, but only within a single # ProcResult; this generalizes it so states can be compared across # multiple commands. Maybe should be rolled into ScriptTest? -def diff_states(start, end, ignore=None): +def diff_states( + start: _FilesState, end: _FilesState, ignore: Optional[List[str]] = None +) -> Dict[str, _FilesState]: """ Differences two "filesystem states" as represented by dictionaries of FoundFile and FoundDir objects. @@ -686,26 +754,30 @@ def diff_states(start, end, ignore=None): """ ignore = ignore or [] - def prefix_match(path, prefix): + def prefix_match(path: str, prefix: str) -> bool: if path == prefix: return True prefix = prefix.rstrip(os.path.sep) + os.path.sep return path.startswith(prefix) - start_keys = {k for k in start.keys() - if not any([prefix_match(k, i) for i in ignore])} - end_keys = {k for k in end.keys() - if not any([prefix_match(k, i) for i in ignore])} + start_keys = { + k for k in start.keys() if not any([prefix_match(k, i) for i in ignore]) + } + end_keys = {k for k in end.keys() if not any([prefix_match(k, i) for i in ignore])} deleted = {k: start[k] for k in start_keys.difference(end_keys)} created = {k: end[k] for k in end_keys.difference(start_keys)} updated = {} for k in start_keys.intersection(end_keys): - if (start[k].size != end[k].size): + if start[k].size != end[k].size: updated[k] = end[k] return dict(deleted=deleted, created=created, updated=updated) -def assert_all_changes(start_state, end_state, expected_changes): +def assert_all_changes( + start_state: Union[_FilesState, TestPipResult], + end_state: Union[_FilesState, TestPipResult], + expected_changes: List[str], +) -> Dict[str, _FilesState]: """ Fails if anything changed that isn't listed in the expected_changes. @@ -726,39 +798,47 @@ def assert_all_changes(start_state, end_state, expected_changes): start_files = start_state.files_before if isinstance(end_state, TestPipResult): end_files = end_state.files_after + start_files = cast(_FilesState, start_files) + end_files = cast(_FilesState, end_files) diff = diff_states(start_files, end_files, ignore=expected_changes) if list(diff.values()) != [{}, {}, {}]: - raise TestFailure('Unexpected changes:\n' + '\n'.join( - [k + ': ' + ', '.join(v.keys()) for k, v in diff.items()])) + raise TestFailure( + "Unexpected changes:\n" + + "\n".join([k + ": " + ", ".join(v.keys()) for k, v in diff.items()]) + ) # Don't throw away this potentially useful information return diff -def _create_main_file(dir_path, name=None, output=None): +def _create_main_file( + dir_path: Path, name: Optional[str] = None, output: Optional[str] = None +) -> None: """ Create a module with a main() function that prints the given output. """ if name is None: - name = 'version_pkg' + name = "version_pkg" if output is None: - output = '0.1' - text = textwrap.dedent("""\ - def main(): - print({!r}) - """.format(output)) - filename = f'{name}.py' + output = "0.1" + text = textwrap.dedent( + f""" + def main(): + print({output!r}) + """ + ) + filename = f"{name}.py" dir_path.joinpath(filename).write_text(text) def _git_commit( - env_or_script, - repo_dir, - message=None, - allow_empty=False, - stage_modified=False, -): + env_or_script: PipTestEnvironment, + repo_dir: str, + message: Optional[str] = None, + allow_empty: bool = False, + stage_modified: bool = False, +) -> None: """ Run git-commit. @@ -768,7 +848,7 @@ def _git_commit( message: an optional commit message. """ if message is None: - message = 'test commit' + message = "test commit" args = [] @@ -779,168 +859,200 @@ def _git_commit( args.append("--all") new_args = [ - 'git', 'commit', '-q', '--author', 'pip ', + "git", + "commit", + "-q", + "--author", + "pip ", ] new_args.extend(args) - new_args.extend(['-m', message]) + new_args.extend(["-m", message]) env_or_script.run(*new_args, cwd=repo_dir) -def _vcs_add(script, version_pkg_path, vcs='git'): - if vcs == 'git': - script.run('git', 'init', cwd=version_pkg_path) - script.run('git', 'add', '.', cwd=version_pkg_path) - _git_commit(script, version_pkg_path, message='initial version') - elif vcs == 'hg': - script.run('hg', 'init', cwd=version_pkg_path) - script.run('hg', 'add', '.', cwd=version_pkg_path) +def _vcs_add( + script: PipTestEnvironment, version_pkg_path: Path, vcs: str = "git" +) -> Path: + if vcs == "git": + script.run("git", "init", cwd=version_pkg_path) + script.run("git", "add", ".", cwd=version_pkg_path) + _git_commit(script, version_pkg_path, message="initial version") + elif vcs == "hg": + script.run("hg", "init", cwd=version_pkg_path) + script.run("hg", "add", ".", cwd=version_pkg_path) script.run( - 'hg', 'commit', '-q', - '--user', 'pip ', - '-m', 'initial version', cwd=version_pkg_path, + "hg", + "commit", + "-q", + "--user", + "pip ", + "-m", + "initial version", + cwd=version_pkg_path, ) - elif vcs == 'svn': + elif vcs == "svn": repo_url = _create_svn_repo(script, version_pkg_path) script.run( - 'svn', 'checkout', repo_url, 'pip-test-package', - cwd=script.scratch_path + "svn", "checkout", repo_url, "pip-test-package", cwd=script.scratch_path ) - checkout_path = script.scratch_path / 'pip-test-package' + checkout_path: str = script.scratch_path / "pip-test-package" # svn internally stores windows drives as uppercase; we'll match that. - checkout_path = checkout_path.replace('c:', 'C:') + checkout_path = Path(checkout_path.replace("c:", "C:")) version_pkg_path = checkout_path - elif vcs == 'bazaar': - script.run('bzr', 'init', cwd=version_pkg_path) - script.run('bzr', 'add', '.', cwd=version_pkg_path) + elif vcs == "bazaar": + script.run("bzr", "init", cwd=version_pkg_path) + script.run("bzr", "add", ".", cwd=version_pkg_path) script.run( - 'bzr', 'whoami', 'pip ', - cwd=version_pkg_path) + "bzr", "whoami", "pip ", cwd=version_pkg_path + ) script.run( - 'bzr', 'commit', '-q', - '--author', 'pip ', - '-m', 'initial version', cwd=version_pkg_path, + "bzr", + "commit", + "-q", + "--author", + "pip ", + "-m", + "initial version", + cwd=version_pkg_path, ) else: - raise ValueError('Unknown vcs: {vcs}'.format(**locals())) + raise ValueError(f"Unknown vcs: {vcs}") return version_pkg_path -def _create_test_package_with_subdirectory(script, subdirectory): +def _create_test_package_with_subdirectory( + script: PipTestEnvironment, subdirectory: str +) -> Path: script.scratch_path.joinpath("version_pkg").mkdir() - version_pkg_path = script.scratch_path / 'version_pkg' + version_pkg_path = script.scratch_path / "version_pkg" _create_main_file(version_pkg_path, name="version_pkg", output="0.1") version_pkg_path.joinpath("setup.py").write_text( - textwrap.dedent(""" - from setuptools import setup, find_packages - setup(name='version_pkg', - version='0.1', - packages=find_packages(), - py_modules=['version_pkg'], - entry_points=dict(console_scripts=['version_pkg=version_pkg:main'])) - """)) + textwrap.dedent( + """ + from setuptools import setup, find_packages + + setup( + name="version_pkg", + version="0.1", + packages=find_packages(), + py_modules=["version_pkg"], + entry_points=dict(console_scripts=["version_pkg=version_pkg:main"]), + ) + """ + ) + ) subdirectory_path = version_pkg_path.joinpath(subdirectory) subdirectory_path.mkdir() _create_main_file(subdirectory_path, name="version_subpkg", output="0.1") - subdirectory_path.joinpath('setup.py').write_text( - textwrap.dedent(""" -from setuptools import setup, find_packages -setup(name='version_subpkg', - version='0.1', - packages=find_packages(), - py_modules=['version_subpkg'], - entry_points=dict(console_scripts=['version_pkg=version_subpkg:main'])) - """)) + subdirectory_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ + from setuptools import find_packages, setup + + setup( + name="version_subpkg", + version="0.1", + packages=find_packages(), + py_modules=["version_subpkg"], + entry_points=dict(console_scripts=["version_pkg=version_subpkg:main"]), + ) + """ + ) + ) - script.run('git', 'init', cwd=version_pkg_path) - script.run('git', 'add', '.', cwd=version_pkg_path) - _git_commit(script, version_pkg_path, message='initial version') + script.run("git", "init", cwd=version_pkg_path) + script.run("git", "add", ".", cwd=version_pkg_path) + _git_commit(script, version_pkg_path, message="initial version") return version_pkg_path -def _create_test_package_with_srcdir(script, name='version_pkg', vcs='git'): +def _create_test_package_with_srcdir( + script: PipTestEnvironment, name: str = "version_pkg", vcs: str = "git" +) -> Path: script.scratch_path.joinpath(name).mkdir() version_pkg_path = script.scratch_path / name - subdir_path = version_pkg_path.joinpath('subdir') + subdir_path = version_pkg_path.joinpath("subdir") subdir_path.mkdir() - src_path = subdir_path.joinpath('src') + src_path = subdir_path.joinpath("src") src_path.mkdir() - pkg_path = src_path.joinpath('pkg') + pkg_path = src_path.joinpath("pkg") pkg_path.mkdir() - pkg_path.joinpath('__init__.py').write_text('') - subdir_path.joinpath("setup.py").write_text(textwrap.dedent(""" - from setuptools import setup, find_packages - setup( - name='{name}', - version='0.1', - packages=find_packages(), - package_dir={{'': 'src'}}, + pkg_path.joinpath("__init__.py").write_text("") + subdir_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ + from setuptools import setup, find_packages + setup( + name="{name}", + version="0.1", + packages=find_packages(), + package_dir={{"": "src"}}, + ) + """.format( + name=name + ) ) - """.format(name=name))) + ) return _vcs_add(script, version_pkg_path, vcs) -def _create_test_package(script, name='version_pkg', vcs='git'): +def _create_test_package( + script: PipTestEnvironment, name: str = "version_pkg", vcs: str = "git" +) -> Path: script.scratch_path.joinpath(name).mkdir() version_pkg_path = script.scratch_path / name - _create_main_file(version_pkg_path, name=name, output='0.1') - version_pkg_path.joinpath("setup.py").write_text(textwrap.dedent(""" - from setuptools import setup, find_packages - setup( - name='{name}', - version='0.1', - packages=find_packages(), - py_modules=['{name}'], - entry_points=dict(console_scripts=['{name}={name}:main']) + _create_main_file(version_pkg_path, name=name, output="0.1") + version_pkg_path.joinpath("setup.py").write_text( + textwrap.dedent( + """ + from setuptools import setup, find_packages + setup( + name="{name}", + version="0.1", + packages=find_packages(), + py_modules=["{name}"], + entry_points=dict(console_scripts=["{name}={name}:main"]), + ) + """.format( + name=name + ) ) - """.format(name=name))) + ) return _vcs_add(script, version_pkg_path, vcs) -def _create_svn_repo(script, version_pkg_path): - repo_url = path_to_url( - script.scratch_path / 'pip-test-package-repo' / 'trunk') - script.run( - 'svnadmin', 'create', 'pip-test-package-repo', - cwd=script.scratch_path - ) +def _create_svn_repo(script: PipTestEnvironment, version_pkg_path: str) -> str: + repo_url = path_to_url(script.scratch_path / "pip-test-package-repo" / "trunk") + script.run("svnadmin", "create", "pip-test-package-repo", cwd=script.scratch_path) script.run( - 'svn', 'import', version_pkg_path, repo_url, - '-m', 'Initial import of pip-test-package', - cwd=script.scratch_path + "svn", + "import", + version_pkg_path, + repo_url, + "-m", + "Initial import of pip-test-package", + cwd=script.scratch_path, ) return repo_url -def _change_test_package_version(script, version_pkg_path): +def _change_test_package_version( + script: PipTestEnvironment, version_pkg_path: Path +) -> None: _create_main_file( - version_pkg_path, name='version_pkg', output='some different version' + version_pkg_path, name="version_pkg", output="some different version" ) # Pass -a to stage the change to the main file. - _git_commit( - script, version_pkg_path, message='messed version', stage_modified=True - ) - - -def assert_raises_regexp(exception, reg, run, *args, **kwargs): - """Like assertRaisesRegexp in unittest""" - __tracebackhide__ = True - - try: - run(*args, **kwargs) - assert False, "{exception} should have been thrown".format(**locals()) - except exception: - e = sys.exc_info()[1] - p = re.compile(reg) - assert p.search(str(e)), str(e) + _git_commit(script, version_pkg_path, message="messed version", stage_modified=True) @contextmanager -def requirements_file(contents, tmpdir): +def requirements_file(contents: str, tmpdir: Path) -> Iterator[Path]: """Return a Path to a requirements file of given contents. As long as the context manager is open, the requirements file will exist. @@ -948,37 +1060,39 @@ def requirements_file(contents, tmpdir): :param tmpdir: A Path to the folder in which to create the file """ - path = tmpdir / 'reqs.txt' + path = tmpdir / "reqs.txt" path.write_text(contents) yield path path.unlink() -def create_test_package_with_setup(script, **setup_kwargs): - assert 'name' in setup_kwargs, setup_kwargs - pkg_path = script.scratch_path / setup_kwargs['name'] +def create_test_package_with_setup( + script: PipTestEnvironment, **setup_kwargs: Any +) -> Path: + assert "name" in setup_kwargs, setup_kwargs + pkg_path = script.scratch_path / setup_kwargs["name"] pkg_path.mkdir() - pkg_path.joinpath("setup.py").write_text(textwrap.dedent(""" - from setuptools import setup - kwargs = {setup_kwargs!r} - setup(**kwargs) - """).format(**locals())) + pkg_path.joinpath("setup.py").write_text( + textwrap.dedent( + f""" + from setuptools import setup + kwargs = {setup_kwargs!r} + setup(**kwargs) + """ + ) + ) return pkg_path -def urlsafe_b64encode_nopad(data): - # type: (bytes) -> str +def urlsafe_b64encode_nopad(data: bytes) -> str: return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") -def create_really_basic_wheel(name, version): - # type: (str, str) -> bytes - def digest(contents): - return "sha256={}".format( - urlsafe_b64encode_nopad(sha256(contents).digest()) - ) +def create_really_basic_wheel(name: str, version: str) -> bytes: + def digest(contents: bytes) -> str: + return "sha256={}".format(urlsafe_b64encode_nopad(sha256(contents).digest())) - def add_file(path, text): + def add_file(path: str, text: str) -> None: contents = text.encode("utf-8") z.writestr(path, contents) records.append((path, digest(contents), str(len(contents)))) @@ -996,7 +1110,9 @@ def add_file(path, text): Metadata-Version: 2.1 Name: {} Version: {} - """.format(name, version) + """.format( + name, version + ) ), ) z.writestr(record_path, "\n".join(",".join(r) for r in records)) @@ -1005,14 +1121,14 @@ def add_file(path, text): def create_basic_wheel_for_package( - script, - name, - version, - depends=None, - extras=None, - requires_python=None, - extra_files=None, -): + script: PipTestEnvironment, + name: str, + version: str, + depends: Optional[List[str]] = None, + extras: Dict[str, List[str]] = None, + requires_python: Optional[str] = None, + extra_files: Optional[Dict[str, Union[bytes, str]]] = None, +) -> Path: if depends is None: depends = [] if extras is None: @@ -1042,7 +1158,7 @@ def hello(): for package in packages ] - metadata_updates = { + metadata_updates: Dict[str, Any] = { "Provides-Extra": list(extras), "Requires-Dist": requires_dist, } @@ -1056,7 +1172,6 @@ def hello(): metadata_updates=metadata_updates, extra_metadata_files={"top_level.txt": name}, extra_files=extra_files, - # Have an empty RECORD because we don't want to be checking hashes. record="", ) @@ -1066,27 +1181,40 @@ def hello(): def create_basic_sdist_for_package( - script, name, version, extra_files=None -): + script: PipTestEnvironment, + name: str, + version: str, + extra_files: Optional[Dict[str, str]] = None, + *, + fails_egg_info: bool = False, + fails_bdist_wheel: bool = False, +) -> Path: files = { - "setup.py": """ + "setup.py": f"""\ + import sys from setuptools import find_packages, setup + + fails_bdist_wheel = {fails_bdist_wheel!r} + fails_egg_info = {fails_egg_info!r} + + if fails_egg_info and "egg_info" in sys.argv: + raise Exception("Simulated failure for generating metadata.") + + if fails_bdist_wheel and "bdist_wheel" in sys.argv: + raise Exception("Simulated failure for building a wheel.") + setup(name={name!r}, version={version!r}) """, } # Some useful shorthands - archive_name = "{name}-{version}.tar.gz".format( - name=name, version=version - ) + archive_name = f"{name}-{version}.tar.gz" # Replace key-values with formatted values for key, value in list(files.items()): del files[key] key = key.format(name=name) - files[key] = textwrap.dedent(value).format( - name=name, version=version - ).strip() + files[key] = textwrap.dedent(value) # Add new files after formatting if extra_files: @@ -1095,12 +1223,12 @@ def create_basic_sdist_for_package( for fname in files: path = script.temp_path / fname path.parent.mkdir(exist_ok=True, parents=True) - path.write_bytes(ensure_binary(files[fname])) + path.write_bytes(files[fname].encode("utf-8")) retval = script.scratch_path / archive_name generated = shutil.make_archive( retval, - 'gztar', + "gztar", root_dir=script.temp_path, base_dir=os.curdir, ) @@ -1112,55 +1240,44 @@ def create_basic_sdist_for_package( return retval -def need_executable(name, check_cmd): - def wrapper(fn): +def need_executable(name: str, check_cmd: Tuple[str, ...]) -> Callable[[_Test], _Test]: + def wrapper(fn: _Test) -> _Test: try: subprocess.check_output(check_cmd) except (OSError, subprocess.CalledProcessError): - return pytest.mark.skip( - reason=f'{name} is not available')(fn) + return pytest.mark.skip(reason=f"{name} is not available")(fn) return fn + return wrapper -def is_bzr_installed(): +def is_bzr_installed() -> bool: try: - subprocess.check_output(('bzr', 'version', '--short')) + subprocess.check_output(("bzr", "version", "--short")) except OSError: return False return True -def is_svn_installed(): +def is_svn_installed() -> bool: try: - subprocess.check_output(('svn', '--version')) + subprocess.check_output(("svn", "--version")) except OSError: return False return True -def need_bzr(fn): - return pytest.mark.bzr(need_executable( - 'Bazaar', ('bzr', 'version', '--short') - )(fn)) - +def need_bzr(fn: _Test) -> _Test: + return pytest.mark.bzr(need_executable("Bazaar", ("bzr", "version", "--short"))(fn)) -def need_svn(fn): - return pytest.mark.svn(need_executable( - 'Subversion', ('svn', '--version') - )(need_executable( - 'Subversion Admin', ('svnadmin', '--version') - )(fn))) - -def need_mercurial(fn): - return pytest.mark.mercurial(need_executable( - 'Mercurial', ('hg', 'version') - )(fn)) +def need_svn(fn: _Test) -> _Test: + return pytest.mark.svn( + need_executable("Subversion", ("svn", "--version"))( + need_executable("Subversion Admin", ("svnadmin", "--version"))(fn) + ) + ) -# Workaround for test failures after new wheel release. -windows_workaround_7667 = pytest.mark.skipif( - "sys.platform == 'win32' and sys.version_info < (3,)", - reason="Workaround for #7667", -) +def need_mercurial(fn: _Test) -> _Test: + return pytest.mark.mercurial(need_executable("Mercurial", ("hg", "version"))(fn)) diff --git a/tests/lib/certs.py b/tests/lib/certs.py index 1f51f2174cf..54b484ac0e7 100644 --- a/tests/lib/certs.py +++ b/tests/lib/certs.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta +from typing import Tuple from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -6,22 +7,16 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -if MYPY_CHECK_RUNNING: - from typing import Tuple - - -def make_tls_cert(hostname): - # type: (str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey] +def make_tls_cert(hostname: str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey]: key = rsa.generate_private_key( - public_exponent=65537, - key_size=2048, - backend=default_backend() + public_exponent=65537, key_size=2048, backend=default_backend() + ) + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COMMON_NAME, hostname), + ] ) - subject = issuer = x509.Name([ - x509.NameAttribute(NameOID.COMMON_NAME, hostname), - ]) cert = ( x509.CertificateBuilder() .subject_name(subject) @@ -39,8 +34,7 @@ def make_tls_cert(hostname): return cert, key -def serialize_key(key): - # type: (rsa.RSAPrivateKey) -> bytes +def serialize_key(key: rsa.RSAPrivateKey) -> bytes: return key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, @@ -48,6 +42,5 @@ def serialize_key(key): ) -def serialize_cert(cert): - # type: (x509.Certificate) -> bytes +def serialize_cert(cert: x509.Certificate) -> bytes: return cert.public_bytes(serialization.Encoding.PEM) diff --git a/tests/lib/compat.py b/tests/lib/compat.py new file mode 100644 index 00000000000..4d44cbddbbc --- /dev/null +++ b/tests/lib/compat.py @@ -0,0 +1,54 @@ +# mypy: no-warn-unused-ignores + +import contextlib +import signal +from typing import Iterable, Iterator + + +@contextlib.contextmanager +def nullcontext() -> Iterator[None]: + """ + Context manager that does no additional processing. + + Used as a stand-in for a normal context manager, when a particular block of + code is only sometimes used with a normal context manager: + + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + + TODO: Replace with contextlib.nullcontext after dropping Python 3.6 + support. + """ + yield + + +# Applies on Windows. +if not hasattr(signal, "pthread_sigmask"): + # We're not relying on this behavior anywhere currently, it's just best + # practice. + blocked_signals = nullcontext +else: + + @contextlib.contextmanager + def blocked_signals() -> Iterator[None]: + """Block all signals for e.g. starting a worker thread.""" + # valid_signals() was added in Python 3.8 (and not using it results + # in a warning on pthread_sigmask() call) + mask: Iterable[int] + try: + mask = signal.valid_signals() + except AttributeError: + mask = set(range(1, signal.NSIG)) + + old_mask = signal.pthread_sigmask( # type: ignore[attr-defined] + signal.SIG_SETMASK, # type: ignore[attr-defined] + mask, + ) + try: + yield + finally: + signal.pthread_sigmask( # type: ignore[attr-defined] + signal.SIG_SETMASK, # type: ignore[attr-defined] + old_mask, + ) diff --git a/tests/lib/configuration_helpers.py b/tests/lib/configuration_helpers.py index 384a424e2d0..67f75e8e7a0 100644 --- a/tests/lib/configuration_helpers.py +++ b/tests/lib/configuration_helpers.py @@ -6,44 +6,44 @@ import os import tempfile import textwrap +from typing import Any, Dict, Iterator import pip._internal.configuration from pip._internal.utils.misc import ensure_dir # This is so that tests don't need to import pip._internal.configuration. +Kind = pip._internal.configuration.Kind kinds = pip._internal.configuration.kinds class ConfigurationMixin: - - def setup(self): + def setup(self) -> None: self.configuration = pip._internal.configuration.Configuration( isolated=False, ) - self._files_to_clear = [] - - def teardown(self): - for fname in self._files_to_clear: - fname.stop() - def patch_configuration(self, variant, di): + def patch_configuration(self, variant: Kind, di: Dict[str, Any]) -> None: old = self.configuration._load_config_files @functools.wraps(old) - def overridden(): + def overridden() -> None: # Manual Overload self.configuration._config[variant].update(di) - self.configuration._parsers[variant].append((None, None)) - return old() + # Configuration._parsers has type: + # Dict[Kind, List[Tuple[str, RawConfigParser]]]. + # As a testing convenience, pass a special value. + self.configuration._parsers[variant].append( + (None, None), # type: ignore[arg-type] + ) + old() - self.configuration._load_config_files = overridden + # https://github.com/python/mypy/issues/2427 + self.configuration._load_config_files = overridden # type: ignore[assignment] @contextlib.contextmanager - def tmpfile(self, contents): + def tmpfile(self, contents: str) -> Iterator[str]: # Create a temporary file - fd, path = tempfile.mkstemp( - prefix="pip_", suffix="_config.ini", text=True - ) + fd, path = tempfile.mkstemp(prefix="pip_", suffix="_config.ini", text=True) os.close(fd) contents = textwrap.dedent(contents).lstrip() @@ -54,8 +54,3 @@ def tmpfile(self, contents): yield path os.remove(path) - - @staticmethod - def get_file_contents(path): - with open(path) as f: - return f.read() diff --git a/tests/lib/direct_url.py b/tests/lib/direct_url.py new file mode 100644 index 00000000000..ec0a32b4d66 --- /dev/null +++ b/tests/lib/direct_url.py @@ -0,0 +1,24 @@ +import re +from typing import Optional + +from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl +from tests.lib import TestPipResult +from tests.lib.path import Path + + +def get_created_direct_url_path(result: TestPipResult, pkg: str) -> Optional[Path]: + direct_url_metadata_re = re.compile( + pkg + r"-[\d\.]+\.dist-info." + DIRECT_URL_METADATA_NAME + r"$" + ) + for filename in result.files_created: + if direct_url_metadata_re.search(filename): + return result.test_env.base_path / filename + return None + + +def get_created_direct_url(result: TestPipResult, pkg: str) -> Optional[DirectUrl]: + direct_url_path = get_created_direct_url_path(result, pkg) + if direct_url_path: + with open(direct_url_path) as f: + return DirectUrl.from_json(f.read()) + return None diff --git a/tests/lib/filesystem.py b/tests/lib/filesystem.py index dc14b323e33..8563783e743 100644 --- a/tests/lib/filesystem.py +++ b/tests/lib/filesystem.py @@ -6,11 +6,12 @@ import sys from functools import partial from itertools import chain +from typing import Iterator, List, Set from .path import Path -def make_socket_file(path): +def make_socket_file(path: str) -> None: # Socket paths are limited to 108 characters (sometimes less) so we # chdir before creating it and use a relative path name. cwd = os.getcwd() @@ -22,20 +23,19 @@ def make_socket_file(path): os.chdir(cwd) -def make_unreadable_file(path): +def make_unreadable_file(path: str) -> None: Path(path).touch() os.chmod(path, 0o000) if sys.platform == "win32": - # Once we drop PY2 we can use `os.getlogin()` instead. - username = os.environ["USERNAME"] + username = os.getlogin() # Remove "Read Data/List Directory" permission for current user, but # leave everything else. args = ["icacls", path, "/deny", username + ":(RD)"] subprocess.check_call(args) -def get_filelist(base): - def join(dirpath, dirnames, filenames): +def get_filelist(base: str) -> Set[str]: + def join(dirpath: str, dirnames: List[str], filenames: List[str]) -> Iterator[str]: relative_dirpath = os.path.relpath(dirpath, base) join_dirpath = partial(os.path.join, relative_dirpath) return chain( @@ -43,6 +43,4 @@ def join(dirpath, dirnames, filenames): (join_dirpath(p) for p in filenames), ) - return set(chain.from_iterable( - join(*dirinfo) for dirinfo in os.walk(base) - )) + return set(chain.from_iterable(join(*dirinfo) for dirinfo in os.walk(base))) diff --git a/tests/lib/git_submodule_helpers.py b/tests/lib/git_submodule_helpers.py index 494d329cac1..32a3c000287 100644 --- a/tests/lib/git_submodule_helpers.py +++ b/tests/lib/git_submodule_helpers.py @@ -1,72 +1,82 @@ import textwrap +from typing import Tuple -from tests.lib import _create_main_file, _git_commit +from tests.lib import PipTestEnvironment, _create_main_file, _git_commit +from tests.lib.path import Path -def _create_test_package_submodule(env): +def _create_test_package_submodule(env: PipTestEnvironment) -> Path: env.scratch_path.joinpath("version_pkg_submodule").mkdir() - submodule_path = env.scratch_path / 'version_pkg_submodule' - env.run('touch', 'testfile', cwd=submodule_path) - env.run('git', 'init', cwd=submodule_path) - env.run('git', 'add', '.', cwd=submodule_path) - _git_commit(env, submodule_path, message='initial version / submodule') + submodule_path = env.scratch_path / "version_pkg_submodule" + env.run("touch", "testfile", cwd=submodule_path) + env.run("git", "init", cwd=submodule_path) + env.run("git", "add", ".", cwd=submodule_path) + _git_commit(env, submodule_path, message="initial version / submodule") return submodule_path -def _change_test_package_submodule(env, submodule_path): +def _change_test_package_submodule( + env: PipTestEnvironment, submodule_path: Path +) -> None: submodule_path.joinpath("testfile").write_text("this is a changed file") submodule_path.joinpath("testfile2").write_text("this is an added file") - env.run('git', 'add', '.', cwd=submodule_path) - _git_commit(env, submodule_path, message='submodule change') + env.run("git", "add", ".", cwd=submodule_path) + _git_commit(env, submodule_path, message="submodule change") -def _pull_in_submodule_changes_to_module(env, module_path, rel_path): +def _pull_in_submodule_changes_to_module( + env: PipTestEnvironment, module_path: Path, rel_path: str +) -> None: """ Args: rel_path: the location of the submodule relative to the superproject. """ submodule_path = module_path / rel_path - env.run('git', 'pull', '-q', 'origin', 'master', cwd=submodule_path) + env.run("git", "pull", "-q", "origin", "master", cwd=submodule_path) # Pass -a to stage the submodule changes that were just pulled in. - _git_commit( - env, module_path, message='submodule change', stage_modified=True - ) + _git_commit(env, module_path, message="submodule change", stage_modified=True) -def _create_test_package_with_submodule(env, rel_path): +def _create_test_package_with_submodule( + env: PipTestEnvironment, rel_path: str +) -> Tuple[Path, Path]: """ Args: rel_path: the location of the submodule relative to the superproject. """ env.scratch_path.joinpath("version_pkg").mkdir() - version_pkg_path = env.scratch_path / 'version_pkg' + version_pkg_path = env.scratch_path / "version_pkg" version_pkg_path.joinpath("testpkg").mkdir() - pkg_path = version_pkg_path / 'testpkg' + pkg_path = version_pkg_path / "testpkg" pkg_path.joinpath("__init__.py").write_text("# hello there") _create_main_file(pkg_path, name="version_pkg", output="0.1") - version_pkg_path.joinpath("setup.py").write_text(textwrap.dedent('''\ + version_pkg_path.joinpath("setup.py").write_text( + textwrap.dedent( + """\ from setuptools import setup, find_packages setup(name='version_pkg', version='0.1', packages=find_packages(), ) - ''')) - env.run('git', 'init', cwd=version_pkg_path) - env.run('git', 'add', '.', cwd=version_pkg_path) - _git_commit(env, version_pkg_path, message='initial version') + """ + ) + ) + env.run("git", "init", cwd=version_pkg_path) + env.run("git", "add", ".", cwd=version_pkg_path) + _git_commit(env, version_pkg_path, message="initial version") submodule_path = _create_test_package_submodule(env) env.run( - 'git', - 'submodule', - 'add', + "git", + "submodule", + "add", submodule_path, rel_path, cwd=version_pkg_path, ) - _git_commit(env, version_pkg_path, message='initial version w submodule') + _git_commit(env, version_pkg_path, message="initial version w submodule") return version_pkg_path, submodule_path diff --git a/tests/lib/index.py b/tests/lib/index.py index e6dc2a58bea..17282bc562f 100644 --- a/tests/lib/index.py +++ b/tests/lib/index.py @@ -1,14 +1,18 @@ +from typing import Optional + from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.link import Link -def make_mock_candidate(version, yanked_reason=None, hex_digest=None): - url = f'https://example.com/pkg-{version}.tar.gz' +def make_mock_candidate( + version: str, yanked_reason: Optional[str] = None, hex_digest: Optional[str] = None +) -> InstallationCandidate: + url = f"https://example.com/pkg-{version}.tar.gz" if hex_digest is not None: assert len(hex_digest) == 64 - url += f'#sha256={hex_digest}' + url += f"#sha256={hex_digest}" link = Link(url, yanked_reason=yanked_reason) - candidate = InstallationCandidate('mypackage', version, link) + candidate = InstallationCandidate("mypackage", version, link) return candidate diff --git a/tests/lib/local_repos.py b/tests/lib/local_repos.py index 63ec50ccb1d..2ead17aca0b 100644 --- a/tests/lib/local_repos.py +++ b/tests/lib/local_repos.py @@ -3,28 +3,25 @@ import urllib.request from pip._internal.utils.misc import hide_url -from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs import vcs from tests.lib import path_to_url +from tests.lib.path import Path -if MYPY_CHECK_RUNNING: - from tests.lib.path import Path - -def _create_svn_initools_repo(initools_dir): +def _create_svn_initools_repo(initools_dir: str) -> None: """ Create the SVN INITools repo. """ directory = os.path.dirname(initools_dir) - subprocess.check_call('svnadmin create INITools'.split(), cwd=directory) + subprocess.check_call("svnadmin create INITools".split(), cwd=directory) filename, _ = urllib.request.urlretrieve( - 'http://bitbucket.org/hltbra/pip-initools-dump/raw/8b55c908a320/' - 'INITools_modified.dump' + "http://bitbucket.org/hltbra/pip-initools-dump/raw/8b55c908a320/" + "INITools_modified.dump" ) with open(filename) as dump: subprocess.check_call( - ['svnadmin', 'load', initools_dir], + ["svnadmin", "load", initools_dir], stdin=dump, stdout=subprocess.DEVNULL, ) @@ -32,36 +29,36 @@ def _create_svn_initools_repo(initools_dir): def local_checkout( - remote_repo, # type: str - temp_path, # type: Path -): - # type: (...) -> str + remote_repo: str, + temp_path: Path, +) -> str: """ :param temp_path: the return value of the tmpdir fixture, which is a temp directory Path object unique to each test function invocation, created as a sub directory of the base temp directory. """ - assert '+' in remote_repo - vcs_name = remote_repo.split('+', 1)[0] + assert "+" in remote_repo + vcs_name = remote_repo.split("+", 1)[0] repository_name = os.path.basename(remote_repo) - directory = temp_path.joinpath('cache') + directory = temp_path.joinpath("cache") repo_url_path = os.path.join(directory, repository_name) assert not os.path.exists(repo_url_path) if not os.path.exists(directory): os.mkdir(directory) - if vcs_name == 'svn': - assert repository_name == 'INITools' + if vcs_name == "svn": + assert repository_name == "INITools" _create_svn_initools_repo(repo_url_path) - repo_url_path = os.path.join(repo_url_path, 'trunk') + repo_url_path = os.path.join(repo_url_path, "trunk") else: vcs_backend = vcs.get_backend(vcs_name) - vcs_backend.obtain(repo_url_path, url=hide_url(remote_repo)) + assert vcs_backend is not None + vcs_backend.obtain(repo_url_path, url=hide_url(remote_repo), verbosity=0) - return '{}+{}'.format(vcs_name, path_to_url(repo_url_path)) + return "{}+{}".format(vcs_name, path_to_url(repo_url_path)) -def local_repo(remote_repo, temp_path): - return local_checkout(remote_repo, temp_path).split('+', 1)[1] +def local_repo(remote_repo: str, temp_path: Path) -> str: + return local_checkout(remote_repo, temp_path).split("+", 1)[1] diff --git a/tests/lib/options_helpers.py b/tests/lib/options_helpers.py index 8cc5e306d52..31f65003545 100644 --- a/tests/lib/options_helpers.py +++ b/tests/lib/options_helpers.py @@ -1,14 +1,18 @@ """Provides helper classes for testing option handling in pip """ +from optparse import Values +from typing import List, Tuple + from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command from pip._internal.commands import CommandInfo, commands_dict class FakeCommand(Command): - - def main(self, args): + def main( # type: ignore[override] + self, args: List[str] + ) -> Tuple[Values, List[str]]: index_opts = cmdoptions.make_option_group( cmdoptions.index_group, self.parser, @@ -18,11 +22,12 @@ def main(self, args): class AddFakeCommandMixin: - - def setup(self): - commands_dict['fake'] = CommandInfo( - 'tests.lib.options_helpers', 'FakeCommand', 'fake summary', + def setup(self) -> None: + commands_dict["fake"] = CommandInfo( + "tests.lib.options_helpers", + "FakeCommand", + "fake summary", ) - def teardown(self): - commands_dict.pop('fake') + def teardown(self) -> None: + commands_dict.pop("fake") diff --git a/tests/lib/path.py b/tests/lib/path.py index a9dc29ad7a5..ea8c819ec74 100644 --- a/tests/lib/path.py +++ b/tests/lib/path.py @@ -1,12 +1,7 @@ -# flake8: noqa # Author: Aziz Köksal import glob import os - -try: - from os import supports_fd -except ImportError: - supports_fd = set() +from typing import Iterable, Iterator, Union class Path(str): @@ -20,12 +15,12 @@ class Path(str): # Separator in the PATH environment variable. pathsep = os.pathsep - def __new__(cls, *paths): + def __new__(cls, *paths: str) -> "Path": if len(paths): - return str.__new__(cls, os.path.join(*paths)) - return str.__new__(cls) + return super().__new__(cls, os.path.join(*paths)) + return super().__new__(cls) - def __div__(self, path): + def __div__(self, path: str) -> "Path": """ Joins this path with another path. @@ -36,7 +31,7 @@ def __div__(self, path): __truediv__ = __div__ - def __rdiv__(self, path): + def __rdiv__(self, path: str) -> "Path": """ Joins this path with another path. @@ -46,7 +41,7 @@ def __rdiv__(self, path): __rtruediv__ = __rdiv__ - def __idiv__(self, path): + def __idiv__(self, path: str) -> "Path": """ Like __div__ but also assigns to the variable. @@ -56,55 +51,52 @@ def __idiv__(self, path): __itruediv__ = __idiv__ - def __add__(self, path): + def __add__(self, path: str) -> "Path": """ >>> Path('/home/a') + 'bc.d' '/home/abc.d' """ return Path(str(self) + path) - def __radd__(self, path): + def __radd__(self, path: str) -> "Path": """ >>> '/home/a' + Path('bc.d') '/home/abc.d' """ return Path(path + str(self)) - def __repr__(self): + def __repr__(self) -> str: return "Path({inner})".format(inner=str.__repr__(self)) - def __hash__(self): - return str.__hash__(self) - @property - def name(self): + def name(self) -> str: """ '/home/a/bc.d' -> 'bc.d' """ return os.path.basename(self) @property - def stem(self): + def stem(self) -> str: """ '/home/a/bc.d' -> 'bc' """ return Path(os.path.splitext(self)[0]).name @property - def suffix(self): + def suffix(self) -> str: """ '/home/a/bc.d' -> '.d' """ return Path(os.path.splitext(self)[1]) - def resolve(self): + def resolve(self) -> "Path": """ Resolves symbolic links. """ return Path(os.path.realpath(self)) @property - def parent(self): + def parent(self) -> "Path": """ Returns the parent directory of this path. @@ -114,13 +106,18 @@ def parent(self): """ return Path(os.path.dirname(self)) - def exists(self): + def exists(self) -> bool: """ Returns True if the path exists. """ return os.path.exists(self) - def mkdir(self, mode=0x1FF, exist_ok=False, parents=False): # 0o777 + def mkdir( + self, + mode: int = 0o777, + exist_ok: bool = False, + parents: bool = False, + ) -> None: """ Creates a directory, if it doesn't exist already. @@ -134,61 +131,60 @@ def mkdir(self, mode=0x1FF, exist_ok=False, parents=False): # 0o777 if not exist_ok or not os.path.isdir(self): raise - def unlink(self): + def unlink(self) -> None: """ Removes a file. """ - return os.remove(self) + os.remove(self) - def rmdir(self): + def rmdir(self) -> None: """ Removes a directory. """ - return os.rmdir(self) + os.rmdir(self) - def rename(self, to): + def rename(self, to: str) -> None: """ Renames a file or directory. May throw an OSError. """ - return os.rename(self, to) + os.rename(self, to) - def glob(self, pattern): + def glob(self, pattern: str) -> Iterator["Path"]: return (Path(i) for i in glob.iglob(self.joinpath(pattern))) - def joinpath(self, *parts): + def joinpath(self, *parts: str) -> "Path": return Path(self, *parts) # TODO: Remove after removing inheritance from str. - def join(self, *parts): - raise RuntimeError('Path.join is invalid, use joinpath instead.') + def join(self, parts: Iterable[str]) -> str: + raise RuntimeError("Path.join is invalid, use joinpath instead.") - def read_bytes(self): - # type: () -> bytes + def read_bytes(self) -> bytes: with open(self, "rb") as fp: return fp.read() - def write_bytes(self, content): - # type: (bytes) -> None + def write_bytes(self, content: bytes) -> None: with open(self, "wb") as f: f.write(content) - def read_text(self): + def read_text(self) -> str: with open(self, "r") as fp: return fp.read() - def write_text(self, content): + def write_text(self, content: str) -> None: with open(self, "w") as fp: fp.write(content) - def touch(self): + def touch(self) -> None: with open(self, "a") as fp: - path = fp.fileno() if os.utime in supports_fd else self - os.utime(path, None) # times is not optional on Python 2.7 + path: Union[int, str] = fp.fileno() if os.utime in os.supports_fd else self + os.utime(path) - def symlink_to(self, target): + def symlink_to(self, target: str) -> None: os.symlink(target, self) - def stat(self): + def stat(self) -> os.stat_result: return os.stat(self) + curdir = Path(os.path.curdir) diff --git a/tests/lib/requests_mocks.py b/tests/lib/requests_mocks.py index b8ae2d232d2..a70a9b2b048 100644 --- a/tests/lib/requests_mocks.py +++ b/tests/lib/requests_mocks.py @@ -2,43 +2,47 @@ """ from io import BytesIO +from typing import Any, Callable, Dict, Iterator, List, Optional +_Hook = Callable[["MockResponse"], None] -class FakeStream: - def __init__(self, contents): +class FakeStream: + def __init__(self, contents: bytes) -> None: self._io = BytesIO(contents) - def read(self, size, decode_content=None): + def read(self, size: int, decode_content: Optional[bool] = None) -> bytes: return self._io.read(size) - def stream(self, size, decode_content=None): + def stream( + self, size: int, decode_content: Optional[bool] = None + ) -> Iterator[bytes]: yield self._io.read(size) - def release_conn(self): + def release_conn(self) -> None: pass class MockResponse: + request: "MockRequest" + connection: "MockConnection" + url: str - def __init__(self, contents): + def __init__(self, contents: bytes) -> None: self.raw = FakeStream(contents) self.content = contents - self.request = None - self.reason = None + self.reason = "OK" self.status_code = 200 - self.connection = None - self.url = None - self.headers = {'Content-Length': len(contents)} - self.history = [] + self.headers = {"Content-Length": str(len(contents))} + self.history: List[MockResponse] = [] + self.from_cache = False class MockConnection: - - def _send(self, req, **kwargs): + def _send(self, req: "MockRequest", **kwargs: Any) -> MockResponse: raise NotImplementedError("_send must be overridden for tests") - def send(self, req, **kwargs): + def send(self, req: "MockRequest", **kwargs: Any) -> MockResponse: resp = self._send(req, **kwargs) for cb in req.hooks.get("response", []): cb(resp) @@ -46,11 +50,10 @@ def send(self, req, **kwargs): class MockRequest: - - def __init__(self, url): + def __init__(self, url: str) -> None: self.url = url - self.headers = {} - self.hooks = {} + self.headers: Dict[str, str] = {} + self.hooks: Dict[str, List[_Hook]] = {} - def register_hook(self, event_name, callback): + def register_hook(self, event_name: str, callback: _Hook) -> None: self.hooks.setdefault(event_name, []).append(callback) diff --git a/tests/lib/server.py b/tests/lib/server.py index cd3c522bfec..95cc6a23e34 100644 --- a/tests/lib/server.py +++ b/tests/lib/server.py @@ -1,61 +1,29 @@ import os -import signal import ssl import threading +from base64 import b64encode from contextlib import contextmanager from textwrap import dedent +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator +from unittest.mock import Mock -from mock import Mock -from pip._vendor.contextlib2 import nullcontext -from werkzeug.serving import WSGIRequestHandler +from werkzeug.serving import BaseWSGIServer, WSGIRequestHandler from werkzeug.serving import make_server as _make_server -from pip._internal.utils.typing import MYPY_CHECK_RUNNING - -if MYPY_CHECK_RUNNING: - from types import TracebackType - from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type - - from werkzeug.serving import BaseWSGIServer - - Environ = Dict[str, str] - Status = str - Headers = Iterable[Tuple[str, str]] - ExcInfo = Tuple[Type[BaseException], BaseException, TracebackType] - Write = Callable[[bytes], None] - StartResponse = Callable[[Status, Headers, Optional[ExcInfo]], Write] - Body = List[bytes] - Responder = Callable[[Environ, StartResponse], Body] - - class MockServer(BaseWSGIServer): - mock = Mock() # type: Mock - -# Applies on Python 2 and Windows. -if not hasattr(signal, "pthread_sigmask"): - # We're not relying on this behavior anywhere currently, it's just best - # practice. - blocked_signals = nullcontext -else: - @contextmanager - def blocked_signals(): - """Block all signals for e.g. starting a worker thread. - """ - # valid_signals() was added in Python 3.8 (and not using it results - # in a warning on pthread_sigmask() call) - try: - mask = signal.valid_signals() - except AttributeError: - mask = set(range(1, signal.NSIG)) +from .compat import blocked_signals + +if TYPE_CHECKING: + from _typeshed.wsgi import StartResponse, WSGIApplication, WSGIEnvironment + +Body = Iterable[bytes] - old_mask = signal.pthread_sigmask(signal.SIG_SETMASK, mask) - try: - yield - finally: - signal.pthread_sigmask(signal.SIG_SETMASK, old_mask) + +class MockServer(BaseWSGIServer): + mock: Mock = Mock() class _RequestHandler(WSGIRequestHandler): - def make_environ(self): + def make_environ(self) -> Dict[str, Any]: environ = super().make_environ() # From pallets/werkzeug#1469, will probably be in release after @@ -79,24 +47,24 @@ def make_environ(self): return environ -def _mock_wsgi_adapter(mock): - # type: (Callable[[Environ, StartResponse], Responder]) -> Responder +def _mock_wsgi_adapter( + mock: Callable[["WSGIEnvironment", "StartResponse"], "WSGIApplication"] +) -> "WSGIApplication": """Uses a mock to record function arguments and provide the actual function that should respond. """ - def adapter(environ, start_response): - # type: (Environ, StartResponse) -> Body + + def adapter(environ: "WSGIEnvironment", start_response: "StartResponse") -> Body: try: responder = mock(environ, start_response) except StopIteration: - raise RuntimeError('Ran out of mocked responses.') + raise RuntimeError("Ran out of mocked responses.") return responder(environ, start_response) return adapter -def make_mock_server(**kwargs): - # type: (Any) -> MockServer +def make_mock_server(**kwargs: Any) -> MockServer: """Creates a mock HTTP(S) server listening on a random port on localhost. The `mock` property of the returned server provides and records all WSGI @@ -136,10 +104,8 @@ def make_mock_server(**kwargs): @contextmanager -def server_running(server): - # type: (BaseWSGIServer) -> None - """Context manager for running the provided server in a separate thread. - """ +def server_running(server: BaseWSGIServer) -> Iterator[None]: + """Context manager for running the provided server in a separate thread.""" thread = threading.Thread(target=server.serve_forever) thread.daemon = True with blocked_signals(): @@ -154,81 +120,92 @@ def server_running(server): # Helper functions for making responses in a declarative way. -def text_html_response(text): - # type: (str) -> Responder - def responder(environ, start_response): - # type: (Environ, StartResponse) -> Body - start_response("200 OK", [ - ("Content-Type", "text/html; charset=UTF-8"), - ]) - return [text.encode('utf-8')] +def text_html_response(text: str) -> "WSGIApplication": + def responder(environ: "WSGIEnvironment", start_response: "StartResponse") -> Body: + start_response( + "200 OK", + [ + ("Content-Type", "text/html; charset=UTF-8"), + ], + ) + return [text.encode("utf-8")] return responder -def html5_page(text): - # type: (str) -> str - return dedent(""" +def html5_page(text: str) -> str: + return ( + dedent( + """ {} - """).strip().format(text) + """ + ) + .strip() + .format(text) + ) -def index_page(spec): - # type: (Dict[str, str]) -> Responder - def link(name, value): - return '{}'.format( - value, name - ) +def index_page(spec: Dict[str, str]) -> "WSGIApplication": + def link(name: str, value: str) -> str: + return '{}'.format(value, name) - links = ''.join(link(*kv) for kv in spec.items()) + links = "".join(link(*kv) for kv in spec.items()) return text_html_response(html5_page(links)) -def package_page(spec): - # type: (Dict[str, str]) -> Responder - def link(name, value): - return '{}'.format( - value, name - ) +def package_page(spec: Dict[str, str]) -> "WSGIApplication": + def link(name: str, value: str) -> str: + return '{}'.format(value, name) - links = ''.join(link(*kv) for kv in spec.items()) + links = "".join(link(*kv) for kv in spec.items()) return text_html_response(html5_page(links)) -def file_response(path): - # type: (str) -> Responder - def responder(environ, start_response): - # type: (Environ, StartResponse) -> Body +def file_response(path: str) -> "WSGIApplication": + def responder(environ: "WSGIEnvironment", start_response: "StartResponse") -> Body: size = os.stat(path).st_size start_response( - "200 OK", [ + "200 OK", + [ ("Content-Type", "application/octet-stream"), ("Content-Length", str(size)), ], ) - with open(path, 'rb') as f: + with open(path, "rb") as f: return [f.read()] return responder -def authorization_response(path): - def responder(environ, start_response): - # type: (Environ, StartResponse) -> Body - - start_response( - "401 Unauthorized", [ - ("WWW-Authenticate", "Basic"), - ], - ) - - with open(path, 'rb') as f: +def authorization_response(path: str) -> "WSGIApplication": + correct_auth = "Basic " + b64encode(b"USERNAME:PASSWORD").decode("ascii") + + def responder(environ: "WSGIEnvironment", start_response: "StartResponse") -> Body: + + if environ.get("HTTP_AUTHORIZATION") == correct_auth: + size = os.stat(path).st_size + start_response( + "200 OK", + [ + ("Content-Type", "application/octet-stream"), + ("Content-Length", str(size)), + ], + ) + else: + start_response( + "401 Unauthorized", + [ + ("WWW-Authenticate", "Basic"), + ], + ) + + with open(path, "rb") as f: return [f.read()] return responder diff --git a/tests/lib/test_lib.py b/tests/lib/test_lib.py index 47b97724f23..118393d03f4 100644 --- a/tests/lib/test_lib.py +++ b/tests/lib/test_lib.py @@ -4,26 +4,27 @@ import sys from contextlib import contextmanager from os.path import isdir, join +from typing import Any, Dict, Iterator, Type import pytest -from tests.lib import SRC_DIR +from tests.lib import SRC_DIR, PipTestEnvironment @contextmanager -def assert_error_startswith(exc_type, expected_start): +def assert_error_startswith( + exc_type: Type[Exception], expected_start: str +) -> Iterator[None]: """ Assert that an exception is raised starting with a certain message. """ with pytest.raises(exc_type) as err: yield - assert str(err.value).startswith(expected_start), ( - f'full message: {err.value}' - ) + assert str(err.value).startswith(expected_start), f"full message: {err.value}" -def test_tmp_dir_exists_in_env(script): +def test_tmp_dir_exists_in_env(script: PipTestEnvironment) -> None: """ Test that $TMPDIR == env.temp_path and path exists and env.assert_no_temp() passes (in fast env) @@ -31,26 +32,28 @@ def test_tmp_dir_exists_in_env(script): # need these tests to ensure the assert_no_temp feature of scripttest is # working script.assert_no_temp() # this fails if env.tmp_path doesn't exist - assert script.environ['TMPDIR'] == script.temp_path + assert script.environ["TMPDIR"] == script.temp_path assert isdir(script.temp_path) -def test_correct_pip_version(script): +def test_correct_pip_version(script: PipTestEnvironment) -> None: """ Check we are running proper version of pip in run_pip. """ # output is like: # pip PIPVERSION from PIPDIRECTORY (python PYVERSION) - result = script.pip('--version') + result = script.pip("--version") # compare the directory tree of the invoked pip with that of this source # distribution - pip_folder_outputed = re.match( - r'pip \d+(\.[\d]+)+(\.?(b|rc|dev|pre|post)\d+)? from (.*) ' - r'\(python \d(.[\d])+\)$', - result.stdout - ).group(4) - pip_folder = join(SRC_DIR, 'src', 'pip') + match = re.match( + r"pip \d+(\.[\d]+)+(\.?(b|rc|dev|pre|post)\d+)? from (.*) " + r"\(python \d+(\.[\d]+)+\)$", + result.stdout, + ) + assert match is not None + pip_folder_outputed = match.group(4) + pip_folder = join(SRC_DIR, "src", "pip") diffs = filecmp.dircmp(pip_folder, pip_folder_outputed) @@ -59,35 +62,40 @@ def test_correct_pip_version(script): # primary resources other than .py files, this code will need # maintenance mismatch_py = [ - x for x in diffs.left_only + diffs.right_only + diffs.diff_files - if x.endswith('.py') + x + for x in diffs.left_only + diffs.right_only + diffs.diff_files + if x.endswith(".py") ] assert not mismatch_py, ( - 'mismatched source files in {pip_folder!r} ' - 'and {pip_folder_outputed!r}: {mismatch_py!r}'.format(**locals()) + f"mismatched source files in {pip_folder!r} " + f"and {pip_folder_outputed!r}: {mismatch_py!r}" ) -def test_as_import(script): - """ test that pip.__init__.py does not shadow +def test_as_import(script: PipTestEnvironment) -> None: + """test that pip.__init__.py does not shadow the command submodule with a dictionary """ import pip._internal.commands.install as inst + assert inst is not None class TestPipTestEnvironment: - - def run_stderr_with_prefix(self, script, prefix, **kwargs): + def run_stderr_with_prefix( + self, script: PipTestEnvironment, prefix: str, **kwargs: Any + ) -> None: """ Call run() that prints stderr with the given prefix. """ - text = f'{prefix}: hello, world\\n' + text = f"{prefix}: hello, world\\n" command = f'import sys; sys.stderr.write("{text}")' - args = [sys.executable, '-c', command] + args = [sys.executable, "-c", command] script.run(*args, **kwargs) - def run_with_log_command(self, script, sub_string, **kwargs): + def run_with_log_command( + self, script: PipTestEnvironment, sub_string: str, **kwargs: Any + ) -> None: """ Call run() on a command that logs a "%"-style format string using the given substring as the string's replacement field. @@ -96,130 +104,153 @@ def run_with_log_command(self, script, sub_string, **kwargs): "import logging; logging.basicConfig(level='INFO'); " "logging.getLogger().info('sub: {}', 'foo')" ).format(sub_string) - args = [sys.executable, '-c', command] + args = [sys.executable, "-c", command] script.run(*args, **kwargs) - @pytest.mark.parametrize('prefix', ( - 'DEBUG', - 'INFO', - 'FOO', - )) - def test_run__allowed_stderr(self, script, prefix): + @pytest.mark.parametrize( + "prefix", + ( + "DEBUG", + "INFO", + "FOO", + ), + ) + def test_run__allowed_stderr(self, script: PipTestEnvironment, prefix: str) -> None: """ Test calling run() with allowed stderr. """ # Check that no error happens. self.run_stderr_with_prefix(script, prefix) - def test_run__allow_stderr_warning(self, script): + def test_run__allow_stderr_warning(self, script: PipTestEnvironment) -> None: """ Test passing allow_stderr_warning=True. """ # Check that no error happens. self.run_stderr_with_prefix( - script, 'WARNING', allow_stderr_warning=True, + script, + "WARNING", + allow_stderr_warning=True, ) # Check that an error still happens with ERROR. - expected_start = 'stderr has an unexpected error' + expected_start = "stderr has an unexpected error" with assert_error_startswith(RuntimeError, expected_start): self.run_stderr_with_prefix( - script, 'ERROR', allow_stderr_warning=True, + script, + "ERROR", + allow_stderr_warning=True, ) - @pytest.mark.parametrize('prefix', ( - 'DEPRECATION', - 'WARNING', - 'ERROR', - )) - def test_run__allow_stderr_error(self, script, prefix): + @pytest.mark.parametrize( + "prefix", + ( + "DEPRECATION", + "WARNING", + "ERROR", + ), + ) + def test_run__allow_stderr_error( + self, script: PipTestEnvironment, prefix: str + ) -> None: """ Test passing allow_stderr_error=True. """ # Check that no error happens. self.run_stderr_with_prefix(script, prefix, allow_stderr_error=True) - @pytest.mark.parametrize('prefix, expected_start', ( - ('DEPRECATION', 'stderr has an unexpected warning'), - ('WARNING', 'stderr has an unexpected warning'), - ('ERROR', 'stderr has an unexpected error'), - )) - def test_run__unexpected_stderr(self, script, prefix, expected_start): + @pytest.mark.parametrize( + "prefix, expected_start", + ( + ("DEPRECATION", "stderr has an unexpected warning"), + ("WARNING", "stderr has an unexpected warning"), + ("ERROR", "stderr has an unexpected error"), + ), + ) + def test_run__unexpected_stderr( + self, script: PipTestEnvironment, prefix: str, expected_start: str + ) -> None: """ Test calling run() with unexpected stderr output. """ with assert_error_startswith(RuntimeError, expected_start): self.run_stderr_with_prefix(script, prefix) - def test_run__logging_error(self, script): + def test_run__logging_error(self, script: PipTestEnvironment) -> None: """ Test calling run() with an unexpected logging error. """ # Pass a good substitution string. - self.run_with_log_command(script, sub_string='%r') + self.run_with_log_command(script, sub_string="%r") - expected_start = 'stderr has a logging error, which is never allowed' + expected_start = "stderr has a logging error, which is never allowed" with assert_error_startswith(RuntimeError, expected_start): # Pass a bad substitution string. Also, pass # allow_stderr_error=True to check that the RuntimeError occurs # even under the stricter test condition of when we are allowing # other types of errors. self.run_with_log_command( - script, sub_string='{!r}', allow_stderr_error=True, + script, + sub_string="{!r}", + allow_stderr_error=True, ) def test_run__allow_stderr_error_false_error_with_expect_error( - self, script, - ): + self, script: PipTestEnvironment + ) -> None: """ Test passing allow_stderr_error=False with expect_error=True. """ - expected_start = ( - 'cannot pass allow_stderr_error=False with expect_error=True' - ) + expected_start = "cannot pass allow_stderr_error=False with expect_error=True" with assert_error_startswith(RuntimeError, expected_start): - script.run('python', allow_stderr_error=False, expect_error=True) + script.run("python", allow_stderr_error=False, expect_error=True) def test_run__allow_stderr_warning_false_error_with_expect_stderr( - self, script, - ): + self, script: PipTestEnvironment + ) -> None: """ Test passing allow_stderr_warning=False with expect_stderr=True. """ expected_start = ( - 'cannot pass allow_stderr_warning=False with expect_stderr=True' + "cannot pass allow_stderr_warning=False with expect_stderr=True" ) with assert_error_startswith(RuntimeError, expected_start): script.run( - 'python', allow_stderr_warning=False, expect_stderr=True, + "python", + allow_stderr_warning=False, + expect_stderr=True, ) - @pytest.mark.parametrize('arg_name', ( - 'expect_error', - 'allow_stderr_error', - )) - def test_run__allow_stderr_warning_false_error(self, script, arg_name): + @pytest.mark.parametrize( + "arg_name", + ( + "expect_error", + "allow_stderr_error", + ), + ) + def test_run__allow_stderr_warning_false_error( + self, script: PipTestEnvironment, arg_name: str + ) -> None: """ Test passing allow_stderr_warning=False when it is not allowed. """ - kwargs = {'allow_stderr_warning': False, arg_name: True} + kwargs: Dict[str, Any] = {"allow_stderr_warning": False, arg_name: True} expected_start = ( - 'cannot pass allow_stderr_warning=False with ' - 'allow_stderr_error=True' + "cannot pass allow_stderr_warning=False with allow_stderr_error=True" ) with assert_error_startswith(RuntimeError, expected_start): - script.run('python', **kwargs) + script.run("python", **kwargs) - def test_run__expect_error_fails_when_zero_returncode(self, script): - expected_start = 'Script passed unexpectedly' + def test_run__expect_error_fails_when_zero_returncode( + self, script: PipTestEnvironment + ) -> None: + expected_start = "Script passed unexpectedly" with assert_error_startswith(AssertionError, expected_start): - script.run( - 'python', expect_error=True - ) + script.run("python", expect_error=True) - def test_run__no_expect_error_fails_when_nonzero_returncode(self, script): - expected_start = 'Script returned code: 1' + def test_run__no_expect_error_fails_when_nonzero_returncode( + self, script: PipTestEnvironment + ) -> None: + expected_start = "Script returned code: 1" with assert_error_startswith(AssertionError, expected_start): - script.run( - 'python', '-c', 'import sys; sys.exit(1)' - ) + script.run("python", "-c", "import sys; sys.exit(1)") diff --git a/tests/lib/test_wheel.py b/tests/lib/test_wheel.py index 15e5a75fe1e..09c2ee3e610 100644 --- a/tests/lib/test_wheel.py +++ b/tests/lib/test_wheel.py @@ -2,13 +2,13 @@ """ import csv from email import message_from_string +from email.message import Message from functools import partial from zipfile import ZipFile -from pip._vendor.six import ensure_text - -from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from tests.lib.path import Path from tests.lib.wheel import ( + File, _default, make_metadata_file, make_wheel, @@ -16,22 +16,18 @@ message_from_dict, ) -if MYPY_CHECK_RUNNING: - from email import Message - -def test_message_from_dict_one_value(): +def test_message_from_dict_one_value() -> None: message = message_from_dict({"a": "1"}) assert set(message.get_all("a")) == {"1"} -def test_message_from_dict_multiple_values(): +def test_message_from_dict_multiple_values() -> None: message = message_from_dict({"a": ["1", "2"]}) assert set(message.get_all("a")) == {"1", "2"} -def message_from_bytes(contents): - # type: (bytes) -> Message +def message_from_bytes(contents: bytes) -> Message: return message_from_string(contents.decode("utf-8")) @@ -45,7 +41,7 @@ def message_from_bytes(contents): ) -def default_metadata_checks(f): +def default_metadata_checks(f: File) -> Message: assert f.name == "simple-0.1.0.dist-info/METADATA" message = message_from_bytes(f.contents) assert message.get_all("Metadata-Version") == ["2.1"] @@ -54,32 +50,37 @@ def default_metadata_checks(f): return message -def test_make_metadata_file_defaults(): +def test_make_metadata_file_defaults() -> None: f = default_make_metadata() + assert f is not None default_metadata_checks(f) -def test_make_metadata_file_custom_value(): +def test_make_metadata_file_custom_value() -> None: f = default_make_metadata(updates={"a": "1"}) + assert f is not None message = default_metadata_checks(f) assert message.get_all("a") == ["1"] -def test_make_metadata_file_custom_value_list(): +def test_make_metadata_file_custom_value_list() -> None: f = default_make_metadata(updates={"a": ["1", "2"]}) + assert f is not None message = default_metadata_checks(f) assert set(message.get_all("a")) == {"1", "2"} -def test_make_metadata_file_custom_value_overrides(): +def test_make_metadata_file_custom_value_overrides() -> None: f = default_make_metadata(updates={"Metadata-Version": "2.2"}) + assert f is not None message = message_from_bytes(f.contents) assert message.get_all("Metadata-Version") == ["2.2"] -def test_make_metadata_file_custom_contents(): +def test_make_metadata_file_custom_contents() -> None: value = b"hello" f = default_make_metadata(value=value) + assert f is not None assert f.contents == value @@ -94,7 +95,7 @@ def test_make_metadata_file_custom_contents(): ) -def default_wheel_metadata_checks(f): +def default_wheel_metadata_checks(f: File) -> Message: assert f.name == "simple-0.1.0.dist-info/WHEEL" message = message_from_bytes(f.contents) assert message.get_all("Wheel-Version") == ["1.0"] @@ -104,43 +105,47 @@ def default_wheel_metadata_checks(f): return message -def test_make_wheel_metadata_file_defaults(): +def test_make_wheel_metadata_file_defaults() -> None: f = default_make_wheel_metadata() + assert f is not None default_wheel_metadata_checks(f) -def test_make_wheel_metadata_file_custom_value(): +def test_make_wheel_metadata_file_custom_value() -> None: f = default_make_wheel_metadata(updates={"a": "1"}) + assert f is not None message = default_wheel_metadata_checks(f) assert message.get_all("a") == ["1"] -def test_make_wheel_metadata_file_custom_value_list(): +def test_make_wheel_metadata_file_custom_value_list() -> None: f = default_make_wheel_metadata(updates={"a": ["1", "2"]}) + assert f is not None message = default_wheel_metadata_checks(f) assert set(message.get_all("a")) == {"1", "2"} -def test_make_wheel_metadata_file_custom_value_override(): +def test_make_wheel_metadata_file_custom_value_override() -> None: f = default_make_wheel_metadata(updates={"Wheel-Version": "1.1"}) + assert f is not None message = message_from_bytes(f.contents) assert message.get_all("Wheel-Version") == ["1.1"] -def test_make_wheel_metadata_file_custom_contents(): +def test_make_wheel_metadata_file_custom_contents() -> None: value = b"hello" f = default_make_wheel_metadata(value=value) - + assert f is not None assert f.name == "simple-0.1.0.dist-info/WHEEL" assert f.contents == value -def test_make_wheel_metadata_file_no_contents(): +def test_make_wheel_metadata_file_no_contents() -> None: f = default_make_wheel_metadata(value=None) assert f is None -def test_make_wheel_basics(tmpdir): +def test_make_wheel_basics(tmpdir: Path) -> None: make_wheel(name="simple", version="0.1.0").save_to_dir(tmpdir) expected_wheel_path = tmpdir / "simple-0.1.0-py2.py3-none-any.whl" @@ -155,7 +160,7 @@ def test_make_wheel_basics(tmpdir): } -def test_make_wheel_default_record(): +def test_make_wheel_default_record() -> None: with make_wheel( name="simple", version="0.1.0", @@ -164,21 +169,22 @@ def test_make_wheel_default_record(): extra_data_files={"purelib/info.txt": "c"}, ).as_zipfile() as z: record_bytes = z.read("simple-0.1.0.dist-info/RECORD") - record_text = ensure_text(record_bytes) + record_text = record_bytes.decode() record_rows = list(csv.reader(record_text.splitlines())) - records = { - row[0]: row[1:] for row in record_rows - } + records = {row[0]: row[1:] for row in record_rows} expected = { "simple/__init__.py": [ - "sha256=ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs", "1" + "sha256=ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs", + "1", ], "simple-0.1.0.data/purelib/info.txt": [ - "sha256=Ln0sA6lQeuJl7PW1NWiFpTOTogKdJBOUmXJloaJa78Y", "1" + "sha256=Ln0sA6lQeuJl7PW1NWiFpTOTogKdJBOUmXJloaJa78Y", + "1", ], "simple-0.1.0.dist-info/LICENSE": [ - "sha256=PiPoFgA5WUoziU9lZOGxNIu9egCI1CxKy3PurtWcAJ0", "1" + "sha256=PiPoFgA5WUoziU9lZOGxNIu9egCI1CxKy3PurtWcAJ0", + "1", ], "simple-0.1.0.dist-info/RECORD": ["", ""], } @@ -196,7 +202,7 @@ def test_make_wheel_default_record(): assert records[name][1] == length, name -def test_make_wheel_extra_files(): +def test_make_wheel_extra_files() -> None: with make_wheel( name="simple", version="0.1.0", @@ -219,7 +225,7 @@ def test_make_wheel_extra_files(): assert z.read("simple-0.1.0.data/info.txt") == b"c" -def test_make_wheel_no_files(): +def test_make_wheel_no_files() -> None: with make_wheel( name="simple", version="0.1.0", @@ -230,7 +236,7 @@ def test_make_wheel_no_files(): assert not z.namelist() -def test_make_wheel_custom_files(): +def test_make_wheel_custom_files() -> None: with make_wheel( name="simple", version="0.1.0", diff --git a/tests/lib/venv.py b/tests/lib/venv.py index 6dbdb4dc75b..a43aead9605 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -1,13 +1,23 @@ import compileall import shutil +import subprocess import sys import textwrap import venv as _venv +from typing import TYPE_CHECKING, Optional import virtualenv as _virtualenv from .path import Path +if TYPE_CHECKING: + # Literal was introduced in Python 3.8. + from typing import Literal + + VirtualEnvironmentType = Literal["virtualenv", "venv"] +else: + VirtualEnvironmentType = str + class VirtualEnvironment: """ @@ -15,56 +25,69 @@ class VirtualEnvironment: virtualenv but in the future it could use pyvenv. """ - def __init__(self, location, template=None, venv_type=None): - assert template is None or venv_type is None - assert venv_type in (None, 'virtualenv', 'venv') + def __init__( + self, + location: str, + template: Optional["VirtualEnvironment"] = None, + venv_type: Optional[VirtualEnvironmentType] = None, + ): self.location = Path(location) - self._venv_type = venv_type or template._venv_type or 'virtualenv' + assert template is None or venv_type is None + self._venv_type: VirtualEnvironmentType + if template is not None: + self._venv_type = template._venv_type + elif venv_type is not None: + self._venv_type = venv_type + else: + self._venv_type = "virtualenv" self._user_site_packages = False self._template = template - self._sitecustomize = None + self._sitecustomize: Optional[str] = None self._update_paths() self._create() - def _update_paths(self): + def _update_paths(self) -> None: home, lib, inc, bin = _virtualenv.path_locations(self.location) self.bin = Path(bin) - self.site = Path(lib) / 'site-packages' + self.site = Path(lib) / "site-packages" # Workaround for https://github.com/pypa/virtualenv/issues/306 if hasattr(sys, "pypy_version_info"): - version_dir = '{0}'.format(*sys.version_info) - self.lib = Path(home, 'lib-python', version_dir) + version_dir = str(sys.version_info.major) + self.lib = Path(home, "lib-python", version_dir) else: self.lib = Path(lib) - def __repr__(self): + def __repr__(self) -> str: return f"" - def _create(self, clear=False): + def _create(self, clear: bool = False) -> None: if clear: shutil.rmtree(self.location) if self._template: # On Windows, calling `_virtualenv.path_locations(target)` # will have created the `target` directory... - if sys.platform == 'win32' and self.location.exists(): + if sys.platform == "win32" and self.location.exists(): self.location.rmdir() # Clone virtual environment from template. - shutil.copytree( - self._template.location, self.location, symlinks=True - ) + shutil.copytree(self._template.location, self.location, symlinks=True) self._sitecustomize = self._template.sitecustomize self._user_site_packages = self._template.user_site_packages else: # Create a new virtual environment. - if self._venv_type == 'virtualenv': - _virtualenv.create_environment( - self.location, - no_pip=True, - no_wheel=True, - no_setuptools=True, + if self._venv_type == "virtualenv": + subprocess.check_call( + [ + sys.executable, + "-m", + "virtualenv", + "--no-pip", + "--no-wheel", + "--no-setuptools", + str(self.location), + ] ) self._fix_virtualenv_site_module() - elif self._venv_type == 'venv': + elif self._venv_type == "venv": builder = _venv.EnvBuilder() context = builder.ensure_directories(self.location) builder.create_configuration(context) @@ -73,48 +96,46 @@ def _create(self, clear=False): self.sitecustomize = self._sitecustomize self.user_site_packages = self._user_site_packages - def _fix_virtualenv_site_module(self): + def _fix_virtualenv_site_module(self) -> None: # Patch `site.py` so user site work as expected. - site_py = self.lib / 'site.py' + site_py = self.lib / "site.py" with open(site_py) as fp: site_contents = fp.read() for pattern, replace in ( ( # Ensure enabling user site does not result in adding # the real site-packages' directory to `sys.path`. + ("\ndef virtual_addsitepackages(known_paths):\n"), ( - '\ndef virtual_addsitepackages(known_paths):\n' - ), - ( - '\ndef virtual_addsitepackages(known_paths):\n' - ' return known_paths\n' + "\ndef virtual_addsitepackages(known_paths):\n" + " return known_paths\n" ), ), ( # Fix sites ordering: user site must be added before system. ( - '\n paths_in_sys = addsitepackages(paths_in_sys)' - '\n paths_in_sys = addusersitepackages(paths_in_sys)\n' + "\n paths_in_sys = addsitepackages(paths_in_sys)" + "\n paths_in_sys = addusersitepackages(paths_in_sys)\n" ), ( - '\n paths_in_sys = addusersitepackages(paths_in_sys)' - '\n paths_in_sys = addsitepackages(paths_in_sys)\n' + "\n paths_in_sys = addusersitepackages(paths_in_sys)" + "\n paths_in_sys = addsitepackages(paths_in_sys)\n" ), ), ): assert pattern in site_contents site_contents = site_contents.replace(pattern, replace) - with open(site_py, 'w') as fp: + with open(site_py, "w") as fp: fp.write(site_contents) # Make sure bytecode is up-to-date too. assert compileall.compile_file(str(site_py), quiet=1, force=True) - def _customize_site(self): - contents = '' - if self._venv_type == 'venv': + def _customize_site(self) -> None: + contents = "" + if self._venv_type == "venv": # Enable user site (before system). contents += textwrap.dedent( - ''' + """ import os, site, sys if not os.environ.get('PYTHONNOUSERSITE', False): @@ -138,43 +159,44 @@ def _customize_site(self): # Third, add back system-sites related paths. for path in site.getsitepackages(): site.addsitedir(path) - ''').strip() + """ + ).strip() if self._sitecustomize is not None: - contents += '\n' + self._sitecustomize + contents += "\n" + self._sitecustomize sitecustomize = self.site / "sitecustomize.py" sitecustomize.write_text(contents) # Make sure bytecode is up-to-date too. assert compileall.compile_file(str(sitecustomize), quiet=1, force=True) - def clear(self): + def clear(self) -> None: self._create(clear=True) - def move(self, location): + def move(self, location: str) -> None: shutil.move(self.location, location) self.location = Path(location) self._update_paths() @property - def sitecustomize(self): + def sitecustomize(self) -> Optional[str]: return self._sitecustomize @sitecustomize.setter - def sitecustomize(self, value): + def sitecustomize(self, value: str) -> None: self._sitecustomize = value self._customize_site() @property - def user_site_packages(self): + def user_site_packages(self) -> bool: return self._user_site_packages @user_site_packages.setter - def user_site_packages(self, value): + def user_site_packages(self, value: bool) -> None: self._user_site_packages = value - if self._venv_type == 'virtualenv': + if self._venv_type == "virtualenv": marker = self.lib / "no-global-site-packages.txt" if self._user_site_packages: marker.unlink() else: marker.touch() - elif self._venv_type == 'venv': + elif self._venv_type == "venv": self._customize_site() diff --git a/tests/lib/wheel.py b/tests/lib/wheel.py index a7403bd7f58..878364cf792 100644 --- a/tests/lib/wheel.py +++ b/tests/lib/wheel.py @@ -10,35 +10,26 @@ from functools import partial from hashlib import sha256 from io import BytesIO, StringIO +from typing import ( + AnyStr, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, +) from zipfile import ZipFile from pip._vendor.requests.structures import CaseInsensitiveDict -from pip._vendor.six import ensure_binary, ensure_text -from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution from tests.lib.path import Path -if MYPY_CHECK_RUNNING: - from typing import ( - AnyStr, - Callable, - Dict, - Iterable, - List, - Optional, - Sequence, - Tuple, - TypeVar, - Union, - ) - - # path, digest, size - RecordLike = Tuple[str, str, str] - RecordCallback = Callable[ - [List["Record"]], Union[str, bytes, List[RecordLike]] - ] - # As would be used in metadata - HeaderValue = Union[str, List[str]] +# As would be used in metadata +HeaderValue = Union[str, List[str]] File = namedtuple("File", ["name", "contents"]) @@ -51,18 +42,19 @@ class Default(Enum): _default = Default.token +T = TypeVar("T") -if MYPY_CHECK_RUNNING: - T = TypeVar("T") +# A type which may be defaulted. +Defaulted = Union[Default, T] - class Defaulted(Union[Default, T]): - """A type which may be defaulted. - """ - pass + +def ensure_binary(value: Union[bytes, str]) -> bytes: + if isinstance(value, bytes): + return value + return value.encode() -def message_from_dict(headers): - # type: (Dict[str, HeaderValue]) -> Message +def message_from_dict(headers: Dict[str, HeaderValue]) -> Message: """Plain key-value pairs are set in the returned message. List values are converted into repeated headers in the result. @@ -77,19 +69,17 @@ def message_from_dict(headers): return message -def dist_info_path(name, version, path): - # type: (str, str, str) -> str +def dist_info_path(name: str, version: str, path: str) -> str: return f"{name}-{version}.dist-info/{path}" def make_metadata_file( - name, # type: str - version, # type: str - value, # type: Defaulted[Optional[AnyStr]] - updates, # type: Defaulted[Dict[str, HeaderValue]] - body, # type: Defaulted[AnyStr] -): - # type: () -> File + name: str, + version: str, + value: Defaulted[Optional[AnyStr]], + updates: Defaulted[Dict[str, HeaderValue]], + body: Defaulted[AnyStr], +) -> Optional[File]: if value is None: return None @@ -98,11 +88,13 @@ def make_metadata_file( if value is not _default: return File(path, ensure_binary(value)) - metadata = CaseInsensitiveDict({ - "Metadata-Version": "2.1", - "Name": name, - "Version": version, - }) + metadata = CaseInsensitiveDict( + { + "Metadata-Version": "2.1", + "Name": name, + "Version": version, + } + ) if updates is not _default: metadata.update(updates) @@ -110,17 +102,16 @@ def make_metadata_file( if body is not _default: message.set_payload(body) - return File(path, ensure_binary(message_from_dict(metadata).as_string())) + return File(path, message_from_dict(metadata).as_bytes()) def make_wheel_metadata_file( - name, # type: str - version, # type: str - value, # type: Defaulted[Optional[AnyStr]] - tags, # type: Sequence[Tuple[str, str, str]] - updates, # type: Defaulted[Dict[str, HeaderValue]] -): - # type: (...) -> Optional[File] + name: str, + version: str, + value: Defaulted[Optional[AnyStr]], + tags: Sequence[Tuple[str, str, str]], + updates: Defaulted[Dict[str, HeaderValue]], +) -> Optional[File]: if value is None: return None @@ -129,26 +120,27 @@ def make_wheel_metadata_file( if value is not _default: return File(path, ensure_binary(value)) - metadata = CaseInsensitiveDict({ - "Wheel-Version": "1.0", - "Generator": "pip-test-suite", - "Root-Is-Purelib": "true", - "Tag": ["-".join(parts) for parts in tags], - }) + metadata = CaseInsensitiveDict( + { + "Wheel-Version": "1.0", + "Generator": "pip-test-suite", + "Root-Is-Purelib": "true", + "Tag": ["-".join(parts) for parts in tags], + } + ) if updates is not _default: metadata.update(updates) - return File(path, ensure_binary(message_from_dict(metadata).as_string())) + return File(path, message_from_dict(metadata).as_bytes()) def make_entry_points_file( - name, # type: str - version, # type: str - entry_points, # type: Defaulted[Dict[str, List[str]]] - console_scripts, # type: Defaulted[List[str]] -): - # type: (...) -> Optional[File] + name: str, + version: str, + entry_points: Defaulted[Dict[str, List[str]]], + console_scripts: Defaulted[List[str]], +) -> Optional[File]: if entry_points is _default and console_scripts is _default: return None @@ -167,20 +159,17 @@ def make_entry_points_file( return File( dist_info_path(name, version, "entry_points.txt"), - ensure_binary("\n".join(lines)), + "\n".join(lines).encode(), ) -def make_files(files): - # type: (Dict[str, AnyStr]) -> List[File] - return [ - File(name, ensure_binary(contents)) - for name, contents in files.items() - ] +def make_files(files: Dict[str, Union[bytes, str]]) -> List[File]: + return [File(name, ensure_binary(contents)) for name, contents in files.items()] -def make_metadata_files(name, version, files): - # type: (str, str, Dict[str, AnyStr]) -> List[File] +def make_metadata_files( + name: str, version: str, files: Dict[str, AnyStr] +) -> List[File]: get_path = partial(dist_info_path, name, version) return [ File(get_path(name), ensure_binary(contents)) @@ -188,8 +177,7 @@ def make_metadata_files(name, version, files): ] -def make_data_files(name, version, files): - # type: (str, str, Dict[str, AnyStr]) -> List[File] +def make_data_files(name: str, version: str, files: Dict[str, AnyStr]) -> List[File]: data_dir = f"{name}-{version}.data" return [ File(f"{data_dir}/{name}", ensure_binary(contents)) @@ -197,32 +185,24 @@ def make_data_files(name, version, files): ] -def urlsafe_b64encode_nopad(data): - # type: (bytes) -> str +def urlsafe_b64encode_nopad(data: bytes) -> str: return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") -def digest(contents): - # type: (bytes) -> str - return "sha256={}".format( - urlsafe_b64encode_nopad(sha256(contents).digest()) - ) +def digest(contents: bytes) -> str: + return "sha256={}".format(urlsafe_b64encode_nopad(sha256(contents).digest())) def record_file_maker_wrapper( - name, # type: str - version, # type: str - files, # type: List[File] - record, # type: Defaulted[Optional[AnyStr]] - record_callback, # type: Defaulted[RecordCallback] -): - # type: (...) -> Iterable[File] - records = [] # type: List[Record] + name: str, + version: str, + files: Iterable[File], + record: Defaulted[Optional[AnyStr]], +) -> Iterable[File]: + records: List[Record] = [] for file in files: records.append( - Record( - file.name, digest(file.contents), str(len(file.contents)) - ) + Record(file.name, digest(file.contents), str(len(file.contents))) ) yield file @@ -237,52 +217,52 @@ def record_file_maker_wrapper( records.append(Record(record_path, "", "")) - if record_callback is not _default: - records = record_callback(records) - with StringIO(newline="") as buf: writer = csv.writer(buf) - for record in records: - writer.writerow(map(ensure_text, record)) + for r in records: + writer.writerow(r) contents = buf.getvalue().encode("utf-8") yield File(record_path, contents) -def wheel_name(name, version, pythons, abis, platforms): - # type: (str, str, str, str, str) -> str - stem = "-".join([ - name, - version, - ".".join(pythons), - ".".join(abis), - ".".join(platforms), - ]) +def wheel_name( + name: str, + version: str, + pythons: Iterable[str], + abis: Iterable[str], + platforms: Iterable[str], +) -> str: + stem = "-".join( + [ + name, + version, + ".".join(pythons), + ".".join(abis), + ".".join(platforms), + ] + ) return f"{stem}.whl" class WheelBuilder: - """A wheel that can be saved or converted to several formats. - """ + """A wheel that can be saved or converted to several formats.""" - def __init__(self, name, files): - # type: (str, List[File]) -> None + def __init__(self, name: str, files: Iterable[File]) -> None: self._name = name self._files = files - def save_to_dir(self, path): - # type: (Union[Path, str]) -> str + def save_to_dir(self, path: Union[Path, str]) -> str: """Generate wheel file with correct name and save into the provided directory. :returns the wheel file path """ - path = Path(path) / self._name - path.write_bytes(self.as_bytes()) - return str(path) + p = Path(path) / self._name + p.write_bytes(self.as_bytes()) + return str(p) - def save_to(self, path): - # type: (Union[Path, str]) -> str + def save_to(self, path: Union[Path, str]) -> str: """Generate wheel file, saving to the provided path. Any parent directories must already exist. @@ -292,36 +272,36 @@ def save_to(self, path): path.write_bytes(self.as_bytes()) return str(path) - def as_bytes(self): - # type: () -> bytes + def as_bytes(self) -> bytes: with BytesIO() as buf: with ZipFile(buf, "w") as z: for file in self._files: z.writestr(file.name, file.contents) return buf.getvalue() - def as_zipfile(self): - # type: () -> ZipFile + def as_zipfile(self) -> ZipFile: return ZipFile(BytesIO(self.as_bytes())) + def as_distribution(self, name: str) -> BaseDistribution: + stream = BytesIO(self.as_bytes()) + return get_wheel_distribution(MemoryWheel(self._name, stream), name) + def make_wheel( - name, # type: str - version, # type: str - wheel_metadata=_default, # type: Defaulted[Optional[AnyStr]] - wheel_metadata_updates=_default, # type: Defaulted[Dict[str, HeaderValue]] - metadata=_default, # type: Defaulted[Optional[AnyStr]] - metadata_body=_default, # type: Defaulted[AnyStr] - metadata_updates=_default, # type: Defaulted[Dict[str, HeaderValue]] - extra_files=_default, # type: Defaulted[Dict[str, AnyStr]] - extra_metadata_files=_default, # type: Defaulted[Dict[str, AnyStr]] - extra_data_files=_default, # type: Defaulted[Dict[str, AnyStr]] - console_scripts=_default, # type: Defaulted[List[str]] - entry_points=_default, # type: Defaulted[Dict[str, List[str]]] - record=_default, # type: Defaulted[Optional[AnyStr]] - record_callback=_default, # type: Defaulted[RecordCallback] -): - # type: (...) -> WheelBuilder + name: str, + version: str, + wheel_metadata: Defaulted[Optional[AnyStr]] = _default, + wheel_metadata_updates: Defaulted[Dict[str, HeaderValue]] = _default, + metadata: Defaulted[Optional[AnyStr]] = _default, + metadata_body: Defaulted[AnyStr] = _default, + metadata_updates: Defaulted[Dict[str, HeaderValue]] = _default, + extra_files: Defaulted[Dict[str, Union[bytes, str]]] = _default, + extra_metadata_files: Defaulted[Dict[str, AnyStr]] = _default, + extra_data_files: Defaulted[Dict[str, AnyStr]] = _default, + console_scripts: Defaulted[List[str]] = _default, + entry_points: Defaulted[Dict[str, List[str]]] = _default, + record: Defaulted[Optional[AnyStr]] = _default, +) -> WheelBuilder: """ Helper function for generating test wheels which are compliant by default. @@ -381,9 +361,6 @@ def make_wheel( :param entry_points: :param record: if provided and None, then no RECORD file is generated; else if a string then sets the content of the RECORD file - :param record_callback: callback function that receives and can edit the - records before they are written to RECORD, ignored if record is - provided """ pythons = ["py2", "py3"] abis = ["none"] @@ -391,9 +368,7 @@ def make_wheel( tags = list(itertools.product(pythons, abis, platforms)) possible_files = [ - make_metadata_file( - name, version, metadata, metadata_updates, metadata_body - ), + make_metadata_file(name, version, metadata, metadata_updates, metadata_body), make_wheel_metadata_file( name, version, wheel_metadata, tags, wheel_metadata_updates ), @@ -404,9 +379,7 @@ def make_wheel( possible_files.extend(make_files(extra_files)) if extra_metadata_files is not _default: - possible_files.extend( - make_metadata_files(name, version, extra_metadata_files) - ) + possible_files.extend(make_metadata_files(name, version, extra_metadata_files)) if extra_data_files is not _default: possible_files.extend(make_data_files(name, version, extra_data_files)) @@ -414,7 +387,7 @@ def make_wheel( actual_files = filter(None, possible_files) files_and_record_file = record_file_maker_wrapper( - name, version, actual_files, record, record_callback + name, version, actual_files, record ) wheel_file_name = wheel_name(name, version, pythons, abis, platforms) diff --git a/tools/requirements/tests-common_wheels.txt b/tests/requirements-common_wheels.txt similarity index 93% rename from tools/requirements/tests-common_wheels.txt rename to tests/requirements-common_wheels.txt index f0edf0b028b..6403ed73898 100644 --- a/tools/requirements/tests-common_wheels.txt +++ b/tests/requirements-common_wheels.txt @@ -5,7 +5,7 @@ # 4. Replacing the `setuptools` entry below with a `file:///...` URL # (Adjust artifact directory used based on preference and operating system) -setuptools >= 40.8.0 +setuptools >= 40.8.0, != 60.6.0 wheel # As required by pytest-cov. coverage >= 4.4 diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000000..64a9757776f --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,12 @@ +cryptography +freezegun +pytest +pytest-cov +pytest-rerunfailures +pytest-xdist +scripttest +setuptools +virtualenv < 20.0 +werkzeug +wheel +tomli-w diff --git a/tests/unit/metadata/test_metadata.py b/tests/unit/metadata/test_metadata.py new file mode 100644 index 00000000000..c519ba89a30 --- /dev/null +++ b/tests/unit/metadata/test_metadata.py @@ -0,0 +1,57 @@ +import logging +from typing import cast +from unittest import mock + +import pytest +from pip._vendor.packaging.utils import NormalizedName + +from pip._internal.metadata import BaseDistribution +from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, ArchiveInfo + + +@mock.patch.object(BaseDistribution, "read_text", side_effect=FileNotFoundError) +def test_dist_get_direct_url_no_metadata(mock_read_text: mock.Mock) -> None: + class FakeDistribution(BaseDistribution): + pass + + dist = FakeDistribution() + assert dist.direct_url is None + mock_read_text.assert_called_once_with(DIRECT_URL_METADATA_NAME) + + +@mock.patch.object(BaseDistribution, "read_text", return_value="{}") +def test_dist_get_direct_url_invalid_json( + mock_read_text: mock.Mock, caplog: pytest.LogCaptureFixture +) -> None: + class FakeDistribution(BaseDistribution): + canonical_name = cast(NormalizedName, "whatever") # Needed for error logging. + + dist = FakeDistribution() + with caplog.at_level(logging.WARNING): + assert dist.direct_url is None + + mock_read_text.assert_called_once_with(DIRECT_URL_METADATA_NAME) + assert ( + caplog.records[-1] + .getMessage() + .startswith( + "Error parsing direct_url.json for whatever:", + ) + ) + + +@mock.patch.object( + BaseDistribution, + "read_text", + return_value='{"url": "https://e.c/p.tgz", "archive_info": {}}', +) +def test_dist_get_direct_url_valid_metadata(mock_read_text: mock.Mock) -> None: + class FakeDistribution(BaseDistribution): + pass + + dist = FakeDistribution() + direct_url = dist.direct_url + assert direct_url is not None + mock_read_text.assert_called_once_with(DIRECT_URL_METADATA_NAME) + assert direct_url.url == "https://e.c/p.tgz" + assert isinstance(direct_url.info, ArchiveInfo) diff --git a/tests/unit/metadata/test_metadata_pkg_resources.py b/tests/unit/metadata/test_metadata_pkg_resources.py new file mode 100644 index 00000000000..6bb67156c9f --- /dev/null +++ b/tests/unit/metadata/test_metadata_pkg_resources.py @@ -0,0 +1,123 @@ +import email.message +import itertools +from typing import List, cast +from unittest import mock + +import pytest +from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.version import parse as parse_version + +from pip._internal.exceptions import UnsupportedWheel +from pip._internal.metadata.pkg_resources import ( + Distribution, + Environment, + WheelMetadata, +) + +pkg_resources = pytest.importorskip("pip._vendor.pkg_resources") + + +def _dist_is_local(dist: mock.Mock) -> bool: + return dist.kind != "global" and dist.kind != "user" + + +def _dist_in_usersite(dist: mock.Mock) -> bool: + return dist.kind == "user" + + +@pytest.fixture(autouse=True) +def patch_distribution_lookups(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Distribution, "local", property(_dist_is_local)) + monkeypatch.setattr(Distribution, "in_usersite", property(_dist_in_usersite)) + + +class _MockWorkingSet(List[mock.Mock]): + def require(self, name: str) -> None: + pass + + +workingset = _MockWorkingSet( + ( + mock.Mock(test_name="global", project_name="global"), + mock.Mock(test_name="editable", project_name="editable"), + mock.Mock(test_name="normal", project_name="normal"), + mock.Mock(test_name="user", project_name="user"), + ) +) + +workingset_stdlib = _MockWorkingSet( + ( + mock.Mock(test_name="normal", project_name="argparse"), + mock.Mock(test_name="normal", project_name="wsgiref"), + ) +) + + +@pytest.mark.parametrize( + "ws, req_name", + itertools.chain( + itertools.product( + [workingset], + (d.project_name for d in workingset), + ), + itertools.product( + [workingset_stdlib], + (d.project_name for d in workingset_stdlib), + ), + ), +) +def test_get_distribution(ws: _MockWorkingSet, req_name: str) -> None: + """Ensure get_distribution() finds all kinds of distributions.""" + dist = Environment(ws).get_distribution(req_name) + assert dist is not None + assert cast(Distribution, dist)._dist.project_name == req_name + + +def test_get_distribution_nonexist() -> None: + dist = Environment(workingset).get_distribution("non-exist") + assert dist is None + + +def test_wheel_metadata_works() -> None: + name = "simple" + version = "0.1.0" + require_a = "a==1.0" + require_b = 'b==1.1; extra == "also_b"' + requires = [require_a, require_b, 'c==1.2; extra == "also_c"'] + extras = ["also_b", "also_c"] + requires_python = ">=3" + + metadata = email.message.Message() + metadata["Name"] = name + metadata["Version"] = version + for require in requires: + metadata["Requires-Dist"] = require + for extra in extras: + metadata["Provides-Extra"] = extra + metadata["Requires-Python"] = requires_python + + dist = Distribution( + pkg_resources.DistInfoDistribution( + location="", + metadata=WheelMetadata({"METADATA": metadata.as_bytes()}, ""), + project_name=name, + ), + ) + + assert name == dist.canonical_name == dist.raw_name + assert parse_version(version) == dist.version + assert set(extras) == set(dist.iter_provided_extras()) + assert [require_a] == [str(r) for r in dist.iter_dependencies()] + assert [require_a, require_b] == [ + str(r) for r in dist.iter_dependencies(["also_b"]) + ] + assert metadata.as_string() == dist.metadata.as_string() + assert SpecifierSet(requires_python) == dist.requires_python + + +def test_wheel_metadata_throws_on_bad_unicode() -> None: + metadata = WheelMetadata({"METADATA": b"\xff"}, "") + + with pytest.raises(UnsupportedWheel) as e: + metadata.get_metadata("METADATA") + assert "METADATA" in str(e.value) diff --git a/news/2b5d1433-ec03-4b33-8cf1-ff76baa3690e.trivial.rst b/tests/unit/resolution_resolvelib/__init__.py similarity index 100% rename from news/2b5d1433-ec03-4b33-8cf1-ff76baa3690e.trivial.rst rename to tests/unit/resolution_resolvelib/__init__.py diff --git a/tests/unit/resolution_resolvelib/conftest.py b/tests/unit/resolution_resolvelib/conftest.py index 9c1c9e5c4b3..6cef3c17cff 100644 --- a/tests/unit/resolution_resolvelib/conftest.py +++ b/tests/unit/resolution_resolvelib/conftest.py @@ -1,3 +1,5 @@ +from typing import Iterator + import pytest from pip._internal.cli.req_command import RequirementCommand @@ -9,46 +11,49 @@ from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences from pip._internal.network.session import PipSession +from pip._internal.operations.build.build_tracker import get_build_tracker +from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req.constructors import install_req_from_line -from pip._internal.req.req_tracker import get_requirement_tracker from pip._internal.resolution.resolvelib.factory import Factory from pip._internal.resolution.resolvelib.provider import PipProvider from pip._internal.utils.temp_dir import TempDirectory, global_tempdir_manager +from tests.lib import TestData @pytest.fixture -def finder(data): +def finder(data: TestData) -> Iterator[PackageFinder]: session = PipSession() scope = SearchScope([str(data.packages)], []) collector = LinkCollector(session, scope) prefs = SelectionPreferences(allow_yanked=False) - finder = PackageFinder.create(collector, prefs) + finder = PackageFinder.create(collector, prefs, use_deprecated_html5lib=False) yield finder @pytest.fixture -def preparer(finder): +def preparer(finder: PackageFinder) -> Iterator[RequirementPreparer]: session = PipSession() rc = InstallCommand("x", "y") o = rc.parse_args([]) with global_tempdir_manager(): with TempDirectory() as tmp: - with get_requirement_tracker() as tracker: + with get_build_tracker() as tracker: preparer = RequirementCommand.make_requirement_preparer( tmp, options=o[0], - req_tracker=tracker, + build_tracker=tracker, session=session, finder=finder, - use_user_site=False + use_user_site=False, + verbosity=0, ) yield preparer @pytest.fixture -def factory(finder, preparer): +def factory(finder: PackageFinder, preparer: RequirementPreparer) -> Iterator[Factory]: yield Factory( finder=finder, preparer=preparer, @@ -58,16 +63,17 @@ def factory(finder, preparer): force_reinstall=False, ignore_installed=False, ignore_requires_python=False, + suppress_build_failures=False, py_version_info=None, ) @pytest.fixture -def provider(factory): +def provider(factory: Factory) -> Iterator[PipProvider]: yield PipProvider( factory=factory, constraints={}, ignore_dependencies=False, upgrade_strategy="to-satisfy-only", - user_requested=set(), + user_requested={}, ) diff --git a/tests/unit/resolution_resolvelib/test_provider.py b/tests/unit/resolution_resolvelib/test_provider.py new file mode 100644 index 00000000000..a62808741d7 --- /dev/null +++ b/tests/unit/resolution_resolvelib/test_provider.py @@ -0,0 +1,78 @@ +from typing import TYPE_CHECKING, List, Optional + +from pip._vendor.resolvelib.resolvers import RequirementInformation + +from pip._internal.models.candidate import InstallationCandidate +from pip._internal.models.link import Link +from pip._internal.req.constructors import install_req_from_req_string +from pip._internal.resolution.resolvelib.factory import Factory +from pip._internal.resolution.resolvelib.provider import PipProvider +from pip._internal.resolution.resolvelib.requirements import SpecifierRequirement + +if TYPE_CHECKING: + from pip._internal.resolution.resolvelib.provider import PreferenceInformation + + +def build_requirement_information( + name: str, parent: Optional[InstallationCandidate] +) -> List["PreferenceInformation"]: + install_requirement = install_req_from_req_string(name) + # RequirementInformation is typed as a tuple, but it is a namedtupled. + # https://github.com/sarugaku/resolvelib/blob/7bc025aa2a4e979597c438ad7b17d2e8a08a364e/src/resolvelib/resolvers.pyi#L20-L22 + requirement_information: "PreferenceInformation" = RequirementInformation( + requirement=SpecifierRequirement(install_requirement), # type: ignore[call-arg] + parent=parent, + ) + return [requirement_information] + + +def test_provider_known_depths(factory: Factory) -> None: + # Root requirement is specified by the user + # therefore has an infered depth of 1 + root_requirement_name = "my-package" + provider = PipProvider( + factory=factory, + constraints={}, + ignore_dependencies=False, + upgrade_strategy="to-satisfy-only", + user_requested={root_requirement_name: 0}, + ) + + root_requirement_information = build_requirement_information( + name=root_requirement_name, parent=None + ) + provider.get_preference( + identifier=root_requirement_name, + resolutions={}, + candidates={}, + information={root_requirement_name: root_requirement_information}, + backtrack_causes=[], + ) + assert provider._known_depths == {root_requirement_name: 1.0} + + # Transative requirement is a dependency of root requirement + # theforefore has an infered depth of 2 + root_package_candidate = InstallationCandidate( + root_requirement_name, + "1.0", + Link("https://{root_requirement_name}.com"), + ) + transative_requirement_name = "my-transitive-package" + + transative_package_information = build_requirement_information( + name=transative_requirement_name, parent=root_package_candidate + ) + provider.get_preference( + identifier=transative_requirement_name, + resolutions={}, + candidates={}, + information={ + root_requirement_name: root_requirement_information, + transative_requirement_name: transative_package_information, + }, + backtrack_causes=[], + ) + assert provider._known_depths == { + transative_requirement_name: 2.0, + root_requirement_name: 1.0, + } diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py index 6149fd1aece..387afbc2304 100644 --- a/tests/unit/resolution_resolvelib/test_requirement.py +++ b/tests/unit/resolution_resolvelib/test_requirement.py @@ -1,8 +1,14 @@ +from typing import Iterator, List, Tuple + import pytest from pip._vendor.resolvelib import BaseReporter, Resolver -from pip._internal.resolution.resolvelib.base import Candidate, Constraint +from pip._internal.resolution.resolvelib.base import Candidate, Constraint, Requirement +from pip._internal.resolution.resolvelib.factory import Factory +from pip._internal.resolution.resolvelib.provider import PipProvider from pip._internal.utils.urls import path_to_url +from tests.lib import TestData +from tests.lib.path import Path # NOTE: All tests are prefixed `test_rlr` (for "test resolvelib resolver"). # This helps select just these tests using pytest's `-k` option, and @@ -18,11 +24,11 @@ @pytest.fixture -def test_cases(data): - def data_file(name): +def test_cases(data: TestData) -> Iterator[List[Tuple[str, str, int]]]: + def data_file(name: str) -> Path: return data.packages.joinpath(name) - def data_url(name): + def data_url(name: str) -> str: return path_to_url(data_file(name)) test_cases = [ @@ -47,39 +53,56 @@ def data_url(name): yield test_cases -def test_new_resolver_requirement_has_name(test_cases, factory): +def test_new_resolver_requirement_has_name( + test_cases: List[Tuple[str, str, int]], factory: Factory +) -> None: """All requirements should have a name""" for spec, name, _ in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) + assert req is not None assert req.name == name -def test_new_resolver_correct_number_of_matches(test_cases, factory): +def test_new_resolver_correct_number_of_matches( + test_cases: List[Tuple[str, str, int]], factory: Factory +) -> None: """Requirements should return the correct number of candidates""" for spec, _, match_count in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) + assert req is not None matches = factory.find_candidates( - [req], Constraint.empty(), prefers_installed=False, + req.name, + {req.name: [req]}, + {}, + Constraint.empty(), + prefers_installed=False, ) assert sum(1 for _ in matches) == match_count -def test_new_resolver_candidates_match_requirement(test_cases, factory): - """Candidates returned from find_candidates should satisfy the requirement - """ +def test_new_resolver_candidates_match_requirement( + test_cases: List[Tuple[str, str, int]], factory: Factory +) -> None: + """Candidates returned from find_candidates should satisfy the requirement""" for spec, _, _ in test_cases: req = factory.make_requirement_from_spec(spec, comes_from=None) + assert req is not None candidates = factory.find_candidates( - [req], Constraint.empty(), prefers_installed=False, + req.name, + {req.name: [req]}, + {}, + Constraint.empty(), + prefers_installed=False, ) for c in candidates: assert isinstance(c, Candidate) assert req.is_satisfied_by(c) -def test_new_resolver_full_resolve(factory, provider): +def test_new_resolver_full_resolve(factory: Factory, provider: PipProvider) -> None: """A very basic full resolve""" req = factory.make_requirement_from_spec("simplewheel", comes_from=None) - r = Resolver(provider, BaseReporter()) + assert req is not None + r: Resolver[Requirement, Candidate, str] = Resolver(provider, BaseReporter()) result = r.resolve([req]) - assert set(result.mapping.keys()) == {'simplewheel'} + assert set(result.mapping.keys()) == {"simplewheel"} diff --git a/tests/unit/resolution_resolvelib/test_resolver.py b/tests/unit/resolution_resolvelib/test_resolver.py index 4a62cefb603..db71f911acd 100644 --- a/tests/unit/resolution_resolvelib/test_resolver.py +++ b/tests/unit/resolution_resolvelib/test_resolver.py @@ -1,9 +1,13 @@ -import mock +from typing import Dict, List, Optional, Set, Tuple, cast +from unittest import mock + import pytest from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.resolvelib.resolvers import Result from pip._vendor.resolvelib.structs import DirectedGraph +from pip._internal.index.package_finder import PackageFinder +from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req.constructors import install_req_from_line from pip._internal.req.req_set import RequirementSet from pip._internal.resolution.resolvelib.resolver import ( @@ -13,30 +17,32 @@ @pytest.fixture() -def resolver(preparer, finder): +def resolver(preparer: RequirementPreparer, finder: PackageFinder) -> Resolver: resolver = Resolver( preparer=preparer, finder=finder, wheel_cache=None, make_install_req=mock.Mock(), - use_user_site="not-used", - ignore_dependencies="not-used", - ignore_installed="not-used", - ignore_requires_python="not-used", - force_reinstall="not-used", + use_user_site=False, + ignore_dependencies=False, + ignore_installed=False, + ignore_requires_python=False, + force_reinstall=False, upgrade_strategy="to-satisfy-only", + suppress_build_failures=False, ) return resolver -def _make_graph(edges): - """Build graph from edge declarations. - """ +def _make_graph( + edges: List[Tuple[Optional[str], Optional[str]]] +) -> "DirectedGraph[Optional[str]]": + """Build graph from edge declarations.""" - graph = DirectedGraph() + graph: "DirectedGraph[Optional[str]]" = DirectedGraph() for parent, child in edges: - parent = canonicalize_name(parent) if parent else None - child = canonicalize_name(child) if child else None + parent = cast(str, canonicalize_name(parent)) if parent else None + child = cast(str, canonicalize_name(child)) if child else None for v in (parent, child): if v not in graph: graph.add(v) @@ -76,12 +82,16 @@ def _make_graph(edges): ), ], ) -def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): +def test_new_resolver_get_installation_order( + resolver: Resolver, + edges: List[Tuple[Optional[str], Optional[str]]], + ordered_reqs: List[str], +) -> None: graph = _make_graph(edges) # Mapping values and criteria are not used in test, so we stub them out. mapping = {vertex: None for vertex in graph if vertex is not None} - resolver._result = Result(mapping, graph, criteria=None) + resolver._result = Result(mapping, graph, criteria=None) # type: ignore reqset = RequirementSet() for r in ordered_reqs: @@ -93,7 +103,7 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): @pytest.mark.parametrize( - "name, edges, expected_weights", + "name, edges, requirement_keys, expected_weights", [ ( # From https://github.com/pypa/pip/pull/8127#discussion_r414564664 @@ -106,7 +116,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("three", "four"), ("four", "five"), ], - {None: 0, "one": 1, "two": 1, "three": 2, "four": 3, "five": 4}, + {"one", "two", "three", "four", "five"}, + {"five": 5, "four": 4, "one": 4, "three": 2, "two": 1}, ), ( "linear", @@ -117,7 +128,20 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("three", "four"), ("four", "five"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + ), + ( + "linear AND restricted", + [ + (None, "one"), + ("one", "two"), + ("two", "three"), + ("three", "four"), + ("four", "five"), + ], + {"one", "three", "five"}, + {"one": 1, "three": 3, "five": 5}, ), ( "linear AND root -> two", @@ -129,7 +153,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), (None, "two"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND root -> three", @@ -141,7 +166,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), (None, "three"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND root -> four", @@ -153,7 +179,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), (None, "four"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND root -> five", @@ -165,7 +192,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), (None, "five"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND one -> four", @@ -177,7 +205,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), ("one", "four"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND two -> four", @@ -189,7 +218,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), ("two", "four"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND four -> one (cycle)", @@ -201,7 +231,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), ("four", "one"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND four -> two (cycle)", @@ -213,7 +244,8 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), ("four", "two"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, ), ( "linear AND four -> three (cycle)", @@ -225,12 +257,44 @@ def test_new_resolver_get_installation_order(resolver, edges, ordered_reqs): ("four", "five"), ("four", "three"), ], - {None: 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + {"one", "two", "three", "four", "five"}, + {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + ), + ( + "linear AND four -> three (cycle) AND restricted 1-2-3", + [ + (None, "one"), + ("one", "two"), + ("two", "three"), + ("three", "four"), + ("four", "five"), + ("four", "three"), + ], + {"one", "two", "three"}, + {"one": 1, "two": 2, "three": 3}, + ), + ( + "linear AND four -> three (cycle) AND restricted 4-5", + [ + (None, "one"), + ("one", "two"), + ("two", "three"), + ("three", "four"), + ("four", "five"), + ("four", "three"), + ], + {"four", "five"}, + {"four": 4, "five": 5}, ), ], ) -def test_new_resolver_topological_weights(name, edges, expected_weights): +def test_new_resolver_topological_weights( + name: str, + edges: List[Tuple[Optional[str], Optional[str]]], + requirement_keys: Set[str], + expected_weights: Dict[Optional[str], int], +) -> None: graph = _make_graph(edges) - weights = get_topological_weights(graph, len(expected_weights)) + weights = get_topological_weights(graph, requirement_keys) assert weights == expected_weights diff --git a/tests/unit/test_appdirs.py b/tests/unit/test_appdirs.py index 623486b2894..fd3ea143bcb 100644 --- a/tests/unit/test_appdirs.py +++ b/tests/unit/test_appdirs.py @@ -1,229 +1,220 @@ -import ntpath +# mypy: no-warn-unused-ignores + import os -import posixpath import sys +from unittest import mock -import pretend -from pip._vendor import appdirs as _appdirs +import pytest +from pip._vendor import platformdirs from pip._internal.utils import appdirs class TestUserCacheDir: - - def test_user_cache_dir_win(self, monkeypatch): - @pretend.call_recorder - def _get_win_folder(base): - return "C:\\Users\\test\\AppData\\Local" + @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test") + def test_user_cache_dir_win(self, monkeypatch: pytest.MonkeyPatch) -> None: + _get_win_folder = mock.Mock(return_value="C:\\Users\\test\\AppData\\Local") monkeypatch.setattr( - _appdirs, - "_get_win_folder", + platformdirs.windows, # type: ignore + "get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(_appdirs, "system", "win32") - monkeypatch.setattr(os, "path", ntpath) - assert (appdirs.user_cache_dir("pip") == - "C:\\Users\\test\\AppData\\Local\\pip\\Cache") - assert _get_win_folder.calls == [pretend.call("CSIDL_LOCAL_APPDATA")] + assert ( + appdirs.user_cache_dir("pip") + == "C:\\Users\\test\\AppData\\Local\\pip\\Cache" + ) + assert _get_win_folder.call_args_list == [mock.call("CSIDL_LOCAL_APPDATA")] - def test_user_cache_dir_osx(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "darwin") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "darwin", reason="MacOS-only test") + def test_user_cache_dir_osx(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "darwin") assert appdirs.user_cache_dir("pip") == "/home/test/Library/Caches/pip" - def test_user_cache_dir_linux(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_user_cache_dir_linux(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("XDG_CACHE_HOME", raising=False) monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.user_cache_dir("pip") == "/home/test/.cache/pip" - def test_user_cache_dir_linux_override(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_user_cache_dir_linux_override( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setenv("XDG_CACHE_HOME", "/home/test/.other-cache") monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.user_cache_dir("pip") == "/home/test/.other-cache/pip" - def test_user_cache_dir_linux_home_slash(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_user_cache_dir_linux_home_slash( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: # Verify that we are not affected by https://bugs.python.org/issue14768 monkeypatch.delenv("XDG_CACHE_HOME", raising=False) monkeypatch.setenv("HOME", "/") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.user_cache_dir("pip") == "/.cache/pip" - def test_user_cache_dir_unicode(self, monkeypatch): - if sys.platform != 'win32': + def test_user_cache_dir_unicode(self, monkeypatch: pytest.MonkeyPatch) -> None: + if sys.platform != "win32": return - def my_get_win_folder(csidl_name): + def my_get_win_folder(csidl_name: str) -> str: return "\u00DF\u00E4\u03B1\u20AC" - monkeypatch.setattr(_appdirs, "_get_win_folder", my_get_win_folder) + monkeypatch.setattr( + platformdirs.windows, # type: ignore + "get_win_folder", + my_get_win_folder, + ) # Do not use the isinstance expression directly in the # assert statement, as the Unicode characters in the result # cause pytest to fail with an internal error on Python 2.7 - result_is_str = isinstance(appdirs.user_cache_dir('test'), str) + result_is_str = isinstance(appdirs.user_cache_dir("test"), str) assert result_is_str, "user_cache_dir did not return a str" # Test against regression #3463 from pip._internal.cli.main_parser import create_main_parser + create_main_parser().print_help() # This should not crash class TestSiteConfigDirs: - - def test_site_config_dirs_win(self, monkeypatch): - @pretend.call_recorder - def _get_win_folder(base): - return "C:\\ProgramData" + @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test") + def test_site_config_dirs_win(self, monkeypatch: pytest.MonkeyPatch) -> None: + _get_win_folder = mock.Mock(return_value="C:\\ProgramData") monkeypatch.setattr( - _appdirs, - "_get_win_folder", + platformdirs.windows, # type: ignore + "get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(_appdirs, "system", "win32") - monkeypatch.setattr(os, "path", ntpath) assert appdirs.site_config_dirs("pip") == ["C:\\ProgramData\\pip"] - assert _get_win_folder.calls == [pretend.call("CSIDL_COMMON_APPDATA")] + assert _get_win_folder.call_args_list == [mock.call("CSIDL_COMMON_APPDATA")] - def test_site_config_dirs_osx(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "darwin") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "darwin", reason="MacOS-only test") + def test_site_config_dirs_osx(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "darwin") - assert appdirs.site_config_dirs("pip") == \ - ["/Library/Application Support/pip"] + assert appdirs.site_config_dirs("pip") == [ + "/Library/Application Support/pip", + ] - def test_site_config_dirs_linux(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_site_config_dirs_linux(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("XDG_CONFIG_DIRS", raising=False) - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.site_config_dirs("pip") == [ - '/etc/xdg/pip', - '/etc' + "/etc/xdg/pip", + "/etc", ] - def test_site_config_dirs_linux_override(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) - monkeypatch.setattr(os, "pathsep", ':') + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_site_config_dirs_linux_override( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(os, "pathsep", ":") monkeypatch.setenv("XDG_CONFIG_DIRS", "/spam:/etc:/etc/xdg") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.site_config_dirs("pip") == [ - '/spam/pip', - '/etc/pip', - '/etc/xdg/pip', - '/etc' + "/spam/pip", + "/etc/pip", + "/etc/xdg/pip", + "/etc", ] - def test_site_config_dirs_linux_empty(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) - monkeypatch.setattr(os, "pathsep", ':') + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_site_config_dirs_linux_empty( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(os, "pathsep", ":") monkeypatch.setenv("XDG_CONFIG_DIRS", "") - monkeypatch.setattr(sys, "platform", "linux2") - assert appdirs.site_config_dirs("pip") == ['/etc/xdg/pip', '/etc'] + assert appdirs.site_config_dirs("pip") == [ + "/etc/xdg/pip", + "/etc", + ] class TestUserConfigDir: - - def test_user_config_dir_win_no_roaming(self, monkeypatch): - @pretend.call_recorder - def _get_win_folder(base): - return "C:\\Users\\test\\AppData\\Local" + @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test") + def test_user_config_dir_win_no_roaming( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + _get_win_folder = mock.Mock(return_value="C:\\Users\\test\\AppData\\Local") monkeypatch.setattr( - _appdirs, - "_get_win_folder", + platformdirs.windows, # type: ignore + "get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(_appdirs, "system", "win32") - monkeypatch.setattr(os, "path", ntpath) assert ( - appdirs.user_config_dir("pip", roaming=False) == - "C:\\Users\\test\\AppData\\Local\\pip" + appdirs.user_config_dir("pip", roaming=False) + == "C:\\Users\\test\\AppData\\Local\\pip" ) - assert _get_win_folder.calls == [pretend.call("CSIDL_LOCAL_APPDATA")] + assert _get_win_folder.call_args_list == [mock.call("CSIDL_LOCAL_APPDATA")] - def test_user_config_dir_win_yes_roaming(self, monkeypatch): - @pretend.call_recorder - def _get_win_folder(base): - return "C:\\Users\\test\\AppData\\Roaming" + @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test") + def test_user_config_dir_win_yes_roaming( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + _get_win_folder = mock.Mock(return_value="C:\\Users\\test\\AppData\\Roaming") monkeypatch.setattr( - _appdirs, - "_get_win_folder", + platformdirs.windows, # type: ignore + "get_win_folder", _get_win_folder, raising=False, ) - monkeypatch.setattr(_appdirs, "system", "win32") - monkeypatch.setattr(os, "path", ntpath) - assert (appdirs.user_config_dir("pip") == - "C:\\Users\\test\\AppData\\Roaming\\pip") - assert _get_win_folder.calls == [pretend.call("CSIDL_APPDATA")] + assert ( + appdirs.user_config_dir("pip") == "C:\\Users\\test\\AppData\\Roaming\\pip" + ) + assert _get_win_folder.call_args_list == [mock.call("CSIDL_APPDATA")] - def test_user_config_dir_osx(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "darwin") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "darwin", reason="MacOS-only test") + def test_user_config_dir_osx(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "darwin") - if os.path.isdir('/home/test/Library/Application Support/'): - assert (appdirs.user_config_dir("pip") == - "/home/test/Library/Application Support/pip") + if os.path.isdir("/home/test/Library/Application Support/"): + assert ( + appdirs.user_config_dir("pip") + == "/home/test/Library/Application Support/pip" + ) else: - assert (appdirs.user_config_dir("pip") == - "/home/test/.config/pip") + assert appdirs.user_config_dir("pip") == "/home/test/.config/pip" - def test_user_config_dir_linux(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_user_config_dir_linux(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.user_config_dir("pip") == "/home/test/.config/pip" - def test_user_config_dir_linux_override(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_user_config_dir_linux_override( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setenv("XDG_CONFIG_HOME", "/home/test/.other-config") monkeypatch.setenv("HOME", "/home/test") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.user_config_dir("pip") == "/home/test/.other-config/pip" - def test_user_config_dir_linux_home_slash(self, monkeypatch): - monkeypatch.setattr(_appdirs, "system", "linux2") - monkeypatch.setattr(os, "path", posixpath) + @pytest.mark.skipif(sys.platform != "linux", reason="Linux-only test") + def test_user_config_dir_linux_home_slash( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: # Verify that we are not affected by https://bugs.python.org/issue14768 monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) monkeypatch.setenv("HOME", "/") - monkeypatch.setattr(sys, "platform", "linux2") assert appdirs.user_config_dir("pip") == "/.config/pip" diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index 857d4f4f300..9a61ccc77b4 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -1,39 +1,45 @@ import logging import os +from optparse import Values +from typing import Callable, Iterator, List, NoReturn, Optional +from unittest.mock import Mock, patch import pytest -from mock import Mock, patch from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import SUCCESS from pip._internal.utils import temp_dir from pip._internal.utils.logging import BrokenStdoutLoggingError from pip._internal.utils.temp_dir import TempDirectory +from tests.lib.path import Path @pytest.fixture -def fixed_time(utc): - with patch('time.time', lambda: 1547704837.040001): +def fixed_time(utc: None) -> Iterator[None]: + with patch("time.time", lambda: 1547704837.040001): yield class FakeCommand(Command): - _name = 'fake' + _name = "fake" - def __init__(self, run_func=None, error=False): + def __init__( + self, run_func: Optional[Callable[[], int]] = None, error: bool = False + ) -> None: if error: - def run_func(): + + def run_func() -> int: raise SystemExit(1) self.run_func = run_func super().__init__(self._name, self._name) - def main(self, args): + def main(self, args: List[str]) -> int: args.append("--disable-pip-version-check") return super().main(args) - def run(self, options, args): + def run(self, options: Values, args: List[str]) -> int: logging.getLogger("pip.tests").info("fake") # Return SUCCESS from run if run_func is not provided if self.run_func: @@ -43,22 +49,21 @@ def run(self, options, args): class FakeCommandWithUnicode(FakeCommand): - _name = 'fake_unicode' + _name = "fake_unicode" - def run(self, options, args): + def run(self, options: Values, args: List[str]) -> int: logging.getLogger("pip.tests").info(b"bytes here \xE9") - logging.getLogger("pip.tests").info( - b"unicode here \xC3\xA9".decode("utf-8") - ) + logging.getLogger("pip.tests").info(b"unicode here \xC3\xA9".decode("utf-8")) + return SUCCESS class TestCommand: - - def call_main(self, capsys, args): + def call_main(self, capsys: pytest.CaptureFixture[str], args: List[str]) -> str: """ Call command.main(), and return the command's stderr. """ - def raise_broken_stdout(): + + def raise_broken_stdout() -> NoReturn: raise BrokenStdoutLoggingError() cmd = FakeCommand(run_func=raise_broken_stdout) @@ -68,26 +73,28 @@ def raise_broken_stdout(): return stderr - def test_raise_broken_stdout(self, capsys): + def test_raise_broken_stdout(self, capsys: pytest.CaptureFixture[str]) -> None: """ Test raising BrokenStdoutLoggingError. """ stderr = self.call_main(capsys, []) - assert stderr.rstrip() == 'ERROR: Pipe to stdout was broken' + assert stderr.rstrip() == "ERROR: Pipe to stdout was broken" - def test_raise_broken_stdout__debug_logging(self, capsys): + def test_raise_broken_stdout__debug_logging( + self, capsys: pytest.CaptureFixture[str] + ) -> None: """ Test raising BrokenStdoutLoggingError with debug logging enabled. """ - stderr = self.call_main(capsys, ['-v']) + stderr = self.call_main(capsys, ["-vv"]) - assert 'ERROR: Pipe to stdout was broken' in stderr - assert 'Traceback (most recent call last):' in stderr + assert "ERROR: Pipe to stdout was broken" in stderr + assert "Traceback (most recent call last):" in stderr -@patch('pip._internal.cli.req_command.Command.handle_pip_version_check') -def test_handle_pip_version_check_called(mock_handle_version_check): +@patch("pip._internal.cli.req_command.Command.handle_pip_version_check") +def test_handle_pip_version_check_called(mock_handle_version_check: Mock) -> None: """ Check that Command.handle_pip_version_check() is called. """ @@ -96,54 +103,55 @@ def test_handle_pip_version_check_called(mock_handle_version_check): mock_handle_version_check.assert_called_once() -def test_log_command_success(fixed_time, tmpdir): +def test_log_command_success(fixed_time: None, tmpdir: Path) -> None: """Test the --log option logs when command succeeds.""" cmd = FakeCommand() - log_path = tmpdir.joinpath('log') - cmd.main(['fake', '--log', log_path]) + log_path = tmpdir.joinpath("log") + cmd.main(["fake", "--log", log_path]) with open(log_path) as f: - assert f.read().rstrip() == '2019-01-17T06:00:37,040 fake' + assert f.read().rstrip() == "2019-01-17T06:00:37,040 fake" -def test_log_command_error(fixed_time, tmpdir): +def test_log_command_error(fixed_time: None, tmpdir: Path) -> None: """Test the --log option logs when command fails.""" cmd = FakeCommand(error=True) - log_path = tmpdir.joinpath('log') - cmd.main(['fake', '--log', log_path]) + log_path = tmpdir.joinpath("log") + cmd.main(["fake", "--log", log_path]) with open(log_path) as f: - assert f.read().startswith('2019-01-17T06:00:37,040 fake') + assert f.read().startswith("2019-01-17T06:00:37,040 fake") -def test_log_file_command_error(fixed_time, tmpdir): +def test_log_file_command_error(fixed_time: None, tmpdir: Path) -> None: """Test the --log-file option logs (when there's an error).""" cmd = FakeCommand(error=True) - log_file_path = tmpdir.joinpath('log_file') - cmd.main(['fake', '--log-file', log_file_path]) + log_file_path = tmpdir.joinpath("log_file") + cmd.main(["fake", "--log-file", log_file_path]) with open(log_file_path) as f: - assert f.read().startswith('2019-01-17T06:00:37,040 fake') + assert f.read().startswith("2019-01-17T06:00:37,040 fake") -def test_log_unicode_messages(fixed_time, tmpdir): +def test_log_unicode_messages(fixed_time: None, tmpdir: Path) -> None: """Tests that logging bytestrings and unicode objects don't break logging. """ cmd = FakeCommandWithUnicode() - log_path = tmpdir.joinpath('log') - cmd.main(['fake_unicode', '--log', log_path]) + log_path = tmpdir.joinpath("log") + cmd.main(["fake_unicode", "--log", log_path]) @pytest.mark.no_auto_tempdir_manager -def test_base_command_provides_tempdir_helpers(): +def test_base_command_provides_tempdir_helpers() -> None: assert temp_dir._tempdir_manager is None assert temp_dir._tempdir_registry is None - def assert_helpers_set(options, args): + def assert_helpers_set(options: Values, args: List[str]) -> int: assert temp_dir._tempdir_manager is not None assert temp_dir._tempdir_registry is not None return SUCCESS c = Command("fake", "fake") - c.run = Mock(side_effect=assert_helpers_set) + # https://github.com/python/mypy/issues/2427 + c.run = Mock(side_effect=assert_helpers_set) # type: ignore[assignment] assert c.main(["fake"]) == SUCCESS c.run.assert_called_once() @@ -151,38 +159,37 @@ def assert_helpers_set(options, args): not_deleted = "not_deleted" -@pytest.mark.parametrize("kind,exists", [ - (not_deleted, True), ("deleted", False) -]) +@pytest.mark.parametrize("kind,exists", [(not_deleted, True), ("deleted", False)]) @pytest.mark.no_auto_tempdir_manager -def test_base_command_global_tempdir_cleanup(kind, exists): +def test_base_command_global_tempdir_cleanup(kind: str, exists: bool) -> None: assert temp_dir._tempdir_manager is None assert temp_dir._tempdir_registry is None class Holder: - value = None + value: str - def create_temp_dirs(options, args): + def create_temp_dirs(options: Values, args: List[str]) -> int: + assert c.tempdir_registry is not None c.tempdir_registry.set_delete(not_deleted, False) Holder.value = TempDirectory(kind=kind, globally_managed=True).path return SUCCESS c = Command("fake", "fake") - c.run = Mock(side_effect=create_temp_dirs) + # https://github.com/python/mypy/issues/2427 + c.run = Mock(side_effect=create_temp_dirs) # type: ignore[assignment] assert c.main(["fake"]) == SUCCESS c.run.assert_called_once() assert os.path.exists(Holder.value) == exists -@pytest.mark.parametrize("kind,exists", [ - (not_deleted, True), ("deleted", False) -]) +@pytest.mark.parametrize("kind,exists", [(not_deleted, True), ("deleted", False)]) @pytest.mark.no_auto_tempdir_manager -def test_base_command_local_tempdir_cleanup(kind, exists): +def test_base_command_local_tempdir_cleanup(kind: str, exists: bool) -> None: assert temp_dir._tempdir_manager is None assert temp_dir._tempdir_registry is None - def create_temp_dirs(options, args): + def create_temp_dirs(options: Values, args: List[str]) -> int: + assert c.tempdir_registry is not None c.tempdir_registry.set_delete(not_deleted, False) with TempDirectory(kind=kind) as d: @@ -192,6 +199,7 @@ def create_temp_dirs(options, args): return SUCCESS c = Command("fake", "fake") - c.run = Mock(side_effect=create_temp_dirs) + # https://github.com/python/mypy/issues/2427 + c.run = Mock(side_effect=create_temp_dirs) # type: ignore[assignment] assert c.main(["fake"]) == SUCCESS c.run.assert_called_once() diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index bab62d4e3a8..acb16034186 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -6,24 +6,25 @@ from pip._internal.models.format_control import FormatControl from pip._internal.models.link import Link from pip._internal.utils.misc import ensure_dir +from tests.lib.path import Path -def test_falsey_path_none(): - wc = WheelCache(False, None) +def test_falsey_path_none() -> None: + wc = WheelCache("", FormatControl()) assert wc.cache_dir is None -def test_subdirectory_fragment(): +def test_subdirectory_fragment() -> None: """ Test the subdirectory URL fragment is part of the cache key. """ - wc = WheelCache("/tmp/.foo/", None) + wc = WheelCache("/tmp/.foo/", FormatControl()) link1 = Link("git+https://g.c/o/r#subdirectory=d1") link2 = Link("git+https://g.c/o/r#subdirectory=d2") assert wc.get_path_for_link(link1) != wc.get_path_for_link(link2) -def test_wheel_name_filter(tmpdir): +def test_wheel_name_filter(tmpdir: Path) -> None: """ Test the wheel cache filters on wheel name when several wheels for different package are stored under the same cache directory. @@ -42,7 +43,7 @@ def test_wheel_name_filter(tmpdir): assert wc.get(link, "package2", [Tag("py3", "none", "any")]) is link -def test_cache_hash(): +def test_cache_hash() -> None: h = _hash_dict({"url": "https://g.c/o/r"}) assert h == "72aa79d3315c181d2cc23239d7109a782de663b6f89982624d8c1e86" h = _hash_dict({"url": "https://g.c/o/r", "subdirectory": "sd"}) @@ -51,7 +52,7 @@ def test_cache_hash(): assert h == "f83b32dfa27a426dec08c21bf006065dd003d0aac78e7fc493d9014d" -def test_get_cache_entry(tmpdir): +def test_get_cache_entry(tmpdir: Path) -> None: wc = WheelCache(tmpdir, FormatControl()) persi_link = Link("https://g.c/o/r/persi") persi_path = wc.get_path_for_link(persi_link) @@ -65,10 +66,12 @@ def test_get_cache_entry(tmpdir): pass other_link = Link("https://g.c/o/r/other") supported_tags = [Tag("py3", "none", "any")] - assert ( - wc.get_cache_entry(persi_link, "persi", supported_tags).persistent - ) - assert ( - not wc.get_cache_entry(ephem_link, "ephem", supported_tags).persistent - ) + entry = wc.get_cache_entry(persi_link, "persi", supported_tags) + assert entry is not None + assert entry.persistent + + entry = wc.get_cache_entry(ephem_link, "ephem", supported_tags) + assert entry is not None + assert not entry.persistent + assert wc.get_cache_entry(other_link, "other", supported_tags) is None diff --git a/tests/unit/test_check.py b/tests/unit/test_check.py deleted file mode 100644 index c53830aa099..00000000000 --- a/tests/unit/test_check.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Unit Tests for pip's dependency checking logic -""" - -import mock - -from pip._internal.operations import check - - -class TestInstalledDistributionsCall: - - def test_passes_correct_default_kwargs(self, monkeypatch): - my_mock = mock.MagicMock(return_value=[]) - monkeypatch.setattr(check, "get_installed_distributions", my_mock) - - check.create_package_set_from_installed() - - my_mock.assert_called_with(local_only=False, skip=()) - - def test_passes_any_given_kwargs(self, monkeypatch): - my_mock = mock.MagicMock(return_value=[]) - monkeypatch.setattr(check, "get_installed_distributions", my_mock) - - obj = object() - check.create_package_set_from_installed(hi=obj) - - my_mock.assert_called_with(hi=obj) diff --git a/tests/unit/test_cmdoptions.py b/tests/unit/test_cmdoptions.py index bac33ce77b1..1e5ef995cd0 100644 --- a/tests/unit/test_cmdoptions.py +++ b/tests/unit/test_cmdoptions.py @@ -1,24 +1,31 @@ +from typing import Optional, Tuple + import pytest from pip._internal.cli.cmdoptions import _convert_python_version -@pytest.mark.parametrize('value, expected', [ - ('', (None, None)), - ('2', ((2,), None)), - ('3', ((3,), None)), - ('3.7', ((3, 7), None)), - ('3.7.3', ((3, 7, 3), None)), - # Test strings without dots of length bigger than 1. - ('34', ((3, 4), None)), - # Test a 2-digit minor version. - ('310', ((3, 10), None)), - # Test some values that fail to parse. - ('ab', ((), 'each version part must be an integer')), - ('3a', ((), 'each version part must be an integer')), - ('3.7.a', ((), 'each version part must be an integer')), - ('3.7.3.1', ((), 'at most three version parts are allowed')), -]) -def test_convert_python_version(value, expected): +@pytest.mark.parametrize( + "value, expected", + [ + ("", (None, None)), + ("2", ((2,), None)), + ("3", ((3,), None)), + ("3.7", ((3, 7), None)), + ("3.7.3", ((3, 7, 3), None)), + # Test strings without dots of length bigger than 1. + ("34", ((3, 4), None)), + # Test a 2-digit minor version. + ("310", ((3, 10), None)), + # Test some values that fail to parse. + ("ab", ((), "each version part must be an integer")), + ("3a", ((), "each version part must be an integer")), + ("3.7.a", ((), "each version part must be an integer")), + ("3.7.3.1", ((), "at most three version parts are allowed")), + ], +) +def test_convert_python_version( + value: str, expected: Tuple[Optional[Tuple[int, ...]], Optional[str]] +) -> None: actual = _convert_python_version(value) - assert actual == expected, f'actual: {actual!r}' + assert actual == expected, f"actual: {actual!r}" diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py index 9fb920b32fe..f77794b55b9 100644 --- a/tests/unit/test_collector.py +++ b/tests/unit/test_collector.py @@ -1,14 +1,14 @@ +import itertools import logging import os.path import re import urllib.request import uuid from textwrap import dedent +from typing import List, Optional, Tuple +from unittest import mock -import mock -import pretend import pytest -from mock import Mock, patch from pip._vendor import html5lib, requests from pip._internal.exceptions import NetworkConnectionError @@ -23,14 +23,15 @@ _make_html_page, _NotHTML, _NotHTTP, - _remove_duplicate_links, - group_locations, parse_links, ) +from pip._internal.index.sources import _FlatDirectorySource, _IndexDirectorySource +from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.index import PyPI from pip._internal.models.link import Link from pip._internal.network.session import PipSession -from tests.lib import make_test_link_collector +from tests.lib import TestData, make_test_link_collector +from tests.lib.path import Path @pytest.mark.parametrize( @@ -40,7 +41,7 @@ "file:///opt/data/pip-18.0.tar.gz", ], ) -def test_get_html_response_archive_to_naive_scheme(url): +def test_get_html_response_archive_to_naive_scheme(url: str) -> None: """ `_get_html_response()` should error on an archive-like URL if the scheme does not allow "poking" without getting data. @@ -57,24 +58,29 @@ def test_get_html_response_archive_to_naive_scheme(url): ], ) @mock.patch("pip._internal.index.collector.raise_for_status") -def test_get_html_response_archive_to_http_scheme(mock_raise_for_status, url, - content_type): +def test_get_html_response_archive_to_http_scheme( + mock_raise_for_status: mock.Mock, url: str, content_type: str +) -> None: """ `_get_html_response()` should send a HEAD request on an archive-like URL if the scheme supports it, and raise `_NotHTML` if the response isn't HTML. """ session = mock.Mock(PipSession) - session.head.return_value = mock.Mock(**{ - "request.method": "HEAD", - "headers": {"Content-Type": content_type}, - }) + session.head.return_value = mock.Mock( + **{ + "request.method": "HEAD", + "headers": {"Content-Type": content_type}, + } + ) with pytest.raises(_NotHTML) as ctx: _get_html_response(url, session=session) - session.assert_has_calls([ - mock.call.head(url, allow_redirects=True), - ]) + session.assert_has_calls( + [ + mock.call.head(url, allow_redirects=True), + ] + ) mock_raise_for_status.assert_called_once_with(session.head.return_value) assert ctx.value.args == (content_type, "HEAD") @@ -86,7 +92,9 @@ def test_get_html_response_archive_to_http_scheme(mock_raise_for_status, url, ("file:///opt/data/pip-18.0.tar.gz"), ], ) -def test_get_html_page_invalid_content_type_archive(caplog, url): +def test_get_html_page_invalid_content_type_archive( + caplog: pytest.LogCaptureFixture, url: str +) -> None: """`_get_html_page()` should warn if an archive URL is not HTML and therefore cannot be used for a HEAD request. """ @@ -96,12 +104,12 @@ def test_get_html_page_invalid_content_type_archive(caplog, url): session = mock.Mock(PipSession) assert _get_html_page(link, session=session) is None - assert ('pip._internal.index.collector', - logging.WARNING, - 'Skipping page {} because it looks like an archive, and cannot ' - 'be checked by a HTTP HEAD request.'.format( - url)) \ - in caplog.record_tuples + assert ( + "pip._internal.index.collector", + logging.WARNING, + "Skipping page {} because it looks like an archive, and cannot " + "be checked by a HTTP HEAD request.".format(url), + ) in caplog.record_tuples @pytest.mark.parametrize( @@ -113,17 +121,19 @@ def test_get_html_page_invalid_content_type_archive(caplog, url): ) @mock.patch("pip._internal.index.collector.raise_for_status") def test_get_html_response_archive_to_http_scheme_is_html( - mock_raise_for_status, url -): + mock_raise_for_status: mock.Mock, url: str +) -> None: """ `_get_html_response()` should work with archive-like URLs if the HEAD request is responded with text/html. """ session = mock.Mock(PipSession) - session.head.return_value = mock.Mock(**{ - "request.method": "HEAD", - "headers": {"Content-Type": "text/html"}, - }) + session.head.return_value = mock.Mock( + **{ + "request.method": "HEAD", + "headers": {"Content-Type": "text/html"}, + } + ) session.get.return_value = mock.Mock(headers={"Content-Type": "text/html"}) resp = _get_html_response(url, session=session) @@ -131,13 +141,17 @@ def test_get_html_response_archive_to_http_scheme_is_html( assert resp is not None assert session.mock_calls == [ mock.call.head(url, allow_redirects=True), - mock.call.get(url, headers={ - "Accept": "text/html", "Cache-Control": "max-age=0", - }), + mock.call.get( + url, + headers={ + "Accept": "text/html", + "Cache-Control": "max-age=0", + }, + ), ] assert mock_raise_for_status.mock_calls == [ mock.call(session.head.return_value), - mock.call(resp) + mock.call(resp), ] @@ -150,7 +164,7 @@ def test_get_html_response_archive_to_http_scheme_is_html( ], ) @mock.patch("pip._internal.index.collector.raise_for_status") -def test_get_html_response_no_head(mock_raise_for_status, url): +def test_get_html_response_no_head(mock_raise_for_status: mock.Mock, url: str) -> None: """ `_get_html_response()` shouldn't send a HEAD request if the URL does not look like an archive, only the GET request that retrieves data. @@ -158,26 +172,35 @@ def test_get_html_response_no_head(mock_raise_for_status, url): session = mock.Mock(PipSession) # Mock the headers dict to ensure it is accessed. - session.get.return_value = mock.Mock(headers=mock.Mock(**{ - "get.return_value": "text/html", - })) + session.get.return_value = mock.Mock( + headers=mock.Mock( + **{ + "get.return_value": "text/html", + } + ) + ) resp = _get_html_response(url, session=session) assert resp is not None assert session.head.call_count == 0 assert session.get.mock_calls == [ - mock.call(url, headers={ - "Accept": "text/html", "Cache-Control": "max-age=0", - }), + mock.call( + url, + headers={ + "Accept": "text/html", + "Cache-Control": "max-age=0", + }, + ), mock.call().headers.get("Content-Type", ""), ] mock_raise_for_status.assert_called_once_with(resp) @mock.patch("pip._internal.index.collector.raise_for_status") -def test_get_html_response_dont_log_clear_text_password(mock_raise_for_status, - caplog): +def test_get_html_response_dont_log_clear_text_password( + mock_raise_for_status: mock.Mock, caplog: pytest.LogCaptureFixture +) -> None: """ `_get_html_response()` should redact the password from the index URL in its DEBUG log message. @@ -185,9 +208,13 @@ def test_get_html_response_dont_log_clear_text_password(mock_raise_for_status, session = mock.Mock(PipSession) # Mock the headers dict to ensure it is accessed. - session.get.return_value = mock.Mock(headers=mock.Mock(**{ - "get.return_value": "text/html", - })) + session.get.return_value = mock.Mock( + headers=mock.Mock( + **{ + "get.return_value": "text/html", + } + ) + ) caplog.set_level(logging.DEBUG) @@ -200,7 +227,7 @@ def test_get_html_response_dont_log_clear_text_password(mock_raise_for_status, assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'DEBUG' + assert record.levelname == "DEBUG" assert record.message.splitlines() == [ "Getting page https://user:****@example.com/simple/", ] @@ -211,87 +238,87 @@ def test_get_html_response_dont_log_clear_text_password(mock_raise_for_status, [ (b"", "https://example.com/", "https://example.com/"), ( - b"" - b"" - b"", + b'', "https://example.com/", "https://foo.example.com/", ), ( b"" - b"" + b'' b"", "https://example.com/", "https://foo.example.com/", ), ], ) -def test_determine_base_url(html, url, expected): +def test_determine_base_url(html: bytes, url: str, expected: str) -> None: document = html5lib.parse( - html, transport_encoding=None, namespaceHTMLElements=False, + html, + transport_encoding=None, + namespaceHTMLElements=False, ) assert _determine_base_url(document, url) == expected @pytest.mark.parametrize( - ('path', 'expected'), + ("path", "expected"), [ # Test a character that needs quoting. - ('a b', 'a%20b'), + ("a b", "a%20b"), # Test an unquoted "@". - ('a @ b', 'a%20@%20b'), + ("a @ b", "a%20@%20b"), # Test multiple unquoted "@". - ('a @ @ b', 'a%20@%20@%20b'), + ("a @ @ b", "a%20@%20@%20b"), # Test a quoted "@". - ('a %40 b', 'a%20%40%20b'), + ("a %40 b", "a%20%40%20b"), # Test a quoted "@" before an unquoted "@". - ('a %40b@ c', 'a%20%40b@%20c'), + ("a %40b@ c", "a%20%40b@%20c"), # Test a quoted "@" after an unquoted "@". - ('a @b%40 c', 'a%20@b%40%20c'), + ("a @b%40 c", "a%20@b%40%20c"), # Test alternating quoted and unquoted "@". - ('a %40@b %40@c %40', 'a%20%40@b%20%40@c%20%40'), + ("a %40@b %40@c %40", "a%20%40@b%20%40@c%20%40"), # Test an unquoted "/". - ('a / b', 'a%20/%20b'), + ("a / b", "a%20/%20b"), # Test multiple unquoted "/". - ('a / / b', 'a%20/%20/%20b'), + ("a / / b", "a%20/%20/%20b"), # Test a quoted "/". - ('a %2F b', 'a%20%2F%20b'), + ("a %2F b", "a%20%2F%20b"), # Test a quoted "/" before an unquoted "/". - ('a %2Fb/ c', 'a%20%2Fb/%20c'), + ("a %2Fb/ c", "a%20%2Fb/%20c"), # Test a quoted "/" after an unquoted "/". - ('a /b%2F c', 'a%20/b%2F%20c'), + ("a /b%2F c", "a%20/b%2F%20c"), # Test alternating quoted and unquoted "/". - ('a %2F/b %2F/c %2F', 'a%20%2F/b%20%2F/c%20%2F'), + ("a %2F/b %2F/c %2F", "a%20%2F/b%20%2F/c%20%2F"), # Test normalizing non-reserved quoted characters "[" and "]" - ('a %5b %5d b', 'a%20%5B%20%5D%20b'), + ("a %5b %5d b", "a%20%5B%20%5D%20b"), # Test normalizing a reserved quoted "/" - ('a %2f b', 'a%20%2F%20b'), - ] + ("a %2f b", "a%20%2F%20b"), + ], ) -@pytest.mark.parametrize('is_local_path', [True, False]) -def test_clean_url_path(path, expected, is_local_path): +@pytest.mark.parametrize("is_local_path", [True, False]) +def test_clean_url_path(path: str, expected: str, is_local_path: bool) -> None: assert _clean_url_path(path, is_local_path=is_local_path) == expected @pytest.mark.parametrize( - ('path', 'expected'), + ("path", "expected"), [ # Test a VCS path with a Windows drive letter and revision. pytest.param( - '/T:/with space/repo.git@1.0', - '///T:/with%20space/repo.git@1.0', + "/T:/with space/repo.git@1.0", + "///T:/with%20space/repo.git@1.0", marks=pytest.mark.skipif("sys.platform != 'win32'"), ), # Test a VCS path with a Windows drive letter and revision, # running on non-windows platform. pytest.param( - '/T:/with space/repo.git@1.0', - '/T%3A/with%20space/repo.git@1.0', + "/T:/with space/repo.git@1.0", + "/T%3A/with%20space/repo.git@1.0", marks=pytest.mark.skipif("sys.platform == 'win32'"), ), - ] + ], ) -def test_clean_url_path_with_local_path(path, expected): +def test_clean_url_path_with_local_path(path: str, expected: str) -> None: actual = _clean_url_path(path, is_local_path=True) assert actual == expected @@ -300,41 +327,63 @@ def test_clean_url_path_with_local_path(path, expected): ("url", "clean_url"), [ # URL with hostname and port. Port separator should not be quoted. - ("https://localhost.localdomain:8181/path/with space/", - "https://localhost.localdomain:8181/path/with%20space/"), + ( + "https://localhost.localdomain:8181/path/with space/", + "https://localhost.localdomain:8181/path/with%20space/", + ), # URL that is already properly quoted. The quoting `%` # characters should not be quoted again. - ("https://localhost.localdomain:8181/path/with%20quoted%20space/", - "https://localhost.localdomain:8181/path/with%20quoted%20space/"), + ( + "https://localhost.localdomain:8181/path/with%20quoted%20space/", + "https://localhost.localdomain:8181/path/with%20quoted%20space/", + ), # URL with IPv4 address and port. - ("https://127.0.0.1:8181/path/with space/", - "https://127.0.0.1:8181/path/with%20space/"), + ( + "https://127.0.0.1:8181/path/with space/", + "https://127.0.0.1:8181/path/with%20space/", + ), # URL with IPv6 address and port. The `[]` brackets around the # IPv6 address should not be quoted. - ("https://[fd00:0:0:236::100]:8181/path/with space/", - "https://[fd00:0:0:236::100]:8181/path/with%20space/"), + ( + "https://[fd00:0:0:236::100]:8181/path/with space/", + "https://[fd00:0:0:236::100]:8181/path/with%20space/", + ), # URL with query. The leading `?` should not be quoted. - ("https://localhost.localdomain:8181/path/with/query?request=test", - "https://localhost.localdomain:8181/path/with/query?request=test"), + ( + "https://localhost.localdomain:8181/path/with/query?request=test", + "https://localhost.localdomain:8181/path/with/query?request=test", + ), # URL with colon in the path portion. - ("https://localhost.localdomain:8181/path:/with:/colon", - "https://localhost.localdomain:8181/path%3A/with%3A/colon"), + ( + "https://localhost.localdomain:8181/path:/with:/colon", + "https://localhost.localdomain:8181/path%3A/with%3A/colon", + ), # URL with something that looks like a drive letter, but is # not. The `:` should be quoted. - ("https://localhost.localdomain/T:/path/", - "https://localhost.localdomain/T%3A/path/"), + ( + "https://localhost.localdomain/T:/path/", + "https://localhost.localdomain/T%3A/path/", + ), # URL with a quoted "/" in the path portion. - ("https://example.com/access%2Ftoken/path/", - "https://example.com/access%2Ftoken/path/"), + ( + "https://example.com/access%2Ftoken/path/", + "https://example.com/access%2Ftoken/path/", + ), # VCS URL containing revision string. - ("git+ssh://example.com/path to/repo.git@1.0#egg=my-package-1.0", - "git+ssh://example.com/path%20to/repo.git@1.0#egg=my-package-1.0"), + ( + "git+ssh://example.com/path to/repo.git@1.0#egg=my-package-1.0", + "git+ssh://example.com/path%20to/repo.git@1.0#egg=my-package-1.0", + ), # VCS URL with a quoted "#" in the revision string. - ("git+https://example.com/repo.git@hash%23symbol#egg=my-package-1.0", - "git+https://example.com/repo.git@hash%23symbol#egg=my-package-1.0"), + ( + "git+https://example.com/repo.git@hash%23symbol#egg=my-package-1.0", + "git+https://example.com/repo.git@hash%23symbol#egg=my-package-1.0", + ), # VCS URL with a quoted "@" in the revision string. - ("git+https://example.com/repo.git@at%40 space#egg=my-package-1.0", - "git+https://example.com/repo.git@at%40%20space#egg=my-package-1.0"), + ( + "git+https://example.com/repo.git@at%40 space#egg=my-package-1.0", + "git+https://example.com/repo.git@at%40%20space#egg=my-package-1.0", + ), # URL with Windows drive letter. The `:` after the drive # letter should not be quoted. The trailing `/` should be # removed. @@ -363,57 +412,97 @@ def test_clean_url_path_with_local_path(path, expected): "git+file:/T%3A/with%20space/repo.git@1.0#egg=my-package-1.0", marks=pytest.mark.skipif("sys.platform == 'win32'"), ), - ] + ], ) -def test_clean_link(url, clean_url): +def test_clean_link(url: str, clean_url: str) -> None: assert _clean_link(url) == clean_url -@pytest.mark.parametrize('anchor_html, expected', [ - # Test not present. - ('', None), - # Test present with no value. - ('', ''), - # Test the empty string. - ('', ''), - # Test a non-empty string. - ('', 'error'), - # Test a value with an escaped character. - ('', - 'version < 1'), - # Test a yanked reason with a non-ascii character. - ('', - 'curlyquote \u2018'), -]) -def test_parse_links__yanked_reason(anchor_html, expected): +def _test_parse_links_data_attribute( + anchor_html: str, attr: str, expected: Optional[str] +) -> None: html = ( - # Mark this as a unicode string for Python 2 since anchor_html - # can contain non-ascii. + "" '' - '{}' + "{}" ).format(anchor_html) - html_bytes = html.encode('utf-8') + html_bytes = html.encode("utf-8") page = HTMLPage( html_bytes, encoding=None, # parse_links() is cached by url, so we inject a random uuid to ensure # the page content isn't cached. - url=f'https://example.com/simple-{uuid.uuid4()}/', + url=f"https://example.com/simple-{uuid.uuid4()}/", ) - links = list(parse_links(page)) - link, = links - actual = link.yanked_reason + links = list(parse_links(page, use_deprecated_html5lib=False)) + (link,) = links + actual = getattr(link, attr) assert actual == expected -def test_parse_links_caches_same_page_by_url(): +@pytest.mark.parametrize( + "anchor_html, expected", + [ + # Test not present. + ('', None), + # Test present with no value. + ('', None), + # Test a value with an escaped character. + ( + '', + ">=3.6", + ), + # Test requires python is unescaped once. + ( + '', + ">=3.6", + ), + ], +) +def test_parse_links__requires_python( + anchor_html: str, expected: Optional[str] +) -> None: + _test_parse_links_data_attribute(anchor_html, "requires_python", expected) + + +@pytest.mark.parametrize( + "anchor_html, expected", + [ + # Test not present. + ('', None), + # Test present with no value. + ('', None), + # Test the empty string. + ('', ""), + # Test a non-empty string. + ('', "error"), + # Test a value with an escaped character. + ('', "version < 1"), + # Test a yanked reason with a non-ascii character. + ( + '', + "curlyquote \u2018", + ), + # Test yanked reason is unescaped once. + ( + '', + "version < 1", + ), + ], +) +def test_parse_links__yanked_reason(anchor_html: str, expected: Optional[str]) -> None: + _test_parse_links_data_attribute(anchor_html, "yanked_reason", expected) + + +def test_parse_links_caches_same_page_by_url() -> None: html = ( + "" '' '' ) - html_bytes = html.encode('utf-8') + html_bytes = html.encode("utf-8") - url = 'https://example.com/simple/' + url = "https://example.com/simple/" page_1 = HTMLPage( html_bytes, @@ -423,7 +512,7 @@ def test_parse_links_caches_same_page_by_url(): # Make a second page with zero content, to ensure that it's not accessed, # because the page was cached by url. page_2 = HTMLPage( - b'', + b"", encoding=None, url=url, ) @@ -431,63 +520,71 @@ def test_parse_links_caches_same_page_by_url(): # cached, even for the same url. We modify the page content slightly to # verify that the result is not cached. page_3 = HTMLPage( - re.sub(b'pkg1', b'pkg2', html_bytes), + re.sub(b"pkg1", b"pkg2", html_bytes), encoding=None, url=url, cache_link_parsing=False, ) - parsed_links_1 = list(parse_links(page_1)) + parsed_links_1 = list(parse_links(page_1, use_deprecated_html5lib=False)) assert len(parsed_links_1) == 1 - assert 'pkg1' in parsed_links_1[0].url + assert "pkg1" in parsed_links_1[0].url - parsed_links_2 = list(parse_links(page_2)) + parsed_links_2 = list(parse_links(page_2, use_deprecated_html5lib=False)) assert parsed_links_2 == parsed_links_1 - parsed_links_3 = list(parse_links(page_3)) + parsed_links_3 = list(parse_links(page_3, use_deprecated_html5lib=False)) assert len(parsed_links_3) == 1 assert parsed_links_3 != parsed_links_1 - assert 'pkg2' in parsed_links_3[0].url + assert "pkg2" in parsed_links_3[0].url + + +def test_parse_link_handles_deprecated_usage_properly() -> None: + html = b'' + url = "https://example.com/simple/" + page = HTMLPage(html, encoding=None, url=url, cache_link_parsing=False) + + parsed_links = list(parse_links(page, use_deprecated_html5lib=True)) + + assert len(parsed_links) == 2 + assert "pkg1-1.0" in parsed_links[0].url + assert "pkg1-2.0" in parsed_links[1].url @mock.patch("pip._internal.index.collector.raise_for_status") -def test_request_http_error(mock_raise_for_status, caplog): +def test_request_http_error( + mock_raise_for_status: mock.Mock, caplog: pytest.LogCaptureFixture +) -> None: caplog.set_level(logging.DEBUG) - link = Link('http://localhost') - session = Mock(PipSession) - session.get.return_value = Mock() - mock_raise_for_status.side_effect = NetworkConnectionError('Http error') + link = Link("http://localhost") + session = mock.Mock(PipSession) + session.get.return_value = mock.Mock() + mock_raise_for_status.side_effect = NetworkConnectionError("Http error") assert _get_html_page(link, session=session) is None - assert ( - 'Could not fetch URL http://localhost: Http error - skipping' - in caplog.text - ) + assert "Could not fetch URL http://localhost: Http error - skipping" in caplog.text -def test_request_retries(caplog): +def test_request_retries(caplog: pytest.LogCaptureFixture) -> None: caplog.set_level(logging.DEBUG) - link = Link('http://localhost') - session = Mock(PipSession) - session.get.side_effect = requests.exceptions.RetryError('Retry error') + link = Link("http://localhost") + session = mock.Mock(PipSession) + session.get.side_effect = requests.exceptions.RetryError("Retry error") assert _get_html_page(link, session=session) is None - assert ( - 'Could not fetch URL http://localhost: Retry error - skipping' - in caplog.text - ) + assert "Could not fetch URL http://localhost: Retry error - skipping" in caplog.text -def test_make_html_page(): - headers = {'Content-Type': 'text/html; charset=UTF-8'} - response = pretend.stub( - content=b'', - url='https://example.com/index.html', +def test_make_html_page() -> None: + headers = {"Content-Type": "text/html; charset=UTF-8"} + response = mock.Mock( + content=b"", + url="https://example.com/index.html", headers=headers, ) actual = _make_html_page(response) - assert actual.content == b'' - assert actual.encoding == 'UTF-8' - assert actual.url == 'https://example.com/index.html' + assert actual.content == b"" + assert actual.encoding == "UTF-8" + assert actual.url == "https://example.com/index.html" @pytest.mark.parametrize( @@ -497,7 +594,9 @@ def test_make_html_page(): ("git+https://github.com/pypa/pip.git", "git"), ], ) -def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme): +def test_get_html_page_invalid_scheme( + caplog: pytest.LogCaptureFixture, url: str, vcs_scheme: str +) -> None: """`_get_html_page()` should error if an invalid scheme is given. Only file:, http:, https:, and ftp: are allowed. @@ -524,47 +623,53 @@ def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme): ], ) @mock.patch("pip._internal.index.collector.raise_for_status") -def test_get_html_page_invalid_content_type(mock_raise_for_status, - caplog, content_type): +def test_get_html_page_invalid_content_type( + mock_raise_for_status: mock.Mock, + caplog: pytest.LogCaptureFixture, + content_type: str, +) -> None: """`_get_html_page()` should warn if an invalid content-type is given. Only text/html is allowed. """ caplog.set_level(logging.DEBUG) - url = 'https://pypi.org/simple/pip' + url = "https://pypi.org/simple/pip" link = Link(url) session = mock.Mock(PipSession) - session.get.return_value = mock.Mock(**{ - "request.method": "GET", - "headers": {"Content-Type": content_type}, - }) + session.get.return_value = mock.Mock( + **{ + "request.method": "GET", + "headers": {"Content-Type": content_type}, + } + ) assert _get_html_page(link, session=session) is None mock_raise_for_status.assert_called_once_with(session.get.return_value) - assert ('pip._internal.index.collector', - logging.WARNING, - 'Skipping page {} because the GET request got Content-Type: {}.' - 'The only supported Content-Type is text/html'.format( - url, content_type)) \ - in caplog.record_tuples + assert ( + "pip._internal.index.collector", + logging.WARNING, + "Skipping page {} because the GET request got Content-Type: {}." + "The only supported Content-Type is text/html".format(url, content_type), + ) in caplog.record_tuples -def make_fake_html_response(url): +def make_fake_html_response(url: str) -> mock.Mock: """ Create a fake requests.Response object. """ - html = dedent("""\ + html = dedent( + """\ abc-1.0.tar.gz - """) - content = html.encode('utf-8') - return pretend.stub(content=content, url=url, headers={}) + """ + ) + content = html.encode("utf-8") + return mock.Mock(content=content, url=url, headers={}) -def test_get_html_page_directory_append_index(tmpdir): - """`_get_html_page()` should append "index.html" to a directory URL. - """ +def test_get_html_page_directory_append_index(tmpdir: Path) -> None: + """`_get_html_page()` should append "index.html" to a directory URL.""" dirpath = tmpdir / "something" dirpath.mkdir() dir_url = "file:///{}".format( @@ -580,71 +685,106 @@ def test_get_html_page_directory_append_index(tmpdir): actual = _get_html_page(Link(dir_url), session=session) assert mock_func.mock_calls == [ mock.call(expected_url, session=session), - ], f'actual calls: {mock_func.mock_calls}' + ], f"actual calls: {mock_func.mock_calls}" + assert actual is not None assert actual.content == fake_response.content assert actual.encoding is None assert actual.url == expected_url -def test_remove_duplicate_links(): - links = [ - # We choose Links that will test that ordering is preserved. - Link('https://example.com/2'), - Link('https://example.com/1'), - Link('https://example.com/2'), - ] - actual = _remove_duplicate_links(links) - assert actual == [ - Link('https://example.com/2'), - Link('https://example.com/1'), - ] - - -def test_group_locations__file_expand_dir(data): +def test_collect_sources__file_expand_dir(data: TestData) -> None: """ - Test that a file:// dir gets listdir run with expand_dir + Test that a file:// dir from --find-links becomes _FlatDirectorySource """ - files, urls = group_locations([data.find_links], expand_dir=True) - assert files and not urls, ( - "files and not urls should have been found " - "at find-links url: {data.find_links}".format(**locals()) + collector = LinkCollector.create( + session=mock.Mock(is_secure_origin=None), # Shouldn't be used. + options=mock.Mock( + index_url="ignored-by-no-index", + extra_index_urls=[], + no_index=True, + find_links=[data.find_links], + ), + ) + sources = collector.collect_sources( + # Shouldn't be used. + project_name=None, # type: ignore[arg-type] + candidates_from_page=None, # type: ignore[arg-type] + ) + assert ( + not sources.index_urls + and len(sources.find_links) == 1 + and isinstance(sources.find_links[0], _FlatDirectorySource) + ), ( + "Directory source should have been found " + f"at find-links url: {data.find_links}" ) -def test_group_locations__file_not_find_link(data): +def test_collect_sources__file_not_find_link(data: TestData) -> None: """ - Test that a file:// url dir that's not a find-link, doesn't get a listdir + Test that a file:// dir from --index-url doesn't become _FlatDirectorySource run """ - files, urls = group_locations([data.index_url("empty_with_pkg")]) - assert urls and not files, "urls, but not files should have been found" + collector = LinkCollector.create( + session=mock.Mock(is_secure_origin=None), # Shouldn't be used. + options=mock.Mock( + index_url=data.index_url("empty_with_pkg"), + extra_index_urls=[], + no_index=False, + find_links=[], + ), + ) + sources = collector.collect_sources( + project_name="", + # Shouldn't be used. + candidates_from_page=None, # type: ignore[arg-type] + ) + assert ( + not sources.find_links + and len(sources.index_urls) == 1 + and isinstance(sources.index_urls[0], _IndexDirectorySource) + ), "Directory specified as index should be treated as a page" -def test_group_locations__non_existing_path(): +def test_collect_sources__non_existing_path() -> None: """ Test that a non-existing path is ignored. """ - files, urls = group_locations([os.path.join('this', 'doesnt', 'exist')]) - assert not urls and not files, "nothing should have been found" + collector = LinkCollector.create( + session=mock.Mock(is_secure_origin=None), # Shouldn't be used. + options=mock.Mock( + index_url="ignored-by-no-index", + extra_index_urls=[], + no_index=True, + find_links=[os.path.join("this", "doesnt", "exist")], + ), + ) + sources = collector.collect_sources( + # Shouldn't be used. + project_name=None, # type: ignore[arg-type] + candidates_from_page=None, # type: ignore[arg-type] + ) + assert not sources.index_urls and sources.find_links == [ + None + ], "Nothing should have been found" -def check_links_include(links, names): +def check_links_include(links: List[Link], names: List[str]) -> None: """ Assert that the given list of Link objects includes, for each of the given names, a link whose URL has a base name matching that name. """ for name in names: - assert any(link.url.endswith(name) for link in links), ( - f'name {name!r} not among links: {links}' - ) + assert any( + link.url.endswith(name) for link in links + ), f"name {name!r} not among links: {links}" class TestLinkCollector: - - @patch('pip._internal.index.collector._get_html_response') - def test_fetch_page(self, mock_get_html_response): - url = 'https://pypi.org/simple/twine/' + @mock.patch("pip._internal.index.collector._get_html_response") + def test_fetch_page(self, mock_get_html_response: mock.Mock) -> None: + url = "https://pypi.org/simple/twine/" fake_response = make_fake_html_response(url) mock_get_html_response.return_value = fake_response @@ -653,6 +793,7 @@ def test_fetch_page(self, mock_get_html_response): link_collector = make_test_link_collector() actual = link_collector.fetch_page(location) + assert actual is not None assert actual.content == fake_response.content assert actual.encoding is None assert actual.url == url @@ -661,10 +802,13 @@ def test_fetch_page(self, mock_get_html_response): # Also check that the right session object was passed to # _get_html_response(). mock_get_html_response.assert_called_once_with( - url, session=link_collector.session, + url, + session=link_collector.session, ) - def test_collect_links(self, caplog, data): + def test_collect_sources( + self, caplog: pytest.LogCaptureFixture, data: TestData + ) -> None: caplog.set_level(logging.DEBUG) link_collector = make_test_link_collector( @@ -673,57 +817,79 @@ def test_collect_links(self, caplog, data): # is skipped. index_urls=[PyPI.simple_url, PyPI.simple_url], ) - actual = link_collector.collect_links('twine') + collected_sources = link_collector.collect_sources( + "twine", + candidates_from_page=lambda link: [ + InstallationCandidate("twine", "1.0", link) + ], + ) - # Spot-check the CollectedLinks return value. - assert len(actual.files) > 20 - check_links_include(actual.files, names=['simple-1.0.tar.gz']) + files_it = itertools.chain.from_iterable( + source.file_links() + for sources in collected_sources + for source in sources + if source is not None + ) + pages_it = itertools.chain.from_iterable( + source.page_candidates() + for sources in collected_sources + for source in sources + if source is not None + ) + files = list(files_it) + pages = list(pages_it) - assert len(actual.find_links) == 1 - check_links_include(actual.find_links, names=['packages']) - # Check that find-links URLs are marked as cacheable. - assert actual.find_links[0].cache_link_parsing + # Spot-check the returned sources. + assert len(files) > 20 + check_links_include(files, names=["simple-1.0.tar.gz"]) - assert actual.project_urls == [Link('https://pypi.org/simple/twine/')] + assert [page.link for page in pages] == [Link("https://pypi.org/simple/twine/")] # Check that index URLs are marked as *un*cacheable. - assert not actual.project_urls[0].cache_link_parsing + assert not pages[0].link.cache_link_parsing - expected_message = dedent("""\ + expected_message = dedent( + """\ 1 location(s) to search for versions of twine: - * https://pypi.org/simple/twine/""") + * https://pypi.org/simple/twine/""" + ) assert caplog.record_tuples == [ - ('pip._internal.index.collector', logging.DEBUG, expected_message), + ("pip._internal.index.collector", logging.DEBUG, expected_message), ] @pytest.mark.parametrize( - 'find_links, no_index, suppress_no_index, expected', [ - (['link1'], False, False, - (['link1'], ['default_url', 'url1', 'url2'])), - (['link1'], False, True, (['link1'], ['default_url', 'url1', 'url2'])), - (['link1'], True, False, (['link1'], [])), + "find_links, no_index, suppress_no_index, expected", + [ + (["link1"], False, False, (["link1"], ["default_url", "url1", "url2"])), + (["link1"], False, True, (["link1"], ["default_url", "url1", "url2"])), + (["link1"], True, False, (["link1"], [])), # Passing suppress_no_index=True suppresses no_index=True. - (['link1'], True, True, (['link1'], ['default_url', 'url1', 'url2'])), + (["link1"], True, True, (["link1"], ["default_url", "url1", "url2"])), # Test options.find_links=False. - (False, False, False, ([], ['default_url', 'url1', 'url2'])), + (False, False, False, ([], ["default_url", "url1", "url2"])), ], ) def test_link_collector_create( - find_links, no_index, suppress_no_index, expected, -): + find_links: List[str], + no_index: bool, + suppress_no_index: bool, + expected: Tuple[List[str], List[str]], +) -> None: """ :param expected: the expected (find_links, index_urls) values. """ expected_find_links, expected_index_urls = expected session = PipSession() - options = pretend.stub( + options = mock.Mock( find_links=find_links, - index_url='default_url', - extra_index_urls=['url1', 'url2'], + index_url="default_url", + extra_index_urls=["url1", "url2"], no_index=no_index, ) link_collector = LinkCollector.create( - session, options=options, suppress_no_index=suppress_no_index, + session, + options=options, + suppress_no_index=suppress_no_index, ) assert link_collector.session is session @@ -733,31 +899,31 @@ def test_link_collector_create( assert search_scope.index_urls == expected_index_urls -@patch('os.path.expanduser') +@mock.patch("os.path.expanduser") def test_link_collector_create_find_links_expansion( - mock_expanduser, tmpdir, -): + mock_expanduser: mock.Mock, tmpdir: Path +) -> None: """ Test "~" expansion in --find-links paths. """ # This is a mock version of expanduser() that expands "~" to the tmpdir. - def expand_path(path): - if path.startswith('~/'): + def expand_path(path: str) -> str: + if path.startswith("~/"): path = os.path.join(tmpdir, path[2:]) return path mock_expanduser.side_effect = expand_path session = PipSession() - options = pretend.stub( - find_links=['~/temp1', '~/temp2'], - index_url='default_url', + options = mock.Mock( + find_links=["~/temp1", "~/temp2"], + index_url="default_url", extra_index_urls=[], no_index=False, ) # Only create temp2 and not temp1 to test that "~" expansion only occurs # when the directory exists. - temp2_dir = os.path.join(tmpdir, 'temp2') + temp2_dir = os.path.join(tmpdir, "temp2") os.mkdir(temp2_dir) link_collector = LinkCollector.create(session, options=options) @@ -765,5 +931,5 @@ def expand_path(path): search_scope = link_collector.search_scope # Only ~/temp2 gets expanded. Also, the path is normalized when expanded. expected_temp2_dir = os.path.normcase(temp2_dir) - assert search_scope.find_links == ['~/temp1', expected_temp2_dir] - assert search_scope.index_urls == ['default_url'] + assert search_scope.find_links == ["~/temp1", expected_temp2_dir] + assert search_scope.index_urls == ["default_url"] diff --git a/tests/unit/test_command_install.py b/tests/unit/test_command_install.py index 66eb8ef3881..69792dd9839 100644 --- a/tests/unit/test_command_install.py +++ b/tests/unit/test_command_install.py @@ -1,9 +1,10 @@ import errno +from unittest import mock import pytest -from mock import patch from pip._vendor.packaging.requirements import Requirement +from pip._internal.commands import install from pip._internal.commands.install import ( create_os_error_message, decide_user_install, @@ -14,38 +15,40 @@ class TestDecideUserInstall: - @patch('site.ENABLE_USER_SITE', True) - @patch('pip._internal.commands.install.site_packages_writable') - def test_prefix_and_target(self, sp_writable): + @mock.patch("site.ENABLE_USER_SITE", True) + @mock.patch("pip._internal.commands.install.site_packages_writable") + def test_prefix_and_target(self, sp_writable: mock.Mock) -> None: sp_writable.return_value = False - assert decide_user_install( - use_user_site=None, prefix_path='foo' - ) is False + assert decide_user_install(use_user_site=None, prefix_path="foo") is False - assert decide_user_install( - use_user_site=None, target_dir='bar' - ) is False + assert decide_user_install(use_user_site=None, target_dir="bar") is False @pytest.mark.parametrize( - "enable_user_site,site_packages_writable,result", [ + "enable_user_site,site_packages_writable,result", + [ (True, True, False), (True, False, True), (False, True, False), (False, False, False), - ]) + ], + ) def test_most_cases( - self, enable_user_site, site_packages_writable, result, monkeypatch, - ): - monkeypatch.setattr('site.ENABLE_USER_SITE', enable_user_site) + self, + enable_user_site: bool, + site_packages_writable: bool, + result: bool, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setattr("site.ENABLE_USER_SITE", enable_user_site) monkeypatch.setattr( - 'pip._internal.commands.install.site_packages_writable', - lambda **kw: site_packages_writable + "pip._internal.commands.install.site_packages_writable", + lambda **kw: site_packages_writable, ) assert decide_user_install(use_user_site=None) is result -def test_rejection_for_pip_install_options(): +def test_rejection_for_pip_install_options() -> None: install_options = ["--prefix=/hello"] with pytest.raises(CommandError) as e: reject_location_related_install_options([], install_options) @@ -53,13 +56,10 @@ def test_rejection_for_pip_install_options(): assert "['--prefix'] from command line" in str(e.value) -def test_rejection_for_location_requirement_options(): - install_options = [] - +def test_rejection_for_location_requirement_options() -> None: bad_named_req_options = ["--home=/wow"] bad_named_req = InstallRequirement( - Requirement("hello"), "requirements.txt", - install_options=bad_named_req_options + Requirement("hello"), "requirements.txt", install_options=bad_named_req_options ) bad_unnamed_req_options = ["--install-lib=/lib"] @@ -69,7 +69,7 @@ def test_rejection_for_location_requirement_options(): with pytest.raises(CommandError) as e: reject_location_related_install_options( - [bad_named_req, bad_unnamed_req], install_options + [bad_named_req, bad_unnamed_req], options=[] ) assert ( @@ -79,37 +79,82 @@ def test_rejection_for_location_requirement_options(): assert "['--home'] from hello (from requirements.txt)" in str(e.value) -@pytest.mark.parametrize('error, show_traceback, using_user_site, expected', [ - # show_traceback = True, using_user_site = True - (OSError("Illegal byte sequence"), True, True, 'Could not install' - ' packages due to an OSError.\n'), - (OSError(errno.EACCES, "No file permission"), True, True, 'Could' - ' not install packages due to an OSError.\nCheck the' - ' permissions.\n'), - # show_traceback = True, using_user_site = False - (OSError("Illegal byte sequence"), True, False, 'Could not' - ' install packages due to an OSError.\n'), - (OSError(errno.EACCES, "No file permission"), True, False, 'Could' - ' not install packages due to an OSError.\nConsider using the' - ' `--user` option or check the permissions.\n'), - # show_traceback = False, using_user_site = True - (OSError("Illegal byte sequence"), False, True, 'Could not' - ' install packages due to an OSError: Illegal byte' - ' sequence\n'), - (OSError(errno.EACCES, "No file permission"), False, True, 'Could' - ' not install packages due to an OSError: [Errno 13] No file' - ' permission\nCheck the permissions.\n'), - # show_traceback = False, using_user_site = False - (OSError("Illegal byte sequence"), False, False, 'Could not' - ' install packages due to an OSError: Illegal byte sequence' - '\n'), - (OSError(errno.EACCES, "No file permission"), False, False, - 'Could not install packages due to an OSError: [Errno 13] No' - ' file permission\nConsider using the `--user` option or check the' - ' permissions.\n'), -]) +@pytest.mark.parametrize( + "error, show_traceback, using_user_site, expected", + [ + # show_traceback = True, using_user_site = True + ( + OSError("Illegal byte sequence"), + True, + True, + "Could not install packages due to an OSError.\n", + ), + ( + OSError(errno.EACCES, "No file permission"), + True, + True, + "Could" + " not install packages due to an OSError.\nCheck the" + " permissions.\n", + ), + # show_traceback = True, using_user_site = False + ( + OSError("Illegal byte sequence"), + True, + False, + "Could not install packages due to an OSError.\n", + ), + ( + OSError(errno.EACCES, "No file permission"), + True, + False, + "Could" + " not install packages due to an OSError.\nConsider using the" + " `--user` option or check the permissions.\n", + ), + # show_traceback = False, using_user_site = True + ( + OSError("Illegal byte sequence"), + False, + True, + "Could not" + " install packages due to an OSError: Illegal byte" + " sequence\n", + ), + ( + OSError(errno.EACCES, "No file permission"), + False, + True, + "Could" + " not install packages due to an OSError: [Errno 13] No file" + " permission\nCheck the permissions.\n", + ), + # show_traceback = False, using_user_site = False + ( + OSError("Illegal byte sequence"), + False, + False, + "Could not" + " install packages due to an OSError: Illegal byte sequence" + "\n", + ), + ( + OSError(errno.EACCES, "No file permission"), + False, + False, + "Could not install packages due to an OSError: [Errno 13] No" + " file permission\nConsider using the `--user` option or check the" + " permissions.\n", + ), + ], +) def test_create_os_error_message( - error, show_traceback, using_user_site, expected -): + monkeypatch: pytest.MonkeyPatch, + error: OSError, + show_traceback: bool, + using_user_site: bool, + expected: str, +) -> None: + monkeypatch.setattr(install, "running_under_virtualenv", lambda: False) msg = create_os_error_message(error, show_traceback, using_user_site) assert msg == expected diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index 59cbf930ffb..7a5c4e8319d 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -1,6 +1,9 @@ +from typing import Callable, List +from unittest import mock + import pytest -from mock import patch +from pip._internal.cli.base_command import Command from pip._internal.cli.req_command import ( IndexGroupCommand, RequirementCommand, @@ -10,68 +13,70 @@ # These are the expected names of the commands whose classes inherit from # IndexGroupCommand. -EXPECTED_INDEX_GROUP_COMMANDS = ['download', 'install', 'list', 'wheel'] +EXPECTED_INDEX_GROUP_COMMANDS = ["download", "index", "install", "list", "wheel"] -def check_commands(pred, expected): +def check_commands(pred: Callable[[Command], bool], expected: List[str]) -> None: """ Check the commands satisfying a predicate. """ commands = [create_command(name) for name in sorted(commands_dict)] actual = [command.name for command in commands if pred(command)] - assert actual == expected, f'actual: {actual}' + assert actual == expected, f"actual: {actual}" -def test_commands_dict__order(): +def test_commands_dict__order() -> None: """ Check the ordering of commands_dict. """ names = list(commands_dict) # A spot-check is sufficient to check that commands_dict encodes an # ordering. - assert names[0] == 'install' - assert names[-1] == 'help' + assert names[0] == "install" + assert names[-1] == "help" -@pytest.mark.parametrize('name', list(commands_dict)) -def test_create_command(name): +@pytest.mark.parametrize("name", list(commands_dict)) +def test_create_command(name: str) -> None: """Test creating an instance of each available command.""" command = create_command(name) assert command.name == name assert command.summary == commands_dict[name].summary -def test_session_commands(): +def test_session_commands() -> None: """ Test which commands inherit from SessionCommandMixin. """ - def is_session_command(command): + + def is_session_command(command: Command) -> bool: return isinstance(command, SessionCommandMixin) - expected = ['download', 'install', 'list', 'search', 'uninstall', 'wheel'] + expected = ["download", "index", "install", "list", "search", "uninstall", "wheel"] check_commands(is_session_command, expected) -def test_index_group_commands(): +def test_index_group_commands() -> None: """ Test the commands inheriting from IndexGroupCommand. """ - def is_index_group_command(command): + + def is_index_group_command(command: Command) -> bool: return isinstance(command, IndexGroupCommand) check_commands(is_index_group_command, EXPECTED_INDEX_GROUP_COMMANDS) # Also check that the commands inheriting from IndexGroupCommand are # exactly the commands with the --no-index option. - def has_option_no_index(command): - return command.parser.has_option('--no-index') + def has_option_no_index(command: Command) -> bool: + return command.parser.has_option("--no-index") check_commands(has_option_no_index, EXPECTED_INDEX_GROUP_COMMANDS) -@pytest.mark.parametrize('command_name', EXPECTED_INDEX_GROUP_COMMANDS) +@pytest.mark.parametrize("command_name", EXPECTED_INDEX_GROUP_COMMANDS) @pytest.mark.parametrize( - 'disable_pip_version_check, no_index, expected_called', + "disable_pip_version_check, no_index, expected_called", [ # pip_self_version_check() is only called when both # disable_pip_version_check and no_index are False. @@ -81,11 +86,14 @@ def has_option_no_index(command): (True, True, False), ], ) -@patch('pip._internal.cli.req_command.pip_self_version_check') +@mock.patch("pip._internal.cli.req_command.pip_self_version_check") def test_index_group_handle_pip_version_check( - mock_version_check, command_name, disable_pip_version_check, no_index, - expected_called, -): + mock_version_check: mock.Mock, + command_name: str, + disable_pip_version_check: bool, + no_index: bool, + expected_called: bool, +) -> None: """ Test whether pip_self_version_check() is called when handle_pip_version_check() is called, for each of the @@ -103,11 +111,12 @@ def test_index_group_handle_pip_version_check( mock_version_check.assert_not_called() -def test_requirement_commands(): +def test_requirement_commands() -> None: """ Test which commands inherit from RequirementCommand. """ - def is_requirement_command(command): + + def is_requirement_command(command: Command) -> bool: return isinstance(command, RequirementCommand) - check_commands(is_requirement_command, ['download', 'install', 'wheel']) + check_commands(is_requirement_command, ["download", "install", "wheel"]) diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py index 655e45ab75e..da58cc8d3b3 100644 --- a/tests/unit/test_compat.py +++ b/tests/unit/test_compat.py @@ -1,20 +1,18 @@ -import locale import os -import sys import pytest -import pip._internal.utils.compat as pip_compat -from pip._internal.utils.compat import console_to_str, get_path_uid, str_to_display +from pip._internal.utils.compat import get_path_uid +from tests.lib.path import Path -def test_get_path_uid(): +def test_get_path_uid() -> None: path = os.getcwd() assert get_path_uid(path) == os.stat(path).st_uid @pytest.mark.skipif("not hasattr(os, 'O_NOFOLLOW')") -def test_get_path_uid_without_NOFOLLOW(monkeypatch): +def test_get_path_uid_without_NOFOLLOW(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr("os.O_NOFOLLOW") path = os.getcwd() assert get_path_uid(path) == os.stat(path).st_uid @@ -23,11 +21,11 @@ def test_get_path_uid_without_NOFOLLOW(monkeypatch): # Skip unconditionally on Windows, as symlinks need admin privs there @pytest.mark.skipif("sys.platform == 'win32'") @pytest.mark.skipif("not hasattr(os, 'symlink')") -def test_get_path_uid_symlink(tmpdir): +def test_get_path_uid_symlink(tmpdir: Path) -> None: f = tmpdir / "symlink" / "somefile" f.parent.mkdir() f.write_text("content") - fs = f + '_link' + fs = f + "_link" os.symlink(f, fs) with pytest.raises(OSError): get_path_uid(fs) @@ -35,90 +33,14 @@ def test_get_path_uid_symlink(tmpdir): @pytest.mark.skipif("not hasattr(os, 'O_NOFOLLOW')") @pytest.mark.skipif("not hasattr(os, 'symlink')") -def test_get_path_uid_symlink_without_NOFOLLOW(tmpdir, monkeypatch): +def test_get_path_uid_symlink_without_NOFOLLOW( + tmpdir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.delattr("os.O_NOFOLLOW") f = tmpdir / "symlink" / "somefile" f.parent.mkdir() f.write_text("content") - fs = f + '_link' + fs = f + "_link" os.symlink(f, fs) with pytest.raises(OSError): get_path_uid(fs) - - -@pytest.mark.parametrize('data, expected', [ - ('abc', 'abc'), - # Test text input with non-ascii characters. - ('déf', 'déf'), -]) -def test_str_to_display(data, expected): - actual = str_to_display(data) - assert actual == expected, ( - # Show the encoding for easier troubleshooting. - f'encoding: {locale.getpreferredencoding()!r}' - ) - - -@pytest.mark.parametrize('data, encoding, expected', [ - # Test str input with non-ascii characters. - ('déf', 'utf-8', 'déf'), - # Test bytes input with non-ascii characters: - ('déf'.encode('utf-8'), 'utf-8', 'déf'), - # Test a Windows encoding. - ('déf'.encode('cp1252'), 'cp1252', 'déf'), - # Test a Windows encoding with incompatibly encoded text. - ('déf'.encode('utf-8'), 'cp1252', 'déf'), -]) -def test_str_to_display__encoding(monkeypatch, data, encoding, expected): - monkeypatch.setattr(locale, 'getpreferredencoding', lambda: encoding) - actual = str_to_display(data) - assert actual == expected, ( - # Show the encoding for easier troubleshooting. - f'encoding: {locale.getpreferredencoding()!r}' - ) - - -def test_str_to_display__decode_error(monkeypatch, caplog): - monkeypatch.setattr(locale, 'getpreferredencoding', lambda: 'utf-8') - # Encode with an incompatible encoding. - data = 'ab'.encode('utf-16') - actual = str_to_display(data) - # Keep the expected value endian safe - if sys.byteorder == "little": - expected = "\\xff\\xfea\x00b\x00" - elif sys.byteorder == "big": - expected = "\\xfe\\xff\x00a\x00b" - - assert actual == expected, ( - # Show the encoding for easier troubleshooting. - f'encoding: {locale.getpreferredencoding()!r}' - ) - assert len(caplog.records) == 1 - record = caplog.records[0] - assert record.levelname == 'WARNING' - assert record.message == ( - 'Bytes object does not appear to be encoded as utf-8' - ) - - -def test_console_to_str(monkeypatch): - some_bytes = b"a\xE9\xC3\xE9b" - encodings = ('ascii', 'utf-8', 'iso-8859-1', 'iso-8859-5', - 'koi8_r', 'cp850') - for e in encodings: - monkeypatch.setattr(locale, 'getpreferredencoding', lambda: e) - result = console_to_str(some_bytes) - assert result.startswith("a") - assert result.endswith("b") - - -def test_console_to_str_warning(monkeypatch): - some_bytes = b"a\xE9b" - - def check_warning(msg, *args, **kwargs): - assert 'does not appear to be encoded as' in msg - assert args[0] == 'Subprocess output' - - monkeypatch.setattr(locale, 'getpreferredencoding', lambda: 'utf-8') - monkeypatch.setattr(pip_compat.logger, 'warning', check_warning) - console_to_str(some_bytes) diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index 0a45fc136d5..788d32b9b76 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -1,8 +1,9 @@ """Tests for all things related to the configuration """ +from unittest.mock import MagicMock + import pytest -from mock import MagicMock from pip._internal.configuration import get_configuration_files, kinds from pip._internal.exceptions import ConfigurationError @@ -10,26 +11,25 @@ class TestConfigurationLoading(ConfigurationMixin): - - def test_global_loading(self): + def test_global_loading(self) -> None: self.patch_configuration(kinds.GLOBAL, {"test.hello": "1"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "1" - def test_user_loading(self): + def test_user_loading(self) -> None: self.patch_configuration(kinds.USER, {"test.hello": "2"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "2" - def test_site_loading(self): + def test_site_loading(self) -> None: self.patch_configuration(kinds.SITE, {"test.hello": "3"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "3" - def test_environment_config_loading(self, monkeypatch): + def test_environment_config_loading(self, monkeypatch: pytest.MonkeyPatch) -> None: contents = """ [test] hello = 4 @@ -39,24 +39,29 @@ def test_environment_config_loading(self, monkeypatch): monkeypatch.setenv("PIP_CONFIG_FILE", config_file) self.configuration.load() - assert self.configuration.get_value("test.hello") == "4", \ - self.configuration._config + assert ( + self.configuration.get_value("test.hello") == "4" + ), self.configuration._config - def test_environment_var_loading(self, monkeypatch): + def test_environment_var_loading(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value(":env:.hello") == "5" @pytest.mark.skipif("sys.platform == 'win32'") - def test_environment_var_does_not_load_lowercase(self, monkeypatch): + def test_environment_var_does_not_load_lowercase( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setenv("pip_hello", "5") self.configuration.load() with pytest.raises(ConfigurationError): self.configuration.get_value(":env:.hello") - def test_environment_var_does_not_load_version(self, monkeypatch): + def test_environment_var_does_not_load_version( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setenv("PIP_VERSION", "True") self.configuration.load() @@ -64,7 +69,9 @@ def test_environment_var_does_not_load_version(self, monkeypatch): with pytest.raises(ConfigurationError): self.configuration.get_value(":env:.version") - def test_environment_config_errors_if_malformed(self, monkeypatch): + def test_environment_config_errors_if_malformed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: contents = """ test] hello = 4 @@ -76,9 +83,8 @@ def test_environment_config_errors_if_malformed(self, monkeypatch): assert "section header" in str(err.value) # error kind assert "1" in str(err.value) # line number - assert ( # file name - config_file in str(err.value) or - repr(config_file) in str(err.value) + assert config_file in str(err.value) or repr(config_file) in str( # file name + err.value ) @@ -86,49 +92,51 @@ class TestConfigurationPrecedence(ConfigurationMixin): # Tests for methods to that determine the order of precedence of # configuration options - def test_env_overides_site(self): + def test_env_overides_site(self) -> None: self.patch_configuration(kinds.SITE, {"test.hello": "1"}) self.patch_configuration(kinds.ENV, {"test.hello": "0"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "0" - def test_env_overides_user(self): + def test_env_overides_user(self) -> None: self.patch_configuration(kinds.USER, {"test.hello": "2"}) self.patch_configuration(kinds.ENV, {"test.hello": "0"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "0" - def test_env_overides_global(self): + def test_env_overides_global(self) -> None: self.patch_configuration(kinds.GLOBAL, {"test.hello": "3"}) self.patch_configuration(kinds.ENV, {"test.hello": "0"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "0" - def test_site_overides_user(self): + def test_site_overides_user(self) -> None: self.patch_configuration(kinds.USER, {"test.hello": "2"}) self.patch_configuration(kinds.SITE, {"test.hello": "1"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "1" - def test_site_overides_global(self): + def test_site_overides_global(self) -> None: self.patch_configuration(kinds.GLOBAL, {"test.hello": "3"}) self.patch_configuration(kinds.SITE, {"test.hello": "1"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "1" - def test_user_overides_global(self): + def test_user_overides_global(self) -> None: self.patch_configuration(kinds.GLOBAL, {"test.hello": "3"}) self.patch_configuration(kinds.USER, {"test.hello": "2"}) self.configuration.load() assert self.configuration.get_value("test.hello") == "2" - def test_env_not_overriden_by_environment_var(self, monkeypatch): + def test_env_not_overriden_by_environment_var( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self.patch_configuration(kinds.ENV, {"test.hello": "1"}) monkeypatch.setenv("PIP_HELLO", "5") @@ -137,7 +145,9 @@ def test_env_not_overriden_by_environment_var(self, monkeypatch): assert self.configuration.get_value("test.hello") == "1" assert self.configuration.get_value(":env:.hello") == "5" - def test_site_not_overriden_by_environment_var(self, monkeypatch): + def test_site_not_overriden_by_environment_var( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self.patch_configuration(kinds.SITE, {"test.hello": "2"}) monkeypatch.setenv("PIP_HELLO", "5") @@ -146,7 +156,9 @@ def test_site_not_overriden_by_environment_var(self, monkeypatch): assert self.configuration.get_value("test.hello") == "2" assert self.configuration.get_value(":env:.hello") == "5" - def test_user_not_overriden_by_environment_var(self, monkeypatch): + def test_user_not_overriden_by_environment_var( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self.patch_configuration(kinds.USER, {"test.hello": "3"}) monkeypatch.setenv("PIP_HELLO", "5") @@ -155,7 +167,9 @@ def test_user_not_overriden_by_environment_var(self, monkeypatch): assert self.configuration.get_value("test.hello") == "3" assert self.configuration.get_value(":env:.hello") == "5" - def test_global_not_overriden_by_environment_var(self, monkeypatch): + def test_global_not_overriden_by_environment_var( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: self.patch_configuration(kinds.GLOBAL, {"test.hello": "4"}) monkeypatch.setenv("PIP_HELLO", "5") @@ -168,7 +182,7 @@ def test_global_not_overriden_by_environment_var(self, monkeypatch): class TestConfigurationModification(ConfigurationMixin): # Tests for methods to that modify the state of a Configuration - def test_no_specific_given_modification(self): + def test_no_specific_given_modification(self) -> None: self.configuration.load() try: @@ -178,30 +192,30 @@ def test_no_specific_given_modification(self): else: assert False, "Should have raised an error." - def test_site_modification(self): + def test_site_modification(self) -> None: self.configuration.load_only = kinds.SITE self.configuration.load() # Mock out the method mymock = MagicMock(spec=self.configuration._mark_as_modified) - self.configuration._mark_as_modified = mymock + # https://github.com/python/mypy/issues/2427 + self.configuration._mark_as_modified = mymock # type: ignore[assignment] self.configuration.set_value("test.hello", "10") # get the path to site config file assert mymock.call_count == 1 - assert mymock.call_args[0][0] == ( - get_configuration_files()[kinds.SITE][0] - ) + assert mymock.call_args[0][0] == (get_configuration_files()[kinds.SITE][0]) - def test_user_modification(self): + def test_user_modification(self) -> None: # get the path to local config file self.configuration.load_only = kinds.USER self.configuration.load() # Mock out the method mymock = MagicMock(spec=self.configuration._mark_as_modified) - self.configuration._mark_as_modified = mymock + # https://github.com/python/mypy/issues/2427 + self.configuration._mark_as_modified = mymock # type: ignore[assignment] self.configuration.set_value("test.hello", "10") @@ -212,19 +226,18 @@ def test_user_modification(self): get_configuration_files()[kinds.USER][1] ) - def test_global_modification(self): + def test_global_modification(self) -> None: # get the path to local config file self.configuration.load_only = kinds.GLOBAL self.configuration.load() # Mock out the method mymock = MagicMock(spec=self.configuration._mark_as_modified) - self.configuration._mark_as_modified = mymock + # https://github.com/python/mypy/issues/2427 + self.configuration._mark_as_modified = mymock # type: ignore[assignment] self.configuration.set_value("test.hello", "10") # get the path to user config file assert mymock.call_count == 1 - assert mymock.call_args[0][0] == ( - get_configuration_files()[kinds.GLOBAL][-1] - ) + assert mymock.call_args[0][0] == (get_configuration_files()[kinds.GLOBAL][-1]) diff --git a/tests/unit/test_direct_url.py b/tests/unit/test_direct_url.py index ee6b7fbf4ea..c81e5129253 100644 --- a/tests/unit/test_direct_url.py +++ b/tests/unit/test_direct_url.py @@ -9,14 +9,15 @@ ) -def test_from_json(): +def test_from_json() -> None: json = '{"url": "file:///home/user/project", "dir_info": {}}' direct_url = DirectUrl.from_json(json) assert direct_url.url == "file:///home/user/project" + assert isinstance(direct_url.info, DirInfo) assert direct_url.info.editable is False -def test_to_json(): +def test_to_json() -> None: direct_url = DirectUrl( url="file:///home/user/archive.tgz", info=ArchiveInfo(), @@ -27,21 +28,21 @@ def test_to_json(): ) -def test_archive_info(): +def test_archive_info() -> None: direct_url_dict = { "url": "file:///home/user/archive.tgz", - "archive_info": { - "hash": "sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220" - }, + "archive_info": {"hash": "sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220"}, } direct_url = DirectUrl.from_dict(direct_url_dict) assert isinstance(direct_url.info, ArchiveInfo) assert direct_url.url == direct_url_dict["url"] - assert direct_url.info.hash == direct_url_dict["archive_info"]["hash"] + assert ( + direct_url.info.hash == direct_url_dict["archive_info"]["hash"] # type: ignore + ) assert direct_url.to_dict() == direct_url_dict -def test_dir_info(): +def test_dir_info() -> None: direct_url_dict = { "url": "file:///home/user/project", "dir_info": {"editable": True}, @@ -54,10 +55,11 @@ def test_dir_info(): # test editable default to False direct_url_dict = {"url": "file:///home/user/project", "dir_info": {}} direct_url = DirectUrl.from_dict(direct_url_dict) + assert isinstance(direct_url.info, DirInfo) assert direct_url.info.editable is False -def test_vcs_info(): +def test_vcs_info() -> None: direct_url_dict = { "url": "https:///g.c/u/p.git", "vcs_info": { @@ -71,58 +73,42 @@ def test_vcs_info(): assert direct_url.url == direct_url_dict["url"] assert direct_url.info.vcs == "git" assert direct_url.info.requested_revision == "master" - assert ( - direct_url.info.commit_id == "1b8c5bc61a86f377fea47b4276c8c8a5842d2220" - ) + assert direct_url.info.commit_id == "1b8c5bc61a86f377fea47b4276c8c8a5842d2220" assert direct_url.to_dict() == direct_url_dict -def test_parsing_validation(): - with pytest.raises( - DirectUrlValidationError, match="url must have a value" - ): +def test_parsing_validation() -> None: + with pytest.raises(DirectUrlValidationError, match="url must have a value"): DirectUrl.from_dict({"dir_info": {}}) with pytest.raises( DirectUrlValidationError, match="missing one of archive_info, dir_info, vcs_info", ): DirectUrl.from_dict({"url": "http://..."}) - with pytest.raises( - DirectUrlValidationError, match="unexpected type for editable" - ): - DirectUrl.from_dict( - {"url": "http://...", "dir_info": {"editable": "false"}} - ) - with pytest.raises( - DirectUrlValidationError, match="unexpected type for hash" - ): + with pytest.raises(DirectUrlValidationError, match="unexpected type for editable"): + DirectUrl.from_dict({"url": "http://...", "dir_info": {"editable": "false"}}) + with pytest.raises(DirectUrlValidationError, match="unexpected type for hash"): DirectUrl.from_dict({"url": "http://...", "archive_info": {"hash": 1}}) - with pytest.raises( - DirectUrlValidationError, match="unexpected type for vcs" - ): + with pytest.raises(DirectUrlValidationError, match="unexpected type for vcs"): DirectUrl.from_dict({"url": "http://...", "vcs_info": {"vcs": None}}) - with pytest.raises( - DirectUrlValidationError, match="commit_id must have a value" - ): + with pytest.raises(DirectUrlValidationError, match="commit_id must have a value"): DirectUrl.from_dict({"url": "http://...", "vcs_info": {"vcs": "git"}}) with pytest.raises( DirectUrlValidationError, match="more than one of archive_info, dir_info, vcs_info", ): - DirectUrl.from_dict( - {"url": "http://...", "dir_info": {}, "archive_info": {}} - ) + DirectUrl.from_dict({"url": "http://...", "dir_info": {}, "archive_info": {}}) -def test_redact_url(): - def _redact_git(url): +def test_redact_url() -> None: + def _redact_git(url: str) -> str: direct_url = DirectUrl( url=url, info=VcsInfo(vcs="git", commit_id="1"), ) return direct_url.redacted_url - def _redact_archive(url): + def _redact_archive(url: str) -> str: direct_url = DirectUrl( url=url, info=ArchiveInfo(), @@ -130,22 +116,16 @@ def _redact_archive(url): return direct_url.redacted_url assert ( - _redact_git("https://user:password@g.c/u/p.git@branch#egg=pkg") == - "https://g.c/u/p.git@branch#egg=pkg" - ) - assert ( - _redact_git("https://${USER}:password@g.c/u/p.git") == - "https://g.c/u/p.git" - ) - assert ( - _redact_archive("file://${U}:${PIP_PASSWORD}@g.c/u/p.tgz") == - "file://${U}:${PIP_PASSWORD}@g.c/u/p.tgz" + _redact_git("https://user:password@g.c/u/p.git@branch#egg=pkg") + == "https://g.c/u/p.git@branch#egg=pkg" ) + assert _redact_git("https://${USER}:password@g.c/u/p.git") == "https://g.c/u/p.git" assert ( - _redact_git("https://${PIP_TOKEN}@g.c/u/p.git") == - "https://${PIP_TOKEN}@g.c/u/p.git" + _redact_archive("file://${U}:${PIP_PASSWORD}@g.c/u/p.tgz") + == "file://${U}:${PIP_PASSWORD}@g.c/u/p.tgz" ) assert ( - _redact_git("ssh://git@g.c/u/p.git") == - "ssh://git@g.c/u/p.git" + _redact_git("https://${PIP_TOKEN}@g.c/u/p.git") + == "https://${PIP_TOKEN}@g.c/u/p.git" ) + assert _redact_git("ssh://git@g.c/u/p.git") == "ssh://git@g.c/u/p.git" diff --git a/tests/unit/test_direct_url_helpers.py b/tests/unit/test_direct_url_helpers.py index b0cb50c6eb9..8d94aeb50b6 100644 --- a/tests/unit/test_direct_url_helpers.py +++ b/tests/unit/test_direct_url_helpers.py @@ -1,61 +1,56 @@ from functools import partial +from unittest import mock -from mock import MagicMock, patch - -from pip._internal.models.direct_url import ( - DIRECT_URL_METADATA_NAME, - ArchiveInfo, - DirectUrl, - DirInfo, - VcsInfo, -) +from pip._internal.models.direct_url import ArchiveInfo, DirectUrl, DirInfo, VcsInfo from pip._internal.models.link import Link from pip._internal.utils.direct_url_helpers import ( direct_url_as_pep440_direct_reference, direct_url_from_link, - dist_get_direct_url, ) from pip._internal.utils.urls import path_to_url +from tests.lib import PipTestEnvironment +from tests.lib.path import Path -def test_as_pep440_requirement_archive(): +def test_as_pep440_requirement_archive() -> None: direct_url = DirectUrl( url="file:///home/user/archive.tgz", info=ArchiveInfo(), ) direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ file:///home/user/archive.tgz" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ file:///home/user/archive.tgz" ) direct_url.subdirectory = "subdir" direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ file:///home/user/archive.tgz#subdirectory=subdir" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ file:///home/user/archive.tgz#subdirectory=subdir" ) + assert isinstance(direct_url.info, ArchiveInfo) direct_url.info.hash = "sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220" direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ file:///home/user/archive.tgz" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ file:///home/user/archive.tgz" "#sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220&subdirectory=subdir" ) -def test_as_pep440_requirement_dir(): +def test_as_pep440_requirement_dir() -> None: direct_url = DirectUrl( url="file:///home/user/project", info=DirInfo(editable=False), ) direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ file:///home/user/project" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ file:///home/user/project" ) -def test_as_pep440_requirement_editable_dir(): +def test_as_pep440_requirement_editable_dir() -> None: # direct_url_as_pep440_direct_reference behaves the same # irrespective of the editable flag. It's the responsibility of # callers to render it as editable @@ -65,35 +60,33 @@ def test_as_pep440_requirement_editable_dir(): ) direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ file:///home/user/project" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ file:///home/user/project" ) -def test_as_pep440_requirement_vcs(): +def test_as_pep440_requirement_vcs() -> None: direct_url = DirectUrl( url="https:///g.c/u/p.git", - info=VcsInfo( - vcs="git", commit_id="1b8c5bc61a86f377fea47b4276c8c8a5842d2220" - ) + info=VcsInfo(vcs="git", commit_id="1b8c5bc61a86f377fea47b4276c8c8a5842d2220"), ) direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ git+https:///g.c/u/p.git" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ git+https:///g.c/u/p.git" "@1b8c5bc61a86f377fea47b4276c8c8a5842d2220" ) direct_url.subdirectory = "subdir" direct_url.validate() assert ( - direct_url_as_pep440_direct_reference(direct_url, "pkg") == - "pkg @ git+https:///g.c/u/p.git" + direct_url_as_pep440_direct_reference(direct_url, "pkg") + == "pkg @ git+https:///g.c/u/p.git" "@1b8c5bc61a86f377fea47b4276c8c8a5842d2220#subdirectory=subdir" ) -@patch("pip._internal.vcs.git.Git.get_revision") -def test_from_link_vcs(mock_get_backend_for_scheme): +@mock.patch("pip._internal.vcs.git.Git.get_revision") +def test_from_link_vcs(mock_get_backend_for_scheme: mock.Mock) -> None: _direct_url_from_link = partial(direct_url_from_link, source_dir="...") direct_url = _direct_url_from_link(Link("git+https://g.c/u/p.git")) assert direct_url.url == "https://g.c/u/p.git" @@ -108,68 +101,63 @@ def test_from_link_vcs(mock_get_backend_for_scheme): assert direct_url.subdirectory == "subdir" direct_url = _direct_url_from_link(Link("git+https://g.c/u/p.git@branch")) assert direct_url.url == "https://g.c/u/p.git" + assert isinstance(direct_url.info, VcsInfo) assert direct_url.info.requested_revision == "branch" - direct_url = _direct_url_from_link( - Link("git+https://g.c/u/p.git@branch#egg=pkg") - ) + direct_url = _direct_url_from_link(Link("git+https://g.c/u/p.git@branch#egg=pkg")) assert direct_url.url == "https://g.c/u/p.git" + assert isinstance(direct_url.info, VcsInfo) assert direct_url.info.requested_revision == "branch" - direct_url = _direct_url_from_link( - Link("git+https://token@g.c/u/p.git") - ) + direct_url = _direct_url_from_link(Link("git+https://token@g.c/u/p.git")) assert direct_url.to_dict()["url"] == "https://g.c/u/p.git" -def test_from_link_vcs_with_source_dir_obtains_commit_id(script, tmpdir): - repo_path = tmpdir / 'test-repo' +def test_from_link_vcs_with_source_dir_obtains_commit_id( + script: PipTestEnvironment, tmpdir: Path +) -> None: + repo_path = tmpdir / "test-repo" repo_path.mkdir() repo_dir = str(repo_path) - script.run('git', 'init', cwd=repo_dir) + script.run("git", "init", cwd=repo_dir) (repo_path / "somefile").touch() - script.run('git', 'add', '.', cwd=repo_dir) - script.run('git', 'commit', '-m', 'commit msg', cwd=repo_dir) - commit_id = script.run( - 'git', 'rev-parse', 'HEAD', cwd=repo_dir - ).stdout.strip() + script.run("git", "add", ".", cwd=repo_dir) + script.run("git", "commit", "-m", "commit msg", cwd=repo_dir) + commit_id = script.run("git", "rev-parse", "HEAD", cwd=repo_dir).stdout.strip() direct_url = direct_url_from_link( Link("git+https://g.c/u/p.git"), source_dir=repo_dir ) assert direct_url.url == "https://g.c/u/p.git" + assert isinstance(direct_url.info, VcsInfo) assert direct_url.info.commit_id == commit_id -def test_from_link_vcs_without_source_dir(script, tmpdir): +def test_from_link_vcs_without_source_dir(script: PipTestEnvironment) -> None: direct_url = direct_url_from_link( Link("git+https://g.c/u/p.git@1"), link_is_in_wheel_cache=True ) assert direct_url.url == "https://g.c/u/p.git" + assert isinstance(direct_url.info, VcsInfo) assert direct_url.info.commit_id == "1" -def test_from_link_archive(): +def test_from_link_archive() -> None: direct_url = direct_url_from_link(Link("https://g.c/archive.tgz")) assert direct_url.url == "https://g.c/archive.tgz" assert isinstance(direct_url.info, ArchiveInfo) direct_url = direct_url_from_link( - Link( - "https://g.c/archive.tgz" - "#sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220" - ) + Link("https://g.c/archive.tgz#sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220") ) assert isinstance(direct_url.info, ArchiveInfo) - assert ( - direct_url.info.hash == "sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220" - ) + assert direct_url.info.hash == "sha1=1b8c5bc61a86f377fea47b4276c8c8a5842d2220" -def test_from_link_dir(tmpdir): +def test_from_link_dir(tmpdir: Path) -> None: dir_url = path_to_url(tmpdir) direct_url = direct_url_from_link(Link(dir_url)) assert direct_url.url == dir_url assert isinstance(direct_url.info, DirInfo) -def test_from_link_hide_user_password(): +def test_from_link_hide_user_password() -> None: # Basic test only here, other variants are covered by # direct_url.redact_url tests. direct_url = direct_url_from_link( @@ -182,30 +170,3 @@ def test_from_link_hide_user_password(): link_is_in_wheel_cache=True, ) assert direct_url.to_dict()["url"] == "ssh://git@g.c/u/p.git" - - -def test_dist_get_direct_url_no_metadata(): - dist = MagicMock() - dist.has_metadata.return_value = False - assert dist_get_direct_url(dist) is None - dist.has_metadata.assert_called() - - -def test_dist_get_direct_url_bad_metadata(): - dist = MagicMock() - dist.has_metadata.return_value = True - dist.get_metadata.return_value = "{}" # invalid direct_url.json - assert dist_get_direct_url(dist) is None - dist.get_metadata.assert_called_with(DIRECT_URL_METADATA_NAME) - - -def test_dist_get_direct_url_valid_metadata(): - dist = MagicMock() - dist.has_metadata.return_value = True - dist.get_metadata.return_value = ( - '{"url": "https://e.c/p.tgz", "archive_info": {}}' - ) - direct_url = dist_get_direct_url(dist) - dist.get_metadata.assert_called_with(DIRECT_URL_METADATA_NAME) - assert direct_url.url == "https://e.c/p.tgz" - assert isinstance(direct_url.info, ArchiveInfo) diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py new file mode 100644 index 00000000000..8f8224dc817 --- /dev/null +++ b/tests/unit/test_exceptions.py @@ -0,0 +1,474 @@ +"""Tests the presentation style of exceptions.""" + +import io +import textwrap + +import pytest +from pip._vendor import rich + +from pip._internal.exceptions import DiagnosticPipError + + +class TestDiagnosticPipErrorCreation: + def test_fails_without_reference(self) -> None: + class DerivedError(DiagnosticPipError): + pass + + with pytest.raises(AssertionError) as exc_info: + DerivedError(message="", context=None, hint_stmt=None) + + assert str(exc_info.value) == "error reference not provided!" + + def test_can_fetch_reference_from_subclass(self) -> None: + class DerivedError(DiagnosticPipError): + reference = "subclass-reference" + + obj = DerivedError(message="", context=None, hint_stmt=None) + assert obj.reference == "subclass-reference" + + def test_can_fetch_reference_from_arguments(self) -> None: + class DerivedError(DiagnosticPipError): + pass + + obj = DerivedError( + message="", context=None, hint_stmt=None, reference="subclass-reference" + ) + assert obj.reference == "subclass-reference" + + @pytest.mark.parametrize( + "name", + [ + "BADNAME", + "BadName", + "bad_name", + "BAD_NAME", + "_bad", + "bad-name-", + "bad--name", + "-bad-name", + "bad-name-due-to-1-number", + ], + ) + def test_rejects_non_kebab_case_names(self, name: str) -> None: + class DerivedError(DiagnosticPipError): + reference = name + + with pytest.raises(AssertionError) as exc_info: + DerivedError(message="", context=None, hint_stmt=None) + + assert str(exc_info.value) == "error reference must be kebab-case!" + + +def rendered_in_ascii(error: DiagnosticPipError, *, color: bool = False) -> str: + with io.BytesIO() as stream: + console = rich.console.Console( + force_terminal=False, + file=io.TextIOWrapper(stream, encoding="ascii", newline=""), + color_system="truecolor" if color else None, + ) + console.print(error) + return stream.getvalue().decode("ascii") + + +class TestDiagnosticPipErrorPresentation_ASCII: + def test_complete(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + Something went wrong + very wrong. + + note: You did something wrong, which is what caused this error. + hint: Do it better next time, by trying harder. + """ + ) + + def test_complete_color(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke.", + context="Something went wrong\nvery wrong.", + note_stmt="You did something wrong.", + hint_stmt="Do it better next time, by trying harder.", + ) + + def esc(code: str = "0") -> str: + return f"\x1b[{code}m" + + assert rendered_in_ascii(err, color=True) == textwrap.dedent( + f"""\ + {esc("1;31")}error{esc("0")}: {esc("1")}test-diagnostic{esc("0")} + + Oh no! + It broke. + + Something went wrong + very wrong. + + {esc("1;35")}note{esc("0")}: You did something wrong. + {esc("1;36")}hint{esc("0")}: Do it better next time, by trying harder. + """ + ) + + def test_no_context(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + note: You did something wrong, which is what caused this error. + hint: Do it better next time, by trying harder. + """ + ) + + def test_no_note(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt=None, + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + Something went wrong + very wrong. + + hint: Do it better next time, by trying harder. + """ + ) + + def test_no_hint(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt=None, + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + Something went wrong + very wrong. + + note: You did something wrong, which is what caused this error. + """ + ) + + def test_no_context_no_hint(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt=None, + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + note: You did something wrong, which is what caused this error. + """ + ) + + def test_no_context_no_note(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + note_stmt=None, + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + hint: Do it better next time, by trying harder. + """ + ) + + def test_no_hint_no_note(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt=None, + hint_stmt=None, + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + + Something went wrong + very wrong. + """ + ) + + def test_no_hint_no_note_no_context(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + hint_stmt=None, + note_stmt=None, + ) + + assert rendered_in_ascii(err) == textwrap.dedent( + """\ + error: test-diagnostic + + Oh no! + It broke. :( + """ + ) + + +def rendered(error: DiagnosticPipError, *, color: bool = False) -> str: + with io.StringIO() as stream: + console = rich.console.Console( + force_terminal=False, + file=stream, + color_system="truecolor" if color else None, + ) + console.print(error) + return stream.getvalue() + + +class TestDiagnosticPipErrorPresentation_Unicode: + def test_complete(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + │ It broke. :( + ╰─> Something went wrong + very wrong. + + note: You did something wrong, which is what caused this error. + hint: Do it better next time, by trying harder. + """ + ) + + def test_complete_color(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke.", + context="Something went wrong\nvery wrong.", + note_stmt="You did something wrong.", + hint_stmt="Do it better next time, by trying harder.", + ) + + def esc(code: str = "0") -> str: + return f"\x1b[{code}m" + + assert rendered(err, color=True) == textwrap.dedent( + f"""\ + {esc("1;31")}error{esc("0")}: {esc("1")}test-diagnostic{esc("0")} + + {esc("31")}×{esc("0")} Oh no! + {esc("31")}│{esc("0")} It broke. + {esc("31")}╰─>{esc("0")} Something went wrong + {esc("31")} {esc("0")} very wrong. + + {esc("1;35")}note{esc("0")}: You did something wrong. + {esc("1;36")}hint{esc("0")}: Do it better next time, by trying harder. + """ + ) + + def test_no_context(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + It broke. :( + + note: You did something wrong, which is what caused this error. + hint: Do it better next time, by trying harder. + """ + ) + + def test_no_note(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt=None, + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + │ It broke. :( + ╰─> Something went wrong + very wrong. + + hint: Do it better next time, by trying harder. + """ + ) + + def test_no_hint(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt=None, + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + │ It broke. :( + ╰─> Something went wrong + very wrong. + + note: You did something wrong, which is what caused this error. + """ + ) + + def test_no_context_no_hint(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + note_stmt="You did something wrong, which is what caused this error.", + hint_stmt=None, + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + It broke. :( + + note: You did something wrong, which is what caused this error. + """ + ) + + def test_no_context_no_note(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + note_stmt=None, + hint_stmt="Do it better next time, by trying harder.", + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + It broke. :( + + hint: Do it better next time, by trying harder. + """ + ) + + def test_no_hint_no_note(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context="Something went wrong\nvery wrong.", + note_stmt=None, + hint_stmt=None, + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + │ It broke. :( + ╰─> Something went wrong + very wrong. + """ + ) + + def test_no_hint_no_note_no_context(self) -> None: + err = DiagnosticPipError( + reference="test-diagnostic", + message="Oh no!\nIt broke. :(", + context=None, + hint_stmt=None, + note_stmt=None, + ) + + assert rendered(err) == textwrap.dedent( + """\ + error: test-diagnostic + + × Oh no! + It broke. :( + """ + ) diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 69ebd5d4b7c..16c24014459 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -1,11 +1,11 @@ import logging -import sys +from typing import Iterable +from unittest.mock import Mock, patch import pytest -from mock import Mock, patch from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.tags import Tag -from pkg_resources import parse_version +from pip._vendor.packaging.version import parse as parse_version import pip._internal.utils.compatibility_tags from pip._internal.exceptions import BestVersionAlreadyInstalled, DistributionNotFound @@ -14,126 +14,120 @@ InstallationCandidate, Link, LinkEvaluator, + LinkType, ) from pip._internal.models.target_python import TargetPython from pip._internal.req.constructors import install_req_from_line -from tests.lib import make_test_finder +from tests.lib import TestData, make_test_finder -def make_no_network_finder( - find_links, - allow_all_prereleases=False, # type: bool -): - """ - Create and return a PackageFinder instance for test purposes that - doesn't make any network requests when _get_pages() is called. - """ - finder = make_test_finder( - find_links=find_links, - allow_all_prereleases=allow_all_prereleases, - ) - # Replace the PackageFinder._link_collector's _get_pages() with a no-op. - link_collector = finder._link_collector - link_collector._get_pages = lambda locations: [] - - return finder - - -def test_no_mpkg(data): +def test_no_mpkg(data: TestData) -> None: """Finder skips zipfiles with "macosx10" in the name.""" finder = make_test_finder(find_links=[data.find_links]) req = install_req_from_line("pkgwithmpkg") found = finder.find_requirement(req, False) - + assert found is not None assert found.link.url.endswith("pkgwithmpkg-1.0.tar.gz"), found -def test_no_partial_name_match(data): +def test_no_partial_name_match(data: TestData) -> None: """Finder requires the full project name to match, not just beginning.""" finder = make_test_finder(find_links=[data.find_links]) req = install_req_from_line("gmpy") found = finder.find_requirement(req, False) - + assert found is not None assert found.link.url.endswith("gmpy-1.15.tar.gz"), found -def test_tilde(): +def test_tilde() -> None: """Finder can accept a path with ~ in it and will normalize it.""" patched_exists = patch( - 'pip._internal.index.collector.os.path.exists', return_value=True + "pip._internal.index.collector.os.path.exists", return_value=True ) with patched_exists: - finder = make_test_finder(find_links=['~/python-pkgs']) + finder = make_test_finder(find_links=["~/python-pkgs"]) req = install_req_from_line("gmpy") with pytest.raises(DistributionNotFound): finder.find_requirement(req, False) -def test_duplicates_sort_ok(data): +def test_duplicates_sort_ok(data: TestData) -> None: """Finder successfully finds one of a set of duplicates in different locations""" finder = make_test_finder(find_links=[data.find_links, data.find_links2]) req = install_req_from_line("duplicate") found = finder.find_requirement(req, False) - + assert found is not None assert found.link.url.endswith("duplicate-1.0.tar.gz"), found -def test_finder_detects_latest_find_links(data): +def test_finder_detects_latest_find_links(data: TestData) -> None: """Test PackageFinder detects latest using find-links""" - req = install_req_from_line('simple', None) + req = install_req_from_line("simple", None) finder = make_test_finder(find_links=[data.find_links]) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url.endswith("simple-3.0.tar.gz") -def test_incorrect_case_file_index(data): +def test_incorrect_case_file_index(data: TestData) -> None: """Test PackageFinder detects latest using wrong case""" - req = install_req_from_line('dinner', None) + req = install_req_from_line("dinner", None) finder = make_test_finder(index_urls=[data.find_links3]) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url.endswith("Dinner-2.0.tar.gz") @pytest.mark.network -def test_finder_detects_latest_already_satisfied_find_links(data): +@pytest.mark.parametrize("use_deprecated_html5lib", [False, True]) +def test_finder_detects_latest_already_satisfied_find_links( + data: TestData, use_deprecated_html5lib: bool +) -> None: """Test PackageFinder detects latest already satisfied using find-links""" - req = install_req_from_line('simple', None) + req = install_req_from_line("simple", None) # the latest simple in local pkgs is 3.0 latest_version = "3.0" satisfied_by = Mock( location="/path", - parsed_version=parse_version(latest_version), - version=latest_version + version=parse_version(latest_version), ) req.satisfied_by = satisfied_by - finder = make_test_finder(find_links=[data.find_links]) + finder = make_test_finder( + find_links=[data.find_links], use_deprecated_html5lib=use_deprecated_html5lib + ) with pytest.raises(BestVersionAlreadyInstalled): finder.find_requirement(req, True) @pytest.mark.network -def test_finder_detects_latest_already_satisfied_pypi_links(): +@pytest.mark.parametrize("use_deprecated_html5lib", [False, True]) +def test_finder_detects_latest_already_satisfied_pypi_links( + use_deprecated_html5lib: bool, +) -> None: """Test PackageFinder detects latest already satisfied using pypi links""" - req = install_req_from_line('initools', None) + req = install_req_from_line("initools", None) # the latest initools on PyPI is 0.3.1 latest_version = "0.3.1" satisfied_by = Mock( location="/path", - parsed_version=parse_version(latest_version), - version=latest_version, + version=parse_version(latest_version), ) req.satisfied_by = satisfied_by - finder = make_test_finder(index_urls=["http://pypi.org/simple/"]) + finder = make_test_finder( + index_urls=["http://pypi.org/simple/"], + use_deprecated_html5lib=use_deprecated_html5lib, + ) with pytest.raises(BestVersionAlreadyInstalled): finder.find_requirement(req, True) class TestWheel: - - def test_skip_invalid_wheel_link(self, caplog, data): + def test_skip_invalid_wheel_link( + self, caplog: pytest.LogCaptureFixture, data: TestData + ) -> None: """ Test if PackageFinder skips invalid wheel filenames """ @@ -145,9 +139,9 @@ def test_skip_invalid_wheel_link(self, caplog, data): with pytest.raises(DistributionNotFound): finder.find_requirement(req, True) - assert 'Skipping link: invalid wheel filename:' in caplog.text + assert "Skipping link: invalid wheel filename:" in caplog.text - def test_not_find_wheel_not_supported(self, data, monkeypatch): + def test_not_find_wheel_not_supported(self, data: TestData) -> None: """ Test not finding an unsupported wheel. """ @@ -163,24 +157,25 @@ def test_not_find_wheel_not_supported(self, data, monkeypatch): with pytest.raises(DistributionNotFound): finder.find_requirement(req, True) - def test_find_wheel_supported(self, data, monkeypatch): + def test_find_wheel_supported( + self, data: TestData, monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test finding supported wheel. """ monkeypatch.setattr( pip._internal.utils.compatibility_tags, "get_supported", - lambda **kw: [('py2', 'none', 'any')], + lambda **kw: [("py2", "none", "any")], ) req = install_req_from_line("simple.dist") finder = make_test_finder(find_links=[data.find_links]) found = finder.find_requirement(req, True) - assert ( - found.link.url.endswith("simple.dist-0.1-py2.py3-none-any.whl") - ), found + assert found is not None + assert found.link.url.endswith("simple.dist-0.1-py2.py3-none-any.whl"), found - def test_wheel_over_sdist_priority(self, data): + def test_wheel_over_sdist_priority(self, data: TestData) -> None: """ Test wheels have priority over sdists. `test_link_sorting` also covers this at lower level @@ -188,20 +183,19 @@ def test_wheel_over_sdist_priority(self, data): req = install_req_from_line("priority") finder = make_test_finder(find_links=[data.find_links]) found = finder.find_requirement(req, True) - assert found.link.url.endswith("priority-1.0-py2.py3-none-any.whl"), \ - found + assert found is not None + assert found.link.url.endswith("priority-1.0-py2.py3-none-any.whl"), found - def test_existing_over_wheel_priority(self, data): + def test_existing_over_wheel_priority(self, data: TestData) -> None: """ Test existing install has priority over wheels. `test_link_sorting` also covers this at a lower level """ - req = install_req_from_line('priority', None) + req = install_req_from_line("priority", None) latest_version = "1.0" satisfied_by = Mock( location="/path", - parsed_version=parse_version(latest_version), - version=latest_version, + version=parse_version(latest_version), ) req.satisfied_by = satisfied_by finder = make_test_finder(find_links=[data.find_links]) @@ -209,49 +203,54 @@ def test_existing_over_wheel_priority(self, data): with pytest.raises(BestVersionAlreadyInstalled): finder.find_requirement(req, True) - def test_link_sorting(self): + +class TestCandidateEvaluator: + def test_link_sorting(self) -> None: """ Test link sorting """ links = [ - InstallationCandidate("simple", "2.0", Link('simple-2.0.tar.gz')), + InstallationCandidate("simple", "2.0", Link("simple-2.0.tar.gz")), InstallationCandidate( "simple", "1.0", - Link('simple-1.0-pyT-none-TEST.whl'), + Link("simple-1.0-pyT-none-TEST.whl"), ), InstallationCandidate( "simple", - '1.0', - Link('simple-1.0-pyT-TEST-any.whl'), + "1.0", + Link("simple-1.0-pyT-TEST-any.whl"), ), InstallationCandidate( "simple", - '1.0', - Link('simple-1.0-pyT-none-any.whl'), + "1.0", + Link("simple-1.0-pyT-none-any.whl"), ), InstallationCandidate( "simple", - '1.0', - Link('simple-1.0.tar.gz'), + "1.0", + Link("simple-1.0.tar.gz"), ), ] valid_tags = [ - Tag('pyT', 'none', 'TEST'), - Tag('pyT', 'TEST', 'any'), - Tag('pyT', 'none', 'any'), + Tag("pyT", "none", "TEST"), + Tag("pyT", "TEST", "any"), + Tag("pyT", "none", "any"), ] specifier = SpecifierSet() evaluator = CandidateEvaluator( - 'my-project', supported_tags=valid_tags, specifier=specifier, + "my-project", + supported_tags=valid_tags, + specifier=specifier, ) sort_key = evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) results2 = sorted(reversed(links), key=sort_key, reverse=True) - assert links == results == results2, results2 + assert links == results, results + assert links == results2, results2 - def test_link_sorting_wheels_with_build_tags(self): + def test_link_sorting_wheels_with_build_tags(self) -> None: """Verify build tags affect sorting.""" links = [ InstallationCandidate( @@ -270,56 +269,102 @@ def test_link_sorting_wheels_with_build_tags(self): Link("simplewheel-1.0-py2.py3-none-any.whl"), ), ] - candidate_evaluator = CandidateEvaluator.create('my-project') + candidate_evaluator = CandidateEvaluator.create("my-project") sort_key = candidate_evaluator._sort_key results = sorted(links, key=sort_key, reverse=True) results2 = sorted(reversed(links), key=sort_key, reverse=True) - assert links == results == results2, results2 + + assert links == results, results + assert links == results2, results2 + + def test_build_tag_is_less_important_than_other_tags(self) -> None: + links = [ + InstallationCandidate( + "simple", + "1.0", + Link("simple-1.0-1-py3-abi3-linux_x86_64.whl"), + ), + InstallationCandidate( + "simple", + "1.0", + Link("simple-1.0-2-py3-abi3-linux_i386.whl"), + ), + InstallationCandidate( + "simple", + "1.0", + Link("simple-1.0-2-py3-any-none.whl"), + ), + InstallationCandidate( + "simple", + "1.0", + Link("simple-1.0.tar.gz"), + ), + ] + valid_tags = [ + Tag("py3", "abi3", "linux_x86_64"), + Tag("py3", "abi3", "linux_i386"), + Tag("py3", "any", "none"), + ] + evaluator = CandidateEvaluator( + "my-project", + supported_tags=valid_tags, + specifier=SpecifierSet(), + ) + sort_key = evaluator._sort_key + results = sorted(links, key=sort_key, reverse=True) + results2 = sorted(reversed(links), key=sort_key, reverse=True) + + assert links == results, results + assert links == results2, results2 -def test_finder_priority_file_over_page(data): +def test_finder_priority_file_over_page(data: TestData) -> None: """Test PackageFinder prefers file links over equivalent page links""" - req = install_req_from_line('gmpy==1.15', None) + req = install_req_from_line("gmpy==1.15", None) finder = make_test_finder( find_links=[data.find_links], index_urls=["http://pypi.org/simple/"], ) all_versions = finder.find_all_candidates(req.name) # 1 file InstallationCandidate followed by all https ones - assert all_versions[0].link.scheme == 'file' - assert all(version.link.scheme == 'https' - for version in all_versions[1:]), all_versions + assert all_versions[0].link.scheme == "file" + assert all( + version.link.scheme == "https" for version in all_versions[1:] + ), all_versions found = finder.find_requirement(req, False) + assert found is not None assert found.link.url.startswith("file://") -def test_finder_priority_nonegg_over_eggfragments(): +def test_finder_priority_nonegg_over_eggfragments() -> None: """Test PackageFinder prefers non-egg links over "#egg=" links""" - req = install_req_from_line('bar==1.0', None) - links = ['http://foo/bar.py#egg=bar-1.0', 'http://foo/bar-1.0.tar.gz'] + req = install_req_from_line("bar==1.0", None) + links = ["http://foo/bar.py#egg=bar-1.0", "http://foo/bar-1.0.tar.gz"] - finder = make_no_network_finder(links) + finder = make_test_finder(links) all_versions = finder.find_all_candidates(req.name) - assert all_versions[0].link.url.endswith('tar.gz') - assert all_versions[1].link.url.endswith('#egg=bar-1.0') + assert all_versions[0].link.url.endswith("tar.gz") + assert all_versions[1].link.url.endswith("#egg=bar-1.0") found = finder.find_requirement(req, False) - assert found.link.url.endswith('tar.gz') + assert found is not None + assert found.link.url.endswith("tar.gz") links.reverse() - finder = make_no_network_finder(links) + finder = make_test_finder(links) all_versions = finder.find_all_candidates(req.name) - assert all_versions[0].link.url.endswith('tar.gz') - assert all_versions[1].link.url.endswith('#egg=bar-1.0') + assert all_versions[0].link.url.endswith("tar.gz") + assert all_versions[1].link.url.endswith("#egg=bar-1.0") found = finder.find_requirement(req, False) - assert found.link.url.endswith('tar.gz') + assert found is not None + assert found.link.url.endswith("tar.gz") -def test_finder_only_installs_stable_releases(data): +def test_finder_only_installs_stable_releases(data: TestData) -> None: """ Test PackageFinder only accepts stable versioned releases by default. """ @@ -329,23 +374,26 @@ def test_finder_only_installs_stable_releases(data): # using a local index (that has pre & dev releases) finder = make_test_finder(index_urls=[data.index_url("pre")]) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url.endswith("bar-1.0.tar.gz"), found.link.url # using find-links links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"] - finder = make_no_network_finder(links) + finder = make_test_finder(links) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url == "https://foo/bar-1.0.tar.gz" links.reverse() - finder = make_no_network_finder(links) + finder = make_test_finder(links) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url == "https://foo/bar-1.0.tar.gz" -def test_finder_only_installs_data_require(data): +def test_finder_only_installs_data_require(data: TestData) -> None: """ Test whether the PackageFinder understand data-python-requires @@ -359,17 +407,10 @@ def test_finder_only_installs_data_require(data): # using a local index (that has pre & dev releases) finder = make_test_finder(index_urls=[data.index_url("datarequire")]) links = finder.find_all_candidates("fakepackage") + assert {str(v.version) for v in links} == {"1.0.0", "3.3.0", "9.9.9"} - expected = ['1.0.0', '9.9.9'] - if (2, 7) < sys.version_info < (3,): - expected.append('2.7.0') - elif sys.version_info > (3, 3): - expected.append('3.3.0') - assert {str(v.version) for v in links} == set(expected) - - -def test_finder_installs_pre_releases(data): +def test_finder_installs_pre_releases(data: TestData) -> None: """ Test PackageFinder finds pre-releases if asked to. """ @@ -382,23 +423,26 @@ def test_finder_installs_pre_releases(data): allow_all_prereleases=True, ) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url.endswith("bar-2.0b1.tar.gz"), found.link.url # using find-links links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"] - finder = make_no_network_finder(links, allow_all_prereleases=True) + finder = make_test_finder(links, allow_all_prereleases=True) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url == "https://foo/bar-2.0b1.tar.gz" links.reverse() - finder = make_no_network_finder(links, allow_all_prereleases=True) + finder = make_test_finder(links, allow_all_prereleases=True) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url == "https://foo/bar-2.0b1.tar.gz" -def test_finder_installs_dev_releases(data): +def test_finder_installs_dev_releases(data: TestData) -> None: """ Test PackageFinder finds dev releases if asked to. """ @@ -411,105 +455,125 @@ def test_finder_installs_dev_releases(data): allow_all_prereleases=True, ) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url.endswith("bar-2.0.dev1.tar.gz"), found.link.url -def test_finder_installs_pre_releases_with_version_spec(): +def test_finder_installs_pre_releases_with_version_spec() -> None: """ Test PackageFinder only accepts stable versioned releases by default. """ req = install_req_from_line("bar>=0.0.dev0", None) links = ["https://foo/bar-1.0.tar.gz", "https://foo/bar-2.0b1.tar.gz"] - finder = make_no_network_finder(links) + finder = make_test_finder(links) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url == "https://foo/bar-2.0b1.tar.gz" links.reverse() - finder = make_no_network_finder(links) + finder = make_test_finder(links) found = finder.find_requirement(req, False) + assert found is not None assert found.link.url == "https://foo/bar-2.0b1.tar.gz" class TestLinkEvaluator: - - def make_test_link_evaluator(self, formats): + def make_test_link_evaluator(self, formats: Iterable[str]) -> LinkEvaluator: target_python = TargetPython() return LinkEvaluator( - project_name='pytest', - canonical_name='pytest', - formats=formats, + project_name="pytest", + canonical_name="pytest", + formats=frozenset(formats), target_python=target_python, allow_yanked=True, ) - @pytest.mark.parametrize('url, expected_version', [ - ('http:/yo/pytest-1.0.tar.gz', '1.0'), - ('http:/yo/pytest-1.0-py2.py3-none-any.whl', '1.0'), - ]) - def test_evaluate_link__match(self, url, expected_version): + @pytest.mark.parametrize( + "url, expected_version", + [ + ("http:/yo/pytest-1.0.tar.gz", "1.0"), + ("http:/yo/pytest-1.0-py2.py3-none-any.whl", "1.0"), + ], + ) + def test_evaluate_link__match(self, url: str, expected_version: str) -> None: """Test that 'pytest' archives match for 'pytest'""" link = Link(url) - evaluator = self.make_test_link_evaluator(formats=['source', 'binary']) + evaluator = self.make_test_link_evaluator(formats=["source", "binary"]) actual = evaluator.evaluate_link(link) - assert actual == (True, expected_version) - - @pytest.mark.parametrize('url, expected_msg', [ - # TODO: Uncomment this test case when #1217 is fixed. - # 'http:/yo/pytest-xdist-1.0.tar.gz', - ('http:/yo/pytest2-1.0.tar.gz', - 'Missing project version for pytest'), - ('http:/yo/pytest_xdist-1.0-py2.py3-none-any.whl', - 'wrong project name (not pytest)'), - ]) - def test_evaluate_link__substring_fails(self, url, expected_msg): + assert actual == (LinkType.candidate, expected_version) + + @pytest.mark.parametrize( + "url, link_type, fail_reason", + [ + # TODO: Uncomment this test case when #1217 is fixed. + # 'http:/yo/pytest-xdist-1.0.tar.gz', + ( + "http:/yo/pytest2-1.0.tar.gz", + LinkType.format_invalid, + "Missing project version for pytest", + ), + ( + "http:/yo/pytest_xdist-1.0-py2.py3-none-any.whl", + LinkType.different_project, + "wrong project name (not pytest)", + ), + ], + ) + def test_evaluate_link__substring_fails( + self, + url: str, + link_type: LinkType, + fail_reason: str, + ) -> None: """Test that 'pytest archives won't match for 'pytest'.""" link = Link(url) - evaluator = self.make_test_link_evaluator(formats=['source', 'binary']) + evaluator = self.make_test_link_evaluator(formats=["source", "binary"]) actual = evaluator.evaluate_link(link) - assert actual == (False, expected_msg) + assert actual == (link_type, fail_reason) -def test_process_project_url(data): - project_name = 'simple' - index_url = data.index_url('simple') - project_url = Link(f'{index_url}/{project_name}') +def test_process_project_url(data: TestData) -> None: + project_name = "simple" + index_url = data.index_url("simple") + project_url = Link(f"{index_url}/{project_name}") finder = make_test_finder(index_urls=[index_url]) link_evaluator = finder.make_link_evaluator(project_name) actual = finder.process_project_url( - project_url, link_evaluator=link_evaluator, + project_url, + link_evaluator=link_evaluator, ) assert len(actual) == 1 package_link = actual[0] - assert package_link.name == 'simple' - assert str(package_link.version) == '1.0' + assert package_link.name == "simple" + assert str(package_link.version) == "1.0" -def test_find_all_candidates_nothing(): +def test_find_all_candidates_nothing() -> None: """Find nothing without anything""" finder = make_test_finder() - assert not finder.find_all_candidates('pip') + assert not finder.find_all_candidates("pip") -def test_find_all_candidates_find_links(data): +def test_find_all_candidates_find_links(data: TestData) -> None: finder = make_test_finder(find_links=[data.find_links]) - versions = finder.find_all_candidates('simple') - assert [str(v.version) for v in versions] == ['3.0', '2.0', '1.0'] + versions = finder.find_all_candidates("simple") + assert [str(v.version) for v in versions] == ["3.0", "2.0", "1.0"] -def test_find_all_candidates_index(data): - finder = make_test_finder(index_urls=[data.index_url('simple')]) - versions = finder.find_all_candidates('simple') - assert [str(v.version) for v in versions] == ['1.0'] +def test_find_all_candidates_index(data: TestData) -> None: + finder = make_test_finder(index_urls=[data.index_url("simple")]) + versions = finder.find_all_candidates("simple") + assert [str(v.version) for v in versions] == ["1.0"] -def test_find_all_candidates_find_links_and_index(data): +def test_find_all_candidates_find_links_and_index(data: TestData) -> None: finder = make_test_finder( find_links=[data.find_links], - index_urls=[data.index_url('simple')], + index_urls=[data.index_url("simple")], ) - versions = finder.find_all_candidates('simple') + versions = finder.find_all_candidates("simple") # first the find-links versions then the page versions - assert [str(v.version) for v in versions] == ['3.0', '2.0', '1.0', '1.0'] + assert [str(v.version) for v in versions] == ["3.0", "2.0", "1.0", "1.0"] diff --git a/tests/unit/test_format_control.py b/tests/unit/test_format_control.py index f8498e8e5d9..33a03729db5 100644 --- a/tests/unit/test_format_control.py +++ b/tests/unit/test_format_control.py @@ -1,57 +1,59 @@ +from optparse import Values +from typing import FrozenSet, List, Set + import pytest from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command +from pip._internal.cli.status_codes import SUCCESS from pip._internal.models.format_control import FormatControl class SimpleCommand(Command): + def __init__(self) -> None: + super().__init__("fake", "fake summary") - def __init__(self): - super().__init__('fake', 'fake summary') - - def add_options(self): + def add_options(self) -> None: self.cmd_opts.add_option(cmdoptions.no_binary()) self.cmd_opts.add_option(cmdoptions.only_binary()) - def run(self, options, args): + def run(self, options: Values, args: List[str]) -> int: self.options = options + return SUCCESS -def test_no_binary_overrides(): +def test_no_binary_overrides() -> None: cmd = SimpleCommand() - cmd.main(['fake', '--only-binary=:all:', '--no-binary=fred']) - format_control = FormatControl({'fred'}, {':all:'}) + cmd.main(["fake", "--only-binary=:all:", "--no-binary=fred"]) + format_control = FormatControl({"fred"}, {":all:"}) assert cmd.options.format_control == format_control -def test_only_binary_overrides(): +def test_only_binary_overrides() -> None: cmd = SimpleCommand() - cmd.main(['fake', '--no-binary=:all:', '--only-binary=fred']) - format_control = FormatControl({':all:'}, {'fred'}) + cmd.main(["fake", "--no-binary=:all:", "--only-binary=fred"]) + format_control = FormatControl({":all:"}, {"fred"}) assert cmd.options.format_control == format_control -def test_none_resets(): +def test_none_resets() -> None: cmd = SimpleCommand() - cmd.main(['fake', '--no-binary=:all:', '--no-binary=:none:']) + cmd.main(["fake", "--no-binary=:all:", "--no-binary=:none:"]) format_control = FormatControl(set(), set()) assert cmd.options.format_control == format_control -def test_none_preserves_other_side(): +def test_none_preserves_other_side() -> None: cmd = SimpleCommand() - cmd.main( - ['fake', '--no-binary=:all:', '--only-binary=fred', - '--no-binary=:none:']) - format_control = FormatControl(set(), {'fred'}) + cmd.main(["fake", "--no-binary=:all:", "--only-binary=fred", "--no-binary=:none:"]) + format_control = FormatControl(set(), {"fred"}) assert cmd.options.format_control == format_control -def test_comma_separated_values(): +def test_comma_separated_values() -> None: cmd = SimpleCommand() - cmd.main(['fake', '--no-binary=1,2,3']) - format_control = FormatControl({'1', '2', '3'}, set()) + cmd.main(["fake", "--no-binary=1,2,3"]) + format_control = FormatControl({"1", "2", "3"}, set()) assert cmd.options.format_control == format_control @@ -61,9 +63,11 @@ def test_comma_separated_values(): ({"fred"}, set(), "fred", frozenset(["source"])), ({"fred"}, {":all:"}, "fred", frozenset(["source"])), (set(), {"fred"}, "fred", frozenset(["binary"])), - ({":all:"}, {"fred"}, "fred", frozenset(["binary"])) - ] + ({":all:"}, {"fred"}, "fred", frozenset(["binary"])), + ], ) -def test_fmt_ctl_matches(no_binary, only_binary, argument, expected): +def test_fmt_ctl_matches( + no_binary: Set[str], only_binary: Set[str], argument: str, expected: FrozenSet[str] +) -> None: fmt = FormatControl(no_binary, only_binary) assert fmt.get_allowed_formats(argument) == expected diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index b6f7f632cc0..7051268545b 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -1,7 +1,9 @@ import logging +from typing import FrozenSet, List, Optional, Set, Tuple import pytest from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.tags import Tag from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import ( @@ -9,6 +11,7 @@ CandidatePreferences, FormatControl, LinkEvaluator, + LinkType, PackageFinder, _check_link_requires_python, _extract_version_from_fragment, @@ -26,49 +29,68 @@ from tests.lib.index import make_mock_candidate -@pytest.mark.parametrize('requires_python, expected', [ - ('== 3.6.4', False), - ('== 3.6.5', True), - # Test an invalid Requires-Python value. - ('invalid', True), -]) -def test_check_link_requires_python(requires_python, expected): +@pytest.mark.parametrize( + "requires_python, expected", + [ + ("== 3.6.4", False), + ("== 3.6.5", True), + # Test an invalid Requires-Python value. + ("invalid", True), + ], +) +def test_check_link_requires_python(requires_python: str, expected: bool) -> None: version_info = (3, 6, 5) - link = Link('https://example.com', requires_python=requires_python) + link = Link("https://example.com", requires_python=requires_python) actual = _check_link_requires_python(link, version_info) assert actual == expected -def check_caplog(caplog, expected_level, expected_message): +def check_caplog( + caplog: pytest.LogCaptureFixture, expected_level: str, expected_message: str +) -> None: assert len(caplog.records) == 1 record = caplog.records[0] assert record.levelname == expected_level assert record.message == expected_message -@pytest.mark.parametrize('ignore_requires_python, expected', [ - (None, ( - False, 'DEBUG', - "Link requires a different Python (3.6.5 not in: '== 3.6.4'): " - "https://example.com" - )), - (True, ( - True, 'DEBUG', - "Ignoring failed Requires-Python check (3.6.5 not in: '== 3.6.4') " - "for link: https://example.com" - )), -]) +@pytest.mark.parametrize( + "ignore_requires_python, expected", + [ + ( + False, + ( + False, + "VERBOSE", + "Link requires a different Python (3.6.5 not in: '== 3.6.4'): " + "https://example.com", + ), + ), + ( + True, + ( + True, + "DEBUG", + "Ignoring failed Requires-Python check (3.6.5 not in: '== 3.6.4') " + "for link: https://example.com", + ), + ), + ], +) def test_check_link_requires_python__incompatible_python( - caplog, ignore_requires_python, expected, -): + caplog: pytest.LogCaptureFixture, + ignore_requires_python: bool, + expected: Tuple[bool, str, str], +) -> None: """ Test an incompatible Python. """ expected_return, expected_level, expected_message = expected - link = Link('https://example.com', requires_python='== 3.6.4') + link = Link("https://example.com", requires_python="== 3.6.4") caplog.set_level(logging.DEBUG) actual = _check_link_requires_python( - link, version_info=(3, 6, 5), + link, + version_info=(3, 6, 5), ignore_requires_python=ignore_requires_python, ) assert actual == expected_return @@ -76,84 +98,123 @@ def test_check_link_requires_python__incompatible_python( check_caplog(caplog, expected_level, expected_message) -def test_check_link_requires_python__invalid_requires(caplog): +def test_check_link_requires_python__invalid_requires( + caplog: pytest.LogCaptureFixture, +) -> None: """ Test the log message for an invalid Requires-Python. """ - link = Link('https://example.com', requires_python='invalid') + link = Link("https://example.com", requires_python="invalid") caplog.set_level(logging.DEBUG) actual = _check_link_requires_python(link, version_info=(3, 6, 5)) assert actual expected_message = ( - "Ignoring invalid Requires-Python ('invalid') for link: " - "https://example.com" + "Ignoring invalid Requires-Python ('invalid') for link: https://example.com" ) - check_caplog(caplog, 'DEBUG', expected_message) + check_caplog(caplog, "DEBUG", expected_message) class TestLinkEvaluator: - @pytest.mark.parametrize( - 'py_version_info,ignore_requires_python,expected', [ - ((3, 6, 5), None, (True, '1.12')), - # Test an incompatible Python. - ((3, 6, 4), None, (False, None)), - # Test an incompatible Python with ignore_requires_python=True. - ((3, 6, 4), True, (True, '1.12')), + "py_version_info, ignore_requires_python, expected", + [ + pytest.param( + (3, 6, 5), + False, + (LinkType.candidate, "1.12"), + id="compatible", + ), + pytest.param( + (3, 6, 4), + False, + ( + LinkType.requires_python_mismatch, + "1.12 Requires-Python == 3.6.5", + ), + id="requires-python-mismatch", + ), + pytest.param( + (3, 6, 4), + True, + (LinkType.candidate, "1.12"), + id="requires-python-mismatch-ignored", + ), ], ) def test_evaluate_link( - self, py_version_info, ignore_requires_python, expected, - ): + self, + py_version_info: Tuple[int, int, int], + ignore_requires_python: bool, + expected: Tuple[LinkType, str], + ) -> None: target_python = TargetPython(py_version_info=py_version_info) evaluator = LinkEvaluator( - project_name='twine', - canonical_name='twine', - formats={'source'}, + project_name="twine", + canonical_name="twine", + formats=frozenset(["source"]), target_python=target_python, allow_yanked=True, ignore_requires_python=ignore_requires_python, ) link = Link( - 'https://example.com/#egg=twine-1.12', - requires_python='== 3.6.5', + "https://example.com/#egg=twine-1.12", + requires_python="== 3.6.5", ) actual = evaluator.evaluate_link(link) assert actual == expected - @pytest.mark.parametrize('yanked_reason, allow_yanked, expected', [ - (None, True, (True, '1.12')), - (None, False, (True, '1.12')), - ('', True, (True, '1.12')), - ('', False, (False, 'yanked for reason: ')), - ('bad metadata', True, (True, '1.12')), - ('bad metadata', False, - (False, 'yanked for reason: bad metadata')), - # Test a unicode string with a non-ascii character. - ('curly quote: \u2018', True, (True, '1.12')), - ('curly quote: \u2018', False, - (False, 'yanked for reason: curly quote: \u2018')), - ]) + @pytest.mark.parametrize( + "yanked_reason, allow_yanked, expected", + [ + (None, True, (LinkType.candidate, "1.12")), + (None, False, (LinkType.candidate, "1.12")), + ("", True, (LinkType.candidate, "1.12")), + ( + "", + False, + (LinkType.yanked, "yanked for reason: "), + ), + ("bad metadata", True, (LinkType.candidate, "1.12")), + ( + "bad metadata", + False, + (LinkType.yanked, "yanked for reason: bad metadata"), + ), + # Test a unicode string with a non-ascii character. + ("curly quote: \u2018", True, (LinkType.candidate, "1.12")), + ( + "curly quote: \u2018", + False, + ( + LinkType.yanked, + "yanked for reason: curly quote: \u2018", + ), + ), + ], + ) def test_evaluate_link__allow_yanked( - self, yanked_reason, allow_yanked, expected, - ): + self, + yanked_reason: str, + allow_yanked: bool, + expected: Tuple[LinkType, str], + ) -> None: target_python = TargetPython(py_version_info=(3, 6, 4)) evaluator = LinkEvaluator( - project_name='twine', - canonical_name='twine', - formats={'source'}, + project_name="twine", + canonical_name="twine", + formats=frozenset(["source"]), target_python=target_python, allow_yanked=allow_yanked, ) link = Link( - 'https://example.com/#egg=twine-1.12', + "https://example.com/#egg=twine-1.12", yanked_reason=yanked_reason, ) actual = evaluator.evaluate_link(link) assert actual == expected - def test_evaluate_link__incompatible_wheel(self): + def test_evaluate_link__incompatible_wheel(self) -> None: """ Test an incompatible wheel. """ @@ -161,38 +222,44 @@ def test_evaluate_link__incompatible_wheel(self): # Set the valid tags to an empty list to make sure nothing matches. target_python._valid_tags = [] evaluator = LinkEvaluator( - project_name='sample', - canonical_name='sample', - formats={'binary'}, + project_name="sample", + canonical_name="sample", + formats=frozenset(["binary"]), target_python=target_python, allow_yanked=True, ) - link = Link('https://example.com/sample-1.0-py2.py3-none-any.whl') + link = Link("https://example.com/sample-1.0-py2.py3-none-any.whl") actual = evaluator.evaluate_link(link) expected = ( - False, "none of the wheel's tags match: py2-none-any, py3-none-any" + LinkType.platform_mismatch, + "none of the wheel's tags (py2-none-any, py3-none-any) are compatible " + "(run pip debug --verbose to show compatible tags)", ) assert actual == expected -@pytest.mark.parametrize('hex_digest, expected_versions', [ - (None, ['1.0', '1.1', '1.2']), - (64 * 'a', ['1.0', '1.1']), - (64 * 'b', ['1.0', '1.2']), - (64 * 'c', ['1.0', '1.1', '1.2']), -]) -def test_filter_unallowed_hashes(hex_digest, expected_versions): +@pytest.mark.parametrize( + "hex_digest, expected_versions", + [ + (64 * "a", ["1.0", "1.1"]), + (64 * "b", ["1.0", "1.2"]), + (64 * "c", ["1.0", "1.1", "1.2"]), + ], +) +def test_filter_unallowed_hashes(hex_digest: str, expected_versions: List[str]) -> None: candidates = [ - make_mock_candidate('1.0'), - make_mock_candidate('1.1', hex_digest=(64 * 'a')), - make_mock_candidate('1.2', hex_digest=(64 * 'b')), + make_mock_candidate("1.0"), + make_mock_candidate("1.1", hex_digest=(64 * "a")), + make_mock_candidate("1.2", hex_digest=(64 * "b")), ] hashes_data = { - 'sha256': [hex_digest], + "sha256": [hex_digest], } hashes = Hashes(hashes_data) actual = filter_unallowed_hashes( - candidates, hashes=hashes, project_name='my-project', + candidates, + hashes=hashes, + project_name="my-project", ) actual_versions = [str(candidate.version) for candidate in actual] @@ -201,15 +268,17 @@ def test_filter_unallowed_hashes(hex_digest, expected_versions): assert actual is not candidates -def test_filter_unallowed_hashes__no_hashes(caplog): +def test_filter_unallowed_hashes__no_hashes(caplog: pytest.LogCaptureFixture) -> None: caplog.set_level(logging.DEBUG) candidates = [ - make_mock_candidate('1.0'), - make_mock_candidate('1.1'), + make_mock_candidate("1.0"), + make_mock_candidate("1.1"), ] actual = filter_unallowed_hashes( - candidates, hashes=Hashes(), project_name='my-project', + candidates, + hashes=Hashes(), + project_name="my-project", ) # Check that the return value is a copy. @@ -220,28 +289,36 @@ def test_filter_unallowed_hashes__no_hashes(caplog): "Given no hashes to check 2 links for project 'my-project': " "discarding no candidates" ) - check_caplog(caplog, 'DEBUG', expected_message) + check_caplog(caplog, "DEBUG", expected_message) -def test_filter_unallowed_hashes__log_message_with_match(caplog): +def test_filter_unallowed_hashes__log_message_with_match( + caplog: pytest.LogCaptureFixture, +) -> None: caplog.set_level(logging.DEBUG) # Test 1 match, 2 non-matches, 3 no hashes so all 3 values will be # different. candidates = [ - make_mock_candidate('1.0'), - make_mock_candidate('1.1',), - make_mock_candidate('1.2',), - make_mock_candidate('1.3', hex_digest=(64 * 'a')), - make_mock_candidate('1.4', hex_digest=(64 * 'b')), - make_mock_candidate('1.5', hex_digest=(64 * 'c')), + make_mock_candidate("1.0"), + make_mock_candidate( + "1.1", + ), + make_mock_candidate( + "1.2", + ), + make_mock_candidate("1.3", hex_digest=(64 * "a")), + make_mock_candidate("1.4", hex_digest=(64 * "b")), + make_mock_candidate("1.5", hex_digest=(64 * "c")), ] hashes_data = { - 'sha256': [64 * 'a', 64 * 'd'], + "sha256": [64 * "a", 64 * "d"], } hashes = Hashes(hashes_data) actual = filter_unallowed_hashes( - candidates, hashes=hashes, project_name='my-project', + candidates, + hashes=hashes, + project_name="my-project", ) assert len(actual) == 4 @@ -253,23 +330,27 @@ def test_filter_unallowed_hashes__log_message_with_match(caplog): " https://example.com/pkg-1.5.tar.gz#sha256=" "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" ) - check_caplog(caplog, 'DEBUG', expected_message) + check_caplog(caplog, "DEBUG", expected_message) -def test_filter_unallowed_hashes__log_message_with_no_match(caplog): +def test_filter_unallowed_hashes__log_message_with_no_match( + caplog: pytest.LogCaptureFixture, +) -> None: caplog.set_level(logging.DEBUG) candidates = [ - make_mock_candidate('1.0'), - make_mock_candidate('1.1', hex_digest=(64 * 'b')), - make_mock_candidate('1.2', hex_digest=(64 * 'c')), + make_mock_candidate("1.0"), + make_mock_candidate("1.1", hex_digest=(64 * "b")), + make_mock_candidate("1.2", hex_digest=(64 * "c")), ] hashes_data = { - 'sha256': [64 * 'a', 64 * 'd'], + "sha256": [64 * "a", 64 * "d"], } hashes = Hashes(hashes_data) actual = filter_unallowed_hashes( - candidates, hashes=hashes, project_name='my-project', + candidates, + hashes=hashes, + project_name="my-project", ) assert len(actual) == 3 @@ -277,23 +358,25 @@ def test_filter_unallowed_hashes__log_message_with_no_match(caplog): "Checked 3 links for project 'my-project' against 2 hashes " "(0 matches, 1 no digest): discarding no candidates" ) - check_caplog(caplog, 'DEBUG', expected_message) + check_caplog(caplog, "DEBUG", expected_message) class TestCandidateEvaluator: - - @pytest.mark.parametrize('allow_all_prereleases, prefer_binary', [ - (False, False), - (False, True), - (True, False), - (True, True), - ]) - def test_create(self, allow_all_prereleases, prefer_binary): + @pytest.mark.parametrize( + "allow_all_prereleases, prefer_binary", + [ + (False, False), + (False, True), + (True, False), + (True, True), + ], + ) + def test_create(self, allow_all_prereleases: bool, prefer_binary: bool) -> None: target_python = TargetPython() - target_python._valid_tags = [('py36', 'none', 'any')] + target_python._valid_tags = [Tag("py36", "none", "any")] specifier = SpecifierSet() evaluator = CandidateEvaluator.create( - project_name='my-project', + project_name="my-project", target_python=target_python, allow_all_prereleases=allow_all_prereleases, prefer_binary=prefer_binary, @@ -302,66 +385,69 @@ def test_create(self, allow_all_prereleases, prefer_binary): assert evaluator._allow_all_prereleases == allow_all_prereleases assert evaluator._prefer_binary == prefer_binary assert evaluator._specifier is specifier - assert evaluator._supported_tags == [('py36', 'none', 'any')] + assert evaluator._supported_tags == [Tag("py36", "none", "any")] - def test_create__target_python_none(self): + def test_create__target_python_none(self) -> None: """ Test passing target_python=None. """ - evaluator = CandidateEvaluator.create('my-project') + evaluator = CandidateEvaluator.create("my-project") expected_tags = get_supported() assert evaluator._supported_tags == expected_tags - def test_create__specifier_none(self): + def test_create__specifier_none(self) -> None: """ Test passing specifier=None. """ - evaluator = CandidateEvaluator.create('my-project') + evaluator = CandidateEvaluator.create("my-project") expected_specifier = SpecifierSet() assert evaluator._specifier == expected_specifier - def test_get_applicable_candidates(self): - specifier = SpecifierSet('<= 1.11') - versions = ['1.10', '1.11', '1.12'] - candidates = [ - make_mock_candidate(version) for version in versions - ] + def test_get_applicable_candidates(self) -> None: + specifier = SpecifierSet("<= 1.11") + versions = ["1.10", "1.11", "1.12"] + candidates = [make_mock_candidate(version) for version in versions] evaluator = CandidateEvaluator.create( - 'my-project', + "my-project", specifier=specifier, ) actual = evaluator.get_applicable_candidates(candidates) expected_applicable = candidates[:2] assert [str(c.version) for c in expected_applicable] == [ - '1.10', - '1.11', + "1.10", + "1.11", ] assert actual == expected_applicable - @pytest.mark.parametrize('specifier, expected_versions', [ - # Test no version constraint. - (SpecifierSet(), ['1.0', '1.2']), - # Test a version constraint that excludes the candidate whose - # hash matches. Then the non-allowed hash is a candidate. - (SpecifierSet('<= 1.1'), ['1.0', '1.1']), - ]) + @pytest.mark.parametrize( + "specifier, expected_versions", + [ + # Test no version constraint. + (SpecifierSet(), ["1.0", "1.2"]), + # Test a version constraint that excludes the candidate whose + # hash matches. Then the non-allowed hash is a candidate. + (SpecifierSet("<= 1.1"), ["1.0", "1.1"]), + ], + ) def test_get_applicable_candidates__hashes( - self, specifier, expected_versions, - ): + self, + specifier: SpecifierSet, + expected_versions: List[str], + ) -> None: """ Test a non-None hashes value. """ candidates = [ - make_mock_candidate('1.0'), - make_mock_candidate('1.1', hex_digest=(64 * 'a')), - make_mock_candidate('1.2', hex_digest=(64 * 'b')), + make_mock_candidate("1.0"), + make_mock_candidate("1.1", hex_digest=(64 * "a")), + make_mock_candidate("1.2", hex_digest=(64 * "b")), ] hashes_data = { - 'sha256': [64 * 'b'], + "sha256": [64 * "b"], } hashes = Hashes(hashes_data) evaluator = CandidateEvaluator.create( - 'my-project', + "my-project", specifier=specifier, hashes=hashes, ) @@ -369,14 +455,12 @@ def test_get_applicable_candidates__hashes( actual_versions = [str(c.version) for c in actual] assert actual_versions == expected_versions - def test_compute_best_candidate(self): - specifier = SpecifierSet('<= 1.11') - versions = ['1.10', '1.11', '1.12'] - candidates = [ - make_mock_candidate(version) for version in versions - ] + def test_compute_best_candidate(self) -> None: + specifier = SpecifierSet("<= 1.11") + versions = ["1.10", "1.11", "1.12"] + candidates = [make_mock_candidate(version) for version in versions] evaluator = CandidateEvaluator.create( - 'my-project', + "my-project", specifier=specifier, ) result = evaluator.compute_best_candidate(candidates) @@ -384,24 +468,22 @@ def test_compute_best_candidate(self): assert result._candidates == candidates expected_applicable = candidates[:2] assert [str(c.version) for c in expected_applicable] == [ - '1.10', - '1.11', + "1.10", + "1.11", ] assert result._applicable_candidates == expected_applicable assert result.best_candidate is expected_applicable[1] - def test_compute_best_candidate__none_best(self): + def test_compute_best_candidate__none_best(self) -> None: """ Test returning a None best candidate. """ - specifier = SpecifierSet('<= 1.10') - versions = ['1.11', '1.12'] - candidates = [ - make_mock_candidate(version) for version in versions - ] + specifier = SpecifierSet("<= 1.10") + versions = ["1.11", "1.12"] + candidates = [make_mock_candidate(version) for version in versions] evaluator = CandidateEvaluator.create( - 'my-project', + "my-project", specifier=specifier, ) result = evaluator.compute_best_candidate(candidates) @@ -410,89 +492,102 @@ def test_compute_best_candidate__none_best(self): assert result._applicable_candidates == [] assert result.best_candidate is None - @pytest.mark.parametrize('hex_digest, expected', [ - # Test a link with no hash. - (None, 0), - # Test a link with an allowed hash. - (64 * 'a', 1), - # Test a link with a hash that isn't allowed. - (64 * 'b', 0), - ]) - def test_sort_key__hash(self, hex_digest, expected): + @pytest.mark.parametrize( + "hex_digest, expected", + [ + # Test a link with no hash. + (None, 0), + # Test a link with an allowed hash. + (64 * "a", 1), + # Test a link with a hash that isn't allowed. + (64 * "b", 0), + ], + ) + def test_sort_key__hash(self, hex_digest: Optional[str], expected: int) -> None: """ Test the effect of the link's hash on _sort_key()'s return value. """ - candidate = make_mock_candidate('1.0', hex_digest=hex_digest) + candidate = make_mock_candidate("1.0", hex_digest=hex_digest) hashes_data = { - 'sha256': [64 * 'a'], + "sha256": [64 * "a"], } hashes = Hashes(hashes_data) - evaluator = CandidateEvaluator.create('my-project', hashes=hashes) + evaluator = CandidateEvaluator.create("my-project", hashes=hashes) sort_value = evaluator._sort_key(candidate) # The hash is reflected in the first element of the tuple. actual = sort_value[0] assert actual == expected - @pytest.mark.parametrize('yanked_reason, expected', [ - # Test a non-yanked file. - (None, 0), - # Test a yanked file (has a lower value than non-yanked). - ('bad metadata', -1), - ]) - def test_sort_key__is_yanked(self, yanked_reason, expected): + @pytest.mark.parametrize( + "yanked_reason, expected", + [ + # Test a non-yanked file. + (None, 0), + # Test a yanked file (has a lower value than non-yanked). + ("bad metadata", -1), + ], + ) + def test_sort_key__is_yanked( + self, yanked_reason: Optional[str], expected: int + ) -> None: """ Test the effect of is_yanked on _sort_key()'s return value. """ - candidate = make_mock_candidate('1.0', yanked_reason=yanked_reason) - evaluator = CandidateEvaluator.create('my-project') + candidate = make_mock_candidate("1.0", yanked_reason=yanked_reason) + evaluator = CandidateEvaluator.create("my-project") sort_value = evaluator._sort_key(candidate) # Yanked / non-yanked is reflected in the second element of the tuple. actual = sort_value[1] assert actual == expected - def test_sort_best_candidate__no_candidates(self): + def test_sort_best_candidate__no_candidates(self) -> None: """ Test passing an empty list. """ - evaluator = CandidateEvaluator.create('my-project') + evaluator = CandidateEvaluator.create("my-project") actual = evaluator.sort_best_candidate([]) assert actual is None def test_sort_best_candidate__best_yanked_but_not_all( - self, caplog, - ): + self, + caplog: pytest.LogCaptureFixture, + ) -> None: """ Test the best candidates being yanked, but not all. """ caplog.set_level(logging.INFO) candidates = [ - make_mock_candidate('4.0', yanked_reason='bad metadata #4'), + make_mock_candidate("4.0", yanked_reason="bad metadata #4"), # Put the best candidate in the middle, to test sorting. - make_mock_candidate('2.0'), - make_mock_candidate('3.0', yanked_reason='bad metadata #3'), - make_mock_candidate('1.0'), + make_mock_candidate("2.0"), + make_mock_candidate("3.0", yanked_reason="bad metadata #3"), + make_mock_candidate("1.0"), ] expected_best = candidates[1] - evaluator = CandidateEvaluator.create('my-project') + evaluator = CandidateEvaluator.create("my-project") actual = evaluator.sort_best_candidate(candidates) assert actual is expected_best - assert str(actual.version) == '2.0' + assert str(actual.version) == "2.0" # Check the log messages. assert len(caplog.records) == 0 class TestPackageFinder: - - @pytest.mark.parametrize('allow_all_prereleases, prefer_binary', [ - (False, False), - (False, True), - (True, False), - (True, True), - ]) + @pytest.mark.parametrize( + "allow_all_prereleases, prefer_binary", + [ + (False, False), + (False, True), + (True, False), + (True, True), + ], + ) def test_create__candidate_prefs( - self, allow_all_prereleases, prefer_binary, - ): + self, + allow_all_prereleases: bool, + prefer_binary: bool, + ) -> None: """ Test that the _candidate_prefs attribute is set correctly. """ @@ -508,12 +603,13 @@ def test_create__candidate_prefs( finder = PackageFinder.create( link_collector=link_collector, selection_prefs=selection_prefs, + use_deprecated_html5lib=False, ) candidate_prefs = finder._candidate_prefs assert candidate_prefs.allow_all_prereleases == allow_all_prereleases assert candidate_prefs.prefer_binary == prefer_binary - def test_create__link_collector(self): + def test_create__link_collector(self) -> None: """ Test that the _link_collector attribute is set correctly. """ @@ -524,11 +620,12 @@ def test_create__link_collector(self): finder = PackageFinder.create( link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), + use_deprecated_html5lib=False, ) assert finder._link_collector is link_collector - def test_create__target_python(self): + def test_create__target_python(self) -> None: """ Test that the _target_python attribute is set correctly. """ @@ -541,6 +638,7 @@ def test_create__target_python(self): link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), target_python=target_python, + use_deprecated_html5lib=False, ) actual_target_python = finder._target_python # The target_python attribute should be set as is. @@ -548,7 +646,7 @@ def test_create__target_python(self): # Check that the attributes weren't reset. assert actual_target_python.py_version_info == (3, 7, 3) - def test_create__target_python_none(self): + def test_create__target_python_none(self) -> None: """ Test passing target_python=None. """ @@ -560,14 +658,15 @@ def test_create__target_python_none(self): link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), target_python=None, + use_deprecated_html5lib=False, ) # Spot-check the default TargetPython object. actual_target_python = finder._target_python assert actual_target_python._given_py_version_info is None assert actual_target_python.py_version_info == CURRENT_PY_VERSION_INFO - @pytest.mark.parametrize('allow_yanked', [False, True]) - def test_create__allow_yanked(self, allow_yanked): + @pytest.mark.parametrize("allow_yanked", [False, True]) + def test_create__allow_yanked(self, allow_yanked: bool) -> None: """ Test that the _allow_yanked attribute is set correctly. """ @@ -579,11 +678,12 @@ def test_create__allow_yanked(self, allow_yanked): finder = PackageFinder.create( link_collector=link_collector, selection_prefs=selection_prefs, + use_deprecated_html5lib=False, ) assert finder._allow_yanked == allow_yanked - @pytest.mark.parametrize('ignore_requires_python', [False, True]) - def test_create__ignore_requires_python(self, ignore_requires_python): + @pytest.mark.parametrize("ignore_requires_python", [False, True]) + def test_create__ignore_requires_python(self, ignore_requires_python: bool) -> None: """ Test that the _ignore_requires_python attribute is set correctly. """ @@ -598,10 +698,11 @@ def test_create__ignore_requires_python(self, ignore_requires_python): finder = PackageFinder.create( link_collector=link_collector, selection_prefs=selection_prefs, + use_deprecated_html5lib=False, ) assert finder._ignore_requires_python == ignore_requires_python - def test_create__format_control(self): + def test_create__format_control(self) -> None: """ Test that the format_control attribute is set correctly. """ @@ -609,7 +710,7 @@ def test_create__format_control(self): session=PipSession(), search_scope=SearchScope([], []), ) - format_control = FormatControl(set(), {':all:'}) + format_control = FormatControl(set(), {":all:"}) selection_prefs = SelectionPreferences( allow_yanked=True, format_control=format_control, @@ -617,28 +718,32 @@ def test_create__format_control(self): finder = PackageFinder.create( link_collector=link_collector, selection_prefs=selection_prefs, + use_deprecated_html5lib=False, ) actual_format_control = finder.format_control assert actual_format_control is format_control # Check that the attributes weren't reset. - assert actual_format_control.only_binary == {':all:'} + assert actual_format_control.only_binary == {":all:"} @pytest.mark.parametrize( - 'allow_yanked, ignore_requires_python, only_binary, expected_formats', + "allow_yanked, ignore_requires_python, only_binary, expected_formats", [ - (False, False, {}, frozenset({'binary', 'source'})), + (False, False, {}, frozenset({"binary", "source"})), # Test allow_yanked=True. - (True, False, {}, frozenset({'binary', 'source'})), + (True, False, {}, frozenset({"binary", "source"})), # Test ignore_requires_python=True. - (False, True, {}, frozenset({'binary', 'source'})), + (False, True, {}, frozenset({"binary", "source"})), # Test a non-trivial only_binary. - (False, False, {'twine'}, frozenset({'binary'})), - ] + (False, False, {"twine"}, frozenset({"binary"})), + ], ) def test_make_link_evaluator( - self, allow_yanked, ignore_requires_python, only_binary, - expected_formats, - ): + self, + allow_yanked: bool, + ignore_requires_python: bool, + only_binary: Set[str], + expected_formats: FrozenSet[str], + ) -> None: # Create a test TargetPython that we can check for. target_python = TargetPython(py_version_info=(3, 7)) format_control = FormatControl(set(), only_binary) @@ -654,13 +759,14 @@ def test_make_link_evaluator( allow_yanked=allow_yanked, format_control=format_control, ignore_requires_python=ignore_requires_python, + use_deprecated_html5lib=False, ) # Pass a project_name that will be different from canonical_name. - link_evaluator = finder.make_link_evaluator('Twine') + link_evaluator = finder.make_link_evaluator("Twine") - assert link_evaluator.project_name == 'Twine' - assert link_evaluator._canonical_name == 'twine' + assert link_evaluator.project_name == "Twine" + assert link_evaluator._canonical_name == "twine" assert link_evaluator._allow_yanked == allow_yanked assert link_evaluator._ignore_requires_python == ignore_requires_python assert link_evaluator._formats == expected_formats @@ -673,17 +779,22 @@ def test_make_link_evaluator( assert actual_target_python._given_py_version_info == (3, 7) assert actual_target_python.py_version_info == (3, 7, 0) - @pytest.mark.parametrize('allow_all_prereleases, prefer_binary', [ - (False, False), - (False, True), - (True, False), - (True, True), - ]) + @pytest.mark.parametrize( + "allow_all_prereleases, prefer_binary", + [ + (False, False), + (False, True), + (True, False), + (True, True), + ], + ) def test_make_candidate_evaluator( - self, allow_all_prereleases, prefer_binary, - ): + self, + allow_all_prereleases: bool, + prefer_binary: bool, + ) -> None: target_python = TargetPython() - target_python._valid_tags = [('py36', 'none', 'any')] + target_python._valid_tags = [Tag("py36", "none", "any")] candidate_prefs = CandidatePreferences( prefer_binary=prefer_binary, allow_all_prereleases=allow_all_prereleases, @@ -697,22 +808,23 @@ def test_make_candidate_evaluator( target_python=target_python, allow_yanked=True, candidate_prefs=candidate_prefs, + use_deprecated_html5lib=False, ) specifier = SpecifierSet() # Pass hashes to check that _hashes is set. - hashes = Hashes({'sha256': [64 * 'a']}) + hashes = Hashes({"sha256": [64 * "a"]}) evaluator = finder.make_candidate_evaluator( - 'my-project', + "my-project", specifier=specifier, hashes=hashes, ) assert evaluator._allow_all_prereleases == allow_all_prereleases assert evaluator._hashes == hashes assert evaluator._prefer_binary == prefer_binary - assert evaluator._project_name == 'my-project' + assert evaluator._project_name == "my-project" assert evaluator._specifier is specifier - assert evaluator._supported_tags == [('py36', 'none', 'any')] + assert evaluator._supported_tags == [Tag("py36", "none", "any")] @pytest.mark.parametrize( @@ -721,31 +833,28 @@ def test_make_candidate_evaluator( # Trivial. ("pip-18.0", "pip", 3), ("zope-interface-4.5.0", "zope-interface", 14), - # Canonicalized name match non-canonicalized egg info. (pypa/pip#5870) ("Jinja2-2.10", "jinja2", 6), ("zope.interface-4.5.0", "zope-interface", 14), ("zope_interface-4.5.0", "zope-interface", 14), - # Should be smart enough to parse ambiguous names from the provided # package name. ("foo-2-2", "foo", 3), ("foo-2-2", "foo-2", 5), - # Should be able to detect collapsed characters in the egg info. ("foo--bar-1.0", "foo-bar", 8), ("foo-_bar-1.0", "foo-bar", 8), - # The package name must not ends with a dash (PEP 508), so the first # dash would be the separator, not the second. ("zope.interface--4.5.0", "zope-interface", 14), ("zope.interface--", "zope-interface", 14), - # The version part is missing, but the split function does not care. ("zope.interface-", "zope-interface", 14), ], ) -def test_find_name_version_sep(fragment, canonical_name, expected): +def test_find_name_version_sep( + fragment: str, canonical_name: str, expected: int +) -> None: index = _find_name_version_sep(fragment, canonical_name) assert index == expected @@ -760,7 +869,7 @@ def test_find_name_version_sep(fragment, canonical_name, expected): ("zope.interface", "zope-interface"), ], ) -def test_find_name_version_sep_failure(fragment, canonical_name): +def test_find_name_version_sep_failure(fragment: str, canonical_name: str) -> None: with pytest.raises(ValueError) as ctx: _find_name_version_sep(fragment, canonical_name) message = f"{fragment} does not match {canonical_name}" @@ -773,23 +882,19 @@ def test_find_name_version_sep_failure(fragment, canonical_name): # Trivial. ("pip-18.0", "pip", "18.0"), ("zope-interface-4.5.0", "zope-interface", "4.5.0"), - # Canonicalized name match non-canonicalized egg info. (pypa/pip#5870) ("Jinja2-2.10", "jinja2", "2.10"), ("zope.interface-4.5.0", "zope-interface", "4.5.0"), ("zope_interface-4.5.0", "zope-interface", "4.5.0"), - # Should be smart enough to parse ambiguous names from the provided # package name. ("foo-2-2", "foo", "2-2"), ("foo-2-2", "foo-2", "2"), ("zope.interface--4.5.0", "zope-interface", "-4.5.0"), ("zope.interface--", "zope-interface", "-"), - # Should be able to detect collapsed characters in the egg info. ("foo--bar-1.0", "foo-bar", "1.0"), ("foo-_bar-1.0", "foo-bar", "1.0"), - # Invalid. ("the-package-name-8.19", "does-not-match", None), ("zope.interface.-4.5.0", "zope.interface", None), @@ -800,6 +905,8 @@ def test_find_name_version_sep_failure(fragment, canonical_name): ("zope.interface", "zope-interface", None), ], ) -def test_extract_version_from_fragment(fragment, canonical_name, expected): +def test_extract_version_from_fragment( + fragment: str, canonical_name: str, expected: Optional[str] +) -> None: version = _extract_version_from_fragment(fragment, canonical_name) assert version == expected diff --git a/tests/unit/test_link.py b/tests/unit/test_link.py index a9e75e38bd3..99ed0aba76e 100644 --- a/tests/unit/test_link.py +++ b/tests/unit/test_link.py @@ -1,140 +1,214 @@ +from typing import Optional + import pytest -from pip._internal.models.link import Link +from pip._internal.models.link import Link, links_equivalent from pip._internal.utils.hashes import Hashes class TestLink: - - @pytest.mark.parametrize('url, expected', [ - ( - 'https://user:password@example.com/path/page.html', - '', - ), - ]) - def test_repr(self, url, expected): + @pytest.mark.parametrize( + "url, expected", + [ + ( + "https://user:password@example.com/path/page.html", + "", + ), + ], + ) + def test_repr(self, url: str, expected: str) -> None: link = Link(url) assert repr(link) == expected - @pytest.mark.parametrize('url, expected', [ - ('http://yo/wheel.whl', 'wheel.whl'), - ('http://yo/wheel', 'wheel'), - ('https://example.com/path/page.html', 'page.html'), - # Test a quoted character. - ('https://example.com/path/page%231.html', 'page#1.html'), - ( - 'http://yo/myproject-1.0%2Bfoobar.0-py2.py3-none-any.whl', - 'myproject-1.0+foobar.0-py2.py3-none-any.whl', - ), - # Test a path that ends in a slash. - ('https://example.com/path/', 'path'), - ('https://example.com/path//', 'path'), - # Test a url with no filename. - ('https://example.com/', 'example.com'), - # Test a url with no filename and with auth information. - ( - 'https://user:password@example.com/', - 'example.com', - ), - ]) - def test_filename(self, url, expected): + @pytest.mark.parametrize( + "url, expected", + [ + ("http://yo/wheel.whl", "wheel.whl"), + ("http://yo/wheel", "wheel"), + ("https://example.com/path/page.html", "page.html"), + # Test a quoted character. + ("https://example.com/path/page%231.html", "page#1.html"), + ( + "http://yo/myproject-1.0%2Bfoobar.0-py2.py3-none-any.whl", + "myproject-1.0+foobar.0-py2.py3-none-any.whl", + ), + # Test a path that ends in a slash. + ("https://example.com/path/", "path"), + ("https://example.com/path//", "path"), + # Test a url with no filename. + ("https://example.com/", "example.com"), + # Test a url with no filename and with auth information. + ( + "https://user:password@example.com/", + "example.com", + ), + ], + ) + def test_filename(self, url: str, expected: str) -> None: link = Link(url) assert link.filename == expected - def test_splitext(self): - assert ('wheel', '.whl') == Link('http://yo/wheel.whl').splitext() + def test_splitext(self) -> None: + assert ("wheel", ".whl") == Link("http://yo/wheel.whl").splitext() - def test_no_ext(self): - assert '' == Link('http://yo/wheel').ext + def test_no_ext(self) -> None: + assert "" == Link("http://yo/wheel").ext - def test_ext(self): - assert '.whl' == Link('http://yo/wheel.whl').ext + def test_ext(self) -> None: + assert ".whl" == Link("http://yo/wheel.whl").ext - def test_ext_fragment(self): - assert '.whl' == Link('http://yo/wheel.whl#frag').ext + def test_ext_fragment(self) -> None: + assert ".whl" == Link("http://yo/wheel.whl#frag").ext - def test_ext_query(self): - assert '.whl' == Link('http://yo/wheel.whl?a=b').ext + def test_ext_query(self) -> None: + assert ".whl" == Link("http://yo/wheel.whl?a=b").ext - def test_is_wheel(self): - assert Link('http://yo/wheel.whl').is_wheel + def test_is_wheel(self) -> None: + assert Link("http://yo/wheel.whl").is_wheel - def test_is_wheel_false(self): - assert not Link('http://yo/not_a_wheel').is_wheel + def test_is_wheel_false(self) -> None: + assert not Link("http://yo/not_a_wheel").is_wheel - def test_fragments(self): - url = 'git+https://example.com/package#egg=eggname' - assert 'eggname' == Link(url).egg_fragment + def test_fragments(self) -> None: + url = "git+https://example.com/package#egg=eggname" + assert "eggname" == Link(url).egg_fragment assert None is Link(url).subdirectory_fragment - url = 'git+https://example.com/package#egg=eggname&subdirectory=subdir' - assert 'eggname' == Link(url).egg_fragment - assert 'subdir' == Link(url).subdirectory_fragment - url = 'git+https://example.com/package#subdirectory=subdir&egg=eggname' - assert 'eggname' == Link(url).egg_fragment - assert 'subdir' == Link(url).subdirectory_fragment - - @pytest.mark.parametrize('yanked_reason, expected', [ - (None, False), - ('', True), - ('there was a mistake', True), - ]) - def test_is_yanked(self, yanked_reason, expected): + url = "git+https://example.com/package#egg=eggname&subdirectory=subdir" + assert "eggname" == Link(url).egg_fragment + assert "subdir" == Link(url).subdirectory_fragment + url = "git+https://example.com/package#subdirectory=subdir&egg=eggname" + assert "eggname" == Link(url).egg_fragment + assert "subdir" == Link(url).subdirectory_fragment + + @pytest.mark.parametrize( + "yanked_reason, expected", + [ + (None, False), + ("", True), + ("there was a mistake", True), + ], + ) + def test_is_yanked(self, yanked_reason: Optional[str], expected: bool) -> None: link = Link( - 'https://example.com/wheel.whl', + "https://example.com/wheel.whl", yanked_reason=yanked_reason, ) assert link.is_yanked == expected - @pytest.mark.parametrize('hash_name, hex_digest, expected', [ - # Test a value that matches but with the wrong hash_name. - ('sha384', 128 * 'a', False), - # Test matching values, including values other than the first. - ('sha512', 128 * 'a', True), - ('sha512', 128 * 'b', True), - # Test a matching hash_name with a value that doesn't match. - ('sha512', 128 * 'c', False), - # Test a link without a hash value. - ('sha512', '', False), - ]) - def test_is_hash_allowed(self, hash_name, hex_digest, expected): - url = ( - 'https://example.com/wheel.whl#{hash_name}={hex_digest}'.format( - hash_name=hash_name, - hex_digest=hex_digest, - ) + @pytest.mark.parametrize( + "hash_name, hex_digest, expected", + [ + # Test a value that matches but with the wrong hash_name. + ("sha384", 128 * "a", False), + # Test matching values, including values other than the first. + ("sha512", 128 * "a", True), + ("sha512", 128 * "b", True), + # Test a matching hash_name with a value that doesn't match. + ("sha512", 128 * "c", False), + # Test a link without a hash value. + ("sha512", "", False), + ], + ) + def test_is_hash_allowed( + self, hash_name: str, hex_digest: str, expected: bool + ) -> None: + url = "https://example.com/wheel.whl#{hash_name}={hex_digest}".format( + hash_name=hash_name, + hex_digest=hex_digest, ) link = Link(url) hashes_data = { - 'sha512': [128 * 'a', 128 * 'b'], + "sha512": [128 * "a", 128 * "b"], } hashes = Hashes(hashes_data) assert link.is_hash_allowed(hashes) == expected - def test_is_hash_allowed__no_hash(self): - link = Link('https://example.com/wheel.whl') + def test_is_hash_allowed__no_hash(self) -> None: + link = Link("https://example.com/wheel.whl") hashes_data = { - 'sha512': [128 * 'a'], + "sha512": [128 * "a"], } hashes = Hashes(hashes_data) assert not link.is_hash_allowed(hashes) - @pytest.mark.parametrize('hashes, expected', [ - (None, False), - # Also test a success case to show the test is correct. - (Hashes({'sha512': [128 * 'a']}), True), - ]) - def test_is_hash_allowed__none_hashes(self, hashes, expected): - url = 'https://example.com/wheel.whl#sha512={}'.format(128 * 'a') + @pytest.mark.parametrize( + "hashes, expected", + [ + (None, False), + # Also test a success case to show the test is correct. + (Hashes({"sha512": [128 * "a"]}), True), + ], + ) + def test_is_hash_allowed__none_hashes( + self, hashes: Optional[Hashes], expected: bool + ) -> None: + url = "https://example.com/wheel.whl#sha512={}".format(128 * "a") link = Link(url) assert link.is_hash_allowed(hashes) == expected - @pytest.mark.parametrize('url, expected', [ - ('git+https://github.com/org/repo', True), - ('bzr+http://bzr.myproject.org/MyProject/trunk/#egg=MyProject', True), - ('hg+file://hg.company.com/repo', True), - ('https://example.com/some.whl', False), - ('file://home/foo/some.whl', False), - ]) - def test_is_vcs(self, url, expected): + @pytest.mark.parametrize( + "url, expected", + [ + ("git+https://github.com/org/repo", True), + ("bzr+http://bzr.myproject.org/MyProject/trunk/#egg=MyProject", True), + ("hg+file://hg.company.com/repo", True), + ("https://example.com/some.whl", False), + ("file://home/foo/some.whl", False), + ], + ) + def test_is_vcs(self, url: str, expected: bool) -> None: link = Link(url) assert link.is_vcs is expected + + +@pytest.mark.parametrize( + "url1, url2", + [ + pytest.param( + "https://example.com/foo#egg=foo", + "https://example.com/foo", + id="drop-egg", + ), + pytest.param( + "https://example.com/foo#subdirectory=bar&egg=foo", + "https://example.com/foo#subdirectory=bar&egg=bar", + id="drop-egg-only", + ), + pytest.param( + "https://example.com/foo#subdirectory=bar&egg=foo", + "https://example.com/foo#egg=foo&subdirectory=bar", + id="fragment-ordering", + ), + pytest.param( + "https://example.com/foo?a=1&b=2", + "https://example.com/foo?b=2&a=1", + id="query-opordering", + ), + ], +) +def test_links_equivalent(url1: str, url2: str) -> None: + assert links_equivalent(Link(url1), Link(url2)) + + +@pytest.mark.parametrize( + "url1, url2", + [ + pytest.param( + "https://example.com/foo#sha512=1234567890abcdef", + "https://example.com/foo#sha512=abcdef1234567890", + id="different-keys", + ), + pytest.param( + "https://example.com/foo#sha512=1234567890abcdef", + "https://example.com/foo#md5=1234567890abcdef", + id="different-values", + ), + pytest.param( + "https://example.com/foo#subdirectory=bar&egg=foo", + "https://example.com/foo#subdirectory=rex", + id="drop-egg-still-different", + ), + ], +) +def test_links_equivalent_false(url1: str, url2: str) -> None: + assert not links_equivalent(Link(url1), Link(url2)) diff --git a/tests/unit/test_locations.py b/tests/unit/test_locations.py index 2ede236f91f..023b6687d41 100644 --- a/tests/unit/test_locations.py +++ b/tests/unit/test_locations.py @@ -6,35 +6,43 @@ import os import shutil import sys +import sysconfig import tempfile +from typing import Any, Dict +from unittest.mock import Mock import pytest -from mock import Mock -from pip._internal.locations import distutils_scheme +from pip._internal.locations import SCHEME_KEYS, _should_use_sysconfig, get_scheme +from tests.lib.path import Path -if sys.platform == 'win32': +if sys.platform == "win32": pwd = Mock() else: import pwd +def _get_scheme_dict(*args: Any, **kwargs: Any) -> Dict[str, str]: + scheme = get_scheme(*args, **kwargs) + return {k: getattr(scheme, k) for k in SCHEME_KEYS} + + class TestLocations: - def setup(self): + def setup(self) -> None: self.tempdir = tempfile.mkdtemp() self.st_uid = 9999 self.username = "example" self.patch() - def teardown(self): + def teardown(self) -> None: self.revert_patch() shutil.rmtree(self.tempdir, ignore_errors=True) - def patch(self): - """ first store and then patch python methods pythons """ + def patch(self) -> None: + """first store and then patch python methods pythons""" self.tempfile_gettempdir = tempfile.gettempdir self.old_os_fstat = os.fstat - if sys.platform != 'win32': + if sys.platform != "win32": # os.geteuid and pwd.getpwuid are not implemented on windows self.old_os_geteuid = os.geteuid self.old_pwd_getpwuid = pwd.getpwuid @@ -43,59 +51,80 @@ def patch(self): # now patch tempfile.gettempdir = lambda: self.tempdir getpass.getuser = lambda: self.username - os.geteuid = lambda: self.st_uid os.fstat = lambda fd: self.get_mock_fstat(fd) + if sys.platform != "win32": + os.geteuid = lambda: self.st_uid + pwd.getpwuid = self.get_mock_getpwuid - if sys.platform != 'win32': - pwd.getpwuid = lambda uid: self.get_mock_getpwuid(uid) - - def revert_patch(self): - """ revert the patches to python methods """ + def revert_patch(self) -> None: + """revert the patches to python methods""" tempfile.gettempdir = self.tempfile_gettempdir getpass.getuser = self.old_getpass_getuser - if sys.platform != 'win32': + if sys.platform != "win32": # os.geteuid and pwd.getpwuid are not implemented on windows os.geteuid = self.old_os_geteuid pwd.getpwuid = self.old_pwd_getpwuid os.fstat = self.old_os_fstat - def get_mock_fstat(self, fd): - """ returns a basic mock fstat call result. - Currently only the st_uid attribute has been set. + def get_mock_fstat(self, fd: int) -> os.stat_result: + """returns a basic mock fstat call result. + Currently only the st_uid attribute has been set. """ result = Mock() result.st_uid = self.st_uid return result - def get_mock_getpwuid(self, uid): - """ returns a basic mock pwd.getpwuid call result. - Currently only the pw_name attribute has been set. + def get_mock_getpwuid(self, uid: int) -> Any: + """returns a basic mock pwd.getpwuid call result. + Currently only the pw_name attribute has been set. """ result = Mock() result.pw_name = self.username return result + def test_default_should_use_sysconfig( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delattr(sysconfig, "_PIP_USE_SYSCONFIG", raising=False) + if sys.version_info[:2] >= (3, 10): + assert _should_use_sysconfig() is True + else: + assert _should_use_sysconfig() is False + + @pytest.mark.parametrize("vendor_value", [True, False, None, "", 0, 1]) + def test_vendor_overriden_should_use_sysconfig( + self, monkeypatch: pytest.MonkeyPatch, vendor_value: Any + ) -> None: + monkeypatch.setattr( + sysconfig, "_PIP_USE_SYSCONFIG", vendor_value, raising=False + ) + assert _should_use_sysconfig() is bool(vendor_value) + class TestDistutilsScheme: - - def test_root_modifies_appropriately(self, monkeypatch): + def test_root_modifies_appropriately(self) -> None: # This deals with nt/posix path differences # root is c:\somewhere\else or /somewhere/else - root = os.path.normcase(os.path.abspath( - os.path.join(os.path.sep, 'somewhere', 'else'))) - norm_scheme = distutils_scheme("example") - root_scheme = distutils_scheme("example", root=root) + root = os.path.normcase( + os.path.abspath(os.path.join(os.path.sep, "somewhere", "else")) + ) + norm_scheme = _get_scheme_dict("example") + root_scheme = _get_scheme_dict("example", root=root) for key, value in norm_scheme.items(): drive, path = os.path.splitdrive(os.path.abspath(value)) expected = os.path.join(root, path[1:]) assert os.path.abspath(root_scheme[key]) == expected + @pytest.mark.incompatible_with_sysconfig @pytest.mark.incompatible_with_venv - def test_distutils_config_file_read(self, tmpdir, monkeypatch): + def test_distutils_config_file_read( + self, tmpdir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: # This deals with nt/posix path differences - install_scripts = os.path.normcase(os.path.abspath( - os.path.join(os.path.sep, 'somewhere', 'else'))) + install_scripts = os.path.normcase( + os.path.abspath(os.path.join(os.path.sep, "somewhere", "else")) + ) f = tmpdir / "config" / "setup.cfg" f.parent.mkdir() f.write_text("[install]\ninstall-scripts=" + install_scripts) @@ -104,20 +133,24 @@ def test_distutils_config_file_read(self, tmpdir, monkeypatch): # patch the function that returns what config files are present monkeypatch.setattr( Distribution, - 'find_config_files', + "find_config_files", lambda self: [f], ) - scheme = distutils_scheme('example') - assert scheme['scripts'] == install_scripts + scheme = _get_scheme_dict("example") + assert scheme["scripts"] == install_scripts + @pytest.mark.incompatible_with_sysconfig @pytest.mark.incompatible_with_venv # when we request install-lib, we should install everything (.py & # .so) into that path; i.e. ensure platlib & purelib are set to - # this path - def test_install_lib_takes_precedence(self, tmpdir, monkeypatch): + # this path. sysconfig does not support this. + def test_install_lib_takes_precedence( + self, tmpdir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: # This deals with nt/posix path differences - install_lib = os.path.normcase(os.path.abspath( - os.path.join(os.path.sep, 'somewhere', 'else'))) + install_lib = os.path.normcase( + os.path.abspath(os.path.join(os.path.sep, "somewhere", "else")) + ) f = tmpdir / "config" / "setup.cfg" f.parent.mkdir() f.write_text("[install]\ninstall-lib=" + install_lib) @@ -126,25 +159,22 @@ def test_install_lib_takes_precedence(self, tmpdir, monkeypatch): # patch the function that returns what config files are present monkeypatch.setattr( Distribution, - 'find_config_files', + "find_config_files", lambda self: [f], ) - scheme = distutils_scheme('example') - assert scheme['platlib'] == install_lib + os.path.sep - assert scheme['purelib'] == install_lib + os.path.sep + scheme = _get_scheme_dict("example") + assert scheme["platlib"] == install_lib + os.path.sep + assert scheme["purelib"] == install_lib + os.path.sep - def test_prefix_modifies_appropriately(self): - prefix = os.path.abspath(os.path.join('somewhere', 'else')) + def test_prefix_modifies_appropriately(self) -> None: + prefix = os.path.abspath(os.path.join("somewhere", "else")) - normal_scheme = distutils_scheme("example") - prefix_scheme = distutils_scheme("example", prefix=prefix) + normal_scheme = _get_scheme_dict("example") + prefix_scheme = _get_scheme_dict("example", prefix=prefix) - def _calculate_expected(value): + def _calculate_expected(value: str) -> str: path = os.path.join(prefix, os.path.relpath(value, sys.prefix)) return os.path.normpath(path) - expected = { - k: _calculate_expected(v) - for k, v in normal_scheme.items() - } + expected = {k: _calculate_expected(v) for k, v in normal_scheme.items()} assert prefix_scheme == expected diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index b3da43cb857..4f0447931dd 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -1,13 +1,13 @@ import logging from threading import Thread +from unittest.mock import patch import pytest -from mock import patch from pip._internal.utils.logging import ( BrokenStdoutLoggingError, - ColorizedStreamHandler, IndentingFormatter, + RichPipStreamHandler, indent_log, ) from pip._internal.utils.misc import captured_stderr, captured_stdout @@ -18,7 +18,7 @@ class TestIndentingFormatter: """Test ``pip._internal.utils.logging.IndentingFormatter``.""" - def make_record(self, msg, level_name): + def make_record(self, msg: str, level_name: str) -> logging.LogRecord: level_number = getattr(logging, level_name) attrs = dict( msg=msg, @@ -31,59 +31,72 @@ def make_record(self, msg, level_name): return record - @pytest.mark.parametrize('level_name, expected', [ - ('DEBUG', 'hello\nworld'), - ('INFO', 'hello\nworld'), - ('WARNING', 'WARNING: hello\nworld'), - ('ERROR', 'ERROR: hello\nworld'), - ('CRITICAL', 'ERROR: hello\nworld'), - ]) - def test_format(self, level_name, expected, utc): + @pytest.mark.parametrize( + "level_name, expected", + [ + ("DEBUG", "hello\nworld"), + ("INFO", "hello\nworld"), + ("WARNING", "WARNING: hello\nworld"), + ("ERROR", "ERROR: hello\nworld"), + ("CRITICAL", "ERROR: hello\nworld"), + ], + ) + def test_format(self, level_name: str, expected: str, utc: None) -> None: """ Args: level_name: a logging level name (e.g. "WARNING"). """ - record = self.make_record('hello\nworld', level_name=level_name) + record = self.make_record("hello\nworld", level_name=level_name) f = IndentingFormatter(fmt="%(message)s") assert f.format(record) == expected - @pytest.mark.parametrize('level_name, expected', [ - ('INFO', - '2019-01-17T06:00:37,040 hello\n' - '2019-01-17T06:00:37,040 world'), - ('WARNING', - '2019-01-17T06:00:37,040 WARNING: hello\n' - '2019-01-17T06:00:37,040 world'), - ]) - def test_format_with_timestamp(self, level_name, expected, utc): - record = self.make_record('hello\nworld', level_name=level_name) + @pytest.mark.parametrize( + "level_name, expected", + [ + ("INFO", "2019-01-17T06:00:37,040 hello\n2019-01-17T06:00:37,040 world"), + ( + "WARNING", + "2019-01-17T06:00:37,040 WARNING: hello\n" + "2019-01-17T06:00:37,040 world", + ), + ], + ) + def test_format_with_timestamp( + self, level_name: str, expected: str, utc: None + ) -> None: + record = self.make_record("hello\nworld", level_name=level_name) f = IndentingFormatter(fmt="%(message)s", add_timestamp=True) assert f.format(record) == expected - @pytest.mark.parametrize('level_name, expected', [ - ('WARNING', 'DEPRECATION: hello\nworld'), - ('ERROR', 'DEPRECATION: hello\nworld'), - ('CRITICAL', 'DEPRECATION: hello\nworld'), - ]) - def test_format_deprecated(self, level_name, expected, utc): + @pytest.mark.parametrize( + "level_name, expected", + [ + ("WARNING", "DEPRECATION: hello\nworld"), + ("ERROR", "DEPRECATION: hello\nworld"), + ("CRITICAL", "DEPRECATION: hello\nworld"), + ], + ) + def test_format_deprecated(self, level_name: str, expected: str, utc: None) -> None: """ Test that logged deprecation warnings coming from deprecated() don't get another prefix. """ record = self.make_record( - 'DEPRECATION: hello\nworld', level_name=level_name, + "DEPRECATION: hello\nworld", + level_name=level_name, ) f = IndentingFormatter(fmt="%(message)s") assert f.format(record) == expected - def test_thread_safety_base(self, utc): + def test_thread_safety_base(self, utc: None) -> None: record = self.make_record( - 'DEPRECATION: hello\nworld', level_name='WARNING', + "DEPRECATION: hello\nworld", + level_name="WARNING", ) f = IndentingFormatter(fmt="%(message)s") results = [] - def thread_function(): + def thread_function() -> None: results.append(f.format(record)) thread_function() @@ -92,14 +105,15 @@ def thread_function(): thread.join() assert results[0] == results[1] - def test_thread_safety_indent_log(self, utc): + def test_thread_safety_indent_log(self, utc: None) -> None: record = self.make_record( - 'DEPRECATION: hello\nworld', level_name='WARNING', + "DEPRECATION: hello\nworld", + level_name="WARNING", ) f = IndentingFormatter(fmt="%(message)s") results = [] - def thread_function(): + def thread_function() -> None: with indent_log(): results.append(f.format(record)) @@ -111,16 +125,15 @@ def thread_function(): class TestColorizedStreamHandler: - - def _make_log_record(self): + def _make_log_record(self) -> logging.LogRecord: attrs = { - 'msg': 'my error', + "msg": "my error", } record = logging.makeLogRecord(attrs) return record - def test_broken_pipe_in_stderr_flush(self): + def test_broken_pipe_in_stderr_flush(self) -> None: """ Test sys.stderr.flush() raising BrokenPipeError. @@ -129,21 +142,21 @@ def test_broken_pipe_in_stderr_flush(self): record = self._make_log_record() with captured_stderr() as stderr: - handler = ColorizedStreamHandler(stream=stderr) - with patch('sys.stderr.flush') as mock_flush: + handler = RichPipStreamHandler(stream=stderr, no_color=True) + with patch("sys.stderr.flush") as mock_flush: mock_flush.side_effect = BrokenPipeError() # The emit() call raises no exception. handler.emit(record) err_text = stderr.getvalue() - assert err_text.startswith('my error') + assert err_text.startswith("my error") # Check that the logging framework tried to log the exception. - assert 'Logging error' in err_text - assert 'BrokenPipeError' in err_text + assert "Logging error" in err_text + assert "BrokenPipeError" in err_text assert "Message: 'my error'" in err_text - def test_broken_pipe_in_stdout_write(self): + def test_broken_pipe_in_stdout_write(self) -> None: """ Test sys.stdout.write() raising BrokenPipeError. @@ -152,13 +165,13 @@ def test_broken_pipe_in_stdout_write(self): record = self._make_log_record() with captured_stdout() as stdout: - handler = ColorizedStreamHandler(stream=stdout) - with patch('sys.stdout.write') as mock_write: + handler = RichPipStreamHandler(stream=stdout, no_color=True) + with patch("sys.stdout.write") as mock_write: mock_write.side_effect = BrokenPipeError() with pytest.raises(BrokenStdoutLoggingError): handler.emit(record) - def test_broken_pipe_in_stdout_flush(self): + def test_broken_pipe_in_stdout_flush(self) -> None: """ Test sys.stdout.flush() raising BrokenPipeError. @@ -167,8 +180,8 @@ def test_broken_pipe_in_stdout_flush(self): record = self._make_log_record() with captured_stdout() as stdout: - handler = ColorizedStreamHandler(stream=stdout) - with patch('sys.stdout.flush') as mock_flush: + handler = RichPipStreamHandler(stream=stdout, no_color=True) + with patch("sys.stdout.flush") as mock_flush: mock_flush.side_effect = BrokenPipeError() with pytest.raises(BrokenStdoutLoggingError): handler.emit(record) @@ -177,4 +190,4 @@ def test_broken_pipe_in_stdout_flush(self): # Sanity check that the log record was written, since flush() happens # after write(). - assert output.startswith('my error') + assert output.startswith("my error") diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 8e2975bd7e2..c5545e37d01 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -4,13 +4,13 @@ from pip._vendor.packaging.version import parse as parse_version from pip._internal.models import candidate, index +from pip._internal.models.link import Link class TestPackageIndex: - """Tests for pip._internal.models.index.PackageIndex - """ + """Tests for pip._internal.models.index.PackageIndex""" - def test_gives_right_urls(self): + def test_gives_right_urls(self) -> None: url = "https://mypypi.internal/path/" file_storage_domain = "files.mypypi.internal" pack_index = index.PackageIndex(url, file_storage_domain) @@ -22,7 +22,7 @@ def test_gives_right_urls(self): assert pack_index.simple_url == url + "simple" assert pack_index.pypi_url == url + "pypi" - def test_PyPI_urls_are_correct(self): + def test_PyPI_urls_are_correct(self) -> None: pack_index = index.PyPI assert pack_index.netloc == "pypi.org" @@ -31,7 +31,7 @@ def test_PyPI_urls_are_correct(self): assert pack_index.pypi_url == "https://pypi.org/pypi" assert pack_index.file_storage_domain == "files.pythonhosted.org" - def test_TestPyPI_urls_are_correct(self): + def test_TestPyPI_urls_are_correct(self) -> None: pack_index = index.TestPyPI assert pack_index.netloc == "test.pypi.org" @@ -42,19 +42,18 @@ def test_TestPyPI_urls_are_correct(self): class TestInstallationCandidate: - - def test_sets_correct_variables(self): + def test_sets_correct_variables(self) -> None: obj = candidate.InstallationCandidate( - "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" + "A", "1.0.0", Link("https://somewhere.com/path/A-1.0.0.tar.gz") ) assert obj.name == "A" assert obj.version == parse_version("1.0.0") - assert obj.link == "https://somewhere.com/path/A-1.0.0.tar.gz" + assert obj.link.url == "https://somewhere.com/path/A-1.0.0.tar.gz" # NOTE: This isn't checking the ordering logic; only the data provided to # it is correct. - def test_sets_the_right_key(self): + def test_sets_the_right_key(self) -> None: obj = candidate.InstallationCandidate( - "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" + "A", "1.0.0", Link("https://somewhere.com/path/A-1.0.0.tar.gz") ) assert obj._compare_key == (obj.name, obj.version, obj.link) diff --git a/tests/unit/test_models_wheel.py b/tests/unit/test_models_wheel.py index e0d45f5c840..c06525e089e 100644 --- a/tests/unit/test_models_wheel.py +++ b/tests/unit/test_models_wheel.py @@ -7,131 +7,122 @@ class TestWheelFile: - - def test_std_wheel_pattern(self): - w = Wheel('simple-1.1.1-py2-none-any.whl') - assert w.name == 'simple' - assert w.version == '1.1.1' - assert w.pyversions == ['py2'] - assert w.abis == ['none'] - assert w.plats == ['any'] - - def test_wheel_pattern_multi_values(self): - w = Wheel('simple-1.1-py2.py3-abi1.abi2-any.whl') - assert w.name == 'simple' - assert w.version == '1.1' - assert w.pyversions == ['py2', 'py3'] - assert w.abis == ['abi1', 'abi2'] - assert w.plats == ['any'] - - def test_wheel_with_build_tag(self): + def test_std_wheel_pattern(self) -> None: + w = Wheel("simple-1.1.1-py2-none-any.whl") + assert w.name == "simple" + assert w.version == "1.1.1" + assert w.pyversions == ["py2"] + assert w.abis == ["none"] + assert w.plats == ["any"] + + def test_wheel_pattern_multi_values(self) -> None: + w = Wheel("simple-1.1-py2.py3-abi1.abi2-any.whl") + assert w.name == "simple" + assert w.version == "1.1" + assert w.pyversions == ["py2", "py3"] + assert w.abis == ["abi1", "abi2"] + assert w.plats == ["any"] + + def test_wheel_with_build_tag(self) -> None: # pip doesn't do anything with build tags, but theoretically, we might # see one, in this case the build tag = '4' - w = Wheel('simple-1.1-4-py2-none-any.whl') - assert w.name == 'simple' - assert w.version == '1.1' - assert w.pyversions == ['py2'] - assert w.abis == ['none'] - assert w.plats == ['any'] - - def test_single_digit_version(self): - w = Wheel('simple-1-py2-none-any.whl') - assert w.version == '1' - - def test_non_pep440_version(self): - w = Wheel('simple-_invalid_-py2-none-any.whl') - assert w.version == '-invalid-' - - def test_missing_version_raises(self): + w = Wheel("simple-1.1-4-py2-none-any.whl") + assert w.name == "simple" + assert w.version == "1.1" + assert w.pyversions == ["py2"] + assert w.abis == ["none"] + assert w.plats == ["any"] + + def test_single_digit_version(self) -> None: + w = Wheel("simple-1-py2-none-any.whl") + assert w.version == "1" + + def test_non_pep440_version(self) -> None: + w = Wheel("simple-_invalid_-py2-none-any.whl") + assert w.version == "-invalid-" + + def test_missing_version_raises(self) -> None: with pytest.raises(InvalidWheelFilename): - Wheel('Cython-cp27-none-linux_x86_64.whl') + Wheel("Cython-cp27-none-linux_x86_64.whl") - def test_invalid_filename_raises(self): + def test_invalid_filename_raises(self) -> None: with pytest.raises(InvalidWheelFilename): - Wheel('invalid.whl') + Wheel("invalid.whl") - def test_supported_single_version(self): + def test_supported_single_version(self) -> None: """ Test single-version wheel is known to be supported """ - w = Wheel('simple-0.1-py2-none-any.whl') - assert w.supported(tags=[Tag('py2', 'none', 'any')]) + w = Wheel("simple-0.1-py2-none-any.whl") + assert w.supported(tags=[Tag("py2", "none", "any")]) - def test_supported_multi_version(self): + def test_supported_multi_version(self) -> None: """ Test multi-version wheel is known to be supported """ - w = Wheel('simple-0.1-py2.py3-none-any.whl') - assert w.supported(tags=[Tag('py3', 'none', 'any')]) + w = Wheel("simple-0.1-py2.py3-none-any.whl") + assert w.supported(tags=[Tag("py3", "none", "any")]) - def test_not_supported_version(self): + def test_not_supported_version(self) -> None: """ Test unsupported wheel is known to be unsupported """ - w = Wheel('simple-0.1-py2-none-any.whl') - assert not w.supported(tags=[Tag('py1', 'none', 'any')]) + w = Wheel("simple-0.1-py2-none-any.whl") + assert not w.supported(tags=[Tag("py1", "none", "any")]) - def test_supported_osx_version(self): + def test_supported_osx_version(self) -> None: """ Wheels built for macOS 10.6 are supported on 10.9 """ tags = compatibility_tags.get_supported( - '27', platforms=['macosx_10_9_intel'], impl='cp' + "27", platforms=["macosx_10_9_intel"], impl="cp" ) - w = Wheel('simple-0.1-cp27-none-macosx_10_6_intel.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_6_intel.whl") assert w.supported(tags=tags) - w = Wheel('simple-0.1-cp27-none-macosx_10_9_intel.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_9_intel.whl") assert w.supported(tags=tags) - def test_not_supported_osx_version(self): + def test_not_supported_osx_version(self) -> None: """ Wheels built for macOS 10.9 are not supported on 10.6 """ tags = compatibility_tags.get_supported( - '27', platforms=['macosx_10_6_intel'], impl='cp' + "27", platforms=["macosx_10_6_intel"], impl="cp" ) - w = Wheel('simple-0.1-cp27-none-macosx_10_9_intel.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_9_intel.whl") assert not w.supported(tags=tags) - @pytest.mark.xfail( - reason=( - "packaging.tags changed behaviour in this area, and @pradyunsg " - "decided as the release manager that this behaviour change is less " - "critical than Big Sur support for pip 20.3. See " - "https://github.com/pypa/packaging/pull/361 for further discussion." - ) - ) - def test_supported_multiarch_darwin(self): + def test_supported_multiarch_darwin(self) -> None: """ Multi-arch wheels (intel) are supported on components (i386, x86_64) """ universal = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_universal'], impl='cp' + "27", platforms=["macosx_10_5_universal"], impl="cp" ) intel = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_intel'], impl='cp' + "27", platforms=["macosx_10_5_intel"], impl="cp" ) x64 = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_x86_64'], impl='cp' + "27", platforms=["macosx_10_5_x86_64"], impl="cp" ) i386 = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_i386'], impl='cp' + "27", platforms=["macosx_10_5_i386"], impl="cp" ) ppc = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_ppc'], impl='cp' + "27", platforms=["macosx_10_5_ppc"], impl="cp" ) ppc64 = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_ppc64'], impl='cp' + "27", platforms=["macosx_10_5_ppc64"], impl="cp" ) - w = Wheel('simple-0.1-cp27-none-macosx_10_5_intel.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_5_intel.whl") assert w.supported(tags=intel) assert w.supported(tags=x64) assert w.supported(tags=i386) assert not w.supported(tags=universal) assert not w.supported(tags=ppc) assert not w.supported(tags=ppc64) - w = Wheel('simple-0.1-cp27-none-macosx_10_5_universal.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_5_universal.whl") assert w.supported(tags=universal) assert w.supported(tags=intel) assert w.supported(tags=x64) @@ -139,50 +130,50 @@ def test_supported_multiarch_darwin(self): assert w.supported(tags=ppc) assert w.supported(tags=ppc64) - def test_not_supported_multiarch_darwin(self): + def test_not_supported_multiarch_darwin(self) -> None: """ Single-arch wheels (x86_64) are not supported on multi-arch (intel) """ universal = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_universal'], impl='cp' + "27", platforms=["macosx_10_5_universal"], impl="cp" ) intel = compatibility_tags.get_supported( - '27', platforms=['macosx_10_5_intel'], impl='cp' + "27", platforms=["macosx_10_5_intel"], impl="cp" ) - w = Wheel('simple-0.1-cp27-none-macosx_10_5_i386.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_5_i386.whl") assert not w.supported(tags=intel) assert not w.supported(tags=universal) - w = Wheel('simple-0.1-cp27-none-macosx_10_5_x86_64.whl') + w = Wheel("simple-0.1-cp27-none-macosx_10_5_x86_64.whl") assert not w.supported(tags=intel) assert not w.supported(tags=universal) - def test_support_index_min(self): + def test_support_index_min(self) -> None: """ Test results from `support_index_min` """ tags = [ - Tag('py2', 'none', 'TEST'), - Tag('py2', 'TEST', 'any'), - Tag('py2', 'none', 'any'), + Tag("py2", "none", "TEST"), + Tag("py2", "TEST", "any"), + Tag("py2", "none", "any"), ] - w = Wheel('simple-0.1-py2-none-any.whl') + w = Wheel("simple-0.1-py2-none-any.whl") assert w.support_index_min(tags=tags) == 2 - w = Wheel('simple-0.1-py2-none-TEST.whl') + w = Wheel("simple-0.1-py2-none-TEST.whl") assert w.support_index_min(tags=tags) == 0 - def test_support_index_min__none_supported(self): + def test_support_index_min__none_supported(self) -> None: """ Test a wheel not supported by the given tags. """ - w = Wheel('simple-0.1-py2-none-any.whl') + w = Wheel("simple-0.1-py2-none-any.whl") with pytest.raises(ValueError): w.support_index_min(tags=[]) - def test_version_underscore_conversion(self): + def test_version_underscore_conversion(self) -> None: """ Test that we convert '_' to '-' for versions parsed out of wheel filenames """ - w = Wheel('simple-0.1_1-py2-none-any.whl') - assert w.version == '0.1-1' + w = Wheel("simple-0.1_1-py2-none-any.whl") + assert w.version == "0.1-1" diff --git a/tests/unit/test_network_auth.py b/tests/unit/test_network_auth.py index 44c739d864f..5c0e5746281 100644 --- a/tests/unit/test_network_auth.py +++ b/tests/unit/test_network_auth.py @@ -1,4 +1,5 @@ import functools +from typing import Any, List, Optional, Tuple import pytest @@ -7,33 +8,38 @@ from tests.lib.requests_mocks import MockConnection, MockRequest, MockResponse -@pytest.mark.parametrize(["input_url", "url", "username", "password"], [ - ( - "http://user%40email.com:password@example.com/path", - "http://example.com/path", - "user@email.com", - "password", - ), - ( - "http://username:password@example.com/path", - "http://example.com/path", - "username", - "password", - ), - ( - "http://token@example.com/path", - "http://example.com/path", - "token", - "", - ), - ( - "http://example.com/path", - "http://example.com/path", - None, - None, - ), -]) -def test_get_credentials_parses_correctly(input_url, url, username, password): +@pytest.mark.parametrize( + ["input_url", "url", "username", "password"], + [ + ( + "http://user%40email.com:password@example.com/path", + "http://example.com/path", + "user@email.com", + "password", + ), + ( + "http://username:password@example.com/path", + "http://example.com/path", + "username", + "password", + ), + ( + "http://token@example.com/path", + "http://example.com/path", + "token", + "", + ), + ( + "http://example.com/path", + "http://example.com/path", + None, + None, + ), + ], +) +def test_get_credentials_parses_correctly( + input_url: str, url: str, username: Optional[str], password: Optional[str] +) -> None: auth = MultiDomainBasicAuth() get = auth._get_url_and_credentials @@ -41,33 +47,57 @@ def test_get_credentials_parses_correctly(input_url, url, username, password): assert get(input_url) == (url, username, password) assert ( # There are no credentials in the URL - (username is None and password is None) or + (username is None and password is None) + or # Credentials were found and "cached" appropriately - auth.passwords['example.com'] == (username, password) + auth.passwords["example.com"] == (username, password) ) -def test_get_credentials_uses_cached_credentials(): +def test_get_credentials_not_to_uses_cached_credentials() -> None: auth = MultiDomainBasicAuth() - auth.passwords['example.com'] = ('user', 'pass') + auth.passwords["example.com"] = ("user", "pass") got = auth._get_url_and_credentials("http://foo:bar@example.com/path") - expected = ('http://example.com/path', 'user', 'pass') + expected = ("http://example.com/path", "foo", "bar") + assert got == expected + + +def test_get_credentials_not_to_uses_cached_credentials_only_username() -> None: + auth = MultiDomainBasicAuth() + auth.passwords["example.com"] = ("user", "pass") + + got = auth._get_url_and_credentials("http://foo@example.com/path") + expected = ("http://example.com/path", "foo", "") + assert got == expected + + +def test_get_credentials_uses_cached_credentials() -> None: + auth = MultiDomainBasicAuth() + auth.passwords["example.com"] = ("user", "pass") + + got = auth._get_url_and_credentials("http://example.com/path") + expected = ("http://example.com/path", "user", "pass") + assert got == expected + + +def test_get_credentials_uses_cached_credentials_only_username() -> None: + auth = MultiDomainBasicAuth() + auth.passwords["example.com"] = ("user", "pass") + + got = auth._get_url_and_credentials("http://user@example.com/path") + expected = ("http://example.com/path", "user", "pass") assert got == expected -def test_get_index_url_credentials(): - auth = MultiDomainBasicAuth(index_urls=[ - "http://foo:bar@example.com/path" - ]) +def test_get_index_url_credentials() -> None: + auth = MultiDomainBasicAuth(index_urls=["http://foo:bar@example.com/path"]) get = functools.partial( - auth._get_new_credentials, - allow_netrc=False, - allow_keyring=False + auth._get_new_credentials, allow_netrc=False, allow_keyring=False ) # Check resolution of indexes - assert get("http://example.com/path/path2") == ('foo', 'bar') + assert get("http://example.com/path/path2") == ("foo", "bar") assert get("http://example.com/path3/path2") == (None, None) @@ -76,126 +106,150 @@ class KeyringModuleV1: was added. """ - def __init__(self): - self.saved_passwords = [] + def __init__(self) -> None: + self.saved_passwords: List[Tuple[str, str, str]] = [] - def get_password(self, system, username): + def get_password(self, system: str, username: str) -> Optional[str]: if system == "example.com" and username: return username + "!netloc" if system == "http://example.com/path2" and username: return username + "!url" return None - def set_password(self, system, username, password): + def set_password(self, system: str, username: str, password: str) -> None: self.saved_passwords.append((system, username, password)) -@pytest.mark.parametrize('url, expect', ( - ("http://example.com/path1", (None, None)), - # path1 URLs will be resolved by netloc - ("http://user@example.com/path1", ("user", "user!netloc")), - ("http://user2@example.com/path1", ("user2", "user2!netloc")), - # path2 URLs will be resolved by index URL - ("http://example.com/path2/path3", (None, None)), - ("http://foo@example.com/path2/path3", ("foo", "foo!url")), -)) -def test_keyring_get_password(monkeypatch, url, expect): +@pytest.mark.parametrize( + "url, expect", + ( + ("http://example.com/path1", (None, None)), + # path1 URLs will be resolved by netloc + ("http://user@example.com/path1", ("user", "user!netloc")), + ("http://user2@example.com/path1", ("user2", "user2!netloc")), + # path2 URLs will be resolved by index URL + ("http://example.com/path2/path3", (None, None)), + ("http://foo@example.com/path2/path3", ("foo", "foo!url")), + ), +) +def test_keyring_get_password( + monkeypatch: pytest.MonkeyPatch, + url: str, + expect: Tuple[Optional[str], Optional[str]], +) -> None: keyring = KeyringModuleV1() - monkeypatch.setattr('pip._internal.network.auth.keyring', keyring) + monkeypatch.setattr("pip._internal.network.auth.keyring", keyring) auth = MultiDomainBasicAuth(index_urls=["http://example.com/path2"]) - actual = auth._get_new_credentials(url, allow_netrc=False, - allow_keyring=True) + actual = auth._get_new_credentials(url, allow_netrc=False, allow_keyring=True) assert actual == expect -def test_keyring_get_password_after_prompt(monkeypatch): +def test_keyring_get_password_after_prompt(monkeypatch: pytest.MonkeyPatch) -> None: keyring = KeyringModuleV1() - monkeypatch.setattr('pip._internal.network.auth.keyring', keyring) + monkeypatch.setattr("pip._internal.network.auth.keyring", keyring) auth = MultiDomainBasicAuth() - def ask_input(prompt): + def ask_input(prompt: str) -> str: assert prompt == "User for example.com: " return "user" - monkeypatch.setattr('pip._internal.network.auth.ask_input', ask_input) + monkeypatch.setattr("pip._internal.network.auth.ask_input", ask_input) actual = auth._prompt_for_password("example.com") assert actual == ("user", "user!netloc", False) -def test_keyring_get_password_after_prompt_when_none(monkeypatch): +def test_keyring_get_password_after_prompt_when_none( + monkeypatch: pytest.MonkeyPatch, +) -> None: keyring = KeyringModuleV1() - monkeypatch.setattr('pip._internal.network.auth.keyring', keyring) + monkeypatch.setattr("pip._internal.network.auth.keyring", keyring) auth = MultiDomainBasicAuth() - def ask_input(prompt): + def ask_input(prompt: str) -> str: assert prompt == "User for unknown.com: " return "user" - def ask_password(prompt): + def ask_password(prompt: str) -> str: assert prompt == "Password: " return "fake_password" - monkeypatch.setattr('pip._internal.network.auth.ask_input', ask_input) - monkeypatch.setattr( - 'pip._internal.network.auth.ask_password', ask_password) + monkeypatch.setattr("pip._internal.network.auth.ask_input", ask_input) + monkeypatch.setattr("pip._internal.network.auth.ask_password", ask_password) actual = auth._prompt_for_password("unknown.com") assert actual == ("user", "fake_password", True) -def test_keyring_get_password_username_in_index(monkeypatch): +def test_keyring_get_password_username_in_index( + monkeypatch: pytest.MonkeyPatch, +) -> None: keyring = KeyringModuleV1() - monkeypatch.setattr('pip._internal.network.auth.keyring', keyring) + monkeypatch.setattr("pip._internal.network.auth.keyring", keyring) auth = MultiDomainBasicAuth(index_urls=["http://user@example.com/path2"]) get = functools.partial( - auth._get_new_credentials, - allow_netrc=False, - allow_keyring=True + auth._get_new_credentials, allow_netrc=False, allow_keyring=True ) assert get("http://example.com/path2/path3") == ("user", "user!url") assert get("http://example.com/path4/path1") == (None, None) -@pytest.mark.parametrize("response_status, creds, expect_save", ( - (403, ("user", "pass", True), False), - (200, ("user", "pass", True), True,), - (200, ("user", "pass", False), False,), -)) -def test_keyring_set_password(monkeypatch, response_status, creds, - expect_save): +@pytest.mark.parametrize( + "response_status, creds, expect_save", + ( + (403, ("user", "pass", True), False), + ( + 200, + ("user", "pass", True), + True, + ), + ( + 200, + ("user", "pass", False), + False, + ), + ), +) +def test_keyring_set_password( + monkeypatch: pytest.MonkeyPatch, + response_status: int, + creds: Tuple[str, str, bool], + expect_save: bool, +) -> None: keyring = KeyringModuleV1() - monkeypatch.setattr('pip._internal.network.auth.keyring', keyring) + monkeypatch.setattr("pip._internal.network.auth.keyring", keyring) auth = MultiDomainBasicAuth(prompting=True) - monkeypatch.setattr(auth, '_get_url_and_credentials', - lambda u: (u, None, None)) - monkeypatch.setattr(auth, '_prompt_for_password', lambda *a: creds) + monkeypatch.setattr(auth, "_get_url_and_credentials", lambda u: (u, None, None)) + monkeypatch.setattr(auth, "_prompt_for_password", lambda *a: creds) if creds[2]: # when _prompt_for_password indicates to save, we should save - def should_save_password_to_keyring(*a): + def should_save_password_to_keyring(*a: Any) -> bool: return True + else: # when _prompt_for_password indicates not to save, we should # never call this function - def should_save_password_to_keyring(*a): - assert False, ("_should_save_password_to_keyring should not be " + - "called") - monkeypatch.setattr(auth, '_should_save_password_to_keyring', - should_save_password_to_keyring) + def should_save_password_to_keyring(*a: Any) -> bool: + assert False, "_should_save_password_to_keyring should not be called" + + monkeypatch.setattr( + auth, "_should_save_password_to_keyring", should_save_password_to_keyring + ) req = MockRequest("https://example.com") resp = MockResponse(b"") resp.url = req.url connection = MockConnection() - def _send(sent_req, **kwargs): + def _send(sent_req: MockRequest, **kwargs: Any) -> MockResponse: assert sent_req is req assert "Authorization" in sent_req.headers r = MockResponse(b"") r.status_code = response_status return r - connection._send = _send + # https://github.com/python/mypy/issues/2427 + connection._send = _send # type: ignore[assignment] resp.request = req resp.status_code = 401 @@ -213,14 +267,14 @@ class KeyringModuleV2: """Represents the current supported API of keyring""" class Credential: - def __init__(self, username, password): + def __init__(self, username: str, password: str) -> None: self.username = username self.password = password - def get_password(self, system, username): + def get_password(self, system: str, username: str) -> None: assert False, "get_password should not ever be called" - def get_credential(self, system, username): + def get_credential(self, system: str, username: str) -> Optional[Credential]: if system == "http://example.com/path2": return self.Credential("username", "url") if system == "example.com": @@ -228,36 +282,39 @@ def get_credential(self, system, username): return None -@pytest.mark.parametrize('url, expect', ( - ("http://example.com/path1", ("username", "netloc")), - ("http://example.com/path2/path3", ("username", "url")), - ("http://user2@example.com/path2/path3", ("username", "url")), -)) -def test_keyring_get_credential(monkeypatch, url, expect): - monkeypatch.setattr( - pip._internal.network.auth, 'keyring', KeyringModuleV2() - ) +@pytest.mark.parametrize( + "url, expect", + ( + ("http://example.com/path1", ("username", "netloc")), + ("http://example.com/path2/path3", ("username", "url")), + ("http://user2@example.com/path2/path3", ("username", "url")), + ), +) +def test_keyring_get_credential( + monkeypatch: pytest.MonkeyPatch, url: str, expect: str +) -> None: + monkeypatch.setattr(pip._internal.network.auth, "keyring", KeyringModuleV2()) auth = MultiDomainBasicAuth(index_urls=["http://example.com/path2"]) - assert auth._get_new_credentials( - url, allow_netrc=False, allow_keyring=True - ) == expect + assert ( + auth._get_new_credentials(url, allow_netrc=False, allow_keyring=True) == expect + ) class KeyringModuleBroken: """Represents the current supported API of keyring, but broken""" - def __init__(self): + def __init__(self) -> None: self._call_count = 0 - def get_credential(self, system, username): + def get_credential(self, system: str, username: str) -> None: self._call_count += 1 raise Exception("This keyring is broken!") -def test_broken_keyring_disables_keyring(monkeypatch): +def test_broken_keyring_disables_keyring(monkeypatch: pytest.MonkeyPatch) -> None: keyring_broken = KeyringModuleBroken() - monkeypatch.setattr(pip._internal.network.auth, 'keyring', keyring_broken) + monkeypatch.setattr(pip._internal.network.auth, "keyring", keyring_broken) auth = MultiDomainBasicAuth(index_urls=["http://example.com/"]) diff --git a/tests/unit/test_network_cache.py b/tests/unit/test_network_cache.py index 5f1d0a0975a..c7e0e382b17 100644 --- a/tests/unit/test_network_cache.py +++ b/tests/unit/test_network_cache.py @@ -1,14 +1,16 @@ import os +from typing import Iterator +from unittest.mock import Mock import pytest -from mock import Mock from pip._vendor.cachecontrol.caches import FileCache from pip._internal.network.cache import SafeFileCache +from tests.lib.path import Path @pytest.fixture(scope="function") -def cache_tmpdir(tmpdir): +def cache_tmpdir(tmpdir: Path) -> Iterator[Path]: cache_dir = tmpdir.joinpath("cache") cache_dir.mkdir(parents=True) yield cache_dir @@ -21,7 +23,7 @@ class TestSafeFileCache: os.geteuid which is absent on Windows. """ - def test_cache_roundtrip(self, cache_tmpdir): + def test_cache_roundtrip(self, cache_tmpdir: Path) -> None: cache = SafeFileCache(cache_tmpdir) assert cache.get("test key") is None @@ -31,7 +33,9 @@ def test_cache_roundtrip(self, cache_tmpdir): assert cache.get("test key") is None @pytest.mark.skipif("sys.platform == 'win32'") - def test_safe_get_no_perms(self, cache_tmpdir, monkeypatch): + def test_safe_get_no_perms( + self, cache_tmpdir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: os.chmod(cache_tmpdir, 000) monkeypatch.setattr(os.path, "exists", lambda x: True) @@ -40,23 +44,21 @@ def test_safe_get_no_perms(self, cache_tmpdir, monkeypatch): cache.get("foo") @pytest.mark.skipif("sys.platform == 'win32'") - def test_safe_set_no_perms(self, cache_tmpdir): + def test_safe_set_no_perms(self, cache_tmpdir: Path) -> None: os.chmod(cache_tmpdir, 000) cache = SafeFileCache(cache_tmpdir) cache.set("foo", b"bar") @pytest.mark.skipif("sys.platform == 'win32'") - def test_safe_delete_no_perms(self, cache_tmpdir): + def test_safe_delete_no_perms(self, cache_tmpdir: Path) -> None: os.chmod(cache_tmpdir, 000) cache = SafeFileCache(cache_tmpdir) cache.delete("foo") - def test_cache_hashes_are_same(self, cache_tmpdir): + def test_cache_hashes_are_same(self, cache_tmpdir: Path) -> None: cache = SafeFileCache(cache_tmpdir) key = "test key" - fake_cache = Mock( - FileCache, directory=cache.directory, encode=FileCache.encode - ) + fake_cache = Mock(FileCache, directory=cache.directory, encode=FileCache.encode) assert cache._get_cache_path(key) == FileCache._fn(fake_cache, key) diff --git a/tests/unit/test_network_download.py b/tests/unit/test_network_download.py index 20f5513a2df..53200f2e511 100644 --- a/tests/unit/test_network_download.py +++ b/tests/unit/test_network_download.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Dict import pytest @@ -12,23 +13,51 @@ from tests.lib.requests_mocks import MockResponse -@pytest.mark.parametrize("url, headers, from_cache, expected", [ - ('http://example.com/foo.tgz', {}, False, - "Downloading http://example.com/foo.tgz"), - ('http://example.com/foo.tgz', {'content-length': 2}, False, - "Downloading http://example.com/foo.tgz (2 bytes)"), - ('http://example.com/foo.tgz', {'content-length': 2}, True, - "Using cached http://example.com/foo.tgz (2 bytes)"), - ('https://files.pythonhosted.org/foo.tgz', {}, False, - "Downloading foo.tgz"), - ('https://files.pythonhosted.org/foo.tgz', {'content-length': 2}, False, - "Downloading foo.tgz (2 bytes)"), - ('https://files.pythonhosted.org/foo.tgz', {'content-length': 2}, True, - "Using cached foo.tgz"), -]) -def test_prepare_download__log(caplog, url, headers, from_cache, expected): +@pytest.mark.parametrize( + "url, headers, from_cache, expected", + [ + ( + "http://example.com/foo.tgz", + {}, + False, + "Downloading http://example.com/foo.tgz", + ), + ( + "http://example.com/foo.tgz", + {"content-length": "2"}, + False, + "Downloading http://example.com/foo.tgz (2 bytes)", + ), + ( + "http://example.com/foo.tgz", + {"content-length": "2"}, + True, + "Using cached http://example.com/foo.tgz (2 bytes)", + ), + ("https://files.pythonhosted.org/foo.tgz", {}, False, "Downloading foo.tgz"), + ( + "https://files.pythonhosted.org/foo.tgz", + {"content-length": "2"}, + False, + "Downloading foo.tgz (2 bytes)", + ), + ( + "https://files.pythonhosted.org/foo.tgz", + {"content-length": "2"}, + True, + "Using cached foo.tgz", + ), + ], +) +def test_prepare_download__log( + caplog: pytest.LogCaptureFixture, + url: str, + headers: Dict[str, str], + from_cache: bool, + expected: str, +) -> None: caplog.set_level(logging.INFO) - resp = MockResponse(b'') + resp = MockResponse(b"") resp.url = url resp.headers = headers if from_cache: @@ -38,55 +67,60 @@ def test_prepare_download__log(caplog, url, headers, from_cache, expected): assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'INFO' + assert record.levelname == "INFO" assert expected in record.message -@pytest.mark.parametrize("filename, expected", [ - ('dir/file', 'file'), - ('../file', 'file'), - ('../../file', 'file'), - ('../', ''), - ('../..', '..'), - ('/', ''), -]) -def test_sanitize_content_filename(filename, expected): +@pytest.mark.parametrize( + "filename, expected", + [ + ("dir/file", "file"), + ("../file", "file"), + ("../../file", "file"), + ("../", ""), + ("../..", ".."), + ("/", ""), + ], +) +def test_sanitize_content_filename(filename: str, expected: str) -> None: """ Test inputs where the result is the same for Windows and non-Windows. """ assert sanitize_content_filename(filename) == expected -@pytest.mark.parametrize("filename, win_expected, non_win_expected", [ - ('dir\\file', 'file', 'dir\\file'), - ('..\\file', 'file', '..\\file'), - ('..\\..\\file', 'file', '..\\..\\file'), - ('..\\', '', '..\\'), - ('..\\..', '..', '..\\..'), - ('\\', '', '\\'), -]) +@pytest.mark.parametrize( + "filename, win_expected, non_win_expected", + [ + ("dir\\file", "file", "dir\\file"), + ("..\\file", "file", "..\\file"), + ("..\\..\\file", "file", "..\\..\\file"), + ("..\\", "", "..\\"), + ("..\\..", "..", "..\\.."), + ("\\", "", "\\"), + ], +) def test_sanitize_content_filename__platform_dependent( - filename, - win_expected, - non_win_expected -): + filename: str, win_expected: str, non_win_expected: str +) -> None: """ Test inputs where the result is different for Windows and non-Windows. """ - if sys.platform == 'win32': + if sys.platform == "win32": expected = win_expected else: expected = non_win_expected assert sanitize_content_filename(filename) == expected -@pytest.mark.parametrize("content_disposition, default_filename, expected", [ - ('attachment;filename="../file"', 'df', 'file'), -]) +@pytest.mark.parametrize( + "content_disposition, default_filename, expected", + [ + ('attachment;filename="../file"', "df", "file"), + ], +) def test_parse_content_disposition( - content_disposition, - default_filename, - expected -): + content_disposition: str, default_filename: str, expected: str +) -> None: actual = parse_content_disposition(content_disposition, default_filename) assert actual == expected diff --git a/tests/unit/test_network_lazy_wheel.py b/tests/unit/test_network_lazy_wheel.py index e6747a18e6a..79e86321793 100644 --- a/tests/unit/test_network_lazy_wheel.py +++ b/tests/unit/test_network_lazy_wheel.py @@ -1,61 +1,66 @@ -from zipfile import BadZipfile +from typing import Iterator -from pip._vendor.pkg_resources import Requirement +from pip._vendor.packaging.version import Version from pytest import fixture, mark, raises +from pip._internal.exceptions import InvalidWheel from pip._internal.network.lazy_wheel import ( HTTPRangeRequestUnsupported, dist_from_wheel_url, ) from pip._internal.network.session import PipSession -from tests.lib.server import file_response +from tests.lib import TestData +from tests.lib.server import MockServer, file_response MYPY_0_782_WHL = ( - 'https://files.pythonhosted.org/packages/9d/65/' - 'b96e844150ce18b9892b155b780248955ded13a2581d31872e7daa90a503/' - 'mypy-0.782-py3-none-any.whl' + "https://files.pythonhosted.org/packages/9d/65/" + "b96e844150ce18b9892b155b780248955ded13a2581d31872e7daa90a503/" + "mypy-0.782-py3-none-any.whl" ) MYPY_0_782_REQS = { - Requirement('typed-ast (<1.5.0,>=1.4.0)'), - Requirement('typing-extensions (>=3.7.4)'), - Requirement('mypy-extensions (<0.5.0,>=0.4.3)'), - Requirement('psutil (>=4.0); extra == "dmypy"'), + "typed-ast<1.5.0,>=1.4.0", + "typing-extensions>=3.7.4", + "mypy-extensions<0.5.0,>=0.4.3", + 'psutil>=4.0; extra == "dmypy"', } @fixture -def session(): +def session() -> PipSession: return PipSession() @fixture -def mypy_whl_no_range(mock_server, shared_data): - mypy_whl = shared_data.packages / 'mypy-0.782-py3-none-any.whl' +def mypy_whl_no_range(mock_server: MockServer, shared_data: TestData) -> Iterator[str]: + mypy_whl = shared_data.packages / "mypy-0.782-py3-none-any.whl" mock_server.set_responses([file_response(mypy_whl)]) mock_server.start() - base_address = f'http://{mock_server.host}:{mock_server.port}' - yield "{}/{}".format(base_address, 'mypy-0.782-py3-none-any.whl') + base_address = f"http://{mock_server.host}:{mock_server.port}" + yield "{}/{}".format(base_address, "mypy-0.782-py3-none-any.whl") mock_server.stop() @mark.network -def test_dist_from_wheel_url(session): +def test_dist_from_wheel_url(session: PipSession) -> None: """Test if the acquired distribution contain correct information.""" - dist = dist_from_wheel_url('mypy', MYPY_0_782_WHL, session) - assert dist.key == 'mypy' - assert dist.version == '0.782' - assert dist.extras == ['dmypy'] - assert set(dist.requires(dist.extras)) == MYPY_0_782_REQS + dist = dist_from_wheel_url("mypy", MYPY_0_782_WHL, session) + assert dist.canonical_name == "mypy" + assert dist.version == Version("0.782") + extras = list(dist.iter_provided_extras()) + assert extras == ["dmypy"] + assert {str(d) for d in dist.iter_dependencies(extras)} == MYPY_0_782_REQS -def test_dist_from_wheel_url_no_range(session, mypy_whl_no_range): +def test_dist_from_wheel_url_no_range( + session: PipSession, mypy_whl_no_range: str +) -> None: """Test handling when HTTP range requests are not supported.""" with raises(HTTPRangeRequestUnsupported): - dist_from_wheel_url('mypy', mypy_whl_no_range, session) + dist_from_wheel_url("mypy", mypy_whl_no_range, session) @mark.network -def test_dist_from_wheel_url_not_zip(session): +def test_dist_from_wheel_url_not_zip(session: PipSession) -> None: """Test handling with the given URL does not point to a ZIP.""" - with raises(BadZipfile): - dist_from_wheel_url('python', 'https://www.python.org/', session) + with raises(InvalidWheel): + dist_from_wheel_url("python", "https://www.python.org/", session) diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index 044d1fb923b..18eb9539f7f 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -1,30 +1,41 @@ import logging +from typing import Any, List, Optional +from urllib.parse import urlparse +from urllib.request import getproxies import pytest +from pip._vendor import requests from pip import __version__ +from pip._internal.models.link import Link from pip._internal.network.session import CI_ENVIRONMENT_VARIABLES, PipSession +from tests.lib.path import Path -def get_user_agent(): +def get_user_agent() -> str: return PipSession().headers["User-Agent"] -def test_user_agent(): +def test_user_agent() -> None: user_agent = get_user_agent() assert user_agent.startswith(f"pip/{__version__}") -@pytest.mark.parametrize('name, expected_like_ci', [ - ('BUILD_BUILDID', True), - ('BUILD_ID', True), - ('CI', True), - ('PIP_IS_CI', True), - # Test a prefix substring of one of the variable names we use. - ('BUILD', False), -]) -def test_user_agent__ci(monkeypatch, name, expected_like_ci): +@pytest.mark.parametrize( + "name, expected_like_ci", + [ + ("BUILD_BUILDID", True), + ("BUILD_ID", True), + ("CI", True), + ("PIP_IS_CI", True), + # Test a prefix substring of one of the variable names we use. + ("BUILD", False), + ], +) +def test_user_agent__ci( + monkeypatch: pytest.MonkeyPatch, name: str, expected_like_ci: bool +) -> None: # Delete the variable names we use to check for CI to prevent the # detection from always returning True in case the tests are being run # under actual CI. It is okay to depend on CI_ENVIRONMENT_VARIABLES @@ -38,41 +49,38 @@ def test_user_agent__ci(monkeypatch, name, expected_like_ci): assert '"ci":null' in user_agent assert '"ci":true' not in user_agent - monkeypatch.setenv(name, 'true') + monkeypatch.setenv(name, "true") user_agent = get_user_agent() assert ('"ci":true' in user_agent) == expected_like_ci assert ('"ci":null' in user_agent) == (not expected_like_ci) -def test_user_agent_user_data(monkeypatch): +def test_user_agent_user_data(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PIP_USER_AGENT_USER_DATA", "some_string") assert "some_string" in PipSession().headers["User-Agent"] class TestPipSession: - - def test_cache_defaults_off(self): + def test_cache_defaults_off(self) -> None: session = PipSession() assert not hasattr(session.adapters["http://"], "cache") assert not hasattr(session.adapters["https://"], "cache") - def test_cache_is_enabled(self, tmpdir): + def test_cache_is_enabled(self, tmpdir: Path) -> None: cache_directory = tmpdir.joinpath("test-cache") session = PipSession(cache=cache_directory) assert hasattr(session.adapters["https://"], "cache") - assert ( - session.adapters["https://"].cache.directory == cache_directory - ) + assert session.adapters["https://"].cache.directory == cache_directory - def test_http_cache_is_not_enabled(self, tmpdir): + def test_http_cache_is_not_enabled(self, tmpdir: Path) -> None: session = PipSession(cache=tmpdir.joinpath("test-cache")) assert not hasattr(session.adapters["http://"], "cache") - def test_trusted_hosts_adapter(self, tmpdir): + def test_trusted_hosts_adapter(self, tmpdir: Path) -> None: session = PipSession( cache=tmpdir.joinpath("test-cache"), trusted_hosts=["example.com"], @@ -82,86 +90,102 @@ def test_trusted_hosts_adapter(self, tmpdir): # Check that the "port wildcard" is present. assert "https://example.com:" in session.adapters # Check that the cache is enabled. + assert hasattr(session.adapters["http://example.com/"], "cache") assert hasattr(session.adapters["https://example.com/"], "cache") - def test_add_trusted_host(self): + def test_add_trusted_host(self) -> None: # Leave a gap to test how the ordering is affected. - trusted_hosts = ['host1', 'host3'] + trusted_hosts = ["host1", "host3"] session = PipSession(trusted_hosts=trusted_hosts) trusted_host_adapter = session._trusted_host_adapter - prefix2 = 'https://host2/' - prefix3 = 'https://host3/' - prefix3_wildcard = 'https://host3:' + prefix2 = "https://host2/" + prefix3 = "https://host3/" + prefix3_wildcard = "https://host3:" + + prefix2_http = "http://host2/" + prefix3_http = "http://host3/" + prefix3_wildcard_http = "http://host3:" # Confirm some initial conditions as a baseline. - assert session.pip_trusted_origins == [ - ('host1', None), ('host3', None) - ] + assert session.pip_trusted_origins == [("host1", None), ("host3", None)] assert session.adapters[prefix3] is trusted_host_adapter assert session.adapters[prefix3_wildcard] is trusted_host_adapter + assert session.adapters[prefix3_http] is trusted_host_adapter + assert session.adapters[prefix3_wildcard_http] is trusted_host_adapter + assert prefix2 not in session.adapters + assert prefix2_http not in session.adapters # Test adding a new host. - session.add_trusted_host('host2') + session.add_trusted_host("host2") assert session.pip_trusted_origins == [ - ('host1', None), ('host3', None), ('host2', None) + ("host1", None), + ("host3", None), + ("host2", None), ] # Check that prefix3 is still present. assert session.adapters[prefix3] is trusted_host_adapter assert session.adapters[prefix2] is trusted_host_adapter + assert session.adapters[prefix2_http] is trusted_host_adapter # Test that adding the same host doesn't create a duplicate. - session.add_trusted_host('host3') + session.add_trusted_host("host3") assert session.pip_trusted_origins == [ - ('host1', None), ('host3', None), ('host2', None) - ], f'actual: {session.pip_trusted_origins}' - - session.add_trusted_host('host4:8080') - prefix4 = 'https://host4:8080/' + ("host1", None), + ("host3", None), + ("host2", None), + ], f"actual: {session.pip_trusted_origins}" + + session.add_trusted_host("host4:8080") + prefix4 = "https://host4:8080/" + prefix4_http = "http://host4:8080/" assert session.pip_trusted_origins == [ - ('host1', None), ('host3', None), - ('host2', None), ('host4', 8080) + ("host1", None), + ("host3", None), + ("host2", None), + ("host4", 8080), ] assert session.adapters[prefix4] is trusted_host_adapter + assert session.adapters[prefix4_http] is trusted_host_adapter - def test_add_trusted_host__logging(self, caplog): + def test_add_trusted_host__logging(self, caplog: pytest.LogCaptureFixture) -> None: """ Test logging when add_trusted_host() is called. """ - trusted_hosts = ['host0', 'host1'] + trusted_hosts = ["host0", "host1"] session = PipSession(trusted_hosts=trusted_hosts) with caplog.at_level(logging.INFO): # Test adding an existing host. - session.add_trusted_host('host1', source='somewhere') - session.add_trusted_host('host2') + session.add_trusted_host("host1", source="somewhere") + session.add_trusted_host("host2") # Test calling add_trusted_host() on the same host twice. - session.add_trusted_host('host2') + session.add_trusted_host("host2") actual = [(r.levelname, r.message) for r in caplog.records] # Observe that "host0" isn't included in the logs. expected = [ - ('INFO', "adding trusted host: 'host1' (from somewhere)"), - ('INFO', "adding trusted host: 'host2'"), - ('INFO', "adding trusted host: 'host2'"), + ("INFO", "adding trusted host: 'host1' (from somewhere)"), + ("INFO", "adding trusted host: 'host2'"), + ("INFO", "adding trusted host: 'host2'"), ] assert actual == expected - def test_iter_secure_origins(self): - trusted_hosts = ['host1', 'host2', 'host3:8080'] + def test_iter_secure_origins(self) -> None: + trusted_hosts = ["host1", "host2", "host3:8080"] session = PipSession(trusted_hosts=trusted_hosts) actual = list(session.iter_secure_origins()) assert len(actual) == 9 # Spot-check that SECURE_ORIGINS is included. - assert actual[0] == ('https', '*', '*') + assert actual[0] == ("https", "*", "*") assert actual[-3:] == [ - ('*', 'host1', '*'), - ('*', 'host2', '*'), - ('*', 'host3', 8080) + ("*", "host1", "*"), + ("*", "host2", "*"), + ("*", "host3", 8080), ] - def test_iter_secure_origins__trusted_hosts_empty(self): + def test_iter_secure_origins__trusted_hosts_empty(self) -> None: """ Test iter_secure_origins() after passing trusted_hosts=[]. """ @@ -170,10 +194,10 @@ def test_iter_secure_origins__trusted_hosts_empty(self): actual = list(session.iter_secure_origins()) assert len(actual) == 6 # Spot-check that SECURE_ORIGINS is included. - assert actual[0] == ('https', '*', '*') + assert actual[0] == ("https", "*", "*") @pytest.mark.parametrize( - 'location, trusted, expected', + "location, trusted, expected", [ ("http://pypi.org/something", [], False), ("https://pypi.org/something", [], True), @@ -191,23 +215,25 @@ def test_iter_secure_origins__trusted_hosts_empty(self): # Test a trusted_host with a port. ("http://example.com:8080/something/", ["example.com:8080"], True), ("http://example.com/something/", ["example.com:8080"], False), - ( - "http://example.com:8888/something/", - ["example.com:8080"], - False - ), + ("http://example.com:8888/something/", ["example.com:8080"], False), ], ) - def test_is_secure_origin(self, caplog, location, trusted, expected): + def test_is_secure_origin( + self, + caplog: pytest.LogCaptureFixture, + location: str, + trusted: List[str], + expected: bool, + ) -> None: class MockLogger: - def __init__(self): + def __init__(self) -> None: self.called = False - def warning(self, *args, **kwargs): + def warning(self, *args: Any, **kwargs: Any) -> None: self.called = True session = PipSession(trusted_hosts=trusted) - actual = session.is_secure_origin(location) + actual = session.is_secure_origin(Link(location)) assert actual == expected log_records = [(r.levelname, r.message) for r in caplog.records] @@ -217,5 +243,33 @@ def warning(self, *args, **kwargs): assert len(log_records) == 1 actual_level, actual_message = log_records[0] - assert actual_level == 'WARNING' - assert 'is not a trusted or secure host' in actual_message + assert actual_level == "WARNING" + assert "is not a trusted or secure host" in actual_message + + @pytest.mark.network + def test_proxy(self, proxy: Optional[str]) -> None: + session = PipSession(trusted_hosts=[]) + + if not proxy: + # if user didn't pass --proxy then try to get it from the system. + env_proxy = getproxies().get("http", None) + proxy = urlparse(env_proxy).netloc if env_proxy else None + + if proxy: + # set proxy scheme to session.proxies + session.proxies = { + "http": f"{proxy}", + "https": f"{proxy}", + "ftp": f"{proxy}", + } + + connection_error = None + try: + session.request("GET", "https://pypi.org", timeout=1) + except requests.exceptions.ConnectionError as e: + connection_error = e + + assert connection_error is None, ( + f"Invalid proxy {proxy} or session.proxies: " + f"{session.proxies} is not correctly passed to session.request." + ) diff --git a/tests/unit/test_network_utils.py b/tests/unit/test_network_utils.py index 09f0684c5ee..cdc10b2ba6e 100644 --- a/tests/unit/test_network_utils.py +++ b/tests/unit/test_network_utils.py @@ -5,30 +5,31 @@ from tests.lib.requests_mocks import MockResponse -@pytest.mark.parametrize(("status_code", "error_type"), [ - (401, "Client Error"), - (501, "Server Error"), -]) -def test_raise_for_status_raises_exception(status_code, error_type): - contents = b'downloaded' +@pytest.mark.parametrize( + ("status_code", "error_type"), + [ + (401, "Client Error"), + (501, "Server Error"), + ], +) +def test_raise_for_status_raises_exception(status_code: int, error_type: str) -> None: + contents = b"downloaded" resp = MockResponse(contents) resp.status_code = status_code resp.url = "http://www.example.com/whatever.tgz" resp.reason = "Network Error" - with pytest.raises(NetworkConnectionError) as exc: + with pytest.raises(NetworkConnectionError) as excinfo: raise_for_status(resp) - assert str(exc.info) == ( - "{} {}: Network Error for url:" - " http://www.example.com/whatever.tgz".format( - status_code, error_type) - ) + assert str(excinfo.value) == ( + "{} {}: Network Error for url:" + " http://www.example.com/whatever.tgz".format(status_code, error_type) + ) -def test_raise_for_status_does_not_raises_exception(): - contents = b'downloaded' +def test_raise_for_status_does_not_raises_exception() -> None: + contents = b"downloaded" resp = MockResponse(contents) resp.status_code = 201 resp.url = "http://www.example.com/whatever.tgz" resp.reason = "No error" - return_value = raise_for_status(resp) - assert return_value is None + raise_for_status(resp) diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index 17fc94929c0..a2ee878708b 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -2,9 +2,10 @@ import shutil from shutil import rmtree from tempfile import mkdtemp +from typing import Any, Dict +from unittest.mock import Mock, patch import pytest -from mock import Mock, patch from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link @@ -13,18 +14,19 @@ from pip._internal.operations.prepare import _copy_source_tree, unpack_url from pip._internal.utils.hashes import Hashes from pip._internal.utils.urls import path_to_url +from tests.lib import TestData from tests.lib.filesystem import get_filelist, make_socket_file, make_unreadable_file from tests.lib.path import Path from tests.lib.requests_mocks import MockResponse -def test_unpack_url_with_urllib_response_without_content_type(data): +def test_unpack_url_with_urllib_response_without_content_type(data: TestData) -> None: """ It should download and unpack files even if no Content-Type header exists """ _real_session = PipSession() - def _fake_session_get(*args, **kwargs): + def _fake_session_get(*args: Any, **kwargs: Any) -> Dict[str, str]: resp = _real_session.get(*args, **kwargs) del resp.headers["Content-Type"] return resp @@ -42,23 +44,29 @@ def _fake_session_get(*args, **kwargs): temp_dir, download=download, download_dir=None, + verbosity=0, ) assert set(os.listdir(temp_dir)) == { - 'PKG-INFO', 'setup.cfg', 'setup.py', 'simple', 'simple.egg-info' + "PKG-INFO", + "setup.cfg", + "setup.py", + "simple", + "simple.egg-info", } finally: rmtree(temp_dir) @patch("pip._internal.network.download.raise_for_status") -def test_download_http_url__no_directory_traversal(mock_raise_for_status, - tmpdir): +def test_download_http_url__no_directory_traversal( + mock_raise_for_status: Mock, tmpdir: Path +) -> None: """ Test that directory traversal doesn't happen on download when the Content-Disposition header contains a filename with a ".." path part. """ - mock_url = 'http://www.example.com/whatever.tgz' - contents = b'downloaded' + mock_url = "http://www.example.com/whatever.tgz" + contents = b"downloaded" link = Link(mock_url) session = Mock() @@ -67,23 +75,23 @@ def test_download_http_url__no_directory_traversal(mock_raise_for_status, resp.headers = { # Set the content-type to a random value to prevent # mimetypes.guess_extension from guessing the extension. - 'content-type': 'random', - 'content-disposition': 'attachment;filename="../out_dir_file"' + "content-type": "random", + "content-disposition": 'attachment;filename="../out_dir_file"', } session.get.return_value = resp download = Downloader(session, progress_bar="on") - download_dir = tmpdir.joinpath('download') + download_dir = tmpdir.joinpath("download") os.mkdir(download_dir) file_path, content_type = download(link, download_dir) # The file should be downloaded to download_dir. actual = os.listdir(download_dir) - assert actual == ['out_dir_file'] + assert actual == ["out_dir_file"] mock_raise_for_status.assert_called_once_with(resp) @pytest.fixture -def clean_project(tmpdir_factory, data): +def clean_project(tmpdir_factory: pytest.TempdirFactory, data: TestData) -> Path: tmpdir = Path(str(tmpdir_factory.mktemp("clean_project"))) new_project_dir = tmpdir.joinpath("FSPkg") path = data.packages.joinpath("FSPkg") @@ -91,7 +99,7 @@ def clean_project(tmpdir_factory, data): return new_project_dir -def test_copy_source_tree(clean_project, tmpdir): +def test_copy_source_tree(clean_project: Path, tmpdir: Path) -> None: target = tmpdir.joinpath("target") expected_files = get_filelist(clean_project) assert len(expected_files) == 3 @@ -102,8 +110,10 @@ def test_copy_source_tree(clean_project, tmpdir): assert expected_files == copied_files -@pytest.mark.skipif("sys.platform == 'win32' or sys.version_info < (3,)") -def test_copy_source_tree_with_socket(clean_project, tmpdir, caplog): +@pytest.mark.skipif("sys.platform == 'win32'") +def test_copy_source_tree_with_socket( + clean_project: Path, tmpdir: Path, caplog: pytest.LogCaptureFixture +) -> None: target = tmpdir.joinpath("target") expected_files = get_filelist(clean_project) socket_path = str(clean_project.joinpath("aaa")) @@ -117,14 +127,14 @@ def test_copy_source_tree_with_socket(clean_project, tmpdir, caplog): # Warning should have been logged. assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'WARNING' + assert record.levelname == "WARNING" assert socket_path in record.message -@pytest.mark.skipif("sys.platform == 'win32' or sys.version_info < (3,)") +@pytest.mark.skipif("sys.platform == 'win32'") def test_copy_source_tree_with_socket_fails_with_no_socket_error( - clean_project, tmpdir -): + clean_project: Path, tmpdir: Path +) -> None: target = tmpdir.joinpath("target") expected_files = get_filelist(clean_project) make_socket_file(clean_project.joinpath("aaa")) @@ -143,7 +153,9 @@ def test_copy_source_tree_with_socket_fails_with_no_socket_error( assert expected_files == copied_files -def test_copy_source_tree_with_unreadable_dir_fails(clean_project, tmpdir): +def test_copy_source_tree_with_unreadable_dir_fails( + clean_project: Path, tmpdir: Path +) -> None: target = tmpdir.joinpath("target") expected_files = get_filelist(clean_project) unreadable_file = clean_project.joinpath("bbb") @@ -162,10 +174,9 @@ def test_copy_source_tree_with_unreadable_dir_fails(clean_project, tmpdir): class Test_unpack_url: - - def prep(self, tmpdir, data): - self.build_dir = tmpdir.joinpath('build') - self.download_dir = tmpdir.joinpath('download') + def prep(self, tmpdir: Path, data: TestData) -> None: + self.build_dir = tmpdir.joinpath("build") + self.download_dir = tmpdir.joinpath("download") os.mkdir(self.build_dir) os.mkdir(self.download_dir) self.dist_file = "simple-1.0.tar.gz" @@ -176,48 +187,50 @@ def prep(self, tmpdir, data): self.dist_url2 = Link(path_to_url(self.dist_path2)) self.no_download = Mock(side_effect=AssertionError) - def test_unpack_url_no_download(self, tmpdir, data): + def test_unpack_url_no_download(self, tmpdir: Path, data: TestData) -> None: self.prep(tmpdir, data) - unpack_url(self.dist_url, self.build_dir, self.no_download) - assert os.path.isdir(os.path.join(self.build_dir, 'simple')) - assert not os.path.isfile( - os.path.join(self.download_dir, self.dist_file)) + unpack_url(self.dist_url, self.build_dir, self.no_download, verbosity=0) + assert os.path.isdir(os.path.join(self.build_dir, "simple")) + assert not os.path.isfile(os.path.join(self.download_dir, self.dist_file)) - def test_unpack_url_bad_hash(self, tmpdir, data, - monkeypatch): + def test_unpack_url_bad_hash(self, tmpdir: Path, data: TestData) -> None: """ Test when the file url hash fragment is wrong """ self.prep(tmpdir, data) - url = f'{self.dist_url.url}#md5=bogus' + url = f"{self.dist_url.url}#md5=bogus" dist_url = Link(url) with pytest.raises(HashMismatch): - unpack_url(dist_url, - self.build_dir, - download=self.no_download, - hashes=Hashes({'md5': ['bogus']})) - - def test_unpack_url_thats_a_dir(self, tmpdir, data): + unpack_url( + dist_url, + self.build_dir, + download=self.no_download, + hashes=Hashes({"md5": ["bogus"]}), + verbosity=0, + ) + + def test_unpack_url_thats_a_dir(self, tmpdir: Path, data: TestData) -> None: self.prep(tmpdir, data) dist_path = data.packages.joinpath("FSPkg") dist_url = Link(path_to_url(dist_path)) - unpack_url(dist_url, self.build_dir, - download=self.no_download, - download_dir=self.download_dir) - assert os.path.isdir(os.path.join(self.build_dir, 'fspkg')) - - -@pytest.mark.parametrize('exclude_dir', [ - '.nox', - '.tox' -]) -def test_unpack_url_excludes_expected_dirs(tmpdir, exclude_dir): - src_dir = tmpdir / 'src' - dst_dir = tmpdir / 'dst' - src_included_file = src_dir.joinpath('file.txt') + unpack_url( + dist_url, + self.build_dir, + download=self.no_download, + download_dir=self.download_dir, + verbosity=0, + ) + assert os.path.isdir(os.path.join(self.build_dir, "fspkg")) + + +@pytest.mark.parametrize("exclude_dir", [".nox", ".tox"]) +def test_unpack_url_excludes_expected_dirs(tmpdir: Path, exclude_dir: str) -> None: + src_dir = tmpdir / "src" + dst_dir = tmpdir / "dst" + src_included_file = src_dir.joinpath("file.txt") src_excluded_dir = src_dir.joinpath(exclude_dir) - src_excluded_file = src_dir.joinpath(exclude_dir, 'file.txt') - src_included_dir = src_dir.joinpath('subdir', exclude_dir) + src_excluded_file = src_dir.joinpath(exclude_dir, "file.txt") + src_included_dir = src_dir.joinpath("subdir", exclude_dir) # set up source directory src_excluded_dir.mkdir(parents=True) @@ -225,17 +238,18 @@ def test_unpack_url_excludes_expected_dirs(tmpdir, exclude_dir): src_included_file.touch() src_excluded_file.touch() - dst_included_file = dst_dir.joinpath('file.txt') + dst_included_file = dst_dir.joinpath("file.txt") dst_excluded_dir = dst_dir.joinpath(exclude_dir) - dst_excluded_file = dst_dir.joinpath(exclude_dir, 'file.txt') - dst_included_dir = dst_dir.joinpath('subdir', exclude_dir) + dst_excluded_file = dst_dir.joinpath(exclude_dir, "file.txt") + dst_included_dir = dst_dir.joinpath("subdir", exclude_dir) src_link = Link(path_to_url(src_dir)) unpack_url( src_link, dst_dir, Mock(side_effect=AssertionError), - download_dir=None + download_dir=None, + verbosity=0, ) assert not os.path.isdir(dst_excluded_dir) assert not os.path.isfile(dst_excluded_file) diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index 6cde2e212fd..ddcc8532cfd 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -1,18 +1,24 @@ import os from contextlib import contextmanager +from optparse import Values from tempfile import NamedTemporaryFile +from typing import Any, Dict, Iterator, List, Tuple, Union, cast import pytest import pip._internal.configuration from pip._internal.cli.main import main from pip._internal.commands import create_command +from pip._internal.commands.configuration import ConfigurationCommand from pip._internal.exceptions import PipError from tests.lib.options_helpers import AddFakeCommandMixin +from tests.lib.path import Path @contextmanager -def assert_option_error(capsys, expected): +def assert_option_error( + capsys: pytest.CaptureFixture[str], expected: str +) -> Iterator[None]: """ Assert that a SystemExit occurred because of a parsing error. @@ -27,10 +33,10 @@ def assert_option_error(capsys, expected): assert expected in stderr -def assert_is_default_cache_dir(value): +def assert_is_default_cache_dir(value: Path) -> None: # This path looks different on different platforms, but the path always # has the substring "pip". - assert 'pip' in value + assert "pip" in value class TestOptionPrecedence(AddFakeCommandMixin): @@ -40,124 +46,153 @@ class TestOptionPrecedence(AddFakeCommandMixin): defaults """ - def get_config_section(self, section): + def get_config_section(self, section: str) -> List[Tuple[str, str]]: config = { - 'global': [('timeout', '-3')], - 'fake': [('timeout', '-2')], + "global": [("timeout", "-3")], + "fake": [("timeout", "-2")], } return config[section] - def get_config_section_global(self, section): - config = { - 'global': [('timeout', '-3')], - 'fake': [], + def get_config_section_global(self, section: str) -> List[Tuple[str, str]]: + config: Dict[str, List[Tuple[str, str]]] = { + "global": [("timeout", "-3")], + "fake": [], } return config[section] - def test_env_override_default_int(self, monkeypatch): + def test_env_override_default_int(self, monkeypatch: pytest.MonkeyPatch) -> None: """ Test that environment variable overrides an int option default. """ - monkeypatch.setenv('PIP_TIMEOUT', '-1') - options, args = main(['fake']) + monkeypatch.setenv("PIP_TIMEOUT", "-1") + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) assert options.timeout == -1 - @pytest.mark.parametrize('values', (['F1'], ['F1', 'F2'])) - def test_env_override_default_append(self, values, monkeypatch): + @pytest.mark.parametrize("values", (["F1"], ["F1", "F2"])) + def test_env_override_default_append( + self, values: List[str], monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test that environment variable overrides an append option default. """ - monkeypatch.setenv('PIP_FIND_LINKS', ' '.join(values)) - options, args = main(['fake']) + monkeypatch.setenv("PIP_FIND_LINKS", " ".join(values)) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) assert options.find_links == values - @pytest.mark.parametrize('choises', (['w'], ['s', 'w'])) - def test_env_override_default_choice(self, choises, monkeypatch): + @pytest.mark.parametrize("choices", (["w"], ["s", "w"])) + def test_env_override_default_choice( + self, choices: List[str], monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test that environment variable overrides a choice option default. """ - monkeypatch.setenv('PIP_EXISTS_ACTION', ' '.join(choises)) - options, args = main(['fake']) - assert options.exists_action == choises + monkeypatch.setenv("PIP_EXISTS_ACTION", " ".join(choices)) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert options.exists_action == choices - @pytest.mark.parametrize('name', ('PIP_LOG_FILE', 'PIP_LOCAL_LOG')) - def test_env_alias_override_default(self, name, monkeypatch): + @pytest.mark.parametrize("name", ("PIP_LOG_FILE", "PIP_LOCAL_LOG")) + def test_env_alias_override_default( + self, name: str, monkeypatch: pytest.MonkeyPatch + ) -> None: """ When an option has multiple long forms, test that the technique of using the env variable, "PIP_" works for all cases. (e.g. PIP_LOG_FILE and PIP_LOCAL_LOG should all work) """ - monkeypatch.setenv(name, 'override.log') - options, args = main(['fake']) - assert options.log == 'override.log' + monkeypatch.setenv(name, "override.log") + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert options.log == "override.log" - def test_cli_override_environment(self, monkeypatch): + def test_cli_override_environment(self, monkeypatch: pytest.MonkeyPatch) -> None: """ Test the cli overrides and environment variable """ - monkeypatch.setenv('PIP_TIMEOUT', '-1') - options, args = main(['fake', '--timeout', '-2']) + monkeypatch.setenv("PIP_TIMEOUT", "-1") + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["fake", "--timeout", "-2"]) + ) assert options.timeout == -2 - @pytest.mark.parametrize('pip_no_cache_dir', [ - # Enabling --no-cache-dir means no cache directory. - '1', - 'true', - 'on', - 'yes', - # For historical / backwards compatibility reasons, we also disable - # the cache directory if provided a value that translates to 0. - '0', - 'false', - 'off', - 'no', - ]) - def test_cache_dir__PIP_NO_CACHE_DIR(self, pip_no_cache_dir, monkeypatch): + @pytest.mark.parametrize( + "pip_no_cache_dir", + [ + # Enabling --no-cache-dir means no cache directory. + "1", + "true", + "on", + "yes", + # For historical / backwards compatibility reasons, we also disable + # the cache directory if provided a value that translates to 0. + "0", + "false", + "off", + "no", + ], + ) + def test_cache_dir__PIP_NO_CACHE_DIR( + self, pip_no_cache_dir: str, monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test setting the PIP_NO_CACHE_DIR environment variable without passing any command-line flags. """ - monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) - options, args = main(['fake']) + monkeypatch.setenv("PIP_NO_CACHE_DIR", pip_no_cache_dir) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) assert options.cache_dir is False - @pytest.mark.parametrize('pip_no_cache_dir', ['yes', 'no']) + @pytest.mark.parametrize("pip_no_cache_dir", ["yes", "no"]) def test_cache_dir__PIP_NO_CACHE_DIR__with_cache_dir( - self, pip_no_cache_dir, monkeypatch, - ): + self, + pip_no_cache_dir: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: """ Test setting PIP_NO_CACHE_DIR while also passing an explicit --cache-dir value. """ - monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) - options, args = main(['--cache-dir', '/cache/dir', 'fake']) + monkeypatch.setenv("PIP_NO_CACHE_DIR", pip_no_cache_dir) + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--cache-dir", "/cache/dir", "fake"]) + ) # The command-line flag takes precedence. - assert options.cache_dir == '/cache/dir' + assert options.cache_dir == "/cache/dir" - @pytest.mark.parametrize('pip_no_cache_dir', ['yes', 'no']) + @pytest.mark.parametrize("pip_no_cache_dir", ["yes", "no"]) def test_cache_dir__PIP_NO_CACHE_DIR__with_no_cache_dir( - self, pip_no_cache_dir, monkeypatch, - ): + self, + pip_no_cache_dir: str, + monkeypatch: pytest.MonkeyPatch, + ) -> None: """ Test setting PIP_NO_CACHE_DIR while also passing --no-cache-dir. """ - monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) - options, args = main(['--no-cache-dir', 'fake']) + monkeypatch.setenv("PIP_NO_CACHE_DIR", pip_no_cache_dir) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["--no-cache-dir", "fake"])) # The command-line flag should take precedence (which has the same # value in this case). assert options.cache_dir is False def test_cache_dir__PIP_NO_CACHE_DIR_invalid__with_no_cache_dir( - self, monkeypatch, capsys, - ): + self, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: """ Test setting PIP_NO_CACHE_DIR to an invalid value while also passing --no-cache-dir. """ - monkeypatch.setenv('PIP_NO_CACHE_DIR', 'maybe') + monkeypatch.setenv("PIP_NO_CACHE_DIR", "maybe") expected_err = "--no-cache-dir error: invalid truth value 'maybe'" with assert_option_error(capsys, expected=expected_err): - main(['--no-cache-dir', 'fake']) + main(["--no-cache-dir", "fake"]) class TestUsePEP517Options: @@ -166,103 +201,115 @@ class TestUsePEP517Options: Test options related to using --use-pep517. """ - def parse_args(self, args): + def parse_args(self, args: List[str]) -> Values: # We use DownloadCommand since that is one of the few Command # classes with the use_pep517 options. - command = create_command('download') + command = create_command("download") options, args = command.parse_args(args) return options - def test_no_option(self): + def test_no_option(self) -> None: """ Test passing no option. """ options = self.parse_args([]) assert options.use_pep517 is None - def test_use_pep517(self): + def test_use_pep517(self) -> None: """ Test passing --use-pep517. """ - options = self.parse_args(['--use-pep517']) + options = self.parse_args(["--use-pep517"]) assert options.use_pep517 is True - def test_no_use_pep517(self): + def test_no_use_pep517(self) -> None: """ Test passing --no-use-pep517. """ - options = self.parse_args(['--no-use-pep517']) + options = self.parse_args(["--no-use-pep517"]) assert options.use_pep517 is False - def test_PIP_USE_PEP517_true(self, monkeypatch): + def test_PIP_USE_PEP517_true(self, monkeypatch: pytest.MonkeyPatch) -> None: """ Test setting PIP_USE_PEP517 to "true". """ - monkeypatch.setenv('PIP_USE_PEP517', 'true') + monkeypatch.setenv("PIP_USE_PEP517", "true") options = self.parse_args([]) # This is an int rather than a boolean because strtobool() in pip's # configuration code returns an int. assert options.use_pep517 == 1 - def test_PIP_USE_PEP517_false(self, monkeypatch): + def test_PIP_USE_PEP517_false(self, monkeypatch: pytest.MonkeyPatch) -> None: """ Test setting PIP_USE_PEP517 to "false". """ - monkeypatch.setenv('PIP_USE_PEP517', 'false') + monkeypatch.setenv("PIP_USE_PEP517", "false") options = self.parse_args([]) # This is an int rather than a boolean because strtobool() in pip's # configuration code returns an int. assert options.use_pep517 == 0 - def test_use_pep517_and_PIP_USE_PEP517_false(self, monkeypatch): + def test_use_pep517_and_PIP_USE_PEP517_false( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test passing --use-pep517 and setting PIP_USE_PEP517 to "false". """ - monkeypatch.setenv('PIP_USE_PEP517', 'false') - options = self.parse_args(['--use-pep517']) + monkeypatch.setenv("PIP_USE_PEP517", "false") + options = self.parse_args(["--use-pep517"]) assert options.use_pep517 is True - def test_no_use_pep517_and_PIP_USE_PEP517_true(self, monkeypatch): + def test_no_use_pep517_and_PIP_USE_PEP517_true( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test passing --no-use-pep517 and setting PIP_USE_PEP517 to "true". """ - monkeypatch.setenv('PIP_USE_PEP517', 'true') - options = self.parse_args(['--no-use-pep517']) + monkeypatch.setenv("PIP_USE_PEP517", "true") + options = self.parse_args(["--no-use-pep517"]) assert options.use_pep517 is False - def test_PIP_NO_USE_PEP517(self, monkeypatch, capsys): + def test_PIP_NO_USE_PEP517( + self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: """ Test setting PIP_NO_USE_PEP517, which isn't allowed. """ - monkeypatch.setenv('PIP_NO_USE_PEP517', 'true') - with assert_option_error(capsys, expected='--no-use-pep517 error'): + monkeypatch.setenv("PIP_NO_USE_PEP517", "true") + with assert_option_error(capsys, expected="--no-use-pep517 error"): self.parse_args([]) class TestOptionsInterspersed(AddFakeCommandMixin): - - def test_general_option_after_subcommand(self): - options, args = main(['fake', '--timeout', '-1']) + def test_general_option_after_subcommand(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["fake", "--timeout", "-1"]) + ) assert options.timeout == -1 - def test_option_after_subcommand_arg(self): - options, args = main(['fake', 'arg', '--timeout', '-1']) + def test_option_after_subcommand_arg(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["fake", "arg", "--timeout", "-1"]) + ) assert options.timeout == -1 - def test_additive_before_after_subcommand(self): - options, args = main(['-v', 'fake', '-v']) + def test_additive_before_after_subcommand(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["-v", "fake", "-v"])) assert options.verbose == 2 - def test_subcommand_option_before_subcommand_fails(self): + def test_subcommand_option_before_subcommand_fails(self) -> None: with pytest.raises(SystemExit): - main(['--find-links', 'F1', 'fake']) + main(["--find-links", "F1", "fake"]) @contextmanager -def tmpconfig(option, value, section='global'): - with NamedTemporaryFile(mode='w', delete=False) as f: - f.write(f'[{section}]\n{option}={value}\n') +def tmpconfig(option: str, value: Any, section: str = "global") -> Iterator[str]: + with NamedTemporaryFile(mode="w", delete=False) as f: + f.write(f"[{section}]\n{option}={value}\n") name = f.name try: yield name @@ -271,93 +318,140 @@ def tmpconfig(option, value, section='global'): class TestCountOptions(AddFakeCommandMixin): - - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', range(4)) - def test_cli_long(self, option, value): - flags = [f'--{option}'] * value - opt1, args1 = main(flags+['fake']) - opt2, args2 = main(['fake']+flags) + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", range(4)) + def test_cli_long(self, option: str, value: int) -> None: + flags = [f"--{option}"] * value + # FakeCommand intentionally returns the wrong type. + opt1, args1 = cast(Tuple[Values, List[str]], main(flags + ["fake"])) + opt2, args2 = cast(Tuple[Values, List[str]], main(["fake"] + flags)) assert getattr(opt1, option) == getattr(opt2, option) == value - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', range(1, 4)) - def test_cli_short(self, option, value): - flag = '-' + option[0]*value - opt1, args1 = main([flag, 'fake']) - opt2, args2 = main(['fake', flag]) + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", range(1, 4)) + def test_cli_short(self, option: str, value: int) -> None: + flag = "-" + option[0] * value + # FakeCommand intentionally returns the wrong type. + opt1, args1 = cast(Tuple[Values, List[str]], main([flag, "fake"])) + opt2, args2 = cast(Tuple[Values, List[str]], main(["fake", flag])) assert getattr(opt1, option) == getattr(opt2, option) == value - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', range(4)) - def test_env_var(self, option, value, monkeypatch): - monkeypatch.setenv('PIP_'+option.upper(), str(value)) - assert getattr(main(['fake'])[0], option) == value - - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', range(3)) - def test_env_var_integrate_cli(self, option, value, monkeypatch): - monkeypatch.setenv('PIP_'+option.upper(), str(value)) - assert getattr(main(['fake', '--'+option])[0], option) == value + 1 - - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', (-1, 'foobar')) - def test_env_var_invalid(self, option, value, monkeypatch, capsys): - monkeypatch.setenv('PIP_'+option.upper(), str(value)) - with assert_option_error(capsys, expected='a non-negative integer'): - main(['fake']) + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", range(4)) + def test_env_var( + self, option: str, value: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("PIP_" + option.upper(), str(value)) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert getattr(options, option) == value + + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", range(3)) + def test_env_var_integrate_cli( + self, option: str, value: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("PIP_" + option.upper(), str(value)) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake", "--" + option])) + assert getattr(options, option) == value + 1 + + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", (-1, "foobar")) + def test_env_var_invalid( + self, + option: str, + value: Any, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: + monkeypatch.setenv("PIP_" + option.upper(), str(value)) + with assert_option_error(capsys, expected="a non-negative integer"): + main(["fake"]) # Undocumented, support for backward compatibility - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', ('no', 'false')) - def test_env_var_false(self, option, value, monkeypatch): - monkeypatch.setenv('PIP_'+option.upper(), str(value)) - assert getattr(main(['fake'])[0], option) == 0 + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", ("no", "false")) + def test_env_var_false( + self, option: str, value: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("PIP_" + option.upper(), str(value)) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert getattr(options, option) == 0 # Undocumented, support for backward compatibility - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', ('yes', 'true')) - def test_env_var_true(self, option, value, monkeypatch): - monkeypatch.setenv('PIP_'+option.upper(), str(value)) - assert getattr(main(['fake'])[0], option) == 1 - - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', range(4)) - def test_config_file(self, option, value, monkeypatch): + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", ("yes", "true")) + def test_env_var_true( + self, option: str, value: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("PIP_" + option.upper(), str(value)) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert getattr(options, option) == 1 + + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", range(4)) + def test_config_file( + self, option: str, value: int, monkeypatch: pytest.MonkeyPatch + ) -> None: with tmpconfig(option, value) as name: - monkeypatch.setenv('PIP_CONFIG_FILE', name) - assert getattr(main(['fake'])[0], option) == value - - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', range(3)) - def test_config_file_integrate_cli(self, option, value, monkeypatch): + monkeypatch.setenv("PIP_CONFIG_FILE", name) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert getattr(options, option) == value + + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", range(3)) + def test_config_file_integrate_cli( + self, option: str, value: int, monkeypatch: pytest.MonkeyPatch + ) -> None: with tmpconfig(option, value) as name: - monkeypatch.setenv('PIP_CONFIG_FILE', name) - assert getattr(main(['fake', '--'+option])[0], option) == value + 1 - - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', (-1, 'foobar')) - def test_config_file_invalid(self, option, value, monkeypatch, capsys): + monkeypatch.setenv("PIP_CONFIG_FILE", name) + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["fake", "--" + option]) + ) + assert getattr(options, option) == value + 1 + + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", (-1, "foobar")) + def test_config_file_invalid( + self, + option: str, + value: Any, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + ) -> None: with tmpconfig(option, value) as name: - monkeypatch.setenv('PIP_CONFIG_FILE', name) - with assert_option_error(capsys, expected='non-negative integer'): - main(['fake']) + monkeypatch.setenv("PIP_CONFIG_FILE", name) + with assert_option_error(capsys, expected="non-negative integer"): + main(["fake"]) # Undocumented, support for backward compatibility - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', ('no', 'false')) - def test_config_file_false(self, option, value, monkeypatch): + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", ("no", "false")) + def test_config_file_false( + self, option: str, value: str, monkeypatch: pytest.MonkeyPatch + ) -> None: with tmpconfig(option, value) as name: - monkeypatch.setenv('PIP_CONFIG_FILE', name) - assert getattr(main(['fake'])[0], option) == 0 + monkeypatch.setenv("PIP_CONFIG_FILE", name) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert getattr(options, option) == 0 # Undocumented, support for backward compatibility - @pytest.mark.parametrize('option', ('verbose', 'quiet')) - @pytest.mark.parametrize('value', ('yes', 'true')) - def test_config_file_true(self, option, value, monkeypatch): + @pytest.mark.parametrize("option", ("verbose", "quiet")) + @pytest.mark.parametrize("value", ("yes", "true")) + def test_config_file_true( + self, option: str, value: str, monkeypatch: pytest.MonkeyPatch + ) -> None: with tmpconfig(option, value) as name: - monkeypatch.setenv('PIP_CONFIG_FILE', name) - assert getattr(main(['fake'])[0], option) == 1 + monkeypatch.setenv("PIP_CONFIG_FILE", name) + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) + assert getattr(options, option) == 1 class TestGeneralOptions(AddFakeCommandMixin): @@ -365,79 +459,128 @@ class TestGeneralOptions(AddFakeCommandMixin): # the reason to specifically test general options is due to the # extra processing they receive, and the number of bugs we've had - def test_cache_dir__default(self): - options, args = main(['fake']) + def test_cache_dir__default(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["fake"])) # With no options the default cache dir should be used. assert_is_default_cache_dir(options.cache_dir) - def test_cache_dir__provided(self): - options, args = main(['--cache-dir', '/cache/dir', 'fake']) - assert options.cache_dir == '/cache/dir' + def test_cache_dir__provided(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--cache-dir", "/cache/dir", "fake"]) + ) + assert options.cache_dir == "/cache/dir" - def test_no_cache_dir__provided(self): - options, args = main(['--no-cache-dir', 'fake']) + def test_no_cache_dir__provided(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast(Tuple[Values, List[str]], main(["--no-cache-dir", "fake"])) assert options.cache_dir is False - def test_require_virtualenv(self): - options1, args1 = main(['--require-virtualenv', 'fake']) - options2, args2 = main(['fake', '--require-virtualenv']) + def test_require_virtualenv(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--require-virtualenv", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--require-virtualenv"]) + ) assert options1.require_venv assert options2.require_venv - def test_log(self): - options1, args1 = main(['--log', 'path', 'fake']) - options2, args2 = main(['fake', '--log', 'path']) - assert options1.log == options2.log == 'path' + def test_log(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--log", "path", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--log", "path"]) + ) + assert options1.log == options2.log == "path" - def test_local_log(self): - options1, args1 = main(['--local-log', 'path', 'fake']) - options2, args2 = main(['fake', '--local-log', 'path']) - assert options1.log == options2.log == 'path' + def test_local_log(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--local-log", "path", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--local-log", "path"]) + ) + assert options1.log == options2.log == "path" - def test_no_input(self): - options1, args1 = main(['--no-input', 'fake']) - options2, args2 = main(['fake', '--no-input']) + def test_no_input(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast(Tuple[Values, List[str]], main(["--no-input", "fake"])) + options2, args2 = cast(Tuple[Values, List[str]], main(["fake", "--no-input"])) assert options1.no_input assert options2.no_input - def test_proxy(self): - options1, args1 = main(['--proxy', 'path', 'fake']) - options2, args2 = main(['fake', '--proxy', 'path']) - assert options1.proxy == options2.proxy == 'path' + def test_proxy(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--proxy", "path", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--proxy", "path"]) + ) + assert options1.proxy == options2.proxy == "path" - def test_retries(self): - options1, args1 = main(['--retries', '-1', 'fake']) - options2, args2 = main(['fake', '--retries', '-1']) + def test_retries(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--retries", "-1", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--retries", "-1"]) + ) assert options1.retries == options2.retries == -1 - def test_timeout(self): - options1, args1 = main(['--timeout', '-1', 'fake']) - options2, args2 = main(['fake', '--timeout', '-1']) + def test_timeout(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--timeout", "-1", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--timeout", "-1"]) + ) assert options1.timeout == options2.timeout == -1 - def test_exists_action(self): - options1, args1 = main(['--exists-action', 'w', 'fake']) - options2, args2 = main(['fake', '--exists-action', 'w']) - assert options1.exists_action == options2.exists_action == ['w'] + def test_exists_action(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--exists-action", "w", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--exists-action", "w"]) + ) + assert options1.exists_action == options2.exists_action == ["w"] - def test_cert(self): - options1, args1 = main(['--cert', 'path', 'fake']) - options2, args2 = main(['fake', '--cert', 'path']) - assert options1.cert == options2.cert == 'path' + def test_cert(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--cert", "path", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--cert", "path"]) + ) + assert options1.cert == options2.cert == "path" - def test_client_cert(self): - options1, args1 = main(['--client-cert', 'path', 'fake']) - options2, args2 = main(['fake', '--client-cert', 'path']) - assert options1.client_cert == options2.client_cert == 'path' + def test_client_cert(self) -> None: + # FakeCommand intentionally returns the wrong type. + options1, args1 = cast( + Tuple[Values, List[str]], main(["--client-cert", "path", "fake"]) + ) + options2, args2 = cast( + Tuple[Values, List[str]], main(["fake", "--client-cert", "path"]) + ) + assert options1.client_cert == options2.client_cert == "path" class TestOptionsConfigFiles: - - def test_venv_config_file_found(self, monkeypatch): + def test_venv_config_file_found(self, monkeypatch: pytest.MonkeyPatch) -> None: # strict limit on the global config files list monkeypatch.setattr( - pip._internal.utils.appdirs, 'site_config_dirs', - lambda _: ['/a/place'] + pip._internal.utils.appdirs, "site_config_dirs", lambda _: ["/a/place"] ) cp = pip._internal.configuration.Configuration(isolated=False) @@ -458,10 +601,15 @@ def test_venv_config_file_found(self, monkeypatch): (["--global", "--user"], PipError), (["--global", "--site"], PipError), (["--global", "--site", "--user"], PipError), - ) + ), ) - def test_config_file_options(self, monkeypatch, args, expect): - cmd = create_command('config') + def test_config_file_options( + self, + monkeypatch: pytest.MonkeyPatch, + args: List[str], + expect: Union[None, str, PipError], + ) -> None: + cmd = cast(ConfigurationCommand, create_command("config")) # Replace a handler with a no-op to avoid side effects monkeypatch.setattr(cmd, "get_name", lambda *a: None) @@ -474,22 +622,37 @@ def test_config_file_options(self, monkeypatch, args, expect): class TestOptionsExpandUser(AddFakeCommandMixin): - def test_cache_dir(self): - options, args = main(['--cache-dir', '~/cache/dir', 'fake']) - assert options.cache_dir == os.path.expanduser('~/cache/dir') + def test_cache_dir(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--cache-dir", "~/cache/dir", "fake"]) + ) + assert options.cache_dir == os.path.expanduser("~/cache/dir") - def test_log(self): - options, args = main(['--log', '~/path', 'fake']) - assert options.log == os.path.expanduser('~/path') + def test_log(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--log", "~/path", "fake"]) + ) + assert options.log == os.path.expanduser("~/path") - def test_local_log(self): - options, args = main(['--local-log', '~/path', 'fake']) - assert options.log == os.path.expanduser('~/path') + def test_local_log(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--local-log", "~/path", "fake"]) + ) + assert options.log == os.path.expanduser("~/path") - def test_cert(self): - options, args = main(['--cert', '~/path', 'fake']) - assert options.cert == os.path.expanduser('~/path') + def test_cert(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--cert", "~/path", "fake"]) + ) + assert options.cert == os.path.expanduser("~/path") - def test_client_cert(self): - options, args = main(['--client-cert', '~/path', 'fake']) - assert options.client_cert == os.path.expanduser('~/path') + def test_client_cert(self) -> None: + # FakeCommand intentionally returns the wrong type. + options, args = cast( + Tuple[Values, List[str]], main(["--client-cert", "~/path", "fake"]) + ) + assert options.client_cert == os.path.expanduser("~/path") diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 448e3806300..88277448c2c 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -1,22 +1,44 @@ +from typing import Optional, Tuple + import pytest from pip._vendor.packaging import specifiers +from pip._vendor.packaging.requirements import Requirement -from pip._internal.utils.packaging import check_requires_python +from pip._internal.utils.packaging import check_requires_python, get_requirement -@pytest.mark.parametrize('version_info, requires_python, expected', [ - ((3, 6, 5), '== 3.6.4', False), - ((3, 6, 5), '== 3.6.5', True), - ((3, 6, 5), None, True), -]) -def test_check_requires_python(version_info, requires_python, expected): +@pytest.mark.parametrize( + "version_info, requires_python, expected", + [ + ((3, 6, 5), "== 3.6.4", False), + ((3, 6, 5), "== 3.6.5", True), + ((3, 6, 5), None, True), + ], +) +def test_check_requires_python( + version_info: Tuple[int, int, int], requires_python: Optional[str], expected: bool +) -> None: actual = check_requires_python(requires_python, version_info) assert actual == expected -def test_check_requires_python__invalid(): +def test_check_requires_python__invalid() -> None: """ Test an invalid Requires-Python value. """ with pytest.raises(specifiers.InvalidSpecifier): - check_requires_python('invalid', (3, 6, 5)) + check_requires_python("invalid", (3, 6, 5)) + + +def test_get_or_create_caching() -> None: + """test caching of get_or_create requirement""" + teststr = "affinegap==1.10" + from_helper = get_requirement(teststr) + freshly_made = Requirement(teststr) + + # Requirement doesn't have an equality operator (yet) so test + # equality of attribute for list of attributes + for iattr in ["name", "url", "extras", "specifier", "marker"]: + assert getattr(from_helper, iattr) == getattr(freshly_made, iattr) + assert get_requirement(teststr) is not Requirement(teststr) + assert get_requirement(teststr) is get_requirement(teststr) diff --git a/tests/unit/test_pep517.py b/tests/unit/test_pep517.py index 18cb178bba7..9305cf2a19b 100644 --- a/tests/unit/test_pep517.py +++ b/tests/unit/test_pep517.py @@ -2,16 +2,21 @@ import pytest -from pip._internal.exceptions import InstallationError +from pip._internal.exceptions import InstallationError, InvalidPyProjectBuildRequires from pip._internal.req import InstallRequirement +from tests.lib import TestData +from tests.lib.path import Path -@pytest.mark.parametrize(('source', 'expected'), [ - ("pep517_setup_and_pyproject", True), - ("pep517_setup_only", False), - ("pep517_pyproject_only", True), -]) -def test_use_pep517(shared_data, source, expected): +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("pep517_setup_and_pyproject", True), + ("pep517_setup_only", False), + ("pep517_pyproject_only", True), + ], +) +def test_use_pep517(shared_data: TestData, source: str, expected: bool) -> None: """ Test that we choose correctly between PEP 517 and legacy code paths """ @@ -22,11 +27,30 @@ def test_use_pep517(shared_data, source, expected): assert req.use_pep517 is expected -@pytest.mark.parametrize(('source', 'msg'), [ - ("pep517_setup_and_pyproject", "specifies a build backend"), - ("pep517_pyproject_only", "does not have a setup.py"), -]) -def test_disabling_pep517_invalid(shared_data, source, msg): +def test_use_pep517_rejects_setup_cfg_only(shared_data: TestData) -> None: + """ + Test that projects with setup.cfg but no pyproject.toml are rejected. + """ + src = shared_data.src.joinpath("pep517_setup_cfg_only") + req = InstallRequirement(None, None) + req.source_dir = src # make req believe it has been unpacked + with pytest.raises(InstallationError) as e: + req.load_pyproject_toml() + err_msg = e.value.args[0] + assert ( + "does not appear to be a Python project: " + "neither 'setup.py' nor 'pyproject.toml' found" in err_msg + ) + + +@pytest.mark.parametrize( + ("source", "msg"), + [ + ("pep517_setup_and_pyproject", "specifies a build backend"), + ("pep517_pyproject_only", "does not have a setup.py"), + ], +) +def test_disabling_pep517_invalid(shared_data: TestData, source: str, msg: str) -> None: """ Test that we fail if we try to disable PEP 517 when it's not acceptable """ @@ -48,19 +72,27 @@ def test_disabling_pep517_invalid(shared_data, source, msg): @pytest.mark.parametrize( ("spec",), [("./foo",), ("git+https://example.com/pkg@dev#egg=myproj",)] ) -def test_pep517_parsing_checks_requirements(tmpdir, spec): - tmpdir.joinpath("pyproject.toml").write_text(dedent( - """ - [build-system] - requires = [{!r}] - build-backend = "foo" - """.format(spec) - )) +def test_pep517_parsing_checks_requirements(tmpdir: Path, spec: str) -> None: + tmpdir.joinpath("pyproject.toml").write_text( + dedent( + f""" + [build-system] + requires = [{spec!r}] + build-backend = "foo" + """ + ) + ) req = InstallRequirement(None, None) req.source_dir = tmpdir # make req believe it has been unpacked - with pytest.raises(InstallationError) as e: + with pytest.raises(InvalidPyProjectBuildRequires) as e: req.load_pyproject_toml() - err_msg = e.value.args[0] - assert "contains an invalid requirement" in err_msg + error = e.value + + assert str(req) in error.message + assert error.context + assert "build-system.requires" in error.context + assert "contains an invalid requirement" in error.context + assert error.hint_stmt + assert "PEP 518" in error.hint_stmt diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index c7be5fe1bac..4a339e4e231 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -1,24 +1,29 @@ import contextlib +import email.message import os import shutil import sys import tempfile from functools import partial +from typing import Iterator, Tuple, cast +from unittest import mock import pytest -from mock import patch -from pip._vendor import pkg_resources from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement from pip._internal.commands import create_command +from pip._internal.commands.install import InstallCommand from pip._internal.exceptions import ( HashErrors, InstallationError, InvalidWheelFilename, PreviousBuildDirError, ) +from pip._internal.index.package_finder import PackageFinder +from pip._internal.metadata.pkg_resources import Distribution from pip._internal.network.session import PipSession +from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import InstallRequirement, RequirementSet from pip._internal.req.constructors import ( @@ -35,13 +40,15 @@ get_line_parser, handle_requirement_line, ) -from pip._internal.req.req_tracker import get_requirement_tracker from pip._internal.resolution.legacy.resolver import Resolver from pip._internal.utils.urls import path_to_url -from tests.lib import assert_raises_regexp, make_test_finder, requirements_file +from tests.lib import TestData, make_test_finder, requirements_file +from tests.lib.path import Path -def get_processed_req_from_line(line, fname='file', lineno=1): +def get_processed_req_from_line( + line: str, fname: str = "file", lineno: int = 1 +) -> InstallRequirement: line_parser = get_line_parser(None) args_str, opts = line_parser(line) parsed_line = ParsedLine( @@ -61,14 +68,16 @@ def get_processed_req_from_line(line, fname='file', lineno=1): class TestRequirementSet: """RequirementSet tests""" - def setup(self): + def setup(self) -> None: self.tempdir = tempfile.mkdtemp() - def teardown(self): + def teardown(self) -> None: shutil.rmtree(self.tempdir, ignore_errors=True) @contextlib.contextmanager - def _basic_resolver(self, finder, require_hashes=False): + def _basic_resolver( + self, finder: PackageFinder, require_hashes: bool = False + ) -> Iterator[Resolver]: make_install_req = partial( install_req_from_req_string, isolated=False, @@ -76,110 +85,108 @@ def _basic_resolver(self, finder, require_hashes=False): ) session = PipSession() - with get_requirement_tracker() as tracker: + with get_build_tracker() as tracker: preparer = RequirementPreparer( - build_dir=os.path.join(self.tempdir, 'build'), - src_dir=os.path.join(self.tempdir, 'src'), + build_dir=os.path.join(self.tempdir, "build"), + src_dir=os.path.join(self.tempdir, "src"), download_dir=None, build_isolation=True, - req_tracker=tracker, + build_tracker=tracker, session=session, - progress_bar='on', + progress_bar="on", finder=finder, require_hashes=require_hashes, use_user_site=False, lazy_wheel=False, + verbosity=0, + in_tree_build=False, ) yield Resolver( preparer=preparer, make_install_req=make_install_req, finder=finder, wheel_cache=None, - use_user_site=False, upgrade_strategy="to-satisfy-only", - ignore_dependencies=False, ignore_installed=False, - ignore_requires_python=False, force_reinstall=False, + use_user_site=False, + upgrade_strategy="to-satisfy-only", + ignore_dependencies=False, + ignore_installed=False, + ignore_requires_python=False, + force_reinstall=False, ) - def test_no_reuse_existing_build_dir(self, data): + def test_no_reuse_existing_build_dir(self, data: TestData) -> None: """Test prepare_files raise exception with previous build dir""" - build_dir = os.path.join(self.tempdir, 'build', 'simple') + build_dir = os.path.join(self.tempdir, "build", "simple") os.makedirs(build_dir) - with open(os.path.join(build_dir, "setup.py"), 'w'): + with open(os.path.join(build_dir, "setup.py"), "w"): pass reqset = RequirementSet() - req = install_req_from_line('simple') + req = install_req_from_line("simple") req.user_supplied = True reqset.add_requirement(req) finder = make_test_finder(find_links=[data.find_links]) with self._basic_resolver(finder) as resolver: - assert_raises_regexp( + with pytest.raises( PreviousBuildDirError, - r"pip can't proceed with [\s\S]*{req}[\s\S]*{build_dir_esc}" - .format( - build_dir_esc=build_dir.replace('\\', '\\\\'), req=req), - resolver.resolve, - reqset.all_requirements, - True, - ) - - # TODO: Update test when Python 2.7 is dropped. - def test_environment_marker_extras(self, data): + match=( + r"pip can't proceed with [\s\S]*{req}[\s\S]*{build_dir_esc}".format( + build_dir_esc=build_dir.replace("\\", "\\\\"), req=req + ) + ), + ): + resolver.resolve(reqset.all_requirements, True) + + def test_environment_marker_extras(self, data: TestData) -> None: """ Test that the environment marker extras are used with non-wheel installs. """ reqset = RequirementSet() - req = install_req_from_editable( - data.packages.joinpath("LocalEnvironMarker") - ) + req = install_req_from_editable(data.packages.joinpath("LocalEnvironMarker")) req.user_supplied = True reqset.add_requirement(req) finder = make_test_finder(find_links=[data.find_links]) with self._basic_resolver(finder) as resolver: reqset = resolver.resolve(reqset.all_requirements, True) - # This is hacky but does test both case in py2 and py3 - if sys.version_info[:2] == (2, 7): - assert reqset.has_requirement('simple') - else: - assert not reqset.has_requirement('simple') + assert not reqset.has_requirement("simple") - def test_missing_hash_with_require_hashes(self, data): + def test_missing_hash_with_require_hashes(self, data: TestData) -> None: """Setting --require-hashes explicitly should raise errors if hashes are missing. """ reqset = RequirementSet() - reqset.add_requirement(get_processed_req_from_line( - 'simple==1.0', lineno=1 - )) + reqset.add_requirement(get_processed_req_from_line("simple==1.0", lineno=1)) finder = make_test_finder(find_links=[data.find_links]) with self._basic_resolver(finder, require_hashes=True) as resolver: - assert_raises_regexp( + with pytest.raises( HashErrors, - r'Hashes are required in --require-hashes mode, but they are ' - r'missing .*\n' - r' simple==1.0 --hash=sha256:393043e672415891885c9a2a0929b1' - r'af95fb866d6ca016b42d2e6ce53619b653$', - resolver.resolve, - reqset.all_requirements, - True, - ) - - def test_missing_hash_with_require_hashes_in_reqs_file(self, data, tmpdir): + match=( + r"Hashes are required in --require-hashes mode, but they are " + r"missing .*\n" + r" simple==1.0 --hash=sha256:393043e672415891885c9a2a0929b1" + r"af95fb866d6ca016b42d2e6ce53619b653$" + ), + ): + resolver.resolve(reqset.all_requirements, True) + + def test_missing_hash_with_require_hashes_in_reqs_file( + self, data: TestData, tmpdir: Path + ) -> None: """--require-hashes in a requirements file should make its way to the RequirementSet. """ finder = make_test_finder(find_links=[data.find_links]) session = finder._link_collector.session - command = create_command('install') - with requirements_file('--require-hashes', tmpdir) as reqs_file: - options, args = command.parse_args(['-r', reqs_file]) + command = cast(InstallCommand, create_command("install")) + with requirements_file("--require-hashes", tmpdir) as reqs_file: + options, args = command.parse_args(["-r", reqs_file]) command.get_requirements(args, options, finder, session) assert options.require_hashes - def test_unsupported_hashes(self, data): + def test_unsupported_hashes(self, data: TestData) -> None: """VCS and dir links should raise errors when --require-hashes is on. @@ -188,112 +195,125 @@ def test_unsupported_hashes(self, data): """ reqset = RequirementSet() - reqset.add_requirement(get_processed_req_from_line( - 'git+git://github.com/pypa/pip-test-package --hash=sha256:123', - lineno=1, - )) - dir_path = data.packages.joinpath('FSPkg') - reqset.add_requirement(get_processed_req_from_line( - 'file://{dir_path}'.format(**locals()), - lineno=2, - )) + reqset.add_requirement( + get_processed_req_from_line( + "git+git://github.com/pypa/pip-test-package --hash=sha256:123", + lineno=1, + ) + ) + dir_path = data.packages.joinpath("FSPkg") + reqset.add_requirement( + get_processed_req_from_line( + f"file://{dir_path}", + lineno=2, + ) + ) finder = make_test_finder(find_links=[data.find_links]) sep = os.path.sep - if sep == '\\': - sep = '\\\\' # This needs to be escaped for the regex + if sep == "\\": + sep = "\\\\" # This needs to be escaped for the regex with self._basic_resolver(finder, require_hashes=True) as resolver: - assert_raises_regexp( + with pytest.raises( HashErrors, - r"Can't verify hashes for these requirements because we don't " - r"have a way to hash version control repositories:\n" - r" git\+git://github\.com/pypa/pip-test-package \(from -r " - r"file \(line 1\)\)\n" - r"Can't verify hashes for these file:// requirements because " - r"they point to directories:\n" - r" file://.*{sep}data{sep}packages{sep}FSPkg " - r"\(from -r file \(line 2\)\)".format(sep=sep), - resolver.resolve, - reqset.all_requirements, - True, - ) - - def test_unpinned_hash_checking(self, data): + match=( + r"Can't verify hashes for these requirements because we don't " + r"have a way to hash version control repositories:\n" + r" git\+git://github\.com/pypa/pip-test-package \(from -r " + r"file \(line 1\)\)\n" + r"Can't verify hashes for these file:// requirements because " + r"they point to directories:\n" + r" file://.*{sep}data{sep}packages{sep}FSPkg " + r"\(from -r file \(line 2\)\)".format(sep=sep) + ), + ): + resolver.resolve(reqset.all_requirements, True) + + def test_unpinned_hash_checking(self, data: TestData) -> None: """Make sure prepare_files() raises an error when a requirement is not version-pinned in hash-checking mode. """ reqset = RequirementSet() # Test that there must be exactly 1 specifier: - reqset.add_requirement(get_processed_req_from_line( - 'simple --hash=sha256:a90427ae31f5d1d0d7ec06ee97d9fcf2d0fc9a786985' - '250c1c83fd68df5911dd', lineno=1, - )) + reqset.add_requirement( + get_processed_req_from_line( + "simple --hash=sha256:a90427ae31f5d1d0d7ec06ee97d9fcf2d0fc9a786985" + "250c1c83fd68df5911dd", + lineno=1, + ) + ) # Test that the operator must be ==: - reqset.add_requirement(get_processed_req_from_line( - 'simple2>1.0 --hash=sha256:3ad45e1e9aa48b4462af0' - '123f6a7e44a9115db1ef945d4d92c123dfe21815a06', - lineno=2, - )) + reqset.add_requirement( + get_processed_req_from_line( + "simple2>1.0 --hash=sha256:3ad45e1e9aa48b4462af0" + "123f6a7e44a9115db1ef945d4d92c123dfe21815a06", + lineno=2, + ) + ) finder = make_test_finder(find_links=[data.find_links]) with self._basic_resolver(finder, require_hashes=True) as resolver: - assert_raises_regexp( + with pytest.raises( HashErrors, # Make sure all failing requirements are listed: - r'versions pinned with ==. These do not:\n' - r' simple .* \(from -r file \(line 1\)\)\n' - r' simple2>1.0 .* \(from -r file \(line 2\)\)', - resolver.resolve, - reqset.all_requirements, - True, - ) - - def test_hash_mismatch(self, data): + match=( + r"versions pinned with ==. These do not:\n" + r" simple .* \(from -r file \(line 1\)\)\n" + r" simple2>1.0 .* \(from -r file \(line 2\)\)" + ), + ): + resolver.resolve(reqset.all_requirements, True) + + def test_hash_mismatch(self, data: TestData) -> None: """A hash mismatch should raise an error.""" - file_url = path_to_url( - (data.packages / 'simple-1.0.tar.gz').resolve()) + file_url = path_to_url((data.packages / "simple-1.0.tar.gz").resolve()) reqset = RequirementSet() - reqset.add_requirement(get_processed_req_from_line( - '{file_url} --hash=sha256:badbad'.format(**locals()), lineno=1, - )) + reqset.add_requirement( + get_processed_req_from_line( + f"{file_url} --hash=sha256:badbad", + lineno=1, + ) + ) finder = make_test_finder(find_links=[data.find_links]) with self._basic_resolver(finder, require_hashes=True) as resolver: - assert_raises_regexp( + with pytest.raises( HashErrors, - r'THESE PACKAGES DO NOT MATCH THE HASHES.*\n' - r' file:///.*/data/packages/simple-1\.0\.tar\.gz .*:\n' - r' Expected sha256 badbad\n' - r' Got 393043e672415891885c9a2a0929b1af95fb' - r'866d6ca016b42d2e6ce53619b653$', - resolver.resolve, - reqset.all_requirements, - True, - ) - - def test_unhashed_deps_on_require_hashes(self, data): + match=( + r"THESE PACKAGES DO NOT MATCH THE HASHES.*\n" + r" file:///.*/data/packages/simple-1\.0\.tar\.gz .*:\n" + r" Expected sha256 badbad\n" + r" Got 393043e672415891885c9a2a0929b1af95fb" + r"866d6ca016b42d2e6ce53619b653$" + ), + ): + resolver.resolve(reqset.all_requirements, True) + + def test_unhashed_deps_on_require_hashes(self, data: TestData) -> None: """Make sure unhashed, unpinned, or otherwise unrepeatable dependencies get complained about when --require-hashes is on.""" reqset = RequirementSet() finder = make_test_finder(find_links=[data.find_links]) - reqset.add_requirement(get_processed_req_from_line( - 'TopoRequires2==0.0.1 ' # requires TopoRequires - '--hash=sha256:eaf9a01242c9f2f42cf2bd82a6a848cd' - 'e3591d14f7896bdbefcf48543720c970', - lineno=1 - )) + reqset.add_requirement( + get_processed_req_from_line( + "TopoRequires2==0.0.1 " # requires TopoRequires + "--hash=sha256:eaf9a01242c9f2f42cf2bd82a6a848cd" + "e3591d14f7896bdbefcf48543720c970", + lineno=1, + ) + ) with self._basic_resolver(finder, require_hashes=True) as resolver: - assert_raises_regexp( + with pytest.raises( HashErrors, - r'In --require-hashes mode, all requirements must have their ' - r'versions pinned.*\n' - r' TopoRequires from .*$', - resolver.resolve, - reqset.all_requirements, - True, - ) - - def test_hashed_deps_on_require_hashes(self): + match=( + r"In --require-hashes mode, all requirements must have their " + r"versions pinned.*\n" + r" TopoRequires from .*$" + ), + ): + resolver.resolve(reqset.all_requirements, True) + + def test_hashed_deps_on_require_hashes(self) -> None: """Make sure hashed dependencies get installed when --require-hashes is on. @@ -303,65 +323,70 @@ def test_hashed_deps_on_require_hashes(self): """ reqset = RequirementSet() - reqset.add_requirement(get_processed_req_from_line( - 'TopoRequires2==0.0.1 ' # requires TopoRequires - '--hash=sha256:eaf9a01242c9f2f42cf2bd82a6a848cd' - 'e3591d14f7896bdbefcf48543720c970', - lineno=1 - )) - reqset.add_requirement(get_processed_req_from_line( - 'TopoRequires==0.0.1 ' - '--hash=sha256:d6dd1e22e60df512fdcf3640ced3039b3b02a56ab2cee81ebcb' - '3d0a6d4e8bfa6', - lineno=2 - )) + reqset.add_requirement( + get_processed_req_from_line( + "TopoRequires2==0.0.1 " # requires TopoRequires + "--hash=sha256:eaf9a01242c9f2f42cf2bd82a6a848cd" + "e3591d14f7896bdbefcf48543720c970", + lineno=1, + ) + ) + reqset.add_requirement( + get_processed_req_from_line( + "TopoRequires==0.0.1 " + "--hash=sha256:d6dd1e22e60df512fdcf3640ced3039b3b02a56ab2cee81ebcb" + "3d0a6d4e8bfa6", + lineno=2, + ) + ) class TestInstallRequirement: - def setup(self): + def setup(self) -> None: self.tempdir = tempfile.mkdtemp() - def teardown(self): + def teardown(self) -> None: shutil.rmtree(self.tempdir, ignore_errors=True) - def test_url_with_query(self): + def test_url_with_query(self) -> None: """InstallRequirement should strip the fragment, but not the query.""" - url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz' - fragment = '#egg=bar' + url = "http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz" + fragment = "#egg=bar" req = install_req_from_line(url + fragment) + assert req.link is not None assert req.link.url == url + fragment, req.link - def test_pep440_wheel_link_requirement(self): - url = 'https://whatever.com/test-0.4-py2.py3-bogus-any.whl' - line = 'test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl' + def test_pep440_wheel_link_requirement(self) -> None: + url = "https://whatever.com/test-0.4-py2.py3-bogus-any.whl" + line = "test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl" req = install_req_from_line(line) - parts = str(req.req).split('@', 1) + parts = str(req.req).split("@", 1) assert len(parts) == 2 - assert parts[0].strip() == 'test' + assert parts[0].strip() == "test" assert parts[1].strip() == url - def test_pep440_url_link_requirement(self): - url = 'git+http://foo.com@ref#egg=foo' - line = 'foo @ git+http://foo.com@ref#egg=foo' + def test_pep440_url_link_requirement(self) -> None: + url = "git+http://foo.com@ref#egg=foo" + line = "foo @ git+http://foo.com@ref#egg=foo" req = install_req_from_line(line) - parts = str(req.req).split('@', 1) + parts = str(req.req).split("@", 1) assert len(parts) == 2 - assert parts[0].strip() == 'foo' + assert parts[0].strip() == "foo" assert parts[1].strip() == url - def test_url_with_authentication_link_requirement(self): - url = 'https://what@whatever.com/test-0.4-py2.py3-bogus-any.whl' - line = 'https://what@whatever.com/test-0.4-py2.py3-bogus-any.whl' + def test_url_with_authentication_link_requirement(self) -> None: + url = "https://what@whatever.com/test-0.4-py2.py3-bogus-any.whl" + line = "https://what@whatever.com/test-0.4-py2.py3-bogus-any.whl" req = install_req_from_line(line) assert req.link is not None assert req.link.is_wheel assert req.link.scheme == "https" assert req.link.url == url - def test_unsupported_wheel_link_requirement_raises(self): + def test_unsupported_wheel_link_requirement_raises(self) -> None: reqset = RequirementSet() req = install_req_from_line( - 'https://whatever.com/peppercorn-0.4-py2.py3-bogus-any.whl', + "https://whatever.com/peppercorn-0.4-py2.py3-bogus-any.whl", ) assert req.link is not None assert req.link.is_wheel @@ -370,10 +395,12 @@ def test_unsupported_wheel_link_requirement_raises(self): with pytest.raises(InstallationError): reqset.add_requirement(req) - def test_unsupported_wheel_local_file_requirement_raises(self, data): + def test_unsupported_wheel_local_file_requirement_raises( + self, data: TestData + ) -> None: reqset = RequirementSet() req = install_req_from_line( - data.packages.joinpath('simple.dist-0.1-py1-none-invalid.whl'), + data.packages.joinpath("simple.dist-0.1-py1-none-invalid.whl"), ) assert req.link is not None assert req.link.is_wheel @@ -382,55 +409,54 @@ def test_unsupported_wheel_local_file_requirement_raises(self, data): with pytest.raises(InstallationError): reqset.add_requirement(req) - def test_installed_version_not_installed(self): - req = install_req_from_line('simple-0.1-py2.py3-none-any.whl') - assert req.installed_version is None + def test_str(self) -> None: + req = install_req_from_line("simple==0.1") + assert str(req) == "simple==0.1" - def test_str(self): - req = install_req_from_line('simple==0.1') - assert str(req) == 'simple==0.1' - - def test_repr(self): - req = install_req_from_line('simple==0.1') - assert repr(req) == ( - '' - ) + def test_repr(self) -> None: + req = install_req_from_line("simple==0.1") + assert repr(req) == ("") - def test_invalid_wheel_requirement_raises(self): + def test_invalid_wheel_requirement_raises(self) -> None: with pytest.raises(InvalidWheelFilename): - install_req_from_line('invalid.whl') + install_req_from_line("invalid.whl") - def test_wheel_requirement_sets_req_attribute(self): - req = install_req_from_line('simple-0.1-py2.py3-none-any.whl') + def test_wheel_requirement_sets_req_attribute(self) -> None: + req = install_req_from_line("simple-0.1-py2.py3-none-any.whl") assert isinstance(req.req, Requirement) - assert str(req.req) == 'simple==0.1' + assert str(req.req) == "simple==0.1" - def test_url_preserved_line_req(self): + def test_url_preserved_line_req(self) -> None: """Confirm the url is preserved in a non-editable requirement""" - url = 'git+http://foo.com@ref#egg=foo' + url = "git+http://foo.com@ref#egg=foo" req = install_req_from_line(url) + assert req.link is not None assert req.link.url == url - def test_url_preserved_editable_req(self): + def test_url_preserved_editable_req(self) -> None: """Confirm the url is preserved in a editable requirement""" - url = 'git+http://foo.com@ref#egg=foo' + url = "git+http://foo.com@ref#egg=foo" req = install_req_from_editable(url) + assert req.link is not None assert req.link.url == url - @pytest.mark.parametrize('path', ( - '/path/to/foo.egg-info'.replace('/', os.path.sep), - # Tests issue fixed by https://github.com/pypa/pip/pull/2530 - '/path/to/foo.egg-info/'.replace('/', os.path.sep), - )) - def test_get_dist(self, path): - req = install_req_from_line('foo') + @pytest.mark.parametrize( + "path", + ( + "/path/to/foo.egg-info".replace("/", os.path.sep), + # Tests issue fixed by https://github.com/pypa/pip/pull/2530 + "/path/to/foo.egg-info/".replace("/", os.path.sep), + ), + ) + def test_get_dist(self, path: str) -> None: + req = install_req_from_line("foo") req.metadata_directory = path dist = req.get_dist() - assert isinstance(dist, pkg_resources.Distribution) - assert dist.project_name == 'foo' - assert dist.location == '/path/to'.replace('/', os.path.sep) + assert isinstance(dist, Distribution) + assert dist.raw_name == dist.canonical_name == "foo" + assert dist.location == "/path/to".replace("/", os.path.sep) - def test_markers(self): + def test_markers(self) -> None: for line in ( # recommended syntax 'mock3; python_version >= "3"', @@ -440,39 +466,43 @@ def test_markers(self): 'mock3;python_version >= "3"', ): req = install_req_from_line(line) - assert req.req.name == 'mock3' - assert str(req.req.specifier) == '' + assert req.req is not None + assert req.req.name == "mock3" + assert str(req.req.specifier) == "" assert str(req.markers) == 'python_version >= "3"' - def test_markers_semicolon(self): + def test_markers_semicolon(self) -> None: # check that the markers can contain a semicolon req = install_req_from_line('semicolon; os_name == "a; b"') - assert req.req.name == 'semicolon' - assert str(req.req.specifier) == '' + assert req.req is not None + assert req.req.name == "semicolon" + assert str(req.req.specifier) == "" assert str(req.markers) == 'os_name == "a; b"' - def test_markers_url(self): + def test_markers_url(self) -> None: # test "URL; markers" syntax - url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz' + url = "http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz" line = f'{url}; python_version >= "3"' req = install_req_from_line(line) - assert req.link.url == url, req.url + assert req.link is not None + assert req.link.url == url, req.link.url assert str(req.markers) == 'python_version >= "3"' # without space, markers are part of the URL - url = 'http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz' + url = "http://foo.com/?p=bar.git;a=snapshot;h=v0.1;sf=tgz" line = f'{url};python_version >= "3"' req = install_req_from_line(line) - assert req.link.url == line, req.url + assert req.link is not None + assert req.link.url == line, req.link.url assert req.markers is None - def test_markers_match_from_line(self): + def test_markers_match_from_line(self) -> None: # match for markers in ( 'python_version >= "1.0"', - 'sys_platform == {sys.platform!r}'.format(**globals()), + f"sys_platform == {sys.platform!r}", ): - line = 'name; ' + markers + line = "name; " + markers req = install_req_from_line(line) assert str(req.markers) == str(Marker(markers)) assert req.match_markers() @@ -480,92 +510,91 @@ def test_markers_match_from_line(self): # don't match for markers in ( 'python_version >= "5.0"', - 'sys_platform != {sys.platform!r}'.format(**globals()), + f"sys_platform != {sys.platform!r}", ): - line = 'name; ' + markers + line = "name; " + markers req = install_req_from_line(line) assert str(req.markers) == str(Marker(markers)) assert not req.match_markers() - def test_markers_match(self): + def test_markers_match(self) -> None: # match for markers in ( 'python_version >= "1.0"', - 'sys_platform == {sys.platform!r}'.format(**globals()), + f"sys_platform == {sys.platform!r}", ): - line = 'name; ' + markers - req = install_req_from_line(line, comes_from='') + line = "name; " + markers + req = install_req_from_line(line, comes_from="") assert str(req.markers) == str(Marker(markers)) assert req.match_markers() # don't match for markers in ( 'python_version >= "5.0"', - 'sys_platform != {sys.platform!r}'.format(**globals()), + f"sys_platform != {sys.platform!r}", ): - line = 'name; ' + markers - req = install_req_from_line(line, comes_from='') + line = "name; " + markers + req = install_req_from_line(line, comes_from="") assert str(req.markers) == str(Marker(markers)) assert not req.match_markers() - def test_extras_for_line_path_requirement(self): - line = 'SomeProject[ex1,ex2]' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + def test_extras_for_line_path_requirement(self) -> None: + line = "SomeProject[ex1,ex2]" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_line(line, comes_from=comes_from) assert len(req.extras) == 2 - assert req.extras == {'ex1', 'ex2'} + assert req.extras == {"ex1", "ex2"} - def test_extras_for_line_url_requirement(self): - line = 'git+https://url#egg=SomeProject[ex1,ex2]' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + def test_extras_for_line_url_requirement(self) -> None: + line = "git+https://url#egg=SomeProject[ex1,ex2]" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_line(line, comes_from=comes_from) assert len(req.extras) == 2 - assert req.extras == {'ex1', 'ex2'} + assert req.extras == {"ex1", "ex2"} - def test_extras_for_editable_path_requirement(self): - url = '.[ex1,ex2]' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + def test_extras_for_editable_path_requirement(self) -> None: + url = ".[ex1,ex2]" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_editable(url, comes_from=comes_from) assert len(req.extras) == 2 - assert req.extras == {'ex1', 'ex2'} + assert req.extras == {"ex1", "ex2"} - def test_extras_for_editable_url_requirement(self): - url = 'git+https://url#egg=SomeProject[ex1,ex2]' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + def test_extras_for_editable_url_requirement(self) -> None: + url = "git+https://url#egg=SomeProject[ex1,ex2]" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_editable(url, comes_from=comes_from) assert len(req.extras) == 2 - assert req.extras == {'ex1', 'ex2'} + assert req.extras == {"ex1", "ex2"} - def test_unexisting_path(self): + def test_unexisting_path(self) -> None: with pytest.raises(InstallationError) as e: - install_req_from_line( - os.path.join('this', 'path', 'does', 'not', 'exist')) + install_req_from_line(os.path.join("this", "path", "does", "not", "exist")) err_msg = e.value.args[0] assert "Invalid requirement" in err_msg assert "It looks like a path." in err_msg - def test_single_equal_sign(self): + def test_single_equal_sign(self) -> None: with pytest.raises(InstallationError) as e: - install_req_from_line('toto=42') + install_req_from_line("toto=42") err_msg = e.value.args[0] assert "Invalid requirement" in err_msg assert "= is not a valid operator. Did you mean == ?" in err_msg - def test_unidentifiable_name(self): - test_name = '-' + def test_unidentifiable_name(self) -> None: + test_name = "-" with pytest.raises(InstallationError) as e: install_req_from_line(test_name) err_msg = e.value.args[0] assert f"Invalid requirement: '{test_name}'" == err_msg - def test_requirement_file(self): - req_file_path = os.path.join(self.tempdir, 'test.txt') - with open(req_file_path, 'w') as req_file: - req_file.write('pip\nsetuptools') + def test_requirement_file(self) -> None: + req_file_path = os.path.join(self.tempdir, "test.txt") + with open(req_file_path, "w") as req_file: + req_file.write("pip\nsetuptools") with pytest.raises(InstallationError) as e: install_req_from_line(req_file_path) err_msg = e.value.args[0] @@ -575,168 +604,209 @@ def test_requirement_file(self): assert "If that is the case, use the '-r' flag to install" in err_msg -@patch('pip._internal.req.req_install.os.path.abspath') -@patch('pip._internal.req.req_install.os.path.exists') -@patch('pip._internal.req.req_install.os.path.isdir') +@mock.patch("pip._internal.req.req_install.os.path.abspath") +@mock.patch("pip._internal.req.req_install.os.path.exists") +@mock.patch("pip._internal.req.req_install.os.path.isdir") def test_parse_editable_local( - isdir_mock, exists_mock, abspath_mock): + isdir_mock: mock.Mock, exists_mock: mock.Mock, abspath_mock: mock.Mock +) -> None: exists_mock.return_value = isdir_mock.return_value = True # mocks needed to support path operations on windows tests abspath_mock.return_value = "/some/path" - assert parse_editable('.') == (None, 'file:///some/path', set()) + assert parse_editable(".") == (None, "file:///some/path", set()) abspath_mock.return_value = "/some/path/foo" - assert parse_editable('foo') == ( - None, 'file:///some/path/foo', set(), + assert parse_editable("foo") == ( + None, + "file:///some/path/foo", + set(), ) -def test_parse_editable_explicit_vcs(): - assert parse_editable('svn+https://foo#egg=foo') == ( - 'foo', - 'svn+https://foo#egg=foo', +def test_parse_editable_explicit_vcs() -> None: + assert parse_editable("svn+https://foo#egg=foo") == ( + "foo", + "svn+https://foo#egg=foo", set(), ) -def test_parse_editable_vcs_extras(): - assert parse_editable('svn+https://foo#egg=foo[extras]') == ( - 'foo[extras]', - 'svn+https://foo#egg=foo[extras]', +def test_parse_editable_vcs_extras() -> None: + assert parse_editable("svn+https://foo#egg=foo[extras]") == ( + "foo[extras]", + "svn+https://foo#egg=foo[extras]", set(), ) -@patch('pip._internal.req.req_install.os.path.abspath') -@patch('pip._internal.req.req_install.os.path.exists') -@patch('pip._internal.req.req_install.os.path.isdir') +@mock.patch("pip._internal.req.req_install.os.path.abspath") +@mock.patch("pip._internal.req.req_install.os.path.exists") +@mock.patch("pip._internal.req.req_install.os.path.isdir") def test_parse_editable_local_extras( - isdir_mock, exists_mock, abspath_mock): + isdir_mock: mock.Mock, exists_mock: mock.Mock, abspath_mock: mock.Mock +) -> None: exists_mock.return_value = isdir_mock.return_value = True abspath_mock.return_value = "/some/path" - assert parse_editable('.[extras]') == ( - None, 'file://' + "/some/path", {'extras'}, + assert parse_editable(".[extras]") == ( + None, + "file:///some/path", + {"extras"}, ) abspath_mock.return_value = "/some/path/foo" - assert parse_editable('foo[bar,baz]') == ( - None, 'file:///some/path/foo', {'bar', 'baz'}, + assert parse_editable("foo[bar,baz]") == ( + None, + "file:///some/path/foo", + {"bar", "baz"}, ) -def test_exclusive_environment_markers(): +def test_exclusive_environment_markers() -> None: """Make sure RequirementSet accepts several excluding env markers""" - eq36 = install_req_from_line( - "Django>=1.6.10,<1.7 ; python_version == '3.6'") + eq36 = install_req_from_line("Django>=1.6.10,<1.7 ; python_version == '3.6'") eq36.user_supplied = True - ne36 = install_req_from_line( - "Django>=1.6.10,<1.8 ; python_version != '3.6'") + ne36 = install_req_from_line("Django>=1.6.10,<1.8 ; python_version != '3.6'") ne36.user_supplied = True req_set = RequirementSet() req_set.add_requirement(eq36) req_set.add_requirement(ne36) - assert req_set.has_requirement('Django') + assert req_set.has_requirement("Django") -def test_mismatched_versions(caplog): +def test_mismatched_versions(caplog: pytest.LogCaptureFixture) -> None: req = InstallRequirement( - req=Requirement('simplewheel==2.0'), + req=Requirement("simplewheel==2.0"), comes_from=None, ) req.source_dir = "/tmp/somewhere" # make req believe it has been unpacked # Monkeypatch! - req._metadata = {"name": "simplewheel", "version": "1.0"} + metadata = email.message.Message() + metadata["name"] = "simplewheel" + metadata["version"] = "1.0" + req._metadata = metadata + req.assert_source_matches_version() assert caplog.records[-1].message == ( - 'Requested simplewheel==2.0, but installing version 1.0' + "Requested simplewheel==2.0, but installing version 1.0" ) -@pytest.mark.parametrize('args, expected', [ - # Test UNIX-like paths - (('/path/to/installable'), True), - # Test relative paths - (('./path/to/installable'), True), - # Test current path - (('.'), True), - # Test url paths - (('https://whatever.com/test-0.4-py2.py3-bogus-any.whl'), True), - # Test pep440 paths - (('test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl'), True), - # Test wheel - (('simple-0.1-py2.py3-none-any.whl'), False), -]) -def test_looks_like_path(args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + # Test UNIX-like paths + (("/path/to/installable"), True), + # Test relative paths + (("./path/to/installable"), True), + # Test current path + (("."), True), + # Test url paths + (("https://whatever.com/test-0.4-py2.py3-bogus-any.whl"), True), + # Test pep440 paths + (("test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl"), True), + # Test wheel + (("simple-0.1-py2.py3-none-any.whl"), False), + ], +) +def test_looks_like_path(args: str, expected: bool) -> None: assert _looks_like_path(args) == expected @pytest.mark.skipif( - not sys.platform.startswith("win"), - reason='Test only available on Windows' + not sys.platform.startswith("win"), reason="Test only available on Windows" ) -@pytest.mark.parametrize('args, expected', [ - # Test relative paths - (('.\\path\\to\\installable'), True), - (('relative\\path'), True), - # Test absolute paths - (('C:\\absolute\\path'), True), -]) -def test_looks_like_path_win(args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + # Test relative paths + ((".\\path\\to\\installable"), True), + (("relative\\path"), True), + # Test absolute paths + (("C:\\absolute\\path"), True), + ], +) +def test_looks_like_path_win(args: str, expected: bool) -> None: assert _looks_like_path(args) == expected -@pytest.mark.parametrize('args, mock_returns, expected', [ - # Test pep440 urls - (('/path/to/foo @ git+http://foo.com@ref#egg=foo', - 'foo @ git+http://foo.com@ref#egg=foo'), (False, False), None), - # Test pep440 urls without spaces - (('/path/to/foo@git+http://foo.com@ref#egg=foo', - 'foo @ git+http://foo.com@ref#egg=foo'), (False, False), None), - # Test pep440 wheel - (('/path/to/test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl', - 'test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl'), - (False, False), None), - # Test name is not a file - (('/path/to/simple==0.1', - 'simple==0.1'), - (False, False), None), -]) -@patch('pip._internal.req.req_install.os.path.isdir') -@patch('pip._internal.req.req_install.os.path.isfile') +@pytest.mark.parametrize( + "args, mock_returns, expected", + [ + # Test pep440 urls + ( + ( + "/path/to/foo @ git+http://foo.com@ref#egg=foo", + "foo @ git+http://foo.com@ref#egg=foo", + ), + (False, False), + None, + ), + # Test pep440 urls without spaces + ( + ( + "/path/to/foo@git+http://foo.com@ref#egg=foo", + "foo @ git+http://foo.com@ref#egg=foo", + ), + (False, False), + None, + ), + # Test pep440 wheel + ( + ( + "/path/to/test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl", + "test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl", + ), + (False, False), + None, + ), + # Test name is not a file + (("/path/to/simple==0.1", "simple==0.1"), (False, False), None), + ], +) +@mock.patch("pip._internal.req.req_install.os.path.isdir") +@mock.patch("pip._internal.req.req_install.os.path.isfile") def test_get_url_from_path( - isdir_mock, isfile_mock, args, mock_returns, expected -): + isdir_mock: mock.Mock, + isfile_mock: mock.Mock, + args: Tuple[str, str], + mock_returns: Tuple[bool, bool], + expected: None, +) -> None: isdir_mock.return_value = mock_returns[0] isfile_mock.return_value = mock_returns[1] assert _get_url_from_path(*args) is expected -@patch('pip._internal.req.req_install.os.path.isdir') -@patch('pip._internal.req.req_install.os.path.isfile') -def test_get_url_from_path__archive_file(isdir_mock, isfile_mock): +@mock.patch("pip._internal.req.req_install.os.path.isdir") +@mock.patch("pip._internal.req.req_install.os.path.isfile") +def test_get_url_from_path__archive_file( + isdir_mock: mock.Mock, isfile_mock: mock.Mock +) -> None: isdir_mock.return_value = False isfile_mock.return_value = True - name = 'simple-0.1-py2.py3-none-any.whl' - path = os.path.join('/path/to/' + name) + name = "simple-0.1-py2.py3-none-any.whl" + path = os.path.join("/path/to/" + name) url = path_to_url(path) assert _get_url_from_path(path, name) == url -@patch('pip._internal.req.req_install.os.path.isdir') -@patch('pip._internal.req.req_install.os.path.isfile') -def test_get_url_from_path__installable_dir(isdir_mock, isfile_mock): +@mock.patch("pip._internal.req.req_install.os.path.isdir") +@mock.patch("pip._internal.req.req_install.os.path.isfile") +def test_get_url_from_path__installable_dir( + isdir_mock: mock.Mock, isfile_mock: mock.Mock +) -> None: isdir_mock.return_value = True isfile_mock.return_value = True - name = 'some/setuptools/project' - path = os.path.join('/path/to/' + name) + name = "some/setuptools/project" + path = os.path.join("/path/to/" + name) url = path_to_url(path) assert _get_url_from_path(path, name) == url -@patch('pip._internal.req.req_install.os.path.isdir') -def test_get_url_from_path__installable_error(isdir_mock): +@mock.patch("pip._internal.req.req_install.os.path.isdir") +def test_get_url_from_path__installable_error(isdir_mock: mock.Mock) -> None: isdir_mock.return_value = True - name = 'some/setuptools/project' - path = os.path.join('/path/to/' + name) + name = "some/setuptools/project" + path = os.path.join("/path/to/" + name) with pytest.raises(InstallationError) as e: _get_url_from_path(path, name) err_msg = e.value.args[0] diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 86f2731e9e3..491877fb973 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -1,15 +1,18 @@ import collections import logging import os +import pathlib import subprocess import textwrap +from optparse import Values +from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Tuple +from unittest import mock import pytest -from mock import patch -from pretend import stub import pip._internal.req.req_file # this will be monkeypatched from pip._internal.exceptions import InstallationError, RequirementsFileParseError +from pip._internal.index.package_finder import PackageFinder from pip._internal.models.format_control import FormatControl from pip._internal.network.session import PipSession from pip._internal.req.constructors import ( @@ -24,161 +27,210 @@ parse_requirements, preprocess, ) -from tests.lib import make_test_finder, requirements_file +from pip._internal.req.req_install import InstallRequirement +from tests.lib import TestData, make_test_finder, requirements_file +from tests.lib.path import Path + +if TYPE_CHECKING: + from typing import Protocol +else: + # Protocol was introduced in Python 3.8. + Protocol = object @pytest.fixture -def session(): +def session() -> PipSession: return PipSession() @pytest.fixture -def finder(session): +def finder(session: PipSession) -> PackageFinder: return make_test_finder(session=session) @pytest.fixture -def options(session): - return stub( +def options(session: PipSession) -> mock.Mock: + return mock.Mock( isolated_mode=False, - index_url='default_url', + index_url="default_url", format_control=FormatControl(set(), set()), features_enabled=[], ) def parse_reqfile( - filename, - session, - finder=None, - options=None, - constraint=False, - isolated=False, -): + filename: str, + session: PipSession, + finder: PackageFinder = None, + options: Values = None, + constraint: bool = False, + isolated: bool = False, +) -> Iterator[InstallRequirement]: # Wrap parse_requirements/install_req_from_parsed_requirement to # avoid having to write the same chunk of code in lots of tests. for parsed_req in parse_requirements( - filename, session, finder=finder, - options=options, constraint=constraint, + filename, + session, + finder=finder, + options=options, + constraint=constraint, ): - yield install_req_from_parsed_requirement( - parsed_req, - isolated=isolated - ) + yield install_req_from_parsed_requirement(parsed_req, isolated=isolated) + + +def test_read_file_url(tmp_path: pathlib.Path, session: PipSession) -> None: + reqs = tmp_path.joinpath("requirements.txt") + reqs.write_text("foo") + result = list(parse_requirements(reqs.as_posix(), session)) + + assert len(result) == 1, result + assert result[0].requirement == "foo" + + # The comes_from value has three parts: -r or -c flag, path, and line. + # The path value in the middle needs some special logic due to our path + # normalization logic. + assert result[0].comes_from[:3] == "-r " + assert result[0].comes_from[-9:] == " (line 1)" + assert os.path.samefile(result[0].comes_from[3:-9], str(reqs)) class TestPreprocess: """tests for `preprocess`""" - def test_comments_and_joins_case1(self): - content = textwrap.dedent("""\ + def test_comments_and_joins_case1(self) -> None: + content = textwrap.dedent( + """\ req1 \\ # comment \\ req2 - """) + """ + ) result = preprocess(content) - assert list(result) == [(1, 'req1'), (3, 'req2')] + assert list(result) == [(1, "req1"), (3, "req2")] - def test_comments_and_joins_case2(self): - content = textwrap.dedent("""\ + def test_comments_and_joins_case2(self) -> None: + content = textwrap.dedent( + """\ req1\\ # comment - """) + """ + ) result = preprocess(content) - assert list(result) == [(1, 'req1')] + assert list(result) == [(1, "req1")] - def test_comments_and_joins_case3(self): - content = textwrap.dedent("""\ + def test_comments_and_joins_case3(self) -> None: + content = textwrap.dedent( + """\ req1 \\ # comment req2 - """) + """ + ) result = preprocess(content) - assert list(result) == [(1, 'req1'), (3, 'req2')] + assert list(result) == [(1, "req1"), (3, "req2")] class TestIgnoreComments: """tests for `ignore_comment`""" - def test_ignore_line(self): - lines = [(1, ''), (2, 'req1'), (3, 'req2')] + def test_ignore_line(self) -> None: + lines = [(1, ""), (2, "req1"), (3, "req2")] result = ignore_comments(lines) - assert list(result) == [(2, 'req1'), (3, 'req2')] + assert list(result) == [(2, "req1"), (3, "req2")] - def test_ignore_comment(self): - lines = [(1, 'req1'), (2, '# comment'), (3, 'req2')] + def test_ignore_comment(self) -> None: + lines = [(1, "req1"), (2, "# comment"), (3, "req2")] result = ignore_comments(lines) - assert list(result) == [(1, 'req1'), (3, 'req2')] + assert list(result) == [(1, "req1"), (3, "req2")] - def test_strip_comment(self): - lines = [(1, 'req1'), (2, 'req # comment'), (3, 'req2')] + def test_strip_comment(self) -> None: + lines = [(1, "req1"), (2, "req # comment"), (3, "req2")] result = ignore_comments(lines) - assert list(result) == [(1, 'req1'), (2, 'req'), (3, 'req2')] + assert list(result) == [(1, "req1"), (2, "req"), (3, "req2")] class TestJoinLines: """tests for `join_lines`""" - def test_join_lines(self): - lines = enumerate([ - 'line 1', - 'line 2:1 \\', - 'line 2:2', - 'line 3:1 \\', - 'line 3:2 \\', - 'line 3:3', - 'line 4' - ], start=1) + def test_join_lines(self) -> None: + lines = enumerate( + [ + "line 1", + "line 2:1 \\", + "line 2:2", + "line 3:1 \\", + "line 3:2 \\", + "line 3:3", + "line 4", + ], + start=1, + ) expect = [ - (1, 'line 1'), - (2, 'line 2:1 line 2:2'), - (4, 'line 3:1 line 3:2 line 3:3'), - (7, 'line 4'), + (1, "line 1"), + (2, "line 2:1 line 2:2"), + (4, "line 3:1 line 3:2 line 3:3"), + (7, "line 4"), ] assert expect == list(join_lines(lines)) - def test_last_line_with_escape(self): - lines = enumerate([ - 'line 1', - 'line 2 \\', - ], start=1) + def test_last_line_with_escape(self) -> None: + lines = enumerate( + [ + "line 1", + "line 2 \\", + ], + start=1, + ) expect = [ - (1, 'line 1'), - (2, 'line 2 '), + (1, "line 1"), + (2, "line 2 "), ] assert expect == list(join_lines(lines)) +class LineProcessor(Protocol): + def __call__( + self, + line: str, + filename: str, + line_number: int, + finder: Optional[PackageFinder] = None, + options: Optional[Values] = None, + session: Optional[PipSession] = None, + constraint: bool = False, + ) -> List[InstallRequirement]: + ... + + @pytest.fixture -def line_processor( - monkeypatch, - tmpdir, -): +def line_processor(monkeypatch: pytest.MonkeyPatch, tmpdir: Path) -> LineProcessor: def process_line( - line, - filename, - line_number, - finder=None, - options=None, - session=None, - constraint=False, - ): + line: str, + filename: str, + line_number: int, + finder: Optional[PackageFinder] = None, + options: Optional[Values] = None, + session: Optional[PipSession] = None, + constraint: bool = False, + ) -> List[InstallRequirement]: if session is None: session = PipSession() - prefix = '\n' * (line_number - 1) + prefix = "\n" * (line_number - 1) path = tmpdir.joinpath(filename) path.parent.mkdir(exist_ok=True) path.write_text(prefix + line) monkeypatch.chdir(str(tmpdir)) - return list(parse_reqfile( - filename, - finder=finder, - options=options, - session=session, - constraint=constraint, - isolated=options.isolated_mode if options else False - )) + return list( + parse_reqfile( + filename, + finder=finder, + options=options, + session=session, + constraint=constraint, + isolated=options.isolated_mode if options else False, + ) + ) return process_line @@ -186,256 +238,277 @@ def process_line( class TestProcessLine: """tests for `process_line`""" - def test_parser_error(self, line_processor): + def test_parser_error(self, line_processor: LineProcessor) -> None: with pytest.raises(RequirementsFileParseError): line_processor("--bogus", "file", 1) - def test_parser_offending_line(self, line_processor): - line = 'pkg==1.0.0 --hash=somehash' + def test_parser_offending_line(self, line_processor: LineProcessor) -> None: + line = "pkg==1.0.0 --hash=somehash" with pytest.raises(RequirementsFileParseError) as err: - line_processor(line, 'file', 1) + line_processor(line, "file", 1) assert line in str(err.value) - def test_parser_non_offending_line(self, line_processor): + def test_parser_non_offending_line(self, line_processor: LineProcessor) -> None: try: - line_processor('pkg==1.0.0 --hash=sha256:somehash', 'file', 1) + line_processor("pkg==1.0.0 --hash=sha256:somehash", "file", 1) except RequirementsFileParseError: - pytest.fail('Reported offending line where it should not.') + pytest.fail("Reported offending line where it should not.") - def test_only_one_req_per_line(self, line_processor): + def test_only_one_req_per_line(self, line_processor: LineProcessor) -> None: # pkg_resources raises the ValueError with pytest.raises(InstallationError): line_processor("req1 req2", "file", 1) - def test_error_message(self, line_processor): + def test_error_message(self, line_processor: LineProcessor) -> None: """ Test the error message if a parsing error occurs (all of path, line number, and hint). """ with pytest.raises(InstallationError) as exc: line_processor( - 'my-package=1.0', - filename='path/requirements.txt', - line_number=3 + "my-package=1.0", filename="path/requirements.txt", line_number=3 ) expected = ( "Invalid requirement: 'my-package=1.0' " - '(from line 3 of path/requirements.txt)\n' - 'Hint: = is not a valid operator. Did you mean == ?' + "(from line 3 of path/requirements.txt)\n" + "Hint: = is not a valid operator. Did you mean == ?" ) assert str(exc.value) == expected - def test_yield_line_requirement(self, line_processor): - line = 'SomeProject' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + def test_yield_line_requirement(self, line_processor: LineProcessor) -> None: + line = "SomeProject" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_line(line, comes_from=comes_from) assert repr(line_processor(line, filename, 1)[0]) == repr(req) - def test_yield_pep440_line_requirement(self, line_processor): - line = 'SomeProject @ https://url/SomeProject-py2-py3-none-any.whl' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + def test_yield_pep440_line_requirement(self, line_processor: LineProcessor) -> None: + line = "SomeProject @ https://url/SomeProject-py2-py3-none-any.whl" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_line(line, comes_from=comes_from) assert repr(line_processor(line, filename, 1)[0]) == repr(req) - def test_yield_line_constraint(self, line_processor): - line = 'SomeProject' - filename = 'filename' - comes_from = '-c {} (line {})'.format(filename, 1) - req = install_req_from_line( - line, comes_from=comes_from, constraint=True) + def test_yield_line_constraint(self, line_processor: LineProcessor) -> None: + line = "SomeProject" + filename = "filename" + comes_from = "-c {} (line {})".format(filename, 1) + req = install_req_from_line(line, comes_from=comes_from, constraint=True) found_req = line_processor(line, filename, 1, constraint=True)[0] assert repr(found_req) == repr(req) assert found_req.constraint is True def test_yield_line_requirement_with_spaces_in_specifier( - self, line_processor - ): - line = 'SomeProject >= 2' - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + self, line_processor: LineProcessor + ) -> None: + line = "SomeProject >= 2" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_line(line, comes_from=comes_from) assert repr(line_processor(line, filename, 1)[0]) == repr(req) - assert str(req.req.specifier) == '>=2' - - def test_yield_editable_requirement(self, line_processor): - url = 'git+https://url#egg=SomeProject' - line = '-e {url}'.format(**locals()) - filename = 'filename' - comes_from = '-r {} (line {})'.format(filename, 1) + assert req.req is not None + assert str(req.req.specifier) == ">=2" + + def test_yield_editable_requirement(self, line_processor: LineProcessor) -> None: + url = "git+https://url#egg=SomeProject" + line = f"-e {url}" + filename = "filename" + comes_from = f"-r {filename} (line 1)" req = install_req_from_editable(url, comes_from=comes_from) assert repr(line_processor(line, filename, 1)[0]) == repr(req) - def test_yield_editable_constraint(self, line_processor): - url = 'git+https://url#egg=SomeProject' - line = f'-e {url}' - filename = 'filename' - comes_from = '-c {} (line {})'.format(filename, 1) - req = install_req_from_editable( - url, comes_from=comes_from, constraint=True) + def test_yield_editable_constraint(self, line_processor: LineProcessor) -> None: + url = "git+https://url#egg=SomeProject" + line = f"-e {url}" + filename = "filename" + comes_from = "-c {} (line {})".format(filename, 1) + req = install_req_from_editable(url, comes_from=comes_from, constraint=True) found_req = line_processor(line, filename, 1, constraint=True)[0] assert repr(found_req) == repr(req) assert found_req.constraint is True - def test_nested_constraints_file(self, monkeypatch, tmpdir): - req_name = 'hello' - req_file = tmpdir / 'parent' / 'req_file.txt' + def test_nested_constraints_file( + self, monkeypatch: pytest.MonkeyPatch, tmpdir: Path, session: PipSession + ) -> None: + req_name = "hello" + req_file = tmpdir / "parent" / "req_file.txt" req_file.parent.mkdir() - req_file.write_text('-c reqs.txt') - req_file.parent.joinpath('reqs.txt').write_text(req_name) + req_file.write_text("-c reqs.txt") + req_file.parent.joinpath("reqs.txt").write_text(req_name) monkeypatch.chdir(str(tmpdir)) - reqs = list( - parse_reqfile('./parent/req_file.txt', session=session) - ) + reqs = list(parse_reqfile("./parent/req_file.txt", session=session)) assert len(reqs) == 1 assert reqs[0].name == req_name assert reqs[0].constraint - def test_options_on_a_requirement_line(self, line_processor): - line = 'SomeProject --install-option=yo1 --install-option yo2 '\ - '--global-option="yo3" --global-option "yo4"' - filename = 'filename' + def test_options_on_a_requirement_line(self, line_processor: LineProcessor) -> None: + line = ( + "SomeProject --install-option=yo1 --install-option yo2 " + '--global-option="yo3" --global-option "yo4"' + ) + filename = "filename" req = line_processor(line, filename, 1)[0] - assert req.global_options == ['yo3', 'yo4'] - assert req.install_options == ['yo1', 'yo2'] + assert req.global_options == ["yo3", "yo4"] + assert req.install_options == ["yo1", "yo2"] - def test_hash_options(self, line_processor): + def test_hash_options(self, line_processor: LineProcessor) -> None: """Test the --hash option: mostly its value storage. Make sure it reads and preserve multiple hashes. """ - line = ('SomeProject --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b1' - '61e5c1fa7425e73043362938b9824 ' - '--hash=sha384:59e1748777448c69de6b800d7a33bbfb9ff1b463e44354c' - '3553bcdb9c666fa90125a3c79f90397bdf5f6a13de828684f ' - '--hash=sha256:486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8' - 'e5a6c65260e9cb8a7') - filename = 'filename' + line = ( + "SomeProject --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b1" + "61e5c1fa7425e73043362938b9824 " + "--hash=sha384:59e1748777448c69de6b800d7a33bbfb9ff1b463e44354c" + "3553bcdb9c666fa90125a3c79f90397bdf5f6a13de828684f " + "--hash=sha256:486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8" + "e5a6c65260e9cb8a7" + ) + filename = "filename" req = line_processor(line, filename, 1)[0] assert req.hash_options == { - 'sha256': ['2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e730433' - '62938b9824', - '486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65' - '260e9cb8a7'], - 'sha384': ['59e1748777448c69de6b800d7a33bbfb9ff1b463e44354c3553bcd' - 'b9c666fa90125a3c79f90397bdf5f6a13de828684f']} - - def test_set_isolated(self, line_processor, options): - line = 'SomeProject' - filename = 'filename' + "sha256": [ + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", + "486ea46224d1bb4fb680f34f7c9ad96a8f24ec88be73ea8e5a6c65260e9cb8a7", + ], + "sha384": [ + "59e1748777448c69de6b800d7a33bbfb9ff1b463e44354c3553bcd" + "b9c666fa90125a3c79f90397bdf5f6a13de828684f" + ], + } + + def test_set_isolated( + self, line_processor: LineProcessor, options: mock.Mock + ) -> None: + line = "SomeProject" + filename = "filename" options.isolated_mode = True result = line_processor(line, filename, 1, options=options) assert result[0].isolated - def test_set_finder_no_index(self, line_processor, finder): + def test_set_finder_no_index( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("--no-index", "file", 1, finder=finder) assert finder.index_urls == [] - def test_set_finder_index_url(self, line_processor, finder, session): - line_processor( - "--index-url=url", "file", 1, finder=finder, session=session) - assert finder.index_urls == ['url'] - assert session.auth.index_urls == ['url'] + def test_set_finder_index_url( + self, line_processor: LineProcessor, finder: PackageFinder, session: PipSession + ) -> None: + line_processor("--index-url=url", "file", 1, finder=finder, session=session) + assert finder.index_urls == ["url"] + assert session.auth.index_urls == ["url"] - def test_set_finder_find_links(self, line_processor, finder): + def test_set_finder_find_links( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("--find-links=url", "file", 1, finder=finder) - assert finder.find_links == ['url'] + assert finder.find_links == ["url"] def test_set_finder_extra_index_urls( - self, line_processor, finder, session): + self, line_processor: LineProcessor, finder: PackageFinder, session: PipSession + ) -> None: line_processor( - "--extra-index-url=url", "file", 1, finder=finder, session=session) - assert finder.index_urls == ['url'] - assert session.auth.index_urls == ['url'] + "--extra-index-url=url", "file", 1, finder=finder, session=session + ) + assert finder.index_urls == ["url"] + assert session.auth.index_urls == ["url"] def test_set_finder_trusted_host( - self, line_processor, caplog, session, finder - ): + self, + line_processor: LineProcessor, + caplog: pytest.LogCaptureFixture, + session: PipSession, + finder: PackageFinder, + ) -> None: with caplog.at_level(logging.INFO): line_processor( "--trusted-host=host1 --trusted-host=host2:8080", - "file.txt", 1, finder=finder, session=session, + "file.txt", + 1, + finder=finder, + session=session, ) - assert list(finder.trusted_hosts) == ['host1', 'host2:8080'] + assert list(finder.trusted_hosts) == ["host1", "host2:8080"] session = finder._link_collector.session - assert ( - session.adapters['https://host1/'] - is session._trusted_host_adapter - ) - assert ( - session.adapters['https://host2:8080/'] - is session._trusted_host_adapter - ) + assert session.adapters["https://host1/"] is session._trusted_host_adapter + assert session.adapters["https://host2:8080/"] is session._trusted_host_adapter # Test the log message. actual = [(r.levelname, r.message) for r in caplog.records] - expected = ( - 'INFO', "adding trusted host: 'host1' (from line 1 of file.txt)" - ) + expected = ("INFO", "adding trusted host: 'host1' (from line 1 of file.txt)") assert expected in actual - def test_set_finder_allow_all_prereleases(self, line_processor, finder): + def test_set_finder_allow_all_prereleases( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("--pre", "file", 1, finder=finder) assert finder.allow_all_prereleases - def test_use_feature(self, line_processor, options): + def test_use_feature( + self, line_processor: LineProcessor, options: mock.Mock + ) -> None: """--use-feature can be set in requirements files.""" - line_processor( - "--use-feature=2020-resolver", "filename", 1, options=options - ) + line_processor("--use-feature=2020-resolver", "filename", 1, options=options) assert "2020-resolver" in options.features_enabled def test_relative_local_find_links( - self, line_processor, finder, monkeypatch, tmpdir - ): + self, + line_processor: LineProcessor, + finder: PackageFinder, + monkeypatch: pytest.MonkeyPatch, + tmpdir: Path, + ) -> None: """ Test a relative find_links path is joined with the req file directory """ - base_path = tmpdir / 'path' + base_path = tmpdir / "path" - def normalize(path): - return os.path.normcase( - os.path.abspath(os.path.normpath(str(path))) - ) + def normalize(path: Path) -> str: + return os.path.normcase(os.path.abspath(os.path.normpath(str(path)))) # Make sure the test also passes on windows - req_file = normalize(base_path / 'req_file.txt') - nested_link = normalize(base_path / 'rel_path') + req_file = normalize(base_path / "req_file.txt") + nested_link = normalize(base_path / "rel_path") exists_ = os.path.exists - def exists(path): + def exists(path: str) -> bool: if path == nested_link: return True else: - exists_(path) + return exists_(path) - monkeypatch.setattr(os.path, 'exists', exists) + monkeypatch.setattr(os.path, "exists", exists) line_processor("--find-links=rel_path", req_file, 1, finder=finder) assert finder.find_links == [nested_link] def test_relative_http_nested_req_files( - self, finder, session, monkeypatch - ): + self, + finder: PackageFinder, + session: PipSession, + monkeypatch: pytest.MonkeyPatch, + ) -> None: """ Test a relative nested req file path is joined with the req file url """ - req_name = 'hello' - req_file = 'http://me.com/me/req_file.txt' + req_name = "hello" + req_file = "http://me.com/me/req_file.txt" - def get_file_content(filename, *args, **kwargs): + def get_file_content( + filename: str, *args: Any, **kwargs: Any + ) -> Tuple[None, str]: if filename == req_file: - return None, '-r reqs.txt' - elif filename == 'http://me.com/me/reqs.txt': + return None, "-r reqs.txt" + elif filename == "http://me.com/me/reqs.txt": return None, req_name - assert False, f'Unexpected file requested {filename}' + assert False, f"Unexpected file requested {filename}" monkeypatch.setattr( - pip._internal.req.req_file, 'get_file_content', get_file_content + pip._internal.req.req_file, "get_file_content", get_file_content ) result = list(parse_reqfile(req_file, session=session)) @@ -444,41 +517,39 @@ def get_file_content(filename, *args, **kwargs): assert not result[0].constraint def test_relative_local_nested_req_files( - self, session, monkeypatch, tmpdir - ): + self, session: PipSession, monkeypatch: pytest.MonkeyPatch, tmpdir: Path + ) -> None: """ Test a relative nested req file path is joined with the req file dir """ - req_name = 'hello' - req_file = tmpdir / 'parent' / 'req_file.txt' + req_name = "hello" + req_file = tmpdir / "parent" / "req_file.txt" req_file.parent.mkdir() - req_file.write_text('-r reqs.txt') - req_file.parent.joinpath('reqs.txt').write_text(req_name) + req_file.write_text("-r reqs.txt") + req_file.parent.joinpath("reqs.txt").write_text(req_name) monkeypatch.chdir(str(tmpdir)) - reqs = list( - parse_reqfile('./parent/req_file.txt', session=session) - ) + reqs = list(parse_reqfile("./parent/req_file.txt", session=session)) assert len(reqs) == 1 assert reqs[0].name == req_name assert not reqs[0].constraint def test_absolute_local_nested_req_files( - self, session, monkeypatch, tmpdir - ): + self, session: PipSession, tmpdir: Path + ) -> None: """ Test an absolute nested req file path """ - req_name = 'hello' - req_file = tmpdir / 'parent' / 'req_file.txt' + req_name = "hello" + req_file = tmpdir / "parent" / "req_file.txt" req_file.parent.mkdir() - other_req_file = tmpdir / 'other' / 'reqs.txt' + other_req_file = tmpdir / "other" / "reqs.txt" other_req_file.parent.mkdir() # POSIX-ify the path, since Windows backslashes aren't supported. - other_req_file_str = str(other_req_file).replace('\\', '/') + other_req_file_str = str(other_req_file).replace("\\", "/") - req_file.write_text(f'-r {other_req_file_str}') + req_file.write_text(f"-r {other_req_file_str}") other_req_file.write_text(req_name) reqs = list(parse_reqfile(str(req_file), session=session)) @@ -487,24 +558,26 @@ def test_absolute_local_nested_req_files( assert not reqs[0].constraint def test_absolute_http_nested_req_file_in_local( - self, session, monkeypatch, tmpdir - ): + self, session: PipSession, monkeypatch: pytest.MonkeyPatch, tmpdir: Path + ) -> None: """ Test a nested req file url in a local req file """ - req_name = 'hello' - req_file = tmpdir / 'req_file.txt' - nested_req_file = 'http://me.com/me/req_file.txt' + req_name = "hello" + req_file = tmpdir / "req_file.txt" + nested_req_file = "http://me.com/me/req_file.txt" - def get_file_content(filename, *args, **kwargs): + def get_file_content( + filename: str, *args: Any, **kwargs: Any + ) -> Tuple[None, str]: if filename == str(req_file): - return None, f'-r {nested_req_file}' + return None, f"-r {nested_req_file}" elif filename == nested_req_file: return None, req_name - assert False, f'Unexpected file requested {filename}' + assert False, f"Unexpected file requested {filename}" monkeypatch.setattr( - pip._internal.req.req_file, 'get_file_content', get_file_content + pip._internal.req.req_file, "get_file_content", get_file_content ) result = list(parse_reqfile(req_file, session=session)) @@ -514,221 +587,280 @@ def get_file_content(filename, *args, **kwargs): class TestBreakOptionsArgs: + def test_no_args(self) -> None: + assert ("", "--option") == break_args_options("--option") - def test_no_args(self): - assert ('', '--option') == break_args_options('--option') - - def test_no_options(self): - assert ('arg arg', '') == break_args_options('arg arg') + def test_no_options(self) -> None: + assert ("arg arg", "") == break_args_options("arg arg") - def test_args_short_options(self): - result = break_args_options('arg arg -s') - assert ('arg arg', '-s') == result + def test_args_short_options(self) -> None: + result = break_args_options("arg arg -s") + assert ("arg arg", "-s") == result - def test_args_long_options(self): - result = break_args_options('arg arg --long') - assert ('arg arg', '--long') == result + def test_args_long_options(self) -> None: + result = break_args_options("arg arg --long") + assert ("arg arg", "--long") == result class TestOptionVariants: # this suite is really just testing optparse, but added it anyway - def test_variant1(self, line_processor, finder): + def test_variant1( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("-i url", "file", 1, finder=finder) - assert finder.index_urls == ['url'] + assert finder.index_urls == ["url"] - def test_variant2(self, line_processor, finder): + def test_variant2( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("-i 'url'", "file", 1, finder=finder) - assert finder.index_urls == ['url'] + assert finder.index_urls == ["url"] - def test_variant3(self, line_processor, finder): + def test_variant3( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("--index-url=url", "file", 1, finder=finder) - assert finder.index_urls == ['url'] + assert finder.index_urls == ["url"] - def test_variant4(self, line_processor, finder): + def test_variant4( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("--index-url url", "file", 1, finder=finder) - assert finder.index_urls == ['url'] + assert finder.index_urls == ["url"] - def test_variant5(self, line_processor, finder): + def test_variant5( + self, line_processor: LineProcessor, finder: PackageFinder + ) -> None: line_processor("--index-url='url'", "file", 1, finder=finder) - assert finder.index_urls == ['url'] + assert finder.index_urls == ["url"] class TestParseRequirements: """tests for `parse_reqfile`""" @pytest.mark.network - def test_remote_reqs_parse(self): + def test_remote_reqs_parse(self) -> None: """ Test parsing a simple remote requirements file """ # this requirements file just contains a comment previously this has # failed in py3: https://github.com/pypa/pip/issues/760 for _ in parse_reqfile( - 'https://raw.githubusercontent.com/pypa/' - 'pip-test-package/master/' - 'tests/req_just_comment.txt', session=PipSession()): + "https://raw.githubusercontent.com/pypa/" + "pip-test-package/master/" + "tests/req_just_comment.txt", + session=PipSession(), + ): pass - def test_multiple_appending_options(self, tmpdir, finder, options): + def test_multiple_appending_options( + self, tmpdir: Path, finder: PackageFinder, options: mock.Mock + ) -> None: with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \n") fp.write("--extra-index-url url2 ") - list(parse_reqfile(tmpdir.joinpath("req1.txt"), finder=finder, - session=PipSession(), options=options)) + list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), + finder=finder, + session=PipSession(), + options=options, + ) + ) - assert finder.index_urls == ['url1', 'url2'] + assert finder.index_urls == ["url1", "url2"] - def test_expand_existing_env_variables(self, tmpdir, finder): - template = ( - 'https://{}:x-oauth-basic@github.com/' - 'user/{}/archive/master.zip' - ) + def test_expand_existing_env_variables( + self, tmpdir: Path, finder: PackageFinder + ) -> None: + template = "https://{}:x-oauth-basic@github.com/user/{}/archive/master.zip" - def make_var(name): - return '${{{name}}}'.format(**locals()) + def make_var(name: str) -> str: + return f"${{{name}}}" - env_vars = collections.OrderedDict([ - ('GITHUB_TOKEN', 'notarealtoken'), - ('DO_12_FACTOR', 'awwyeah'), - ]) + env_vars = collections.OrderedDict( + [ + ("GITHUB_TOKEN", "notarealtoken"), + ("DO_12_FACTOR", "awwyeah"), + ] + ) - with open(tmpdir.joinpath('req1.txt'), 'w') as fp: + with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write(template.format(*map(make_var, env_vars))) - with patch('pip._internal.req.req_file.os.getenv') as getenv: + # Construct the session outside the monkey-patch, since it access the + # env + session = PipSession() + with mock.patch("pip._internal.req.req_file.os.getenv") as getenv: getenv.side_effect = lambda n: env_vars[n] - reqs = list(parse_reqfile( - tmpdir.joinpath('req1.txt'), - finder=finder, - session=PipSession() - )) + reqs = list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), finder=finder, session=session + ) + ) - assert len(reqs) == 1, \ - 'parsing requirement file with env variable failed' + assert len(reqs) == 1, "parsing requirement file with env variable failed" expected_url = template.format(*env_vars.values()) - assert reqs[0].link.url == expected_url, \ - 'variable expansion in req file failed' + assert reqs[0].link is not None + assert reqs[0].link.url == expected_url, "variable expansion in req file failed" - def test_expand_missing_env_variables(self, tmpdir, finder): + def test_expand_missing_env_variables( + self, tmpdir: Path, finder: PackageFinder + ) -> None: req_url = ( - 'https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@' - '%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip' + "https://${NON_EXISTENT_VARIABLE}:$WRONG_FORMAT@" + "%WINDOWS_FORMAT%github.com/user/repo/archive/master.zip" ) - with open(tmpdir.joinpath('req1.txt'), 'w') as fp: + with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write(req_url) - with patch('pip._internal.req.req_file.os.getenv') as getenv: - getenv.return_value = '' + # Construct the session outside the monkey-patch, since it access the + # env + session = PipSession() + with mock.patch("pip._internal.req.req_file.os.getenv") as getenv: + getenv.return_value = "" - reqs = list(parse_reqfile( - tmpdir.joinpath('req1.txt'), - finder=finder, - session=PipSession() - )) + reqs = list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), finder=finder, session=session + ) + ) - assert len(reqs) == 1, \ - 'parsing requirement file with env variable failed' - assert reqs[0].link.url == req_url, \ - 'ignoring invalid env variable in req file failed' + assert len(reqs) == 1, "parsing requirement file with env variable failed" + assert reqs[0].link is not None + assert ( + reqs[0].link.url == req_url + ), "ignoring invalid env variable in req file failed" - def test_join_lines(self, tmpdir, finder): + def test_join_lines(self, tmpdir: Path, finder: PackageFinder) -> None: with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("--extra-index-url url1 \\\n--extra-index-url url2") - list(parse_reqfile(tmpdir.joinpath("req1.txt"), finder=finder, - session=PipSession())) + list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), finder=finder, session=PipSession() + ) + ) - assert finder.index_urls == ['url1', 'url2'] + assert finder.index_urls == ["url1", "url2"] - def test_req_file_parse_no_only_binary(self, data, finder): - list(parse_reqfile( - data.reqfiles.joinpath("supported_options2.txt"), - finder=finder, - session=PipSession())) - expected = FormatControl({'fred'}, {'wilma'}) + def test_req_file_parse_no_only_binary( + self, data: TestData, finder: PackageFinder + ) -> None: + list( + parse_reqfile( + data.reqfiles.joinpath("supported_options2.txt"), + finder=finder, + session=PipSession(), + ) + ) + expected = FormatControl({"fred"}, {"wilma"}) assert finder.format_control == expected - def test_req_file_parse_comment_start_of_line(self, tmpdir, finder): + def test_req_file_parse_comment_start_of_line( + self, tmpdir: Path, finder: PackageFinder + ) -> None: """ Test parsing comments in a requirements file """ with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("# Comment ") - reqs = list(parse_reqfile(tmpdir.joinpath("req1.txt"), - finder=finder, - session=PipSession())) + reqs = list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), finder=finder, session=PipSession() + ) + ) assert not reqs - def test_req_file_parse_comment_end_of_line_with_url(self, tmpdir, finder): + def test_req_file_parse_comment_end_of_line_with_url( + self, tmpdir: Path, finder: PackageFinder + ) -> None: """ Test parsing comments in a requirements file """ with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("https://example.com/foo.tar.gz # Comment ") - reqs = list(parse_reqfile(tmpdir.joinpath("req1.txt"), - finder=finder, - session=PipSession())) + reqs = list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), finder=finder, session=PipSession() + ) + ) assert len(reqs) == 1 + assert reqs[0].link is not None assert reqs[0].link.url == "https://example.com/foo.tar.gz" - def test_req_file_parse_egginfo_end_of_line_with_url(self, tmpdir, finder): + def test_req_file_parse_egginfo_end_of_line_with_url( + self, tmpdir: Path, finder: PackageFinder + ) -> None: """ Test parsing comments in a requirements file """ with open(tmpdir.joinpath("req1.txt"), "w") as fp: fp.write("https://example.com/foo.tar.gz#egg=wat") - reqs = list(parse_reqfile(tmpdir.joinpath("req1.txt"), - finder=finder, - session=PipSession())) + reqs = list( + parse_reqfile( + tmpdir.joinpath("req1.txt"), finder=finder, session=PipSession() + ) + ) assert len(reqs) == 1 assert reqs[0].name == "wat" - def test_req_file_no_finder(self, tmpdir): + def test_req_file_no_finder(self, tmpdir: Path) -> None: """ Test parsing a requirements file without a finder """ with open(tmpdir.joinpath("req.txt"), "w") as fp: - fp.write(""" + fp.write( + """ --find-links https://example.com/ --index-url https://example.com/ --extra-index-url https://two.example.com/ --no-use-wheel --no-index - """) + """ + ) parse_reqfile(tmpdir.joinpath("req.txt"), session=PipSession()) - def test_install_requirements_with_options(self, tmpdir, finder, session, - options): - global_option = '--dry-run' - install_option = '--prefix=/opt' - - content = ''' + def test_install_requirements_with_options( + self, + tmpdir: Path, + finder: PackageFinder, + session: PipSession, + options: mock.Mock, + ) -> None: + global_option = "--dry-run" + install_option = "--prefix=/opt" + + content = """ --only-binary :all: INITools==2.0 --global-option="{global_option}" \ --install-option "{install_option}" - '''.format(global_option=global_option, install_option=install_option) + """.format( + global_option=global_option, install_option=install_option + ) with requirements_file(content, tmpdir) as reqs_file: - req = next(parse_reqfile(reqs_file.resolve(), - finder=finder, - options=options, - session=session)) + req = next( + parse_reqfile( + reqs_file.resolve(), finder=finder, options=options, session=session + ) + ) req.source_dir = os.curdir - with patch.object(subprocess, 'Popen') as popen: + with mock.patch.object(subprocess, "Popen") as popen: popen.return_value.stdout.readline.return_value = b"" try: req.install([]) @@ -738,8 +870,10 @@ def test_install_requirements_with_options(self, tmpdir, finder, session, last_call = popen.call_args_list[-1] args = last_call[0][0] assert ( - 0 < args.index(global_option) < args.index('install') < - args.index(install_option) + 0 + < args.index(global_option) + < args.index("install") + < args.index(install_option) ) - assert options.format_control.no_binary == {':all:'} + assert options.format_control.no_binary == {":all:"} assert options.format_control.only_binary == set() diff --git a/tests/unit/test_req_install.py b/tests/unit/test_req_install.py index d8eee8d13d4..ac2c0cdbb89 100644 --- a/tests/unit/test_req_install.py +++ b/tests/unit/test_req_install.py @@ -10,22 +10,24 @@ install_req_from_req_string, ) from pip._internal.req.req_install import InstallRequirement +from tests.lib.path import Path class TestInstallRequirementBuildDirectory: # no need to test symlinks on Windows @pytest.mark.skipif("sys.platform == 'win32'") - def test_tmp_build_directory(self): + def test_tmp_build_directory(self) -> None: # when req is None, we can produce a temporary directory # Make sure we're handling it correctly with real path. requirement = InstallRequirement(None, None) - tmp_dir = tempfile.mkdtemp('-build', 'pip-') + tmp_dir = tempfile.mkdtemp("-build", "pip-") tmp_build_dir = requirement.ensure_build_location( - tmp_dir, autodelete=False, parallel_builds=False, + tmp_dir, + autodelete=False, + parallel_builds=False, ) - assert ( - os.path.dirname(tmp_build_dir) == - os.path.realpath(os.path.dirname(tmp_dir)) + assert os.path.dirname(tmp_build_dir) == os.path.realpath( + os.path.dirname(tmp_dir) ) # are we on a system where /tmp is a symlink if os.path.realpath(tmp_dir) != os.path.abspath(tmp_dir): @@ -35,14 +37,14 @@ def test_tmp_build_directory(self): os.rmdir(tmp_dir) assert not os.path.exists(tmp_dir) - def test_forward_slash_results_in_a_link(self, tmpdir): + def test_forward_slash_results_in_a_link(self, tmpdir: Path) -> None: install_dir = tmpdir / "foo" / "bar" # Just create a file for letting the logic work setup_py_path = install_dir / "setup.py" os.makedirs(str(install_dir)) - with open(setup_py_path, 'w') as f: - f.write('') + with open(setup_py_path, "w") as f: + f.write("") requirement = install_req_from_line( str(install_dir).replace(os.sep, os.altsep or os.sep) @@ -52,8 +54,7 @@ def test_forward_slash_results_in_a_link(self, tmpdir): class TestInstallRequirementFrom: - - def test_install_req_from_string_invalid_requirement(self): + def test_install_req_from_string_invalid_requirement(self) -> None: """ Requirement strings that cannot be parsed by packaging.requirements.Requirement raise an InstallationError. @@ -61,50 +62,53 @@ def test_install_req_from_string_invalid_requirement(self): with pytest.raises(InstallationError) as excinfo: install_req_from_req_string("http:/this/is/invalid") - assert str(excinfo.value) == ( - "Invalid requirement: 'http:/this/is/invalid'" - ) + assert str(excinfo.value) == ("Invalid requirement: 'http:/this/is/invalid'") - def test_install_req_from_string_without_comes_from(self): + def test_install_req_from_string_without_comes_from(self) -> None: """ Test to make sure that install_req_from_string succeeds when called with URL (PEP 508) but without comes_from. """ # Test with a PEP 508 url install string: - wheel_url = ("https://download.pytorch.org/whl/cu90/" - "torch-1.0.0-cp36-cp36m-win_amd64.whl") + wheel_url = ( + "https://download.pytorch.org/whl/cu90/" + "torch-1.0.0-cp36-cp36m-win_amd64.whl" + ) install_str = "torch@ " + wheel_url install_req = install_req_from_req_string(install_str) assert isinstance(install_req, InstallRequirement) + assert install_req.link is not None assert install_req.link.url == wheel_url + assert install_req.req is not None assert install_req.req.url == wheel_url assert install_req.comes_from is None assert install_req.is_wheel - def test_install_req_from_string_with_comes_from_without_link(self): + def test_install_req_from_string_with_comes_from_without_link(self) -> None: """ Test to make sure that install_req_from_string succeeds when called with URL (PEP 508) and comes_from does not have a link. """ # Test with a PEP 508 url install string: - wheel_url = ("https://download.pytorch.org/whl/cu90/" - "torch-1.0.0-cp36-cp36m-win_amd64.whl") + wheel_url = ( + "https://download.pytorch.org/whl/cu90/" + "torch-1.0.0-cp36-cp36m-win_amd64.whl" + ) install_str = "torch@ " + wheel_url # Dummy numpy "comes_from" requirement without link: - comes_from = InstallRequirement( - Requirement("numpy>=1.15.0"), comes_from=None - ) + comes_from = InstallRequirement(Requirement("numpy>=1.15.0"), comes_from=None) # Attempt install from install string comes: - install_req = install_req_from_req_string( - install_str, comes_from=comes_from - ) + install_req = install_req_from_req_string(install_str, comes_from=comes_from) assert isinstance(install_req, InstallRequirement) + assert isinstance(install_req.comes_from, InstallRequirement) assert install_req.comes_from.link is None + assert install_req.link is not None assert install_req.link.url == wheel_url + assert install_req.req is not None assert install_req.req.url == wheel_url assert install_req.is_wheel diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index 90bf0d50fbc..b3049e1037c 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -1,8 +1,9 @@ import os import sys +from typing import Iterator, List, Optional, Tuple +from unittest.mock import Mock import pytest -from mock import Mock import pip._internal.req.req_uninstall from pip._internal.req.req_uninstall import ( @@ -15,34 +16,38 @@ uninstallation_paths, ) from tests.lib import create_file +from tests.lib.path import Path # Pretend all files are local, so UninstallPathSet accepts files in the tmpdir, # outside the virtualenv -def mock_is_local(path): +def mock_is_local(path: str) -> bool: return True -def test_uninstallation_paths(): +def test_uninstallation_paths() -> None: class dist: - def get_metadata_lines(self, record): - return ['file.py,,', - 'file.pyc,,', - 'file.so,,', - 'nopyc.py'] - location = '' + def iter_declared_entries(self) -> Optional[Iterator[str]]: + yield "file.py" + yield "file.pyc" + yield "file.so" + yield "nopyc.py" + + location = "" d = dist() paths = list(uninstallation_paths(d)) - expected = ['file.py', - 'file.pyc', - 'file.pyo', - 'file.so', - 'nopyc.py', - 'nopyc.pyc', - 'nopyc.pyo'] + expected = [ + "file.py", + "file.pyc", + "file.pyo", + "file.so", + "nopyc.py", + "nopyc.pyc", + "nopyc.pyo", + ] assert paths == expected @@ -52,30 +57,30 @@ def get_metadata_lines(self, record): assert paths2 == paths -def test_compressed_listing(tmpdir): - def in_tmpdir(paths): +def test_compressed_listing(tmpdir: Path) -> None: + def in_tmpdir(paths: List[str]) -> List[str]: li = [] for path in paths: - li.append( - str(os.path.join(tmpdir, path.replace("/", os.path.sep))) - ) + li.append(str(os.path.join(tmpdir, path.replace("/", os.path.sep)))) return li - sample = in_tmpdir([ - "lib/mypkg.dist-info/METADATA", - "lib/mypkg.dist-info/PKG-INFO", - "lib/mypkg/would_be_removed.txt", - "lib/mypkg/would_be_skipped.skip.txt", - "lib/mypkg/__init__.py", - "lib/mypkg/my_awesome_code.py", - "lib/mypkg/__pycache__/my_awesome_code-magic.pyc", - "lib/mypkg/support/support_file.py", - "lib/mypkg/support/more_support.py", - "lib/mypkg/support/would_be_skipped.skip.py", - "lib/mypkg/support/__pycache__/support_file-magic.pyc", - "lib/random_other_place/file_without_a_dot_pyc", - "bin/mybin", - ]) + sample = in_tmpdir( + [ + "lib/mypkg.dist-info/METADATA", + "lib/mypkg.dist-info/PKG-INFO", + "lib/mypkg/would_be_removed.txt", + "lib/mypkg/would_be_skipped.skip.txt", + "lib/mypkg/__init__.py", + "lib/mypkg/my_awesome_code.py", + "lib/mypkg/__pycache__/my_awesome_code-magic.pyc", + "lib/mypkg/support/support_file.py", + "lib/mypkg/support/more_support.py", + "lib/mypkg/support/would_be_skipped.skip.py", + "lib/mypkg/support/__pycache__/support_file-magic.pyc", + "lib/random_other_place/file_without_a_dot_pyc", + "bin/mybin", + ] + ) # Create the required files for fname in sample: @@ -84,30 +89,36 @@ def in_tmpdir(paths): # Remove the files to be skipped from the paths sample = [path for path in sample if ".skip." not in path] - expected_remove = in_tmpdir([ - "bin/mybin", - "lib/mypkg.dist-info/*", - "lib/mypkg/*", - "lib/random_other_place/file_without_a_dot_pyc", - ]) - - expected_skip = in_tmpdir([ - "lib/mypkg/would_be_skipped.skip.txt", - "lib/mypkg/support/would_be_skipped.skip.py", - ]) - - expected_rename = in_tmpdir([ - "bin/", - "lib/mypkg.dist-info/", - "lib/mypkg/would_be_removed.txt", - "lib/mypkg/__init__.py", - "lib/mypkg/my_awesome_code.py", - "lib/mypkg/__pycache__/", - "lib/mypkg/support/support_file.py", - "lib/mypkg/support/more_support.py", - "lib/mypkg/support/__pycache__/", - "lib/random_other_place/", - ]) + expected_remove = in_tmpdir( + [ + "bin/mybin", + "lib/mypkg.dist-info/*", + "lib/mypkg/*", + "lib/random_other_place/file_without_a_dot_pyc", + ] + ) + + expected_skip = in_tmpdir( + [ + "lib/mypkg/would_be_skipped.skip.txt", + "lib/mypkg/support/would_be_skipped.skip.py", + ] + ) + + expected_rename = in_tmpdir( + [ + "bin/", + "lib/mypkg.dist-info/", + "lib/mypkg/would_be_removed.txt", + "lib/mypkg/__init__.py", + "lib/mypkg/my_awesome_code.py", + "lib/mypkg/__pycache__/", + "lib/mypkg/support/support_file.py", + "lib/mypkg/support/more_support.py", + "lib/mypkg/support/__pycache__/", + "lib/random_other_place/", + ] + ) will_remove, will_skip = compress_for_output_listing(sample) will_rename = compress_for_rename(sample) @@ -117,42 +128,37 @@ def in_tmpdir(paths): class TestUninstallPathSet: - def test_add(self, tmpdir, monkeypatch): - monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - mock_is_local) + def test_add(self, tmpdir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pip._internal.req.req_uninstall, "is_local", mock_is_local) # Fix case for windows tests - file_extant = os.path.normcase(os.path.join(tmpdir, 'foo')) - file_nonexistent = os.path.normcase( - os.path.join(tmpdir, 'nonexistent')) - with open(file_extant, 'w'): + file_extant = os.path.normcase(os.path.join(tmpdir, "foo")) + file_nonexistent = os.path.normcase(os.path.join(tmpdir, "nonexistent")) + with open(file_extant, "w"): pass ups = UninstallPathSet(dist=Mock()) - assert ups.paths == set() + assert ups._paths == set() ups.add(file_extant) - assert ups.paths == {file_extant} + assert ups._paths == {file_extant} ups.add(file_nonexistent) - assert ups.paths == {file_extant} + assert ups._paths == {file_extant} - def test_add_pth(self, tmpdir, monkeypatch): - monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - mock_is_local) + def test_add_pth(self, tmpdir: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pip._internal.req.req_uninstall, "is_local", mock_is_local) # Fix case for windows tests tmpdir = os.path.normcase(tmpdir) - on_windows = sys.platform == 'win32' - pth_file = os.path.join(tmpdir, 'foo.pth') - relative = '../../example' + on_windows = sys.platform == "win32" + pth_file = os.path.join(tmpdir, "foo.pth") + relative = "../../example" if on_windows: - share = '\\\\example\\share\\' - share_com = '\\\\example.com\\share\\' + share = "\\\\example\\share\\" + share_com = "\\\\example.com\\share\\" # Create a .pth file for testing - with open(pth_file, 'w') as f: - f.writelines([tmpdir, '\n', - relative, '\n']) + with open(pth_file, "w") as f: + f.writelines([tmpdir, "\n", relative, "\n"]) if on_windows: - f.writelines([share, '\n', - share_com, '\n']) + f.writelines([share, "\n", share_com, "\n"]) # Add paths to be removed pth = UninstallPthEntries(pth_file) pth.add(tmpdir) @@ -162,61 +168,61 @@ def test_add_pth(self, tmpdir, monkeypatch): pth.add(share_com) # Check that the paths were added to entries if on_windows: - check = set([tmpdir, relative, share, share_com]) + check = {tmpdir, relative, share, share_com} else: - check = set([tmpdir, relative]) + check = {tmpdir, relative} assert pth.entries == check @pytest.mark.skipif("sys.platform == 'win32'") - def test_add_symlink(self, tmpdir, monkeypatch): - monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - mock_is_local) - f = os.path.join(tmpdir, 'foo') - with open(f, 'w'): + def test_add_symlink(self, tmpdir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pip._internal.req.req_uninstall, "is_local", mock_is_local) + f = os.path.join(tmpdir, "foo") + with open(f, "w"): pass - foo_link = os.path.join(tmpdir, 'foo_link') + foo_link = os.path.join(tmpdir, "foo_link") os.symlink(f, foo_link) ups = UninstallPathSet(dist=Mock()) ups.add(foo_link) - assert ups.paths == {foo_link} + assert ups._paths == {foo_link} - def test_compact_shorter_path(self, monkeypatch): - monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - mock_is_local) - monkeypatch.setattr('os.path.exists', lambda p: True) + def test_compact_shorter_path(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pip._internal.req.req_uninstall, "is_local", mock_is_local) + monkeypatch.setattr("os.path.exists", lambda p: True) # This deals with nt/posix path differences - short_path = os.path.normcase(os.path.abspath( - os.path.join(os.path.sep, 'path'))) + short_path = os.path.normcase( + os.path.abspath(os.path.join(os.path.sep, "path")) + ) ups = UninstallPathSet(dist=Mock()) ups.add(short_path) - ups.add(os.path.join(short_path, 'longer')) - assert compact(ups.paths) == {short_path} + ups.add(os.path.join(short_path, "longer")) + assert compact(ups._paths) == {short_path} @pytest.mark.skipif("sys.platform == 'win32'") - def test_detect_symlink_dirs(self, monkeypatch, tmpdir): - monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - mock_is_local) + def test_detect_symlink_dirs( + self, monkeypatch: pytest.MonkeyPatch, tmpdir: Path + ) -> None: + monkeypatch.setattr(pip._internal.req.req_uninstall, "is_local", mock_is_local) # construct 2 paths: # tmpdir/dir/file # tmpdir/dirlink/file (where dirlink is a link to dir) - d = tmpdir.joinpath('dir') + d = tmpdir.joinpath("dir") d.mkdir() - dlink = tmpdir.joinpath('dirlink') + dlink = tmpdir.joinpath("dirlink") os.symlink(d, dlink) - d.joinpath('file').touch() - path1 = str(d.joinpath('file')) - path2 = str(dlink.joinpath('file')) + d.joinpath("file").touch() + path1 = str(d.joinpath("file")) + path2 = str(dlink.joinpath("file")) ups = UninstallPathSet(dist=Mock()) ups.add(path1) ups.add(path2) - assert ups.paths == {path1} + assert ups._paths == {path1} class TestStashedUninstallPathSet: - WALK_RESULT = [ + WALK_RESULT: List[Tuple[str, List[str], List[str]]] = [ ("A", ["B", "C"], ["a.py"]), ("A/B", ["D"], ["b.py"]), ("A/B/D", [], ["c.py"]), @@ -228,35 +234,43 @@ class TestStashedUninstallPathSet: ] @classmethod - def mock_walk(cls, root): + def mock_walk(cls, root: str) -> Iterator[Tuple[str, List[str], List[str]]]: for dirname, subdirs, files in cls.WALK_RESULT: dirname = os.path.sep.join(dirname.split("/")) if dirname.startswith(root): - yield dirname[len(root) + 1:], subdirs, files - - def test_compress_for_rename(self, monkeypatch): - paths = [os.path.sep.join(p.split("/")) for p in [ - "A/B/b.py", - "A/B/D/c.py", - "A/C/d.py", - "A/E/f.py", - "A/G/g.py", - ]] - - expected_paths = [os.path.sep.join(p.split("/")) for p in [ - "A/B/", # selected everything below A/B - "A/C/d.py", # did not select everything below A/C - "A/E/", # only empty folders remain under A/E - "A/G/g.py", # non-empty folder remains under A/G - ]] - - monkeypatch.setattr('os.walk', self.mock_walk) + yield dirname[len(root) + 1 :], subdirs, files + + def test_compress_for_rename(self, monkeypatch: pytest.MonkeyPatch) -> None: + paths = [ + os.path.sep.join(p.split("/")) + for p in [ + "A/B/b.py", + "A/B/D/c.py", + "A/C/d.py", + "A/E/f.py", + "A/G/g.py", + ] + ] + + expected_paths = [ + os.path.sep.join(p.split("/")) + for p in [ + "A/B/", # selected everything below A/B + "A/C/d.py", # did not select everything below A/C + "A/E/", # only empty folders remain under A/E + "A/G/g.py", # non-empty folder remains under A/G + ] + ] + + monkeypatch.setattr("os.walk", self.mock_walk) actual_paths = compress_for_rename(paths) assert set(expected_paths) == set(actual_paths) @classmethod - def make_stash(cls, tmpdir, paths): + def make_stash( + cls, tmpdir: Path, paths: List[str] + ) -> Tuple[StashedUninstallPathSet, List[Tuple[str, str]]]: for dirname, subdirs, files in cls.WALK_RESULT: root = os.path.join(tmpdir, *dirname.split("/")) if not os.path.exists(root): @@ -269,15 +283,21 @@ def make_stash(cls, tmpdir, paths): pathset = StashedUninstallPathSet() - paths = [os.path.join(tmpdir, *p.split('/')) for p in paths] + paths = [os.path.join(tmpdir, *p.split("/")) for p in paths] stashed_paths = [(p, pathset.stash(p)) for p in paths] return pathset, stashed_paths - def test_stash(self, tmpdir): - pathset, stashed_paths = self.make_stash(tmpdir, [ - "A/B/", "A/C/d.py", "A/E/", "A/G/g.py", - ]) + def test_stash(self, tmpdir: Path) -> None: + pathset, stashed_paths = self.make_stash( + tmpdir, + [ + "A/B/", + "A/C/d.py", + "A/E/", + "A/G/g.py", + ], + ) for old_path, new_path in stashed_paths: assert not os.path.exists(old_path) @@ -285,10 +305,16 @@ def test_stash(self, tmpdir): assert stashed_paths == pathset._moves - def test_commit(self, tmpdir): - pathset, stashed_paths = self.make_stash(tmpdir, [ - "A/B/", "A/C/d.py", "A/E/", "A/G/g.py", - ]) + def test_commit(self, tmpdir: Path) -> None: + pathset, stashed_paths = self.make_stash( + tmpdir, + [ + "A/B/", + "A/C/d.py", + "A/E/", + "A/G/g.py", + ], + ) pathset.commit() @@ -296,10 +322,16 @@ def test_commit(self, tmpdir): assert not os.path.exists(old_path) assert not os.path.exists(new_path) - def test_rollback(self, tmpdir): - pathset, stashed_paths = self.make_stash(tmpdir, [ - "A/B/", "A/C/d.py", "A/E/", "A/G/g.py", - ]) + def test_rollback(self, tmpdir: Path) -> None: + pathset, stashed_paths = self.make_stash( + tmpdir, + [ + "A/B/", + "A/C/d.py", + "A/E/", + "A/G/g.py", + ], + ) pathset.rollback() @@ -308,7 +340,7 @@ def test_rollback(self, tmpdir): assert not os.path.exists(new_path) @pytest.mark.skipif("sys.platform == 'win32'") - def test_commit_symlinks(self, tmpdir): + def test_commit_symlinks(self, tmpdir: Path) -> None: adir = tmpdir / "dir" adir.mkdir() dirlink = tmpdir / "dirlink" @@ -340,7 +372,7 @@ def test_commit_symlinks(self, tmpdir): assert os.path.isfile(afile) @pytest.mark.skipif("sys.platform == 'win32'") - def test_rollback_symlinks(self, tmpdir): + def test_rollback_symlinks(self, tmpdir: Path) -> None: adir = tmpdir / "dir" adir.mkdir() dirlink = tmpdir / "dirlink" diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 90e98a691d2..7b5ba6372cc 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -1,55 +1,53 @@ +import email.message import logging +from typing import List, Optional, Type, TypeVar, cast +from unittest import mock -import mock import pytest -from pip._vendor import pkg_resources +from pip._vendor.packaging.specifiers import SpecifierSet +from pip._vendor.packaging.utils import NormalizedName from pip._internal.exceptions import NoneMetadataError, UnsupportedPythonVersion +from pip._internal.metadata import BaseDistribution +from pip._internal.models.candidate import InstallationCandidate from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.legacy.resolver import ( Resolver, _check_dist_requires_python, ) -from pip._internal.utils.packaging import get_requires_python from tests.lib import make_test_finder from tests.lib.index import make_mock_candidate +T = TypeVar("T") -# We need to inherit from DistInfoDistribution for the `isinstance()` -# check inside `packaging.get_metadata()` to work. -class FakeDist(pkg_resources.DistInfoDistribution): - def __init__(self, metadata, metadata_name=None): - """ - :param metadata: The value that dist.get_metadata() should return - for the `metadata_name` metadata. - :param metadata_name: The name of the metadata to store - (can be "METADATA" or "PKG-INFO"). Defaults to "METADATA". - """ - if metadata_name is None: - metadata_name = 'METADATA' +class FakeDist(BaseDistribution): + def __init__(self, metadata: email.message.Message) -> None: + self._canonical_name = cast(NormalizedName, "my-project") + self._metadata = metadata - self.project_name = 'my-project' - self.metadata_name = metadata_name - self.metadata = metadata + def __str__(self) -> str: + return f"" - def __str__(self): - return f'' + @property + def canonical_name(self) -> NormalizedName: + return self._canonical_name - def has_metadata(self, name): - return (name == self.metadata_name) + @property + def metadata(self) -> email.message.Message: + return self._metadata - def get_metadata(self, name): - assert name == self.metadata_name - return self.metadata - -def make_fake_dist(requires_python=None, metadata_name=None): - metadata = 'Name: test\n' +def make_fake_dist( + *, klass: Type[BaseDistribution] = FakeDist, requires_python: Optional[str] = None +) -> BaseDistribution: + metadata = email.message.Message() + metadata["Name"] = "my-project" if requires_python is not None: - metadata += f'Requires-Python:{requires_python}' + metadata["Requires-Python"] = requires_python - return FakeDist(metadata, metadata_name=metadata_name) + # Too many arguments for "BaseDistribution" + return klass(metadata) # type: ignore[call-arg] class TestCheckDistRequiresPython: @@ -58,12 +56,12 @@ class TestCheckDistRequiresPython: Test _check_dist_requires_python(). """ - def test_compatible(self, caplog): + def test_compatible(self, caplog: pytest.LogCaptureFixture) -> None: """ Test a Python version compatible with the dist's Requires-Python. """ caplog.set_level(logging.DEBUG) - dist = make_fake_dist('== 3.6.5') + dist = make_fake_dist(requires_python="== 3.6.5") _check_dist_requires_python( dist, @@ -72,11 +70,11 @@ def test_compatible(self, caplog): ) assert not len(caplog.records) - def test_incompatible(self): + def test_incompatible(self) -> None: """ Test a Python version incompatible with the dist's Requires-Python. """ - dist = make_fake_dist('== 3.6.4') + dist = make_fake_dist(requires_python="== 3.6.4") with pytest.raises(UnsupportedPythonVersion) as exc: _check_dist_requires_python( dist, @@ -85,16 +83,18 @@ def test_incompatible(self): ) assert str(exc.value) == ( "Package 'my-project' requires a different Python: " - "3.6.5 not in '== 3.6.4'" + "3.6.5 not in '==3.6.4'" ) - def test_incompatible_with_ignore_requires(self, caplog): + def test_incompatible_with_ignore_requires( + self, caplog: pytest.LogCaptureFixture + ) -> None: """ Test a Python version incompatible with the dist's Requires-Python while passing ignore_requires_python=True. """ caplog.set_level(logging.DEBUG) - dist = make_fake_dist('== 3.6.4') + dist = make_fake_dist(requires_python="== 3.6.4") _check_dist_requires_python( dist, version_info=(3, 6, 5), @@ -102,20 +102,20 @@ def test_incompatible_with_ignore_requires(self, caplog): ) assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'DEBUG' + assert record.levelname == "DEBUG" assert record.message == ( "Ignoring failed Requires-Python check for package 'my-project': " - "3.6.5 not in '== 3.6.4'" + "3.6.5 not in '==3.6.4'" ) - def test_none_requires_python(self, caplog): + def test_none_requires_python(self, caplog: pytest.LogCaptureFixture) -> None: """ Test a dist with Requires-Python None. """ caplog.set_level(logging.DEBUG) dist = make_fake_dist() # Make sure our test setup is correct. - assert get_requires_python(dist) is None + assert dist.requires_python == SpecifierSet() assert len(caplog.records) == 0 # Then there is no exception and no log message. @@ -126,12 +126,12 @@ def test_none_requires_python(self, caplog): ) assert len(caplog.records) == 0 - def test_invalid_requires_python(self, caplog): + def test_invalid_requires_python(self, caplog: pytest.LogCaptureFixture) -> None: """ Test a dist with an invalid Requires-Python. """ caplog.set_level(logging.DEBUG) - dist = make_fake_dist('invalid') + dist = make_fake_dist(requires_python="invalid") _check_dist_requires_python( dist, version_info=(3, 6, 5), @@ -139,27 +139,28 @@ def test_invalid_requires_python(self, caplog): ) assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'WARNING' + assert record.levelname == "WARNING" assert record.message == ( "Package 'my-project' has an invalid Requires-Python: " "Invalid specifier: 'invalid'" ) - @pytest.mark.parametrize('metadata_name', [ - 'METADATA', - 'PKG-INFO', - ]) - def test_empty_metadata_error(self, caplog, metadata_name): - """ - Test dist.has_metadata() returning True and dist.get_metadata() - returning None. - """ - dist = make_fake_dist(metadata_name=metadata_name) - dist.metadata = None + @pytest.mark.parametrize( + "metadata_name", + [ + "METADATA", + "PKG-INFO", + ], + ) + def test_empty_metadata_error(self, metadata_name: str) -> None: + """Test dist.metadata raises FileNotFoundError.""" - # Make sure our test setup is correct. - assert dist.has_metadata(metadata_name) - assert dist.get_metadata(metadata_name) is None + class NotWorkingFakeDist(FakeDist): + @property + def metadata(self) -> email.message.Message: + raise FileNotFoundError(metadata_name) + + dist = make_fake_dist(klass=NotWorkingFakeDist) with pytest.raises(NoneMetadataError) as exc: _check_dist_requires_python( @@ -177,8 +178,13 @@ class TestYankedWarning: """ Test _populate_link() emits warning if one or more candidates are yanked. """ - def _make_test_resolver(self, monkeypatch, mock_candidates): - def _find_candidates(project_name): + + def _make_test_resolver( + self, + monkeypatch: pytest.MonkeyPatch, + mock_candidates: List[InstallationCandidate], + ) -> Resolver: + def _find_candidates(project_name: str) -> List[InstallationCandidate]: return mock_candidates finder = make_test_finder() @@ -197,13 +203,19 @@ def _find_candidates(project_name): upgrade_strategy="to-satisfy-only", ) - def test_sort_best_candidate__has_non_yanked(self, caplog, monkeypatch): + def test_sort_best_candidate__has_non_yanked( + self, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test unyanked candidate preferred over yanked. """ + # Ignore spurious DEBUG level messages + # TODO: Probably better to work out why they are occurring, but IMO the + # tests are at fault here for being to dependent on exact output. + caplog.set_level(logging.WARNING) candidates = [ - make_mock_candidate('1.0'), - make_mock_candidate('2.0', yanked_reason='bad metadata #2'), + make_mock_candidate("1.0"), + make_mock_candidate("2.0", yanked_reason="bad metadata #2"), ] ireq = install_req_from_line("pkg") @@ -213,15 +225,21 @@ def test_sort_best_candidate__has_non_yanked(self, caplog, monkeypatch): assert ireq.link == candidates[0].link assert len(caplog.records) == 0 - def test_sort_best_candidate__all_yanked(self, caplog, monkeypatch): + def test_sort_best_candidate__all_yanked( + self, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch + ) -> None: """ Test all candidates yanked. """ + # Ignore spurious DEBUG level messages + # TODO: Probably better to work out why they are occurring, but IMO the + # tests are at fault here for being to dependent on exact output. + caplog.set_level(logging.WARNING) candidates = [ - make_mock_candidate('1.0', yanked_reason='bad metadata #1'), + make_mock_candidate("1.0", yanked_reason="bad metadata #1"), # Put the best candidate in the middle, to test sorting. - make_mock_candidate('3.0', yanked_reason='bad metadata #3'), - make_mock_candidate('2.0', yanked_reason='bad metadata #2'), + make_mock_candidate("3.0", yanked_reason="bad metadata #3"), + make_mock_candidate("2.0", yanked_reason="bad metadata #2"), ] ireq = install_req_from_line("pkg") @@ -233,28 +251,39 @@ def test_sort_best_candidate__all_yanked(self, caplog, monkeypatch): # Check the log messages. assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'WARNING' + assert record.levelname == "WARNING" assert record.message == ( - 'The candidate selected for download or install is a yanked ' + "The candidate selected for download or install is a yanked " "version: 'mypackage' candidate " - '(version 3.0 at https://example.com/pkg-3.0.tar.gz)\n' - 'Reason for being yanked: bad metadata #3' + "(version 3.0 at https://example.com/pkg-3.0.tar.gz)\n" + "Reason for being yanked: bad metadata #3" ) - @pytest.mark.parametrize('yanked_reason, expected_reason', [ - # Test no reason given. - ('', ''), - # Test a unicode string with a non-ascii character. - ('curly quote: \u2018', 'curly quote: \u2018'), - ]) + @pytest.mark.parametrize( + "yanked_reason, expected_reason", + [ + # Test no reason given. + ("", ""), + # Test a unicode string with a non-ascii character. + ("curly quote: \u2018", "curly quote: \u2018"), + ], + ) def test_sort_best_candidate__yanked_reason( - self, caplog, monkeypatch, yanked_reason, expected_reason, - ): + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + yanked_reason: str, + expected_reason: str, + ) -> None: """ Test the log message with various reason strings. """ + # Ignore spurious DEBUG level messages + # TODO: Probably better to work out why they are occurring, but IMO the + # tests are at fault here for being to dependent on exact output. + caplog.set_level(logging.WARNING) candidates = [ - make_mock_candidate('1.0', yanked_reason=yanked_reason), + make_mock_candidate("1.0", yanked_reason=yanked_reason), ] ireq = install_req_from_line("pkg") @@ -265,11 +294,11 @@ def test_sort_best_candidate__yanked_reason( assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'WARNING' + assert record.levelname == "WARNING" expected_message = ( - 'The candidate selected for download or install is a yanked ' + "The candidate selected for download or install is a yanked " "version: 'mypackage' candidate " - '(version 1.0 at https://example.com/pkg-1.0.tar.gz)\n' - 'Reason for being yanked: ' + "(version 1.0 at https://example.com/pkg-1.0.tar.gz)\n" + "Reason for being yanked: " ) + expected_reason assert record.message == expected_message diff --git a/tests/unit/test_search_scope.py b/tests/unit/test_search_scope.py index e7f4e3f16b5..ef21c10b820 100644 --- a/tests/unit/test_search_scope.py +++ b/tests/unit/test_search_scope.py @@ -3,39 +3,37 @@ class TestSearchScope: - - def test_get_formatted_locations_basic_auth(self): + def test_get_formatted_locations_basic_auth(self) -> None: """ Test that basic authentication credentials defined in URL is not included in formatted output. """ index_urls = [ - 'https://pypi.org/simple', - 'https://repo-user:repo-pass@repo.domain.com', - ] - find_links = [ - 'https://links-user:links-pass@page.domain.com' + "https://pypi.org/simple", + "https://repo-user:repo-pass@repo.domain.com", ] + find_links = ["https://links-user:links-pass@page.domain.com"] search_scope = SearchScope( - find_links=find_links, index_urls=index_urls, + find_links=find_links, + index_urls=index_urls, ) result = search_scope.get_formatted_locations() - assert 'repo-user:****@repo.domain.com' in result - assert 'repo-pass' not in result - assert 'links-user:****@page.domain.com' in result - assert 'links-pass' not in result + assert "repo-user:****@repo.domain.com" in result + assert "repo-pass" not in result + assert "links-user:****@page.domain.com" in result + assert "links-pass" not in result - def test_get_index_urls_locations(self): + def test_get_index_urls_locations(self) -> None: """Check that the canonical name is on all indexes""" search_scope = SearchScope( find_links=[], - index_urls=['file://index1/', 'file://index2'], - ) - actual = search_scope.get_index_urls_locations( - install_req_from_line('Complex_Name').name + index_urls=["file://index1/", "file://index2"], ) + req = install_req_from_line("Complex_Name") + assert req.name is not None + actual = search_scope.get_index_urls_locations(req.name) assert actual == [ - 'file://index1/complex-name/', - 'file://index2/complex-name/', + "file://index1/complex-name/", + "file://index2/complex-name/", ] diff --git a/tests/unit/test_self_check_outdated.py b/tests/unit/test_self_check_outdated.py index 42c4c452726..d313f3fd019 100644 --- a/tests/unit/test_self_check_outdated.py +++ b/tests/unit/test_self_check_outdated.py @@ -1,14 +1,19 @@ import datetime +import functools import json import os import sys +from typing import Any, Optional, cast +from unittest import mock -import freezegun -import pretend +import freezegun # type: ignore import pytest +from pip._vendor.packaging.version import parse as parse_version from pip._internal import self_outdated_check from pip._internal.models.candidate import InstallationCandidate +from pip._internal.models.link import Link +from pip._internal.network.session import PipSession from pip._internal.self_outdated_check import ( SelfCheckState, logger, @@ -18,95 +23,118 @@ class MockBestCandidateResult: - def __init__(self, best): + def __init__(self, best: InstallationCandidate) -> None: self.best_candidate = best class MockPackageFinder: - BASE_URL = 'https://pypi.org/simple/pip-{0}.tar.gz' - PIP_PROJECT_NAME = 'pip' + BASE_URL = "https://pypi.org/simple/pip-{0}.tar.gz" + PIP_PROJECT_NAME = "pip" INSTALLATION_CANDIDATES = [ - InstallationCandidate(PIP_PROJECT_NAME, '6.9.0', - BASE_URL.format('6.9.0')), - InstallationCandidate(PIP_PROJECT_NAME, '3.3.1', - BASE_URL.format('3.3.1')), - InstallationCandidate(PIP_PROJECT_NAME, '1.0', - BASE_URL.format('1.0')), + InstallationCandidate( + PIP_PROJECT_NAME, + "6.9.0", + Link(BASE_URL.format("6.9.0")), + ), + InstallationCandidate( + PIP_PROJECT_NAME, + "3.3.1", + Link(BASE_URL.format("3.3.1")), + ), + InstallationCandidate( + PIP_PROJECT_NAME, + "1.0", + Link(BASE_URL.format("1.0")), + ), ] @classmethod - def create(cls, *args, **kwargs): + def create(cls, *args: Any, **kwargs: Any) -> "MockPackageFinder": return cls() - def find_best_candidate(self, project_name): + def find_best_candidate(self, project_name: str) -> MockBestCandidateResult: return MockBestCandidateResult(self.INSTALLATION_CANDIDATES[0]) class MockDistribution: - def __init__(self, installer): + def __init__(self, installer: str, version: str) -> None: self.installer = installer + self.version = parse_version(version) - def has_metadata(self, name): - return name == 'INSTALLER' - def get_metadata_lines(self, name): - if self.has_metadata(name): - yield self.installer - else: - raise NotImplementedError('nope') - - -def _options(): - ''' Some default options that we pass to - self_outdated_check.pip_self_version_check ''' - return pretend.stub( - find_links=[], index_url='default_url', extra_index_urls=[], - no_index=False, pre=False, cache_dir='', +class MockEnvironment: + def __init__(self, installer: str, installed_version: Optional[str]) -> None: + self.installer = installer + self.installed_version = installed_version + + def get_distribution(self, name: str) -> Optional[MockDistribution]: + if self.installed_version is None: + return None + return MockDistribution(self.installer, self.installed_version) + + +def _options() -> mock.Mock: + """Some default options that we pass to + self_outdated_check.pip_self_version_check""" + return mock.Mock( + find_links=[], + index_url="default_url", + extra_index_urls=[], + no_index=False, + pre=False, + cache_dir="", + deprecated_features_enabled=[], ) @pytest.mark.parametrize( [ - 'stored_time', - 'installed_ver', - 'new_ver', - 'installer', - 'check_if_upgrade_required', - 'check_warn_logs', + "stored_time", + "installed_ver", + "new_ver", + "installer", + "check_if_upgrade_required", + "check_warn_logs", ], [ # Test we return None when installed version is None - ('1970-01-01T10:00:00Z', None, '1.0', 'pip', False, False), + ("1970-01-01T10:00:00Z", None, "1.0", "pip", False, False), # Need an upgrade - upgrade warning should print - ('1970-01-01T10:00:00Z', '1.0', '6.9.0', 'pip', True, True), + ("1970-01-01T10:00:00Z", "1.0", "6.9.0", "pip", True, True), # Upgrade available, pip installed via rpm - warning should not print - ('1970-01-01T10:00:00Z', '1.0', '6.9.0', 'rpm', True, False), + ("1970-01-01T10:00:00Z", "1.0", "6.9.0", "rpm", True, False), # No upgrade - upgrade warning should not print - ('1970-01-9T10:00:00Z', '6.9.0', '6.9.0', 'pip', False, False), - ] + ("1970-01-9T10:00:00Z", "6.9.0", "6.9.0", "pip", False, False), + ], ) -def test_pip_self_version_check(monkeypatch, stored_time, installed_ver, - new_ver, installer, - check_if_upgrade_required, check_warn_logs): - monkeypatch.setattr(self_outdated_check, 'get_installed_version', - lambda name: installed_ver) - monkeypatch.setattr(self_outdated_check, 'PackageFinder', - MockPackageFinder) - monkeypatch.setattr(logger, 'warning', - pretend.call_recorder(lambda *a, **kw: None)) - monkeypatch.setattr(logger, 'debug', - pretend.call_recorder(lambda s, exc_info=None: None)) - monkeypatch.setattr(self_outdated_check, 'get_distribution', - lambda name: MockDistribution(installer)) - - fake_state = pretend.stub( - state={"last_check": stored_time, 'pypi_version': installed_ver}, - save=pretend.call_recorder(lambda v, t: None), +def test_pip_self_version_check( + monkeypatch: pytest.MonkeyPatch, + stored_time: str, + installed_ver: Optional[str], + new_ver: str, + installer: str, + check_if_upgrade_required: bool, + check_warn_logs: bool, +) -> None: + monkeypatch.setattr( + self_outdated_check, + "get_default_environment", + functools.partial(MockEnvironment, installer, installed_ver), ) monkeypatch.setattr( - self_outdated_check, 'SelfCheckState', lambda **kw: fake_state + self_outdated_check, + "PackageFinder", + MockPackageFinder, + ) + monkeypatch.setattr(logger, "warning", mock.Mock()) + monkeypatch.setattr(logger, "debug", mock.Mock()) + + fake_state = mock.Mock( + state={"last_check": stored_time, "pypi_version": installed_ver}, + save=mock.Mock(), ) + monkeypatch.setattr(self_outdated_check, "SelfCheckState", lambda **kw: fake_state) with freezegun.freeze_time( "1970-01-09 10:00:00", @@ -114,61 +142,57 @@ def test_pip_self_version_check(monkeypatch, stored_time, installed_ver, "six.moves", "pip._vendor.six.moves", "pip._vendor.requests.packages.urllib3.packages.six.moves", - ] + ], ): - latest_pypi_version = pip_self_version_check(None, _options()) + pip_self_version_check(PipSession(), _options()) - # See we return None if not installed_version - if not installed_ver: - assert not latest_pypi_version # See that we saved the correct version - elif check_if_upgrade_required: - assert fake_state.save.calls == [ - pretend.call(new_ver, datetime.datetime(1970, 1, 9, 10, 00, 00)), + if check_if_upgrade_required: + assert fake_state.save.call_args_list == [ + mock.call(new_ver, datetime.datetime(1970, 1, 9, 10, 00, 00)), ] - else: + elif installed_ver: # Make sure no Exceptions - assert not logger.debug.calls + assert not cast(mock.Mock, logger.debug).call_args_list # See that save was not called - assert fake_state.save.calls == [] + assert fake_state.save.call_args_list == [] # Ensure we warn the user or not if check_warn_logs: - assert len(logger.warning.calls) == 1 + assert cast(mock.Mock, logger.warning).call_count == 1 else: - assert len(logger.warning.calls) == 0 + assert cast(mock.Mock, logger.warning).call_count == 0 -statefile_name_case_1 = ( - "fcd2d5175dd33d5df759ee7b045264230205ef837bf9f582f7c3ada7" -) +statefile_name_case_1 = "fcd2d5175dd33d5df759ee7b045264230205ef837bf9f582f7c3ada7" -statefile_name_case_2 = ( - "902cecc0745b8ecf2509ba473f3556f0ba222fedc6df433acda24aa5" -) +statefile_name_case_2 = "902cecc0745b8ecf2509ba473f3556f0ba222fedc6df433acda24aa5" -@pytest.mark.parametrize("key,expected", [ - ("/hello/world/venv", statefile_name_case_1), - ("C:\\Users\\User\\Desktop\\venv", statefile_name_case_2), -]) -def test_get_statefile_name_known_values(key, expected): +@pytest.mark.parametrize( + "key,expected", + [ + ("/hello/world/venv", statefile_name_case_1), + ("C:\\Users\\User\\Desktop\\venv", statefile_name_case_2), + ], +) +def test_get_statefile_name_known_values(key: str, expected: str) -> None: assert expected == self_outdated_check._get_statefile_name(key) -def _get_statefile_path(cache_dir, key): +def _get_statefile_path(cache_dir: str, key: str) -> str: return os.path.join( cache_dir, "selfcheck", self_outdated_check._get_statefile_name(key) ) -def test_self_check_state_no_cache_dir(): - state = SelfCheckState(cache_dir=False) +def test_self_check_state_no_cache_dir() -> None: + state = SelfCheckState(cache_dir="") assert state.state == {} assert state.statefile_path is None -def test_self_check_state_key_uses_sys_prefix(monkeypatch): +def test_self_check_state_key_uses_sys_prefix(monkeypatch: pytest.MonkeyPatch) -> None: key = "helloworld" monkeypatch.setattr(sys, "prefix", key) @@ -177,7 +201,9 @@ def test_self_check_state_key_uses_sys_prefix(monkeypatch): assert state.key == key -def test_self_check_state_reads_expected_statefile(monkeypatch, tmpdir): +def test_self_check_state_reads_expected_statefile( + monkeypatch: pytest.MonkeyPatch, tmpdir: Path +) -> None: cache_dir = tmpdir / "cache_dir" cache_dir.mkdir() key = "helloworld" @@ -203,7 +229,9 @@ def test_self_check_state_reads_expected_statefile(monkeypatch, tmpdir): assert state.state["pypi_version"] == pypi_version -def test_self_check_state_writes_expected_statefile(monkeypatch, tmpdir): +def test_self_check_state_writes_expected_statefile( + monkeypatch: pytest.MonkeyPatch, tmpdir: Path +) -> None: cache_dir = tmpdir / "cache_dir" cache_dir.mkdir() key = "helloworld" @@ -223,8 +251,7 @@ def test_self_check_state_writes_expected_statefile(monkeypatch, tmpdir): expected = { "key": key, - "last_check": last_check.strftime( - self_outdated_check.SELFCHECK_DATE_FMT), + "last_check": last_check.strftime(self_outdated_check.SELFCHECK_DATE_FMT), "pypi_version": pypi_version, } assert expected == saved diff --git a/tests/unit/test_target_python.py b/tests/unit/test_target_python.py index a314988ebc0..d3e27e39ae8 100644 --- a/tests/unit/test_target_python.py +++ b/tests/unit/test_target_python.py @@ -1,22 +1,31 @@ +from typing import Any, Dict, Optional, Tuple +from unittest import mock + import pytest -from mock import patch +from pip._vendor.packaging.tags import Tag from pip._internal.models.target_python import TargetPython from tests.lib import CURRENT_PY_VERSION_INFO, pyversion class TestTargetPython: - - @pytest.mark.parametrize('py_version_info, expected', [ - ((), ((0, 0, 0), '0.0')), - ((2, ), ((2, 0, 0), '2.0')), - ((3, ), ((3, 0, 0), '3.0')), - ((3, 7), ((3, 7, 0), '3.7')), - ((3, 7, 3), ((3, 7, 3), '3.7')), - # Check a minor version with two digits. - ((3, 10, 1), ((3, 10, 1), '3.10')), - ]) - def test_init__py_version_info(self, py_version_info, expected): + @pytest.mark.parametrize( + "py_version_info, expected", + [ + ((), ((0, 0, 0), "0.0")), + ((2,), ((2, 0, 0), "2.0")), + ((3,), ((3, 0, 0), "3.0")), + ((3, 7), ((3, 7, 0), "3.7")), + ((3, 7, 3), ((3, 7, 3), "3.7")), + # Check a minor version with two digits. + ((3, 10, 1), ((3, 10, 1), "3.10")), + ], + ) + def test_init__py_version_info( + self, + py_version_info: Tuple[int, ...], + expected: Tuple[Tuple[int, int, int], str], + ) -> None: """ Test passing the py_version_info argument. """ @@ -30,7 +39,7 @@ def test_init__py_version_info(self, py_version_info, expected): assert target_python.py_version_info == expected_py_version_info assert target_python.py_version == expected_py_version - def test_init__py_version_info_none(self): + def test_init__py_version_info_none(self) -> None: """ Test passing py_version_info=None. """ @@ -41,61 +50,75 @@ def test_init__py_version_info_none(self): assert target_python.py_version_info == CURRENT_PY_VERSION_INFO assert target_python.py_version == pyversion - @pytest.mark.parametrize('kwargs, expected', [ - ({}, ''), - (dict(py_version_info=(3, 6)), "version_info='3.6'"), - ( - dict(platforms=['darwin'], py_version_info=(3, 6)), - "platforms=['darwin'] version_info='3.6'", - ), - ( - dict( - platforms=['darwin'], py_version_info=(3, 6), abis=['cp36m'], - implementation='cp' + @pytest.mark.parametrize( + "kwargs, expected", + [ + ({}, ""), + (dict(py_version_info=(3, 6)), "version_info='3.6'"), + ( + dict(platforms=["darwin"], py_version_info=(3, 6)), + "platforms=['darwin'] version_info='3.6'", ), ( - "platforms=['darwin'] version_info='3.6' abis=['cp36m'] " - "implementation='cp'" + dict( + platforms=["darwin"], + py_version_info=(3, 6), + abis=["cp36m"], + implementation="cp", + ), + ( + "platforms=['darwin'] version_info='3.6' abis=['cp36m'] " + "implementation='cp'" + ), ), - ), - ]) - def test_format_given(self, kwargs, expected): + ], + ) + def test_format_given(self, kwargs: Dict[str, Any], expected: str) -> None: target_python = TargetPython(**kwargs) actual = target_python.format_given() assert actual == expected - @pytest.mark.parametrize('py_version_info, expected_version', [ - ((), ''), - ((2, ), '2'), - ((3, ), '3'), - ((3, 7), '37'), - ((3, 7, 3), '37'), - # Check a minor version with two digits. - ((3, 10, 1), '310'), - # Check that versions=None is passed to get_tags(). - (None, None), - ]) - @patch('pip._internal.models.target_python.get_supported') + @pytest.mark.parametrize( + "py_version_info, expected_version", + [ + ((), ""), + ((2,), "2"), + ((3,), "3"), + ((3, 7), "37"), + ((3, 7, 3), "37"), + # Check a minor version with two digits. + ((3, 10, 1), "310"), + # Check that versions=None is passed to get_tags(). + (None, None), + ], + ) + @mock.patch("pip._internal.models.target_python.get_supported") def test_get_tags( - self, mock_get_supported, py_version_info, expected_version, - ): - mock_get_supported.return_value = ['tag-1', 'tag-2'] + self, + mock_get_supported: mock.Mock, + py_version_info: Optional[Tuple[int, ...]], + expected_version: Optional[str], + ) -> None: + mock_get_supported.return_value = ["tag-1", "tag-2"] target_python = TargetPython(py_version_info=py_version_info) actual = target_python.get_tags() - assert actual == ['tag-1', 'tag-2'] + assert actual == ["tag-1", "tag-2"] - actual = mock_get_supported.call_args[1]['version'] + actual = mock_get_supported.call_args[1]["version"] assert actual == expected_version # Check that the value was cached. - assert target_python._valid_tags == ['tag-1', 'tag-2'] + assert target_python._valid_tags == ["tag-1", "tag-2"] - def test_get_tags__uses_cached_value(self): + def test_get_tags__uses_cached_value(self) -> None: """ Test that get_tags() uses the cached value. """ target_python = TargetPython(py_version_info=None) - target_python._valid_tags = ['tag-1', 'tag-2'] + target_python._valid_tags = [ + Tag("py2", "none", "any"), + Tag("py3", "none", "any"), + ] actual = target_python.get_tags() - assert actual == ['tag-1', 'tag-2'] + assert actual == [Tag("py2", "none", "any"), Tag("py3", "none", "any")] diff --git a/tests/unit/test_urls.py b/tests/unit/test_urls.py index 607023fd28e..56ee80aa802 100644 --- a/tests/unit/test_urls.py +++ b/tests/unit/test_urls.py @@ -1,50 +1,57 @@ import os import sys import urllib.request +from typing import Optional import pytest from pip._internal.utils.urls import get_url_scheme, path_to_url, url_to_path -@pytest.mark.parametrize("url,expected", [ - ('http://localhost:8080/', 'http'), - ('file:c:/path/to/file', 'file'), - ('file:/dev/null', 'file'), - ('', None), -]) -def test_get_url_scheme(url, expected): +@pytest.mark.parametrize( + "url,expected", + [ + ("http://localhost:8080/", "http"), + ("file:c:/path/to/file", "file"), + ("file:/dev/null", "file"), + ("", None), + ], +) +def test_get_url_scheme(url: str, expected: Optional[str]) -> None: assert get_url_scheme(url) == expected @pytest.mark.skipif("sys.platform == 'win32'") -def test_path_to_url_unix(): - assert path_to_url('/tmp/file') == 'file:///tmp/file' - path = os.path.join(os.getcwd(), 'file') - assert path_to_url('file') == 'file://' + urllib.request.pathname2url(path) +def test_path_to_url_unix() -> None: + assert path_to_url("/tmp/file") == "file:///tmp/file" + path = os.path.join(os.getcwd(), "file") + assert path_to_url("file") == "file://" + urllib.request.pathname2url(path) @pytest.mark.skipif("sys.platform != 'win32'") -def test_path_to_url_win(): - assert path_to_url('c:/tmp/file') == 'file:///C:/tmp/file' - assert path_to_url('c:\\tmp\\file') == 'file:///C:/tmp/file' - assert path_to_url(r'\\unc\as\path') == 'file://unc/as/path' - path = os.path.join(os.getcwd(), 'file') - assert path_to_url('file') == 'file:' + urllib.request.pathname2url(path) +def test_path_to_url_win() -> None: + assert path_to_url("c:/tmp/file") == "file:///C:/tmp/file" + assert path_to_url("c:\\tmp\\file") == "file:///C:/tmp/file" + assert path_to_url(r"\\unc\as\path") == "file://unc/as/path" + path = os.path.join(os.getcwd(), "file") + assert path_to_url("file") == "file:" + urllib.request.pathname2url(path) -@pytest.mark.parametrize("url,win_expected,non_win_expected", [ - ('file:tmp', 'tmp', 'tmp'), - ('file:c:/path/to/file', r'C:\path\to\file', 'c:/path/to/file'), - ('file:/path/to/file', r'\path\to\file', '/path/to/file'), - ('file://localhost/tmp/file', r'\tmp\file', '/tmp/file'), - ('file://localhost/c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), - ('file://somehost/tmp/file', r'\\somehost\tmp\file', None), - ('file:///tmp/file', r'\tmp\file', '/tmp/file'), - ('file:///c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), -]) -def test_url_to_path(url, win_expected, non_win_expected): - if sys.platform == 'win32': +@pytest.mark.parametrize( + "url,win_expected,non_win_expected", + [ + ("file:tmp", "tmp", "tmp"), + ("file:c:/path/to/file", r"C:\path\to\file", "c:/path/to/file"), + ("file:/path/to/file", r"\path\to\file", "/path/to/file"), + ("file://localhost/tmp/file", r"\tmp\file", "/tmp/file"), + ("file://localhost/c:/tmp/file", r"C:\tmp\file", "/c:/tmp/file"), + ("file://somehost/tmp/file", r"\\somehost\tmp\file", None), + ("file:///tmp/file", r"\tmp\file", "/tmp/file"), + ("file:///c:/tmp/file", r"C:\tmp\file", "/c:/tmp/file"), + ], +) +def test_url_to_path(url: str, win_expected: str, non_win_expected: str) -> None: + if sys.platform == "win32": expected_path = win_expected else: expected_path = non_win_expected @@ -57,9 +64,9 @@ def test_url_to_path(url, win_expected, non_win_expected): @pytest.mark.skipif("sys.platform != 'win32'") -def test_url_to_path_path_to_url_symmetry_win(): - path = r'C:\tmp\file' +def test_url_to_path_path_to_url_symmetry_win() -> None: + path = r"C:\tmp\file" assert url_to_path(path_to_url(path)) == path - unc_path = r'\\unc\share\path' + unc_path = r"\\unc\share\path" assert url_to_path(path_to_url(unc_path)) == unc_path diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9c43d553143..2d0a82bddf7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -3,19 +3,20 @@ """ import codecs -import itertools import os import shutil import stat import sys import time from io import BytesIO +from typing import Any, Callable, Iterator, List, NoReturn, Optional, Tuple, Type +from unittest.mock import Mock, patch import pytest -from mock import Mock, patch from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError from pip._internal.utils.deprecation import PipDeprecationWarning, deprecated +from pip._internal.utils.egg_link import egg_link_path_from_location from pip._internal.utils.encoding import BOMS, auto_decode from pip._internal.utils.glibc import ( glibc_version_string, @@ -27,10 +28,7 @@ HiddenText, build_netloc, build_url_from_netloc, - egg_link_path, format_size, - get_distribution, - get_installed_distributions, get_prog, hide_url, hide_value, @@ -38,7 +36,6 @@ normalize_path, normalize_version_info, parse_netloc, - path_to_display, redact_auth_from_url, redact_netloc, remove_auth_from_url, @@ -49,321 +46,207 @@ tabulate, ) from pip._internal.utils.setuptools_build import make_setuptools_shim_args +from tests.lib.path import Path class Tests_EgglinkPath: - "util.egg_link_path() tests" + "util.egg_link_path_from_location() tests" - def setup(self): + def setup(self) -> None: - project = 'foo' + project = "foo" self.mock_dist = Mock(project_name=project) - self.site_packages = 'SITE_PACKAGES' - self.user_site = 'USER_SITE' - self.user_site_egglink = os.path.join( - self.user_site, - f'{project}.egg-link' - ) + self.site_packages = "SITE_PACKAGES" + self.user_site = "USER_SITE" + self.user_site_egglink = os.path.join(self.user_site, f"{project}.egg-link") self.site_packages_egglink = os.path.join( self.site_packages, - f'{project}.egg-link', + f"{project}.egg-link", ) # patches - from pip._internal.utils import misc as utils + from pip._internal.utils import egg_link as utils + self.old_site_packages = utils.site_packages - self.mock_site_packages = utils.site_packages = 'SITE_PACKAGES' + self.mock_site_packages = utils.site_packages = "SITE_PACKAGES" self.old_running_under_virtualenv = utils.running_under_virtualenv - self.mock_running_under_virtualenv = utils.running_under_virtualenv = \ - Mock() + self.mock_running_under_virtualenv = utils.running_under_virtualenv = Mock() self.old_virtualenv_no_global = utils.virtualenv_no_global self.mock_virtualenv_no_global = utils.virtualenv_no_global = Mock() self.old_user_site = utils.user_site self.mock_user_site = utils.user_site = self.user_site from os import path + self.old_isfile = path.isfile self.mock_isfile = path.isfile = Mock() - def teardown(self): - from pip._internal.utils import misc as utils + def teardown(self) -> None: + from pip._internal.utils import egg_link as utils + utils.site_packages = self.old_site_packages utils.running_under_virtualenv = self.old_running_under_virtualenv utils.virtualenv_no_global = self.old_virtualenv_no_global utils.user_site = self.old_user_site from os import path + path.isfile = self.old_isfile - def eggLinkInUserSite(self, egglink): + def eggLinkInUserSite(self, egglink: str) -> bool: return egglink == self.user_site_egglink - def eggLinkInSitePackages(self, egglink): + def eggLinkInSitePackages(self, egglink: str) -> bool: return egglink == self.site_packages_egglink # ####################### # # # egglink in usersite # # # ####################### # - def test_egglink_in_usersite_notvenv(self): + def test_egglink_in_usersite_notvenv(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = False self.mock_isfile.side_effect = self.eggLinkInUserSite - assert egg_link_path(self.mock_dist) == self.user_site_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.user_site_egglink + ) - def test_egglink_in_usersite_venv_noglobal(self): + def test_egglink_in_usersite_venv_noglobal(self) -> None: self.mock_virtualenv_no_global.return_value = True self.mock_running_under_virtualenv.return_value = True self.mock_isfile.side_effect = self.eggLinkInUserSite - assert egg_link_path(self.mock_dist) is None + assert egg_link_path_from_location(self.mock_dist.project_name) is None - def test_egglink_in_usersite_venv_global(self): + def test_egglink_in_usersite_venv_global(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = True self.mock_isfile.side_effect = self.eggLinkInUserSite - assert egg_link_path(self.mock_dist) == self.user_site_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.user_site_egglink + ) # ####################### # # # egglink in sitepkgs # # # ####################### # - def test_egglink_in_sitepkgs_notvenv(self): + def test_egglink_in_sitepkgs_notvenv(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = False self.mock_isfile.side_effect = self.eggLinkInSitePackages - assert egg_link_path(self.mock_dist) == self.site_packages_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.site_packages_egglink + ) - def test_egglink_in_sitepkgs_venv_noglobal(self): + def test_egglink_in_sitepkgs_venv_noglobal(self) -> None: self.mock_virtualenv_no_global.return_value = True self.mock_running_under_virtualenv.return_value = True self.mock_isfile.side_effect = self.eggLinkInSitePackages - assert egg_link_path(self.mock_dist) == self.site_packages_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.site_packages_egglink + ) - def test_egglink_in_sitepkgs_venv_global(self): + def test_egglink_in_sitepkgs_venv_global(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = True self.mock_isfile.side_effect = self.eggLinkInSitePackages - assert egg_link_path(self.mock_dist) == self.site_packages_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.site_packages_egglink + ) # ################################## # # # egglink in usersite & sitepkgs # # # ################################## # - def test_egglink_in_both_notvenv(self): + def test_egglink_in_both_notvenv(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = False self.mock_isfile.return_value = True - assert egg_link_path(self.mock_dist) == self.user_site_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.user_site_egglink + ) - def test_egglink_in_both_venv_noglobal(self): + def test_egglink_in_both_venv_noglobal(self) -> None: self.mock_virtualenv_no_global.return_value = True self.mock_running_under_virtualenv.return_value = True self.mock_isfile.return_value = True - assert egg_link_path(self.mock_dist) == self.site_packages_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.site_packages_egglink + ) - def test_egglink_in_both_venv_global(self): + def test_egglink_in_both_venv_global(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = True self.mock_isfile.return_value = True - assert egg_link_path(self.mock_dist) == self.site_packages_egglink + assert ( + egg_link_path_from_location(self.mock_dist.project_name) + == self.site_packages_egglink + ) # ############## # # # no egglink # # # ############## # - def test_noegglink_in_sitepkgs_notvenv(self): + def test_noegglink_in_sitepkgs_notvenv(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = False self.mock_isfile.return_value = False - assert egg_link_path(self.mock_dist) is None + assert egg_link_path_from_location(self.mock_dist.project_name) is None - def test_noegglink_in_sitepkgs_venv_noglobal(self): + def test_noegglink_in_sitepkgs_venv_noglobal(self) -> None: self.mock_virtualenv_no_global.return_value = True self.mock_running_under_virtualenv.return_value = True self.mock_isfile.return_value = False - assert egg_link_path(self.mock_dist) is None + assert egg_link_path_from_location(self.mock_dist.project_name) is None - def test_noegglink_in_sitepkgs_venv_global(self): + def test_noegglink_in_sitepkgs_venv_global(self) -> None: self.mock_virtualenv_no_global.return_value = False self.mock_running_under_virtualenv.return_value = True self.mock_isfile.return_value = False - assert egg_link_path(self.mock_dist) is None + assert egg_link_path_from_location(self.mock_dist.project_name) is None -@patch('pip._internal.utils.misc.dist_in_usersite') -@patch('pip._internal.utils.misc.dist_is_local') -@patch('pip._internal.utils.misc.dist_is_editable') -class TestsGetDistributions: - """Test get_installed_distributions() and get_distribution(). - """ - class MockWorkingSet(list): - def require(self, name): - pass - - workingset = MockWorkingSet(( - Mock(test_name="global", key="global"), - Mock(test_name="editable", key="editable"), - Mock(test_name="normal", key="normal"), - Mock(test_name="user", key="user"), - )) - - workingset_stdlib = MockWorkingSet(( - Mock(test_name='normal', key='argparse'), - Mock(test_name='normal', key='wsgiref') - )) - - workingset_freeze = MockWorkingSet(( - Mock(test_name='normal', key='pip'), - Mock(test_name='normal', key='setuptools'), - Mock(test_name='normal', key='distribute') - )) - - def dist_is_editable(self, dist): - return dist.test_name == "editable" - - def dist_is_local(self, dist): - return dist.test_name != "global" and dist.test_name != 'user' - - def dist_in_usersite(self, dist): - return dist.test_name == "user" - - @patch('pip._vendor.pkg_resources.working_set', workingset) - def test_editables_only(self, mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dists = get_installed_distributions(editables_only=True) - assert len(dists) == 1, dists - assert dists[0].test_name == "editable" - - @patch('pip._vendor.pkg_resources.working_set', workingset) - def test_exclude_editables(self, mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dists = get_installed_distributions(include_editables=False) - assert len(dists) == 1 - assert dists[0].test_name == "normal" - - @patch('pip._vendor.pkg_resources.working_set', workingset) - def test_include_globals(self, mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dists = get_installed_distributions(local_only=False) - assert len(dists) == 4 - - @patch('pip._vendor.pkg_resources.working_set', workingset) - def test_user_only(self, mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dists = get_installed_distributions(local_only=False, - user_only=True) - assert len(dists) == 1 - assert dists[0].test_name == "user" - - @patch('pip._vendor.pkg_resources.working_set', workingset_stdlib) - def test_gte_py27_excludes(self, mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dists = get_installed_distributions() - assert len(dists) == 0 - - @patch('pip._vendor.pkg_resources.working_set', workingset_freeze) - def test_freeze_excludes(self, mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dists = get_installed_distributions( - skip=('setuptools', 'pip', 'distribute')) - assert len(dists) == 0 - - @pytest.mark.parametrize( - "working_set, req_name", - itertools.chain( - itertools.product([workingset], (d.key for d in workingset)), - itertools.product( - [workingset_stdlib], (d.key for d in workingset_stdlib), - ), - ), - ) - def test_get_distribution( - self, - mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite, - working_set, - req_name, - ): - """Ensure get_distribution() finds all kinds of distributions. - """ - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - with patch("pip._vendor.pkg_resources.working_set", working_set): - dist = get_distribution(req_name) - assert dist is not None - assert dist.key == req_name - - @patch('pip._vendor.pkg_resources.working_set', workingset) - def test_get_distribution_nonexist( - self, - mock_dist_is_editable, - mock_dist_is_local, - mock_dist_in_usersite, - ): - mock_dist_is_editable.side_effect = self.dist_is_editable - mock_dist_is_local.side_effect = self.dist_is_local - mock_dist_in_usersite.side_effect = self.dist_in_usersite - dist = get_distribution("non-exist") - assert dist is None - - -def test_rmtree_errorhandler_nonexistent_directory(tmpdir): +def test_rmtree_errorhandler_nonexistent_directory(tmpdir: Path) -> None: """ Test rmtree_errorhandler ignores the given non-existing directory. """ - nonexistent_path = str(tmpdir / 'foo') + nonexistent_path = str(tmpdir / "foo") mock_func = Mock() - rmtree_errorhandler(mock_func, nonexistent_path, None) + # Argument 3 to "rmtree_errorhandler" has incompatible type "None"; expected + # "Tuple[Type[BaseException], BaseException, TracebackType]" + rmtree_errorhandler(mock_func, nonexistent_path, None) # type: ignore[arg-type] mock_func.assert_not_called() -def test_rmtree_errorhandler_readonly_directory(tmpdir): +def test_rmtree_errorhandler_readonly_directory(tmpdir: Path) -> None: """ Test rmtree_errorhandler makes the given read-only directory writable. """ # Create read only directory - subdir_path = tmpdir / 'subdir' + subdir_path = tmpdir / "subdir" subdir_path.mkdir() path = str(subdir_path) os.chmod(path, stat.S_IREAD) # Make sure mock_func is called with the given path mock_func = Mock() - rmtree_errorhandler(mock_func, path, None) + # Argument 3 to "rmtree_errorhandler" has incompatible type "None"; expected + # "Tuple[Type[BaseException], BaseException, TracebackType]" + rmtree_errorhandler(mock_func, path, None) # type: ignore[arg-type] mock_func.assert_called_with(path) # Make sure the path is now writable assert os.stat(path).st_mode & stat.S_IWRITE -def test_rmtree_errorhandler_reraises_error(tmpdir): +def test_rmtree_errorhandler_reraises_error(tmpdir: Path) -> None: """ Test rmtree_errorhandler reraises an exception by the given unreadable directory. """ # Create directory without read permission - subdir_path = tmpdir / 'subdir' + subdir_path = tmpdir / "subdir" subdir_path.mkdir() path = str(subdir_path) os.chmod(path, stat.S_IWRITE) @@ -371,111 +254,95 @@ def test_rmtree_errorhandler_reraises_error(tmpdir): mock_func = Mock() try: - raise RuntimeError('test message') + raise RuntimeError("test message") except RuntimeError: # Make sure the handler reraises an exception - with pytest.raises(RuntimeError, match='test message'): - rmtree_errorhandler(mock_func, path, None) + with pytest.raises(RuntimeError, match="test message"): + # Argument 3 to "rmtree_errorhandler" has incompatible type "None"; expected + # "Tuple[Type[BaseException], BaseException, TracebackType]" + rmtree_errorhandler(mock_func, path, None) # type: ignore[arg-type] mock_func.assert_not_called() -def test_rmtree_skips_nonexistent_directory(): +def test_rmtree_skips_nonexistent_directory() -> None: """ Test wrapped rmtree doesn't raise an error by the given nonexistent directory. """ - rmtree.__wrapped__('nonexistent-subdir') + rmtree.__wrapped__("nonexistent-subdir") # type: ignore[attr-defined] class Failer: - def __init__(self, duration=1): + def __init__(self, duration: int = 1) -> None: self.succeed_after = time.time() + duration - def call(self, *args, **kw): + def call(self, *args: Any, **kw: Any) -> None: """Fail with OSError self.max_fails times""" if time.time() < self.succeed_after: raise OSError("Failed") -def test_rmtree_retries(tmpdir, monkeypatch): +def test_rmtree_retries(monkeypatch: pytest.MonkeyPatch) -> None: """ Test pip._internal.utils.rmtree will retry failures """ - monkeypatch.setattr(shutil, 'rmtree', Failer(duration=1).call) - rmtree('foo') + monkeypatch.setattr(shutil, "rmtree", Failer(duration=1).call) + rmtree("foo") -def test_rmtree_retries_for_3sec(tmpdir, monkeypatch): +def test_rmtree_retries_for_3sec(monkeypatch: pytest.MonkeyPatch) -> None: """ Test pip._internal.utils.rmtree will retry failures for no more than 3 sec """ - monkeypatch.setattr(shutil, 'rmtree', Failer(duration=5).call) + monkeypatch.setattr(shutil, "rmtree", Failer(duration=5).call) with pytest.raises(OSError): - rmtree('foo') + rmtree("foo") if sys.byteorder == "little": expected_byte_string = ( - "b'\\xff\\xfe/\\x00p\\x00a\\x00t\\x00h\\x00/" - "\\x00d\\x00\\xe9\\x00f\\x00'" + "b'\\xff\\xfe/\\x00p\\x00a\\x00t\\x00h\\x00/\\x00d\\x00\\xe9\\x00f\\x00'" ) elif sys.byteorder == "big": expected_byte_string = ( - "b'\\xfe\\xff\\x00/\\x00p\\x00a\\x00t\\x00h\\" - "x00/\\x00d\\x00\\xe9\\x00f'" + "b'\\xfe\\xff\\x00/\\x00p\\x00a\\x00t\\x00h\\x00/\\x00d\\x00\\xe9\\x00f'" ) -@pytest.mark.parametrize('path, fs_encoding, expected', [ - (None, None, None), - # Test passing a text (unicode) string. - ('/path/déf', None, '/path/déf'), - # Test a bytes object with a non-ascii character. - ('/path/déf'.encode('utf-8'), 'utf-8', '/path/déf'), - # Test a bytes object with a character that can't be decoded. - ('/path/déf'.encode('utf-8'), 'ascii', "b'/path/d\\xc3\\xa9f'"), - ('/path/déf'.encode('utf-16'), 'utf-8', expected_byte_string), -]) -def test_path_to_display(monkeypatch, path, fs_encoding, expected): - monkeypatch.setattr(sys, 'getfilesystemencoding', lambda: fs_encoding) - actual = path_to_display(path) - assert actual == expected, f'actual: {actual!r}' - - class Test_normalize_path: # Technically, symlinks are possible on Windows, but you need a special # permission bit to create them, and Python 2 doesn't support it anyway, so # it's easiest just to skip this test on Windows altogether. @pytest.mark.skipif("sys.platform == 'win32'") - def test_resolve_symlinks(self, tmpdir): + def test_resolve_symlinks(self, tmpdir: Path) -> None: print(type(tmpdir)) print(dir(tmpdir)) orig_working_dir = os.getcwd() os.chdir(tmpdir) try: - d = os.path.join('foo', 'bar') - f = os.path.join(d, 'file1') + d = os.path.join("foo", "bar") + f = os.path.join(d, "file1") os.makedirs(d) - with open(f, 'w'): # Create the file + with open(f, "w"): # Create the file pass - os.symlink(d, 'dir_link') - os.symlink(f, 'file_link') + os.symlink(d, "dir_link") + os.symlink(f, "file_link") assert normalize_path( - 'dir_link/file1', resolve_symlinks=True + "dir_link/file1", resolve_symlinks=True ) == os.path.join(tmpdir, f) assert normalize_path( - 'dir_link/file1', resolve_symlinks=False - ) == os.path.join(tmpdir, 'dir_link', 'file1') - - assert normalize_path( - 'file_link', resolve_symlinks=True - ) == os.path.join(tmpdir, f) - assert normalize_path( - 'file_link', resolve_symlinks=False - ) == os.path.join(tmpdir, 'file_link') + "dir_link/file1", resolve_symlinks=False + ) == os.path.join(tmpdir, "dir_link", "file1") + + assert normalize_path("file_link", resolve_symlinks=True) == os.path.join( + tmpdir, f + ) + assert normalize_path("file_link", resolve_symlinks=False) == os.path.join( + tmpdir, "file_link" + ) finally: os.chdir(orig_working_dir) @@ -483,352 +350,426 @@ def test_resolve_symlinks(self, tmpdir): class TestHashes: """Tests for pip._internal.utils.hashes""" - @pytest.mark.parametrize('hash_name, hex_digest, expected', [ - # Test a value that matches but with the wrong hash_name. - ('sha384', 128 * 'a', False), - # Test matching values, including values other than the first. - ('sha512', 128 * 'a', True), - ('sha512', 128 * 'b', True), - # Test a matching hash_name with a value that doesn't match. - ('sha512', 128 * 'c', False), - ]) - def test_is_hash_allowed(self, hash_name, hex_digest, expected): + @pytest.mark.parametrize( + "hash_name, hex_digest, expected", + [ + # Test a value that matches but with the wrong hash_name. + ("sha384", 128 * "a", False), + # Test matching values, including values other than the first. + ("sha512", 128 * "a", True), + ("sha512", 128 * "b", True), + # Test a matching hash_name with a value that doesn't match. + ("sha512", 128 * "c", False), + ], + ) + def test_is_hash_allowed( + self, hash_name: str, hex_digest: str, expected: bool + ) -> None: hashes_data = { - 'sha512': [128 * 'a', 128 * 'b'], + "sha512": [128 * "a", 128 * "b"], } hashes = Hashes(hashes_data) assert hashes.is_hash_allowed(hash_name, hex_digest) == expected - def test_success(self, tmpdir): + def test_success(self, tmpdir: Path) -> None: """Make sure no error is raised when at least one hash matches. Test check_against_path because it calls everything else. """ - file = tmpdir / 'to_hash' - file.write_text('hello') - hashes = Hashes({ - 'sha256': ['2cf24dba5fb0a30e26e83b2ac5b9e29e' - '1b161e5c1fa7425e73043362938b9824'], - 'sha224': ['wrongwrong'], - 'md5': ['5d41402abc4b2a76b9719d911017c592']}) + file = tmpdir / "to_hash" + file.write_text("hello") + hashes = Hashes( + { + "sha256": [ + "2cf24dba5fb0a30e26e83b2ac5b9e29e" + "1b161e5c1fa7425e73043362938b9824" + ], + "sha224": ["wrongwrong"], + "md5": ["5d41402abc4b2a76b9719d911017c592"], + } + ) hashes.check_against_path(file) - def test_failure(self): + def test_failure(self) -> None: """Hashes should raise HashMismatch when no hashes match.""" - hashes = Hashes({'sha256': ['wrongwrong']}) + hashes = Hashes({"sha256": ["wrongwrong"]}) with pytest.raises(HashMismatch): - hashes.check_against_file(BytesIO(b'hello')) + hashes.check_against_file(BytesIO(b"hello")) - def test_missing_hashes(self): + def test_missing_hashes(self) -> None: """MissingHashes should raise HashMissing when any check is done.""" with pytest.raises(HashMissing): - MissingHashes().check_against_file(BytesIO(b'hello')) + MissingHashes().check_against_file(BytesIO(b"hello")) - def test_unknown_hash(self): + def test_unknown_hash(self) -> None: """Hashes should raise InstallationError when it encounters an unknown hash.""" - hashes = Hashes({'badbad': ['dummy']}) + hashes = Hashes({"badbad": ["dummy"]}) with pytest.raises(InstallationError): - hashes.check_against_file(BytesIO(b'hello')) + hashes.check_against_file(BytesIO(b"hello")) - def test_non_zero(self): + def test_non_zero(self) -> None: """Test that truthiness tests tell whether any known-good hashes exist.""" - assert Hashes({'sha256': 'dummy'}) + assert Hashes({"sha256": ["dummy"]}) assert not Hashes() assert not Hashes({}) - def test_equality(self): + def test_equality(self) -> None: assert Hashes() == Hashes() - assert Hashes({'sha256': ['abcd']}) == Hashes({'sha256': ['abcd']}) - assert Hashes({'sha256': ['ab', 'cd']}) == Hashes({'sha256': ['cd', 'ab']}) + assert Hashes({"sha256": ["abcd"]}) == Hashes({"sha256": ["abcd"]}) + assert Hashes({"sha256": ["ab", "cd"]}) == Hashes({"sha256": ["cd", "ab"]}) - def test_hash(self): + def test_hash(self) -> None: cache = {} - cache[Hashes({'sha256': ['ab', 'cd']})] = 42 - assert cache[Hashes({'sha256': ['ab', 'cd']})] == 42 + cache[Hashes({"sha256": ["ab", "cd"]})] = 42 + assert cache[Hashes({"sha256": ["ab", "cd"]})] == 42 class TestEncoding: """Tests for pip._internal.utils.encoding""" - def test_auto_decode_utf_16_le(self): + def test_auto_decode_utf_16_le(self) -> None: data = ( - b'\xff\xfeD\x00j\x00a\x00n\x00g\x00o\x00=\x00' - b'=\x001\x00.\x004\x00.\x002\x00' + b"\xff\xfeD\x00j\x00a\x00n\x00g\x00o\x00=\x00" + b"=\x001\x00.\x004\x00.\x002\x00" ) assert data.startswith(codecs.BOM_UTF16_LE) assert auto_decode(data) == "Django==1.4.2" - def test_auto_decode_utf_16_be(self): + def test_auto_decode_utf_16_be(self) -> None: data = ( - b'\xfe\xff\x00D\x00j\x00a\x00n\x00g\x00o\x00=' - b'\x00=\x001\x00.\x004\x00.\x002' + b"\xfe\xff\x00D\x00j\x00a\x00n\x00g\x00o\x00=" + b"\x00=\x001\x00.\x004\x00.\x002" ) assert data.startswith(codecs.BOM_UTF16_BE) assert auto_decode(data) == "Django==1.4.2" - def test_auto_decode_no_bom(self): - assert auto_decode(b'foobar') == 'foobar' + def test_auto_decode_no_bom(self) -> None: + assert auto_decode(b"foobar") == "foobar" - def test_auto_decode_pep263_headers(self): - latin1_req = '# coding=latin1\n# Pas trop de café' - assert auto_decode(latin1_req.encode('latin1')) == latin1_req + def test_auto_decode_pep263_headers(self) -> None: + latin1_req = "# coding=latin1\n# Pas trop de café" + assert auto_decode(latin1_req.encode("latin1")) == latin1_req - def test_auto_decode_no_preferred_encoding(self): + def test_auto_decode_no_preferred_encoding(self) -> None: om, em = Mock(), Mock() - om.return_value = 'ascii' + om.return_value = "ascii" em.return_value = None - data = 'data' - with patch('sys.getdefaultencoding', om): - with patch('locale.getpreferredencoding', em): + data = "data" + with patch("sys.getdefaultencoding", om): + with patch("locale.getpreferredencoding", em): ret = auto_decode(data.encode(sys.getdefaultencoding())) assert ret == data - @pytest.mark.parametrize('encoding', [encoding for bom, encoding in BOMS]) - def test_all_encodings_are_valid(self, encoding): + @pytest.mark.parametrize("encoding", [encoding for bom, encoding in BOMS]) + def test_all_encodings_are_valid(self, encoding: str) -> None: # we really only care that there is no LookupError - assert ''.encode(encoding).decode(encoding) == '' + assert "".encode(encoding).decode(encoding) == "" -def raises(error): +def raises(error: Type[Exception]) -> NoReturn: raise error class TestGlibc: @pytest.mark.skipif("sys.platform == 'win32'") - def test_glibc_version_string(self, monkeypatch): + def test_glibc_version_string(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( - os, "confstr", lambda x: "glibc 2.20", raising=False, + os, + "confstr", + lambda x: "glibc 2.20", + raising=False, ) assert glibc_version_string() == "2.20" @pytest.mark.skipif("sys.platform == 'win32'") - def test_glibc_version_string_confstr(self, monkeypatch): + def test_glibc_version_string_confstr( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setattr( - os, "confstr", lambda x: "glibc 2.20", raising=False, + os, + "confstr", + lambda x: "glibc 2.20", + raising=False, ) assert glibc_version_string_confstr() == "2.20" - @pytest.mark.parametrize("failure", [ - lambda x: raises(ValueError), - lambda x: raises(OSError), - lambda x: "XXX", - ]) - def test_glibc_version_string_confstr_fail(self, monkeypatch, failure): + @pytest.mark.parametrize( + "failure", + [ + lambda x: raises(ValueError), + lambda x: raises(OSError), + lambda x: "XXX", + ], + ) + def test_glibc_version_string_confstr_fail( + self, monkeypatch: pytest.MonkeyPatch, failure: Callable[[Any], Any] + ) -> None: monkeypatch.setattr(os, "confstr", failure, raising=False) assert glibc_version_string_confstr() is None - def test_glibc_version_string_confstr_missing(self, monkeypatch): + def test_glibc_version_string_confstr_missing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.delattr(os, "confstr", raising=False) assert glibc_version_string_confstr() is None - def test_glibc_version_string_ctypes_missing(self, monkeypatch): + def test_glibc_version_string_ctypes_missing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: monkeypatch.setitem(sys.modules, "ctypes", None) assert glibc_version_string_ctypes() is None -@pytest.mark.parametrize('version_info, expected', [ - ((), (0, 0, 0)), - ((3, ), (3, 0, 0)), - ((3, 6), (3, 6, 0)), - ((3, 6, 2), (3, 6, 2)), - ((3, 6, 2, 4), (3, 6, 2)), -]) -def test_normalize_version_info(version_info, expected): +@pytest.mark.parametrize( + "version_info, expected", + [ + ((), (0, 0, 0)), + ((3,), (3, 0, 0)), + ((3, 6), (3, 6, 0)), + ((3, 6, 2), (3, 6, 2)), + ((3, 6, 2, 4), (3, 6, 2)), + ], +) +def test_normalize_version_info( + version_info: Tuple[int, ...], expected: Tuple[int, int, int] +) -> None: actual = normalize_version_info(version_info) assert actual == expected class TestGetProg: - @pytest.mark.parametrize( ("argv", "executable", "expected"), [ - ('/usr/bin/pip', '', 'pip'), - ('-c', '/usr/bin/python', '/usr/bin/python -m pip'), - ('__main__.py', '/usr/bin/python', '/usr/bin/python -m pip'), - ('/usr/bin/pip3', '', 'pip3'), - ] + ("/usr/bin/pip", "", "pip"), + ("-c", "/usr/bin/python", "/usr/bin/python -m pip"), + ("__main__.py", "/usr/bin/python", "/usr/bin/python -m pip"), + ("/usr/bin/pip3", "", "pip3"), + ], ) - def test_get_prog(self, monkeypatch, argv, executable, expected): - monkeypatch.setattr('pip._internal.utils.misc.sys.argv', [argv]) - monkeypatch.setattr( - 'pip._internal.utils.misc.sys.executable', - executable - ) + def test_get_prog( + self, monkeypatch: pytest.MonkeyPatch, argv: str, executable: str, expected: str + ) -> None: + monkeypatch.setattr("pip._internal.utils.misc.sys.argv", [argv]) + monkeypatch.setattr("pip._internal.utils.misc.sys.executable", executable) assert get_prog() == expected -@pytest.mark.parametrize('host_port, expected_netloc', [ - # Test domain name. - (('example.com', None), 'example.com'), - (('example.com', 5000), 'example.com:5000'), - # Test IPv4 address. - (('127.0.0.1', None), '127.0.0.1'), - (('127.0.0.1', 5000), '127.0.0.1:5000'), - # Test bare IPv6 address. - (('2001:db6::1', None), '2001:db6::1'), - # Test IPv6 with port. - (('2001:db6::1', 5000), '[2001:db6::1]:5000'), -]) -def test_build_netloc(host_port, expected_netloc): +@pytest.mark.parametrize( + "host_port, expected_netloc", + [ + # Test domain name. + (("example.com", None), "example.com"), + (("example.com", 5000), "example.com:5000"), + # Test IPv4 address. + (("127.0.0.1", None), "127.0.0.1"), + (("127.0.0.1", 5000), "127.0.0.1:5000"), + # Test bare IPv6 address. + (("2001:db6::1", None), "2001:db6::1"), + # Test IPv6 with port. + (("2001:db6::1", 5000), "[2001:db6::1]:5000"), + ], +) +def test_build_netloc( + host_port: Tuple[str, Optional[int]], expected_netloc: str +) -> None: assert build_netloc(*host_port) == expected_netloc -@pytest.mark.parametrize('netloc, expected_url, expected_host_port', [ - # Test domain name. - ('example.com', 'https://example.com', ('example.com', None)), - ('example.com:5000', 'https://example.com:5000', ('example.com', 5000)), - # Test IPv4 address. - ('127.0.0.1', 'https://127.0.0.1', ('127.0.0.1', None)), - ('127.0.0.1:5000', 'https://127.0.0.1:5000', ('127.0.0.1', 5000)), - # Test bare IPv6 address. - ('2001:db6::1', 'https://[2001:db6::1]', ('2001:db6::1', None)), - # Test IPv6 with port. - ( - '[2001:db6::1]:5000', - 'https://[2001:db6::1]:5000', - ('2001:db6::1', 5000) - ), - # Test netloc with auth. - ( - 'user:password@localhost:5000', - 'https://user:password@localhost:5000', - ('localhost', 5000) - ) -]) +@pytest.mark.parametrize( + "netloc, expected_url, expected_host_port", + [ + # Test domain name. + ("example.com", "https://example.com", ("example.com", None)), + ("example.com:5000", "https://example.com:5000", ("example.com", 5000)), + # Test IPv4 address. + ("127.0.0.1", "https://127.0.0.1", ("127.0.0.1", None)), + ("127.0.0.1:5000", "https://127.0.0.1:5000", ("127.0.0.1", 5000)), + # Test bare IPv6 address. + ("2001:db6::1", "https://[2001:db6::1]", ("2001:db6::1", None)), + # Test IPv6 with port. + ("[2001:db6::1]:5000", "https://[2001:db6::1]:5000", ("2001:db6::1", 5000)), + # Test netloc with auth. + ( + "user:password@localhost:5000", + "https://user:password@localhost:5000", + ("localhost", 5000), + ), + ], +) def test_build_url_from_netloc_and_parse_netloc( - netloc, expected_url, expected_host_port, -): + netloc: str, + expected_url: str, + expected_host_port: Tuple[str, Optional[int]], +) -> None: assert build_url_from_netloc(netloc) == expected_url assert parse_netloc(netloc) == expected_host_port -@pytest.mark.parametrize('netloc, expected', [ - # Test a basic case. - ('example.com', ('example.com', (None, None))), - # Test with username and no password. - ('user@example.com', ('example.com', ('user', None))), - # Test with username and password. - ('user:pass@example.com', ('example.com', ('user', 'pass'))), - # Test with username and empty password. - ('user:@example.com', ('example.com', ('user', ''))), - # Test the password containing an @ symbol. - ('user:pass@word@example.com', ('example.com', ('user', 'pass@word'))), - # Test the password containing a : symbol. - ('user:pass:word@example.com', ('example.com', ('user', 'pass:word'))), - # Test URL-encoded reserved characters. - ('user%3Aname:%23%40%5E@example.com', - ('example.com', ('user:name', '#@^'))), -]) -def test_split_auth_from_netloc(netloc, expected): +@pytest.mark.parametrize( + "netloc, expected", + [ + # Test a basic case. + ("example.com", ("example.com", (None, None))), + # Test with username and no password. + ("user@example.com", ("example.com", ("user", None))), + # Test with username and password. + ("user:pass@example.com", ("example.com", ("user", "pass"))), + # Test with username and empty password. + ("user:@example.com", ("example.com", ("user", ""))), + # Test the password containing an @ symbol. + ("user:pass@word@example.com", ("example.com", ("user", "pass@word"))), + # Test the password containing a : symbol. + ("user:pass:word@example.com", ("example.com", ("user", "pass:word"))), + # Test URL-encoded reserved characters. + ("user%3Aname:%23%40%5E@example.com", ("example.com", ("user:name", "#@^"))), + ], +) +def test_split_auth_from_netloc( + netloc: str, expected: Tuple[str, Tuple[Optional[str], Optional[str]]] +) -> None: actual = split_auth_from_netloc(netloc) assert actual == expected -@pytest.mark.parametrize('url, expected', [ - # Test a basic case. - ('http://example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', (None, None))), - # Test with username and no password. - ('http://user@example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', ('user', None))), - # Test with username and password. - ('http://user:pass@example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', ('user', 'pass'))), - # Test with username and empty password. - ('http://user:@example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', ('user', ''))), - # Test the password containing an @ symbol. - ('http://user:pass@word@example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', ('user', 'pass@word'))), - # Test the password containing a : symbol. - ('http://user:pass:word@example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', ('user', 'pass:word'))), - # Test URL-encoded reserved characters. - ('http://user%3Aname:%23%40%5E@example.com/path#anchor', - ('http://example.com/path#anchor', 'example.com', ('user:name', '#@^'))), -]) -def test_split_auth_netloc_from_url(url, expected): +@pytest.mark.parametrize( + "url, expected", + [ + # Test a basic case. + ( + "http://example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", (None, None)), + ), + # Test with username and no password. + ( + "http://user@example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", ("user", None)), + ), + # Test with username and password. + ( + "http://user:pass@example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", ("user", "pass")), + ), + # Test with username and empty password. + ( + "http://user:@example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", ("user", "")), + ), + # Test the password containing an @ symbol. + ( + "http://user:pass@word@example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", ("user", "pass@word")), + ), + # Test the password containing a : symbol. + ( + "http://user:pass:word@example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", ("user", "pass:word")), + ), + # Test URL-encoded reserved characters. + ( + "http://user%3Aname:%23%40%5E@example.com/path#anchor", + ("http://example.com/path#anchor", "example.com", ("user:name", "#@^")), + ), + ], +) +def test_split_auth_netloc_from_url( + url: str, expected: Tuple[str, str, Tuple[Optional[str], Optional[str]]] +) -> None: actual = split_auth_netloc_from_url(url) assert actual == expected -@pytest.mark.parametrize('netloc, expected', [ - # Test a basic case. - ('example.com', 'example.com'), - # Test with username and no password. - ('accesstoken@example.com', '****@example.com'), - # Test with username and password. - ('user:pass@example.com', 'user:****@example.com'), - # Test with username and empty password. - ('user:@example.com', 'user:****@example.com'), - # Test the password containing an @ symbol. - ('user:pass@word@example.com', 'user:****@example.com'), - # Test the password containing a : symbol. - ('user:pass:word@example.com', 'user:****@example.com'), - # Test URL-encoded reserved characters. - ('user%3Aname:%23%40%5E@example.com', 'user%3Aname:****@example.com'), -]) -def test_redact_netloc(netloc, expected): +@pytest.mark.parametrize( + "netloc, expected", + [ + # Test a basic case. + ("example.com", "example.com"), + # Test with username and no password. + ("accesstoken@example.com", "****@example.com"), + # Test with username and password. + ("user:pass@example.com", "user:****@example.com"), + # Test with username and empty password. + ("user:@example.com", "user:****@example.com"), + # Test the password containing an @ symbol. + ("user:pass@word@example.com", "user:****@example.com"), + # Test the password containing a : symbol. + ("user:pass:word@example.com", "user:****@example.com"), + # Test URL-encoded reserved characters. + ("user%3Aname:%23%40%5E@example.com", "user%3Aname:****@example.com"), + ], +) +def test_redact_netloc(netloc: str, expected: str) -> None: actual = redact_netloc(netloc) assert actual == expected -@pytest.mark.parametrize('auth_url, expected_url', [ - ('https://user:pass@domain.tld/project/tags/v0.2', - 'https://domain.tld/project/tags/v0.2'), - ('https://domain.tld/project/tags/v0.2', - 'https://domain.tld/project/tags/v0.2',), - ('https://user:pass@domain.tld/svn/project/trunk@8181', - 'https://domain.tld/svn/project/trunk@8181'), - ('https://domain.tld/project/trunk@8181', - 'https://domain.tld/project/trunk@8181',), - ('git+https://pypi.org/something', - 'git+https://pypi.org/something'), - ('git+https://user:pass@pypi.org/something', - 'git+https://pypi.org/something'), - ('git+ssh://git@pypi.org/something', - 'git+ssh://pypi.org/something'), -]) -def test_remove_auth_from_url(auth_url, expected_url): +@pytest.mark.parametrize( + "auth_url, expected_url", + [ + ( + "https://user:pass@domain.tld/project/tags/v0.2", + "https://domain.tld/project/tags/v0.2", + ), + ( + "https://domain.tld/project/tags/v0.2", + "https://domain.tld/project/tags/v0.2", + ), + ( + "https://user:pass@domain.tld/svn/project/trunk@8181", + "https://domain.tld/svn/project/trunk@8181", + ), + ( + "https://domain.tld/project/trunk@8181", + "https://domain.tld/project/trunk@8181", + ), + ("git+https://pypi.org/something", "git+https://pypi.org/something"), + ("git+https://user:pass@pypi.org/something", "git+https://pypi.org/something"), + ("git+ssh://git@pypi.org/something", "git+ssh://pypi.org/something"), + ], +) +def test_remove_auth_from_url(auth_url: str, expected_url: str) -> None: url = remove_auth_from_url(auth_url) assert url == expected_url -@pytest.mark.parametrize('auth_url, expected_url', [ - ('https://accesstoken@example.com/abc', 'https://****@example.com/abc'), - ('https://user:password@example.com', 'https://user:****@example.com'), - ('https://user:@example.com', 'https://user:****@example.com'), - ('https://example.com', 'https://example.com'), - # Test URL-encoded reserved characters. - ('https://user%3Aname:%23%40%5E@example.com', - 'https://user%3Aname:****@example.com'), -]) -def test_redact_auth_from_url(auth_url, expected_url): +@pytest.mark.parametrize( + "auth_url, expected_url", + [ + ("https://accesstoken@example.com/abc", "https://****@example.com/abc"), + ("https://user:password@example.com", "https://user:****@example.com"), + ("https://user:@example.com", "https://user:****@example.com"), + ("https://example.com", "https://example.com"), + # Test URL-encoded reserved characters. + ( + "https://user%3Aname:%23%40%5E@example.com", + "https://user%3Aname:****@example.com", + ), + ], +) +def test_redact_auth_from_url(auth_url: str, expected_url: str) -> None: url = redact_auth_from_url(auth_url) assert url == expected_url class TestHiddenText: - - def test_basic(self): + def test_basic(self) -> None: """ Test str(), repr(), and attribute access. """ - hidden = HiddenText('my-secret', redacted='######') + hidden = HiddenText("my-secret", redacted="######") assert repr(hidden) == "" - assert str(hidden) == '######' - assert hidden.redacted == '######' - assert hidden.secret == 'my-secret' + assert str(hidden) == "######" + assert hidden.redacted == "######" + assert hidden.secret == "my-secret" - def test_equality_with_str(self): + def test_equality_with_str(self) -> None: """ Test equality (and inequality) with str objects. """ - hidden = HiddenText('secret', redacted='****') + hidden = HiddenText("secret", redacted="****") # Test that the object doesn't compare equal to either its original # or redacted forms. @@ -838,50 +779,51 @@ def test_equality_with_str(self): assert hidden != hidden.redacted assert hidden.redacted != hidden - def test_equality_same_secret(self): + def test_equality_same_secret(self) -> None: """ Test equality with an object having the same secret. """ # Choose different redactions for the two objects. - hidden1 = HiddenText('secret', redacted='****') - hidden2 = HiddenText('secret', redacted='####') + hidden1 = HiddenText("secret", redacted="****") + hidden2 = HiddenText("secret", redacted="####") assert hidden1 == hidden2 # Also test __ne__. assert not hidden1 != hidden2 - def test_equality_different_secret(self): + def test_equality_different_secret(self) -> None: """ Test equality with an object having a different secret. """ - hidden1 = HiddenText('secret-1', redacted='****') - hidden2 = HiddenText('secret-2', redacted='****') + hidden1 = HiddenText("secret-1", redacted="****") + hidden2 = HiddenText("secret-2", redacted="****") assert hidden1 != hidden2 # Also test __eq__. assert not hidden1 == hidden2 -def test_hide_value(): - hidden = hide_value('my-secret') +def test_hide_value() -> None: + hidden = hide_value("my-secret") assert repr(hidden) == "" - assert str(hidden) == '****' - assert hidden.redacted == '****' - assert hidden.secret == 'my-secret' + assert str(hidden) == "****" + assert hidden.redacted == "****" + assert hidden.secret == "my-secret" -def test_hide_url(): - hidden_url = hide_url('https://user:password@example.com') +def test_hide_url() -> None: + hidden_url = hide_url("https://user:password@example.com") assert repr(hidden_url) == "" - assert str(hidden_url) == 'https://user:****@example.com' - assert hidden_url.redacted == 'https://user:****@example.com' - assert hidden_url.secret == 'https://user:password@example.com' + assert str(hidden_url) == "https://user:****@example.com" + assert hidden_url.redacted == "https://user:****@example.com" + assert hidden_url.secret == "https://user:password@example.com" @pytest.fixture() -def patch_deprecation_check_version(): +def patch_deprecation_check_version() -> Iterator[None]: # We do this, so that the deprecation tests are easier to write. import pip._internal.utils.deprecation as d + old_version = d.current_version d.current_version = "1.0" yield @@ -892,21 +834,29 @@ def patch_deprecation_check_version(): @pytest.mark.parametrize("replacement", [None, "a magic 8 ball"]) @pytest.mark.parametrize("gone_in", [None, "2.0"]) @pytest.mark.parametrize("issue", [None, 988]) -def test_deprecated_message_contains_information(gone_in, replacement, issue): +@pytest.mark.parametrize("feature_flag", [None, "magic-8-ball"]) +def test_deprecated_message_contains_information( + gone_in: Optional[str], + replacement: Optional[str], + issue: Optional[int], + feature_flag: Optional[str], +) -> None: with pytest.warns(PipDeprecationWarning) as record: deprecated( - "Stop doing this!", + reason="Stop doing this!", replacement=replacement, gone_in=gone_in, + feature_flag=feature_flag, issue=issue, ) assert len(record) == 1 + assert isinstance(record[0].message, PipDeprecationWarning) message = record[0].message.args[0] assert "DEPRECATION: Stop doing this!" in message # Ensure non-None values are mentioned. - for item in [gone_in, replacement, issue]: + for item in [gone_in, replacement, issue, feature_flag]: if item is not None: assert str(item) in message @@ -914,12 +864,16 @@ def test_deprecated_message_contains_information(gone_in, replacement, issue): @pytest.mark.usefixtures("patch_deprecation_check_version") @pytest.mark.parametrize("replacement", [None, "a magic 8 ball"]) @pytest.mark.parametrize("issue", [None, 988]) -def test_deprecated_raises_error_if_too_old(replacement, issue): +@pytest.mark.parametrize("feature_flag", [None, "magic-8-ball"]) +def test_deprecated_raises_error_if_too_old( + replacement: Optional[str], issue: Optional[int], feature_flag: Optional[str] +) -> None: with pytest.raises(PipDeprecationWarning) as exception: deprecated( - "Stop doing this!", + reason="Stop doing this!", gone_in="1.0", # this matches the patched version. replacement=replacement, + feature_flag=feature_flag, issue=issue, ) @@ -927,6 +881,7 @@ def test_deprecated_raises_error_if_too_old(replacement, issue): assert "DEPRECATION: Stop doing this!" in message assert "1.0" in message + assert str(feature_flag) not in message # Ensure non-None values are mentioned. for item in [replacement, issue]: if item is not None: @@ -934,50 +889,75 @@ def test_deprecated_raises_error_if_too_old(replacement, issue): @pytest.mark.usefixtures("patch_deprecation_check_version") -def test_deprecated_message_reads_well(): +def test_deprecated_message_reads_well_past() -> None: with pytest.raises(PipDeprecationWarning) as exception: deprecated( - "Stop doing this!", + reason="Stop doing this!", gone_in="1.0", # this matches the patched version. replacement="to be nicer", - issue="100000", # I hope we never reach this number. + feature_flag="magic-8-ball", + issue=100000, ) message = exception.value.args[0] assert message == ( "DEPRECATION: Stop doing this! " - "pip 1.0 will remove support for this functionality. " + "Since pip 1.0, this is no longer supported. " "A possible replacement is to be nicer. " - "You can find discussion regarding this at " - "https://github.com/pypa/pip/issues/100000." + "Discussion can be found at https://github.com/pypa/pip/issues/100000" ) -def test_make_setuptools_shim_args(): +@pytest.mark.usefixtures("patch_deprecation_check_version") +def test_deprecated_message_reads_well_future() -> None: + with pytest.warns(PipDeprecationWarning) as record: + deprecated( + reason="Stop doing this!", + gone_in="2.0", # this is greater than the patched version. + replacement="to be nicer", + feature_flag="crisis", + issue=100000, + ) + + assert len(record) == 1 + assert isinstance(record[0].message, PipDeprecationWarning) + message = record[0].message.args[0] + + assert message == ( + "DEPRECATION: Stop doing this! " + "pip 2.0 will enforce this behaviour change. " + "A possible replacement is to be nicer. " + "You can use the flag --use-feature=crisis to test the upcoming behaviour. " + "Discussion can be found at https://github.com/pypa/pip/issues/100000" + ) + + +def test_make_setuptools_shim_args() -> None: # Test all arguments at once, including the overall ordering. args = make_setuptools_shim_args( - '/dir/path/setup.py', - global_options=['--some', '--option'], + "/dir/path/setup.py", + global_options=["--some", "--option"], no_user_config=True, unbuffered_output=True, ) - assert args[1:3] == ['-u', '-c'] + assert args[1:3] == ["-u", "-c"] + assert args[4:] == ["--some", "--option", "--no-user-cfg"] + + shim = args[3] # Spot-check key aspects of the command string. - assert "sys.argv[0] = '/dir/path/setup.py'" in args[3] - assert "__file__='/dir/path/setup.py'" in args[3] - assert args[4:] == ['--some', '--option', '--no-user-cfg'] + assert "import setuptools" in shim + assert "'/dir/path/setup.py'" in args[3] + assert "sys.argv[0] = __file__" in args[3] -@pytest.mark.parametrize('global_options', [ - None, - [], - ['--some', '--option'] -]) -def test_make_setuptools_shim_args__global_options(global_options): +@pytest.mark.parametrize("global_options", [None, [], ["--some", "--option"]]) +def test_make_setuptools_shim_args__global_options( + global_options: Optional[List[str]], +) -> None: args = make_setuptools_shim_args( - '/dir/path/setup.py', + "/dir/path/setup.py", global_options=global_options, ) @@ -989,61 +969,79 @@ def test_make_setuptools_shim_args__global_options(global_options): assert len(args) == 3 -@pytest.mark.parametrize('no_user_config', [False, True]) -def test_make_setuptools_shim_args__no_user_config(no_user_config): +@pytest.mark.parametrize("no_user_config", [False, True]) +def test_make_setuptools_shim_args__no_user_config(no_user_config: bool) -> None: args = make_setuptools_shim_args( - '/dir/path/setup.py', + "/dir/path/setup.py", no_user_config=no_user_config, ) - assert ('--no-user-cfg' in args) == no_user_config + assert ("--no-user-cfg" in args) == no_user_config -@pytest.mark.parametrize('unbuffered_output', [False, True]) -def test_make_setuptools_shim_args__unbuffered_output(unbuffered_output): +@pytest.mark.parametrize("unbuffered_output", [False, True]) +def test_make_setuptools_shim_args__unbuffered_output(unbuffered_output: bool) -> None: args = make_setuptools_shim_args( - '/dir/path/setup.py', - unbuffered_output=unbuffered_output + "/dir/path/setup.py", unbuffered_output=unbuffered_output ) - assert ('-u' in args) == unbuffered_output + assert ("-u" in args) == unbuffered_output -@pytest.mark.parametrize('isatty,no_stdin,expected', [ - (True, False, True), - (False, False, False), - (True, True, False), - (False, True, False), -]) -def test_is_console_interactive(monkeypatch, isatty, no_stdin, expected): - monkeypatch.setattr(sys.stdin, 'isatty', Mock(return_value=isatty)) +@pytest.mark.parametrize( + "isatty,no_stdin,expected", + [ + (True, False, True), + (False, False, False), + (True, True, False), + (False, True, False), + ], +) +def test_is_console_interactive( + monkeypatch: pytest.MonkeyPatch, isatty: bool, no_stdin: bool, expected: bool +) -> None: + monkeypatch.setattr(sys.stdin, "isatty", Mock(return_value=isatty)) if no_stdin: - monkeypatch.setattr(sys, 'stdin', None) + monkeypatch.setattr(sys, "stdin", None) assert is_console_interactive() is expected -@pytest.mark.parametrize('size,expected', [ - (123, "123 bytes"), - (1234, "1.2 kB"), - (123456, "123 kB"), - (1234567890, "1234.6 MB"), -]) -def test_format_size(size, expected): +@pytest.mark.parametrize( + "size,expected", + [ + (123, "123 bytes"), + (1234, "1.2 kB"), + (123456, "123 kB"), + (1234567890, "1234.6 MB"), + ], +) +def test_format_size(size: int, expected: str) -> None: assert format_size(size) == expected @pytest.mark.parametrize( - ('rows', 'table', 'sizes'), - [([], [], []), - ([('I?', 'version', 'sdist', 'wheel'), - ('', '1.18.2', 'zip', 'cp38-cp38m-win_amd64'), - ('v', 1.18, 'zip')], - ['I? version sdist wheel', - ' 1.18.2 zip cp38-cp38m-win_amd64', - 'v 1.18 zip'], - [2, 7, 5, 20]), - ([('I?', 'version', 'sdist', 'wheel'), (), ('v', '1.18.1', 'zip')], - ['I? version sdist wheel', '', 'v 1.18.1 zip'], - [2, 7, 5, 5])]) -def test_tabulate(rows, table, sizes): + ("rows", "table", "sizes"), + [ + ([], [], []), + ( + [ + ("I?", "version", "sdist", "wheel"), + ("", "1.18.2", "zip", "cp38-cp38m-win_amd64"), + ("v", 1.18, "zip"), + ], + [ + "I? version sdist wheel", + " 1.18.2 zip cp38-cp38m-win_amd64", + "v 1.18 zip", + ], + [2, 7, 5, 20], + ), + ( + [("I?", "version", "sdist", "wheel"), (), ("v", "1.18.1", "zip")], + ["I? version sdist wheel", "", "v 1.18.1 zip"], + [2, 7, 5, 5], + ), + ], +) +def test_tabulate(rows: List[Tuple[str]], table: List[str], sizes: List[int]) -> None: assert tabulate(rows) == (table, sizes) diff --git a/tests/unit/test_utils_compatibility_tags.py b/tests/unit/test_utils_compatibility_tags.py index 735f024c122..f09c451b8ee 100644 --- a/tests/unit/test_utils_compatibility_tags.py +++ b/tests/unit/test_utils_compatibility_tags.py @@ -1,101 +1,108 @@ import sysconfig +from typing import Any, Callable, Dict, List, Tuple +from unittest.mock import patch import pytest -from mock import patch from pip._internal.utils import compatibility_tags -@pytest.mark.parametrize('version_info, expected', [ - ((2,), '2'), - ((2, 8), '28'), - ((3,), '3'), - ((3, 6), '36'), - # Test a tuple of length 3. - ((3, 6, 5), '36'), - # Test a 2-digit minor version. - ((3, 10), '310'), -]) -def test_version_info_to_nodot(version_info, expected): +@pytest.mark.parametrize( + "version_info, expected", + [ + ((2,), "2"), + ((2, 8), "28"), + ((3,), "3"), + ((3, 6), "36"), + # Test a tuple of length 3. + ((3, 6, 5), "36"), + # Test a 2-digit minor version. + ((3, 10), "310"), + ], +) +def test_version_info_to_nodot(version_info: Tuple[int], expected: str) -> None: actual = compatibility_tags.version_info_to_nodot(version_info) assert actual == expected class Testcompatibility_tags: - - def mock_get_config_var(self, **kwd): + def mock_get_config_var(self, **kwd: str) -> Callable[[str], Any]: """ Patch sysconfig.get_config_var for arbitrary keys. """ get_config_var = sysconfig.get_config_var - def _mock_get_config_var(var): + def _mock_get_config_var(var: str) -> Any: if var in kwd: return kwd[var] return get_config_var(var) + return _mock_get_config_var - def test_no_hyphen_tag(self): + def test_no_hyphen_tag(self) -> None: """ Test that no tag contains a hyphen. """ import pip._internal.utils.compatibility_tags - mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + mock_gcf = self.mock_get_config_var(SOABI="cpython-35m-darwin") - with patch('sysconfig.get_config_var', mock_gcf): + with patch("sysconfig.get_config_var", mock_gcf): supported = pip._internal.utils.compatibility_tags.get_supported() for tag in supported: - assert '-' not in tag.interpreter - assert '-' not in tag.abi - assert '-' not in tag.platform + assert "-" not in tag.interpreter + assert "-" not in tag.abi + assert "-" not in tag.platform class TestManylinux2010Tags: - - @pytest.mark.parametrize("manylinux2010,manylinux1", [ - ("manylinux2010_x86_64", "manylinux1_x86_64"), - ("manylinux2010_i686", "manylinux1_i686"), - ]) - def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): + @pytest.mark.parametrize( + "manylinux2010,manylinux1", + [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), + ], + ) + def test_manylinux2010_implies_manylinux1( + self, manylinux2010: str, manylinux1: str + ) -> None: """ Specifying manylinux2010 implies manylinux1. """ - groups = {} + groups: Dict[Tuple[str, str], List[str]] = {} supported = compatibility_tags.get_supported(platforms=[manylinux2010]) for tag in supported: - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) for arches in groups.values(): - if arches == ['any']: + if arches == ["any"]: continue assert arches[:2] == [manylinux2010, manylinux1] class TestManylinux2014Tags: - - @pytest.mark.parametrize("manylinuxA,manylinuxB", [ - ("manylinux2014_x86_64", ["manylinux2010_x86_64", - "manylinux1_x86_64"]), - ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), - ]) - def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): + @pytest.mark.parametrize( + "manylinuxA,manylinuxB", + [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ], + ) + def test_manylinuxA_implies_manylinuxB( + self, manylinuxA: str, manylinuxB: List[str] + ) -> None: """ Specifying manylinux2014 implies manylinux2010/manylinux1. """ - groups = {} + groups: Dict[Tuple[str, str], List[str]] = {} supported = compatibility_tags.get_supported(platforms=[manylinuxA]) for tag in supported: - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) expected_arches = [manylinuxA] expected_arches.extend(manylinuxB) for arches in groups.values(): - if arches == ['any']: + if arches == ["any"]: continue assert arches[:3] == expected_arches diff --git a/tests/unit/test_utils_distutils_args.py b/tests/unit/test_utils_distutils_args.py index 96cdb180424..e63c565a12f 100644 --- a/tests/unit/test_utils_distutils_args.py +++ b/tests/unit/test_utils_distutils_args.py @@ -3,30 +3,30 @@ from pip._internal.utils.distutils_args import parse_distutils_args -def test_unknown_option_is_ok(): +def test_unknown_option_is_ok() -> None: result = parse_distutils_args(["--foo"]) assert not result -def test_option_is_returned(): +def test_option_is_returned() -> None: result = parse_distutils_args(["--prefix=hello"]) assert result["prefix"] == "hello" -def test_options_are_clobbered(): +def test_options_are_clobbered() -> None: # Matches the current setuptools behavior that the last argument # wins. result = parse_distutils_args(["--prefix=hello", "--prefix=world"]) assert result["prefix"] == "world" -def test_multiple_options_work(): +def test_multiple_options_work() -> None: result = parse_distutils_args(["--prefix=hello", "--root=world"]) assert result["prefix"] == "hello" assert result["root"] == "world" -def test_multiple_invocations_do_not_keep_options(): +def test_multiple_invocations_do_not_keep_options() -> None: result = parse_distutils_args(["--prefix=hello1"]) assert len(result) == 1 assert result["prefix"] == "hello1" @@ -36,25 +36,28 @@ def test_multiple_invocations_do_not_keep_options(): assert result["root"] == "world1" -@pytest.mark.parametrize("name,value", [ - ("exec-prefix", "1"), - ("home", "2"), - ("install-base", "3"), - ("install-data", "4"), - ("install-headers", "5"), - ("install-lib", "6"), - ("install-platlib", "7"), - ("install-purelib", "8"), - ("install-scripts", "9"), - ("prefix", "10"), - ("root", "11"), -]) -def test_all_value_options_work(name, value): +@pytest.mark.parametrize( + "name,value", + [ + ("exec-prefix", "1"), + ("home", "2"), + ("install-base", "3"), + ("install-data", "4"), + ("install-headers", "5"), + ("install-lib", "6"), + ("install-platlib", "7"), + ("install-purelib", "8"), + ("install-scripts", "9"), + ("prefix", "10"), + ("root", "11"), + ], +) +def test_all_value_options_work(name: str, value: str) -> None: result = parse_distutils_args([f"--{name}={value}"]) key_name = name.replace("-", "_") assert result[key_name] == value -def test_user_option_works(): +def test_user_option_works() -> None: result = parse_distutils_args(["--user"]) assert result["user"] == 1 diff --git a/tests/unit/test_utils_filesystem.py b/tests/unit/test_utils_filesystem.py index 3ef814dce4b..b15c3141ad0 100644 --- a/tests/unit/test_utils_filesystem.py +++ b/tests/unit/test_utils_filesystem.py @@ -1,5 +1,6 @@ import os import shutil +from typing import Callable, Type import pytest @@ -8,21 +9,21 @@ from tests.lib.path import Path -def make_file(path): +def make_file(path: str) -> None: Path(path).touch() -def make_valid_symlink(path): +def make_valid_symlink(path: str) -> None: target = path + "1" make_file(target) os.symlink(target, path) -def make_broken_symlink(path): +def make_broken_symlink(path: str) -> None: os.symlink("foo", path) -def make_dir(path): +def make_dir(path: str) -> None: os.mkdir(path) @@ -30,27 +31,33 @@ def make_dir(path): @skip_on_windows -@pytest.mark.parametrize("create,result", [ - (make_socket_file, True), - (make_file, False), - (make_valid_symlink, False), - (make_broken_symlink, False), - (make_dir, False), -]) -def test_is_socket(create, result, tmpdir): +@pytest.mark.parametrize( + "create,result", + [ + (make_socket_file, True), + (make_file, False), + (make_valid_symlink, False), + (make_broken_symlink, False), + (make_dir, False), + ], +) +def test_is_socket(create: Callable[[str], None], result: bool, tmpdir: Path) -> None: target = tmpdir.joinpath("target") create(target) assert os.path.lexists(target) assert is_socket(target) == result -@pytest.mark.parametrize("create,error_type", [ - pytest.param( - make_socket_file, shutil.SpecialFileError, marks=skip_on_windows - ), - (make_unreadable_file, OSError), -]) -def test_copy2_fixed_raises_appropriate_errors(create, error_type, tmpdir): +@pytest.mark.parametrize( + "create,error_type", + [ + pytest.param(make_socket_file, shutil.SpecialFileError, marks=skip_on_windows), + (make_unreadable_file, OSError), + ], +) +def test_copy2_fixed_raises_appropriate_errors( + create: Callable[[str], None], error_type: Type[Exception], tmpdir: Path +) -> None: src = tmpdir.joinpath("src") create(src) dest = tmpdir.joinpath("dest") diff --git a/tests/unit/test_utils_parallel.py b/tests/unit/test_utils_parallel.py deleted file mode 100644 index 5a23f7d655f..00000000000 --- a/tests/unit/test_utils_parallel.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Test multiprocessing/multithreading higher-order functions.""" - -from contextlib import contextmanager -from importlib import import_module -from math import factorial -from sys import modules - -from pytest import mark - -DUNDER_IMPORT = 'builtins.__import__' -FUNC, ITERABLE = factorial, range(42) -MAPS = 'map_multiprocess', 'map_multithread' -_import = __import__ - - -def unload_parallel(): - try: - del modules['pip._internal.utils.parallel'] - except KeyError: - pass - - -@contextmanager -def tmp_import_parallel(): - unload_parallel() - try: - yield import_module('pip._internal.utils.parallel') - finally: - unload_parallel() - - -def lack_sem_open(name, *args, **kwargs): - """Raise ImportError on import of multiprocessing.synchronize.""" - if name.endswith('synchronize'): - raise ImportError - return _import(name, *args, **kwargs) - - -def have_sem_open(name, *args, **kwargs): - """Make sure multiprocessing.synchronize import is successful.""" - # We don't care about the return value - # since we don't use the pool with this import. - if name.endswith('synchronize'): - return - return _import(name, *args, **kwargs) - - -@mark.parametrize('name', MAPS) -def test_lack_sem_open(name, monkeypatch): - """Test fallback when sem_open is not available. - - If so, multiprocessing[.dummy].Pool will fail to be created and - map_async should fallback to map. - """ - monkeypatch.setattr(DUNDER_IMPORT, lack_sem_open) - with tmp_import_parallel() as parallel: - assert getattr(parallel, name) is parallel._map_fallback - - -@mark.parametrize('name', MAPS) -def test_have_sem_open(name, monkeypatch): - """Test fallback when sem_open is available.""" - monkeypatch.setattr(DUNDER_IMPORT, have_sem_open) - with tmp_import_parallel() as parallel: - assert getattr(parallel, name) is getattr(parallel, f'_{name}') - - -@mark.parametrize('name', MAPS) -def test_map(name): - """Test correctness of result of asynchronous maps.""" - map_async = getattr(import_module('pip._internal.utils.parallel'), name) - assert set(map_async(FUNC, ITERABLE)) == set(map(FUNC, ITERABLE)) diff --git a/tests/unit/test_utils_pkg_resources.py b/tests/unit/test_utils_pkg_resources.py deleted file mode 100644 index ae7357ba1cc..00000000000 --- a/tests/unit/test_utils_pkg_resources.py +++ /dev/null @@ -1,54 +0,0 @@ -from email.message import Message - -import pytest -from pip._vendor.pkg_resources import DistInfoDistribution, Requirement -from pip._vendor.six import ensure_binary - -from pip._internal.utils.packaging import get_metadata, get_requires_python -from pip._internal.utils.pkg_resources import DictMetadata - - -def test_dict_metadata_works(): - name = "simple" - version = "0.1.0" - require_a = "a==1.0" - require_b = "b==1.1; extra == 'also_b'" - requires = [require_a, require_b, "c==1.2; extra == 'also_c'"] - extras = ["also_b", "also_c"] - requires_python = ">=3" - - metadata = Message() - metadata["Name"] = name - metadata["Version"] = version - for require in requires: - metadata["Requires-Dist"] = require - for extra in extras: - metadata["Provides-Extra"] = extra - metadata["Requires-Python"] = requires_python - - inner_metadata = DictMetadata({ - "METADATA": ensure_binary(metadata.as_string()) - }) - dist = DistInfoDistribution( - location="", metadata=inner_metadata, project_name=name - ) - - assert name == dist.project_name - assert version == dist.version - assert set(extras) == set(dist.extras) - assert [Requirement.parse(require_a)] == dist.requires([]) - assert [ - Requirement.parse(require_a), Requirement.parse(require_b) - ] == dist.requires(["also_b"]) - assert metadata.as_string() == get_metadata(dist).as_string() - assert requires_python == get_requires_python(dist) - - -def test_dict_metadata_throws_on_bad_unicode(): - metadata = DictMetadata({ - "METADATA": b"\xff" - }) - - with pytest.raises(UnicodeDecodeError) as e: - metadata.get_metadata("METADATA") - assert "METADATA" in str(e.value) diff --git a/tests/unit/test_utils_subprocess.py b/tests/unit/test_utils_subprocess.py index ecae2295c88..ea3a7b26175 100644 --- a/tests/unit/test_utils_subprocess.py +++ b/tests/unit/test_utils_subprocess.py @@ -1,198 +1,90 @@ import locale import sys from logging import DEBUG, ERROR, INFO, WARNING -from textwrap import dedent +from typing import List, Optional, Tuple, Type import pytest from pip._internal.cli.spinners import SpinnerInterface from pip._internal.exceptions import InstallationSubprocessError +from pip._internal.utils.logging import VERBOSE from pip._internal.utils.misc import hide_value from pip._internal.utils.subprocess import ( + CommandArgs, call_subprocess, format_command_args, make_command, - make_subprocess_output_error, subprocess_logger, ) -@pytest.mark.parametrize('args, expected', [ - (['pip', 'list'], 'pip list'), - (['foo', 'space space', 'new\nline', 'double"quote', "single'quote"], - """foo 'space space' 'new\nline' 'double"quote' 'single'"'"'quote'"""), - # Test HiddenText arguments. - (make_command(hide_value('secret1'), 'foo', hide_value('secret2')), - "'****' foo '****'"), -]) -def test_format_command_args(args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + (["pip", "list"], "pip list"), + ( + ["foo", "space space", "new\nline", 'double"quote', "single'quote"], + """foo 'space space' 'new\nline' 'double"quote' 'single'"'"'quote'""", + ), + # Test HiddenText arguments. + ( + make_command(hide_value("secret1"), "foo", hide_value("secret2")), + "'****' foo '****'", + ), + ], +) +def test_format_command_args(args: CommandArgs, expected: str) -> None: actual = format_command_args(args) assert actual == expected -def test_make_subprocess_output_error(): - cmd_args = ['test', 'has space'] - cwd = '/path/to/cwd' - lines = ['line1\n', 'line2\n', 'line3\n'] - actual = make_subprocess_output_error( - cmd_args=cmd_args, - cwd=cwd, - lines=lines, - exit_status=3, - ) - expected = dedent("""\ - Command errored out with exit status 3: - command: test 'has space' - cwd: /path/to/cwd - Complete output (3 lines): - line1 - line2 - line3 - ----------------------------------------""") - assert actual == expected, f'actual: {actual}' - - -def test_make_subprocess_output_error__non_ascii_command_arg(monkeypatch): - """ - Test a command argument with a non-ascii character. - """ - cmd_args = ['foo', 'déf'] - if sys.version_info[0] == 2: - # Check in Python 2 that the str (bytes object) with the non-ascii - # character has the encoding we expect. (This comes from the source - # code encoding at the top of the file.) - assert cmd_args[1].decode('utf-8') == 'déf' - - # We need to monkeypatch so the encoding will be correct on Windows. - monkeypatch.setattr(locale, 'getpreferredencoding', lambda: 'utf-8') - actual = make_subprocess_output_error( - cmd_args=cmd_args, - cwd='/path/to/cwd', - lines=[], - exit_status=1, - ) - expected = dedent("""\ - Command errored out with exit status 1: - command: foo 'déf' - cwd: /path/to/cwd - Complete output (0 lines): - ----------------------------------------""") - assert actual == expected, f'actual: {actual}' - - -@pytest.mark.skipif("sys.version_info < (3,)") -def test_make_subprocess_output_error__non_ascii_cwd_python_3(monkeypatch): - """ - Test a str (text) cwd with a non-ascii character in Python 3. - """ - cmd_args = ['test'] - cwd = '/path/to/cwd/déf' - actual = make_subprocess_output_error( - cmd_args=cmd_args, - cwd=cwd, - lines=[], - exit_status=1, - ) - expected = dedent("""\ - Command errored out with exit status 1: - command: test - cwd: /path/to/cwd/déf - Complete output (0 lines): - ----------------------------------------""") - assert actual == expected, f'actual: {actual}' - - -@pytest.mark.parametrize('encoding', [ - 'utf-8', - # Test a Windows encoding. - 'cp1252', -]) -@pytest.mark.skipif("sys.version_info >= (3,)") -def test_make_subprocess_output_error__non_ascii_cwd_python_2( - monkeypatch, encoding, -): - """ - Test a str (bytes object) cwd with a non-ascii character in Python 2. - """ - cmd_args = ['test'] - cwd = '/path/to/cwd/déf'.encode(encoding) - monkeypatch.setattr(sys, 'getfilesystemencoding', lambda: encoding) - actual = make_subprocess_output_error( - cmd_args=cmd_args, - cwd=cwd, - lines=[], - exit_status=1, - ) - expected = dedent("""\ - Command errored out with exit status 1: - command: test - cwd: /path/to/cwd/déf - Complete output (0 lines): - ----------------------------------------""") - assert actual == expected, f'actual: {actual}' - - -# This test is mainly important for checking unicode in Python 2. -def test_make_subprocess_output_error__non_ascii_line(): - """ - Test a line with a non-ascii character. - """ - lines = ['curly-quote: \u2018\n'] - actual = make_subprocess_output_error( - cmd_args=['test'], - cwd='/path/to/cwd', - lines=lines, - exit_status=1, - ) - expected = dedent("""\ - Command errored out with exit status 1: - command: test - cwd: /path/to/cwd - Complete output (1 lines): - curly-quote: \u2018 - ----------------------------------------""") - assert actual == expected, f'actual: {actual}' - - @pytest.mark.parametrize( - ('stdout_only', 'expected'), + ("stdout_only", "expected"), [ (True, ("out\n", "out\r\n")), (False, ("out\nerr\n", "out\r\nerr\r\n", "err\nout\n", "err\r\nout\r\n")), ], ) -def test_call_subprocess_stdout_only(capfd, monkeypatch, stdout_only, expected): +def test_call_subprocess_stdout_only( + capfd: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + stdout_only: bool, + expected: Tuple[str, ...], +) -> None: log = [] - monkeypatch.setattr(subprocess_logger, "debug", lambda *args: log.append(args[0])) + monkeypatch.setattr( + subprocess_logger, + "log", + lambda level, *args: log.append(args[0]), + ) out = call_subprocess( [ sys.executable, "-c", - "import sys; " - "sys.stdout.write('out\\n'); " - "sys.stderr.write('err\\n')" + "import sys; sys.stdout.write('out\\n'); sys.stderr.write('err\\n')", ], + command_desc="test stdout_only", stdout_only=stdout_only, ) assert out in expected captured = capfd.readouterr() assert captured.err == "" - assert ( - log == ["Running command %s", "out", "err"] - or log == ["Running command %s", "err", "out"] - ) + assert log == ["Running command %s", "out", "err"] or log == [ + "Running command %s", + "err", + "out", + ] class FakeSpinner(SpinnerInterface): - - def __init__(self): + def __init__(self) -> None: self.spin_count = 0 - self.final_status = None + self.final_status: Optional[str] = None - def spin(self): + def spin(self) -> None: self.spin_count += 1 - def finish(self, final_status): + def finish(self, final_status: str) -> None: self.final_status = final_status @@ -203,9 +95,15 @@ class TestCallSubprocess: """ def check_result( - self, capfd, caplog, log_level, spinner, result, expected, - expected_spinner, - ): + self, + capfd: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, + log_level: int, + spinner: FakeSpinner, + result: Optional[str], + expected: Tuple[Optional[List[str]], List[Tuple[str, int, str]]], + expected_spinner: Tuple[int, Optional[str]], + ) -> None: """ Check the result of calling call_subprocess(). @@ -227,15 +125,16 @@ def check_result( if expected_proc is None: assert result is None else: + assert result is not None assert result.splitlines() == expected_proc # Confirm that stdout and stderr haven't been written to. captured = capfd.readouterr() - assert (captured.out, captured.err) == ('', '') + assert (captured.out, captured.err) == ("", "") records = caplog.record_tuples if len(records) != len(expected_records): - raise RuntimeError(f'{records} != {expected_records}') + raise RuntimeError(f"{records} != {expected_records}") for record, expected_record in zip(records, expected_records): # Check the logger_name and log level parts exactly. @@ -250,53 +149,88 @@ def check_result( assert (spinner.spin_count, spinner.final_status) == expected_spinner - def prepare_call(self, caplog, log_level, command=None): + def prepare_call( + self, + caplog: pytest.LogCaptureFixture, + log_level: int, + command: Optional[str] = None, + ) -> Tuple[List[str], FakeSpinner]: if command is None: command = 'print("Hello"); print("world")' caplog.set_level(log_level) spinner = FakeSpinner() - args = [sys.executable, '-c', command] + args = [sys.executable, "-c", command] return (args, spinner) - def test_debug_logging(self, capfd, caplog): + def test_debug_logging( + self, capfd: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture + ) -> None: """ Test DEBUG logging (and without passing show_stdout=True). """ log_level = DEBUG args, spinner = self.prepare_call(caplog, log_level) - result = call_subprocess(args, spinner=spinner) + result = call_subprocess( + args, + command_desc="test debug logging", + spinner=spinner, + ) - expected = (['Hello', 'world'], [ - ('pip.subprocessor', DEBUG, 'Running command '), - ('pip.subprocessor', DEBUG, 'Hello'), - ('pip.subprocessor', DEBUG, 'world'), - ]) + expected = ( + ["Hello", "world"], + [ + ("pip.subprocessor", VERBOSE, "Running "), + ("pip.subprocessor", VERBOSE, "Hello"), + ("pip.subprocessor", VERBOSE, "world"), + ], + ) # The spinner shouldn't spin in this case since the subprocess # output is already being logged to the console. self.check_result( - capfd, caplog, log_level, spinner, result, expected, + capfd, + caplog, + log_level, + spinner, + result, + expected, expected_spinner=(0, None), ) - def test_info_logging(self, capfd, caplog): + def test_info_logging( + self, capfd: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture + ) -> None: """ Test INFO logging (and without passing show_stdout=True). """ log_level = INFO args, spinner = self.prepare_call(caplog, log_level) - result = call_subprocess(args, spinner=spinner) + result = call_subprocess( + args, + command_desc="test info logging", + spinner=spinner, + ) - expected = (['Hello', 'world'], []) + expected: Tuple[List[str], List[Tuple[str, int, str]]] = ( + ["Hello", "world"], + [], + ) # The spinner should spin twice in this case since the subprocess # output isn't being written to the console. self.check_result( - capfd, caplog, log_level, spinner, result, expected, - expected_spinner=(2, 'done'), + capfd, + caplog, + log_level, + spinner, + result, + expected, + expected_spinner=(2, "done"), ) - def test_info_logging__subprocess_error(self, capfd, caplog): + def test_info_logging__subprocess_error( + self, capfd: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture + ) -> None: """ Test INFO logging of a subprocess with an error (and without passing show_stdout=True). @@ -306,79 +240,85 @@ def test_info_logging__subprocess_error(self, capfd, caplog): args, spinner = self.prepare_call(caplog, log_level, command=command) with pytest.raises(InstallationSubprocessError) as exc: - call_subprocess(args, spinner=spinner) + call_subprocess( + args, + command_desc="test info logging with subprocess error", + spinner=spinner, + ) result = None - exc_message = str(exc.value) - assert exc_message.startswith( - 'Command errored out with exit status 1: ' + exception = exc.value + assert exception.reference == "subprocess-exited-with-error" + assert "exit code: 1" in exception.message + assert exception.note_stmt + assert "not a problem with pip" in exception.note_stmt + # Check that the process output is captured, and would be shown. + assert exception.context + assert "Hello\n" in exception.context + assert "fail\n" in exception.context + assert "world\n" in exception.context + + expected = ( + None, + [ + # pytest's caplog overrides th formatter, which means that we + # won't see the message formatted through our formatters. + ("pip.subprocessor", ERROR, "[present-diagnostic]"), + ], ) - assert exc_message.endswith('Check the logs for full command output.') - - expected = (None, [ - ('pip.subprocessor', ERROR, 'Complete output (3 lines):\n'), - ]) # The spinner should spin three times in this case since the # subprocess output isn't being written to the console. self.check_result( - capfd, caplog, log_level, spinner, result, expected, - expected_spinner=(3, 'error'), + capfd, + caplog, + log_level, + spinner, + result, + expected, + expected_spinner=(3, "error"), ) - # Do some further checking on the captured log records to confirm - # that the subprocess output was logged. - last_record = caplog.record_tuples[-1] - last_message = last_record[2] - lines = last_message.splitlines() - - # We have to sort before comparing the lines because we can't - # guarantee the order in which stdout and stderr will appear. - # For example, we observed the stderr lines coming before stdout - # in CI for PyPy 2.7 even though stdout happens first chronologically. - actual = sorted(lines) - # Test the "command" line separately because we can't test an - # exact match. - command_line = actual.pop(1) - assert actual == [ - ' cwd: None', - '----------------------------------------', - 'Command errored out with exit status 1:', - 'Complete output (3 lines):', - 'Hello', - 'fail', - 'world', - ], f'lines: {actual}' # Show the full output on failure. - - assert command_line.startswith(' command: ') - assert command_line.endswith('print("world"); exit("fail")\'') - - def test_info_logging_with_show_stdout_true(self, capfd, caplog): + def test_info_logging_with_show_stdout_true( + self, capfd: pytest.CaptureFixture[str], caplog: pytest.LogCaptureFixture + ) -> None: """ Test INFO logging with show_stdout=True. """ log_level = INFO args, spinner = self.prepare_call(caplog, log_level) - result = call_subprocess(args, spinner=spinner, show_stdout=True) + result = call_subprocess( + args, + command_desc="test info logging with show_stdout", + spinner=spinner, + show_stdout=True, + ) - expected = (['Hello', 'world'], [ - ('pip.subprocessor', INFO, 'Running command '), - ('pip.subprocessor', INFO, 'Hello'), - ('pip.subprocessor', INFO, 'world'), - ]) + expected = ( + ["Hello", "world"], + [ + ("pip.subprocessor", INFO, "Running "), + ("pip.subprocessor", INFO, "Hello"), + ("pip.subprocessor", INFO, "world"), + ], + ) # The spinner shouldn't spin in this case since the subprocess # output is already being written to the console. self.check_result( - capfd, caplog, log_level, spinner, result, expected, + capfd, + caplog, + log_level, + spinner, + result, + expected, expected_spinner=(0, None), ) - @pytest.mark.parametrize(( - 'exit_status', 'show_stdout', 'extra_ok_returncodes', 'log_level', - 'expected'), + @pytest.mark.parametrize( + ("exit_status", "show_stdout", "extra_ok_returncodes", "log_level", "expected"), [ # The spinner should show here because show_stdout=False means # the subprocess should get logged at DEBUG level, but the passed # log level is only INFO. - (0, False, None, INFO, (None, 'done', 2)), + (0, False, None, INFO, (None, "done", 2)), # Test some cases where the spinner should not be shown. (0, False, None, DEBUG, (None, None, 0)), # Test show_stdout=True. @@ -387,16 +327,22 @@ def test_info_logging_with_show_stdout_true(self, capfd, caplog): # The spinner should show here because show_stdout=True means # the subprocess should get logged at INFO level, but the passed # log level is only WARNING. - (0, True, None, WARNING, (None, 'done', 2)), + (0, True, None, WARNING, (None, "done", 2)), # Test a non-zero exit status. - (3, False, None, INFO, (InstallationSubprocessError, 'error', 2)), + (3, False, None, INFO, (InstallationSubprocessError, "error", 2)), # Test a non-zero exit status also in extra_ok_returncodes. - (3, False, (3, ), INFO, (None, 'done', 2)), - ]) + (3, False, (3,), INFO, (None, "done", 2)), + ], + ) def test_spinner_finish( - self, exit_status, show_stdout, extra_ok_returncodes, log_level, - caplog, expected, - ): + self, + exit_status: int, + show_stdout: bool, + extra_ok_returncodes: Optional[Tuple[int, ...]], + log_level: int, + caplog: pytest.LogCaptureFixture, + expected: Tuple[Optional[Type[Exception]], Optional[str], int], + ) -> None: """ Test that the spinner finishes correctly. """ @@ -404,13 +350,13 @@ def test_spinner_finish( expected_final_status = expected[1] expected_spin_count = expected[2] - command = ( - f'print("Hello"); print("world"); exit({exit_status})' - ) + command = f'print("Hello"); print("world"); exit({exit_status})' args, spinner = self.prepare_call(caplog, log_level, command=command) + exc_type: Optional[Type[Exception]] try: call_subprocess( args, + command_desc="spinner go spinny", show_stdout=show_stdout, extra_ok_returncodes=extra_ok_returncodes, spinner=spinner, @@ -424,9 +370,29 @@ def test_spinner_finish( assert spinner.final_status == expected_final_status assert spinner.spin_count == expected_spin_count - def test_closes_stdin(self): + def test_closes_stdin(self) -> None: with pytest.raises(InstallationSubprocessError): call_subprocess( - [sys.executable, '-c', 'input()'], + [sys.executable, "-c", "input()"], show_stdout=True, + command_desc="stdin reader", ) + + +def test_unicode_decode_error(caplog: pytest.LogCaptureFixture) -> None: + if locale.getpreferredencoding() != "UTF-8": + pytest.skip("locale.getpreferredencoding() is not UTF-8") + caplog.set_level(INFO) + call_subprocess( + [ + sys.executable, + "-c", + "import sys; sys.stdout.buffer.write(b'\\xff')", + ], + command_desc="invalid decode output", + show_stdout=True, + ) + + assert len(caplog.records) == 2 + # First log record is "Running ..." + assert caplog.record_tuples[1] == ("pip.subprocessor", INFO, "\\xff") diff --git a/tests/unit/test_utils_temp_dir.py b/tests/unit/test_utils_temp_dir.py index 0d1b0a5ea20..6b3571ff71c 100644 --- a/tests/unit/test_utils_temp_dir.py +++ b/tests/unit/test_utils_temp_dir.py @@ -2,6 +2,7 @@ import os import stat import tempfile +from typing import Any, Iterator, Optional, Union import pytest @@ -10,46 +11,41 @@ from pip._internal.utils.temp_dir import ( AdjacentTempDirectory, TempDirectory, + _Default, _default, global_tempdir_manager, tempdir_registry, ) +from tests.lib.path import Path # No need to test symlinked directories on Windows @pytest.mark.skipif("sys.platform == 'win32'") -def test_symlinked_path(): +def test_symlinked_path() -> None: with TempDirectory() as tmp_dir: assert os.path.exists(tmp_dir.path) alt_tmp_dir = tempfile.mkdtemp(prefix="pip-test-") - assert ( - os.path.dirname(tmp_dir.path) == - os.path.dirname(os.path.realpath(alt_tmp_dir)) + assert os.path.dirname(tmp_dir.path) == os.path.dirname( + os.path.realpath(alt_tmp_dir) ) # are we on a system where /tmp is a symlink if os.path.realpath(alt_tmp_dir) != os.path.abspath(alt_tmp_dir): - assert ( - os.path.dirname(tmp_dir.path) != - os.path.dirname(alt_tmp_dir) - ) + assert os.path.dirname(tmp_dir.path) != os.path.dirname(alt_tmp_dir) else: - assert ( - os.path.dirname(tmp_dir.path) == - os.path.dirname(alt_tmp_dir) - ) + assert os.path.dirname(tmp_dir.path) == os.path.dirname(alt_tmp_dir) os.rmdir(tmp_dir.path) assert not os.path.exists(tmp_dir.path) -def test_deletes_readonly_files(): - def create_file(*args): +def test_deletes_readonly_files() -> None: + def create_file(*args: str) -> None: fpath = os.path.join(*args) ensure_dir(os.path.dirname(fpath)) with open(fpath, "w") as f: f.write("Holla!") - def readonly_file(*args): + def readonly_file(*args: str) -> None: fpath = os.path.join(*args) os.chmod(fpath, stat.S_IREAD) @@ -63,7 +59,7 @@ def readonly_file(*args): readonly_file(tmp_dir.path, "subfolder", "readonly-file") -def test_path_access_after_context_raises(): +def test_path_access_after_context_raises() -> None: with TempDirectory() as tmp_dir: path = tmp_dir.path @@ -73,7 +69,7 @@ def test_path_access_after_context_raises(): assert path in str(e.value) -def test_path_access_after_clean_raises(): +def test_path_access_after_clean_raises() -> None: tmp_dir = TempDirectory() path = tmp_dir.path tmp_dir.cleanup() @@ -84,7 +80,7 @@ def test_path_access_after_clean_raises(): assert path in str(e.value) -def test_create_and_cleanup_work(): +def test_create_and_cleanup_work() -> None: tmp_dir = TempDirectory() created_path = tmp_dir.path @@ -95,18 +91,21 @@ def test_create_and_cleanup_work(): assert not os.path.exists(created_path) -@pytest.mark.parametrize("name", [ - "ABC", - "ABC.dist-info", - "_+-", - "_package", - "A......B", - "AB", - "A", - "2", -]) -def test_adjacent_directory_names(name): - def names(): +@pytest.mark.parametrize( + "name", + [ + "ABC", + "ABC.dist-info", + "_+-", + "_package", + "A......B", + "AB", + "A", + "2", + ], +) +def test_adjacent_directory_names(name: str) -> None: + def names() -> Iterator[str]: return AdjacentTempDirectory._generate_names(name) chars = AdjacentTempDirectory.LEADING_CHARS @@ -132,15 +131,12 @@ def names(): assert len(some_names) > 0.9 * len(set(some_names)) # Ensure the first few names are the same length as the original - same_len = list(itertools.takewhile( - lambda x: len(x) == len(name), - some_names - )) + same_len = list(itertools.takewhile(lambda x: len(x) == len(name), some_names)) assert len(same_len) > 10 # Check the first group are correct - expected_names = ['~' + name[1:]] - expected_names.extend('~' + c + name[2:] for c in chars) + expected_names = ["~" + name[1:]] + expected_names.extend("~" + c + name[2:] for c in chars) for x, y in zip(some_names, expected_names): assert x == y @@ -159,16 +155,20 @@ def names(): assert all(x.endswith(name) for x in some_names) -@pytest.mark.parametrize("name", [ - "A", - "ABC", - "ABC.dist-info", - "_+-", - "_package", -]) -def test_adjacent_directory_exists(name, tmpdir): +@pytest.mark.parametrize( + "name", + [ + "A", + "ABC", + "ABC.dist-info", + "_+-", + "_package", + ], +) +def test_adjacent_directory_exists(name: str, tmpdir: Path) -> None: block_name, expect_name = itertools.islice( - AdjacentTempDirectory._generate_names(name), 2) + AdjacentTempDirectory._generate_names(name), 2 + ) original = os.path.join(tmpdir, name) blocker = os.path.join(tmpdir, block_name) @@ -180,10 +180,10 @@ def test_adjacent_directory_exists(name, tmpdir): assert expect_name == os.path.split(atmp_dir.path)[1] -def test_adjacent_directory_permission_error(monkeypatch): +def test_adjacent_directory_permission_error(monkeypatch: pytest.MonkeyPatch) -> None: name = "ABC" - def raising_mkdir(*args, **kwargs): + def raising_mkdir(*args: Any, **kwargs: Any) -> None: raise OSError("Unknown OSError") with TempDirectory() as tmp_dir: @@ -197,7 +197,7 @@ def raising_mkdir(*args, **kwargs): pass -def test_global_tempdir_manager(): +def test_global_tempdir_manager() -> None: with global_tempdir_manager(): d = TempDirectory(globally_managed=True) path = d.path @@ -205,7 +205,7 @@ def test_global_tempdir_manager(): assert not os.path.exists(path) -def test_tempdirectory_asserts_global_tempdir(monkeypatch): +def test_tempdirectory_asserts_global_tempdir(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(temp_dir, "_tempdir_manager", None) with pytest.raises(AssertionError): TempDirectory(globally_managed=True) @@ -215,21 +215,26 @@ def test_tempdirectory_asserts_global_tempdir(monkeypatch): not_deleted_kind = "not-deleted" -@pytest.mark.parametrize("delete,kind,exists", [ - (None, deleted_kind, False), - (_default, deleted_kind, False), - (True, deleted_kind, False), - (False, deleted_kind, True), - (None, not_deleted_kind, True), - (_default, not_deleted_kind, True), - (True, not_deleted_kind, False), - (False, not_deleted_kind, True), - (None, "unspecified", False), - (_default, "unspecified", False), - (True, "unspecified", False), - (False, "unspecified", True), -]) -def test_tempdir_registry(kind, delete, exists): +@pytest.mark.parametrize( + "delete,kind,exists", + [ + (None, deleted_kind, False), + (_default, deleted_kind, False), + (True, deleted_kind, False), + (False, deleted_kind, True), + (None, not_deleted_kind, True), + (_default, not_deleted_kind, True), + (True, not_deleted_kind, False), + (False, not_deleted_kind, True), + (None, "unspecified", False), + (_default, "unspecified", False), + (True, "unspecified", False), + (False, "unspecified", True), + ], +) +def test_tempdir_registry( + delete: Union[bool, _Default], kind: str, exists: bool +) -> None: with tempdir_registry() as registry: registry.set_delete(deleted_kind, True) registry.set_delete(not_deleted_kind, False) @@ -240,12 +245,10 @@ def test_tempdir_registry(kind, delete, exists): assert os.path.exists(path) == exists -@pytest.mark.parametrize("delete,exists", [ - (_default, True), (None, False) -]) +@pytest.mark.parametrize("delete,exists", [(_default, True), (None, False)]) def test_temp_dir_does_not_delete_explicit_paths_by_default( - tmpdir, delete, exists -): + tmpdir: Path, delete: Optional[_Default], exists: bool +) -> None: path = tmpdir / "example" path.mkdir() @@ -259,7 +262,7 @@ def test_temp_dir_does_not_delete_explicit_paths_by_default( @pytest.mark.parametrize("should_delete", [True, False]) -def test_tempdir_registry_lazy(should_delete): +def test_tempdir_registry_lazy(should_delete: bool) -> None: """ Test the registry entry can be updated after a temp dir is created, to change whether a kind should be deleted or not. diff --git a/tests/unit/test_utils_unpacking.py b/tests/unit/test_utils_unpacking.py index aea70efbc07..ccb7a304925 100644 --- a/tests/unit/test_utils_unpacking.py +++ b/tests/unit/test_utils_unpacking.py @@ -1,3 +1,4 @@ +import io import os import shutil import stat @@ -6,11 +7,14 @@ import tempfile import time import zipfile +from typing import List, Tuple import pytest from pip._internal.exceptions import InstallationError from pip._internal.utils.unpacking import is_within_directory, untar_file, unzip_file +from tests.lib import TestData +from tests.lib.path import Path class TestUnpackArchives: @@ -33,71 +37,71 @@ class TestUnpackArchives: """ - def setup(self): + def setup(self) -> None: self.tempdir = tempfile.mkdtemp() self.old_mask = os.umask(0o022) self.symlink_expected_mode = None - def teardown(self): + def teardown(self) -> None: os.umask(self.old_mask) shutil.rmtree(self.tempdir, ignore_errors=True) - def mode(self, path): + def mode(self, path: str) -> int: return stat.S_IMODE(os.stat(path).st_mode) - def confirm_files(self): + def confirm_files(self) -> None: # expectations based on 022 umask set above and the unpack logic that # sets execute permissions, not preservation for fname, expected_mode, test, expected_contents in [ - ('file.txt', 0o644, os.path.isfile, b'file\n'), + ("file.txt", 0o644, os.path.isfile, b"file\n"), # We don't test the "symlink.txt" contents for now. - ('symlink.txt', 0o644, os.path.isfile, None), - ('script_owner.sh', 0o755, os.path.isfile, b'file\n'), - ('script_group.sh', 0o755, os.path.isfile, b'file\n'), - ('script_world.sh', 0o755, os.path.isfile, b'file\n'), - ('dir', 0o755, os.path.isdir, None), - (os.path.join('dir', 'dirfile'), 0o644, os.path.isfile, b''), + ("symlink.txt", 0o644, os.path.isfile, None), + ("script_owner.sh", 0o755, os.path.isfile, b"file\n"), + ("script_group.sh", 0o755, os.path.isfile, b"file\n"), + ("script_world.sh", 0o755, os.path.isfile, b"file\n"), + ("dir", 0o755, os.path.isdir, None), + (os.path.join("dir", "dirfile"), 0o644, os.path.isfile, b""), ]: path = os.path.join(self.tempdir, fname) - if path.endswith('symlink.txt') and sys.platform == 'win32': + if path.endswith("symlink.txt") and sys.platform == "win32": # no symlinks created on windows continue assert test(path), path if expected_contents is not None: - with open(path, mode='rb') as f: + with open(path, mode="rb") as f: contents = f.read() - assert contents == expected_contents, f'fname: {fname}' - if sys.platform == 'win32': + assert contents == expected_contents, f"fname: {fname}" + if sys.platform == "win32": # the permissions tests below don't apply in windows # due to os.chmod being a noop continue mode = self.mode(path) - assert mode == expected_mode, ( - f"mode: {mode}, expected mode: {expected_mode}" - ) + assert ( + mode == expected_mode + ), f"mode: {mode}, expected mode: {expected_mode}" - def make_zip_file(self, filename, file_list): + def make_zip_file(self, filename: str, file_list: List[str]) -> str: """ Create a zip file for test case """ test_zip = os.path.join(self.tempdir, filename) - with zipfile.ZipFile(test_zip, 'w') as myzip: + with zipfile.ZipFile(test_zip, "w") as myzip: for item in file_list: - myzip.writestr(item, 'file content') + myzip.writestr(item, "file content") return test_zip - def make_tar_file(self, filename, file_list): + def make_tar_file(self, filename: str, file_list: List[str]) -> str: """ Create a tar file for test case """ test_tar = os.path.join(self.tempdir, filename) - with tarfile.open(test_tar, 'w') as mytar: + with tarfile.open(test_tar, "w") as mytar: for item in file_list: file_tarinfo = tarfile.TarInfo(item) - mytar.addfile(file_tarinfo, 'file content') + mytar.addfile(file_tarinfo, io.BytesIO(b"file content")) return test_tar - def test_unpack_tgz(self, data): + def test_unpack_tgz(self, data: TestData) -> None: """ Test unpacking a *.tgz, and setting execute permissions """ @@ -105,11 +109,11 @@ def test_unpack_tgz(self, data): untar_file(test_file, self.tempdir) self.confirm_files() # Check the timestamp of an extracted file - file_txt_path = os.path.join(self.tempdir, 'file.txt') + file_txt_path = os.path.join(self.tempdir, "file.txt") mtime = time.gmtime(os.stat(file_txt_path).st_mtime) assert mtime[0:6] == (2013, 8, 16, 5, 13, 37), mtime - def test_unpack_zip(self, data): + def test_unpack_zip(self, data: TestData) -> None: """ Test unpacking a *.zip, and setting execute permissions """ @@ -117,69 +121,89 @@ def test_unpack_zip(self, data): unzip_file(test_file, self.tempdir) self.confirm_files() - def test_unpack_zip_failure(self): + def test_unpack_zip_failure(self) -> None: """ Test unpacking a *.zip with file containing .. path and expect exception """ - files = ['regular_file.txt', os.path.join('..', 'outside_file.txt')] - test_zip = self.make_zip_file('test_zip.zip', files) + files = ["regular_file.txt", os.path.join("..", "outside_file.txt")] + test_zip = self.make_zip_file("test_zip.zip", files) with pytest.raises(InstallationError) as e: unzip_file(test_zip, self.tempdir) - assert 'trying to install outside target directory' in str(e.value) + assert "trying to install outside target directory" in str(e.value) - def test_unpack_zip_success(self): + def test_unpack_zip_success(self) -> None: """ Test unpacking a *.zip with regular files, no file will be installed outside target directory after unpack so no exception raised """ files = [ - 'regular_file1.txt', - os.path.join('dir', 'dir_file1.txt'), - os.path.join('dir', '..', 'dir_file2.txt'), + "regular_file1.txt", + os.path.join("dir", "dir_file1.txt"), + os.path.join("dir", "..", "dir_file2.txt"), ] - test_zip = self.make_zip_file('test_zip.zip', files) + test_zip = self.make_zip_file("test_zip.zip", files) unzip_file(test_zip, self.tempdir) - def test_unpack_tar_failure(self): + def test_unpack_tar_failure(self) -> None: """ Test unpacking a *.tar with file containing .. path and expect exception """ - files = ['regular_file.txt', os.path.join('..', 'outside_file.txt')] - test_tar = self.make_tar_file('test_tar.tar', files) + files = ["regular_file.txt", os.path.join("..", "outside_file.txt")] + test_tar = self.make_tar_file("test_tar.tar", files) with pytest.raises(InstallationError) as e: untar_file(test_tar, self.tempdir) - assert 'trying to install outside target directory' in str(e.value) + assert "trying to install outside target directory" in str(e.value) - def test_unpack_tar_success(self): + def test_unpack_tar_success(self) -> None: """ Test unpacking a *.tar with regular files, no file will be installed outside target directory after unpack so no exception raised """ files = [ - 'regular_file1.txt', - os.path.join('dir', 'dir_file1.txt'), - os.path.join('dir', '..', 'dir_file2.txt'), + "regular_file1.txt", + os.path.join("dir", "dir_file1.txt"), + os.path.join("dir", "..", "dir_file2.txt"), ] - test_tar = self.make_tar_file('test_tar.tar', files) + test_tar = self.make_tar_file("test_tar.tar", files) untar_file(test_tar, self.tempdir) -@pytest.mark.parametrize('args, expected', [ - # Test the second containing the first. - (('parent/sub', 'parent/'), False), - # Test the first not ending in a trailing slash. - (('parent', 'parent/foo'), True), - # Test target containing `..` but still inside the parent. - (('parent/', 'parent/foo/../bar'), True), - # Test target within the parent - (('parent/', 'parent/sub'), True), - # Test target outside parent - (('parent/', 'parent/../sub'), False), -]) -def test_is_within_directory(args, expected): +def test_unpack_tar_unicode(tmpdir: Path) -> None: + test_tar = tmpdir / "test.tar" + # tarfile tries to decode incoming + with tarfile.open(test_tar, "w", format=tarfile.PAX_FORMAT, encoding="utf-8") as f: + metadata = tarfile.TarInfo("dir/åäö_日本語.py") + f.addfile(metadata, io.BytesIO(b"hello world")) + + output_dir = tmpdir / "output" + output_dir.mkdir() + + untar_file(test_tar, str(output_dir)) + + output_dir_name = str(output_dir) + contents = os.listdir(output_dir_name) + assert "åäö_日本語.py" in contents + + +@pytest.mark.parametrize( + "args, expected", + [ + # Test the second containing the first. + (("parent/sub", "parent/"), False), + # Test the first not ending in a trailing slash. + (("parent", "parent/foo"), True), + # Test target containing `..` but still inside the parent. + (("parent/", "parent/foo/../bar"), True), + # Test target within the parent + (("parent/", "parent/sub"), True), + # Test target outside parent + (("parent/", "parent/../sub"), False), + ], +) +def test_is_within_directory(args: Tuple[str, str], expected: bool) -> None: result = is_within_directory(*args) assert result == expected diff --git a/tests/unit/test_utils_virtualenv.py b/tests/unit/test_utils_virtualenv.py index 625539d7617..8f517d24d36 100644 --- a/tests/unit/test_utils_virtualenv.py +++ b/tests/unit/test_utils_virtualenv.py @@ -1,25 +1,34 @@ import logging import site import sys +from typing import List, Optional import pytest from pip._internal.utils import virtualenv +from tests.lib.path import Path -@pytest.mark.parametrize("real_prefix, base_prefix, expected", [ - (None, None, False), # Python 2 base interpreter - (None, sys.prefix, False), # Python 3 base interpreter - (None, "not_sys_prefix", True), # PEP405 venv - (sys.prefix, None, True), # Unknown case - (sys.prefix, sys.prefix, True), # Unknown case - (sys.prefix, "not_sys_prefix", True), # Unknown case - ("not_sys_prefix", None, True), # Python 2 virtualenv - ("not_sys_prefix", sys.prefix, True), # Python 3 virtualenv - ("not_sys_prefix", "not_sys_prefix", True), # Unknown case -]) +@pytest.mark.parametrize( + "real_prefix, base_prefix, expected", + [ + (None, None, False), # Python 2 base interpreter + (None, sys.prefix, False), # Python 3 base interpreter + (None, "not_sys_prefix", True), # PEP405 venv + (sys.prefix, None, True), # Unknown case + (sys.prefix, sys.prefix, True), # Unknown case + (sys.prefix, "not_sys_prefix", True), # Unknown case + ("not_sys_prefix", None, True), # Python 2 virtualenv + ("not_sys_prefix", sys.prefix, True), # Python 3 virtualenv + ("not_sys_prefix", "not_sys_prefix", True), # Unknown case + ], +) def test_running_under_virtualenv( - monkeypatch, real_prefix, base_prefix, expected): + monkeypatch: pytest.MonkeyPatch, + real_prefix: Optional[str], + base_prefix: Optional[str], + expected: bool, +) -> None: # Use raising=False to prevent AttributeError on missing attribute if real_prefix is None: monkeypatch.delattr(sys, "real_prefix", raising=False) @@ -33,7 +42,8 @@ def test_running_under_virtualenv( @pytest.mark.parametrize( - "under_virtualenv, no_global_file, expected", [ + "under_virtualenv, no_global_file, expected", + [ (False, False, False), (False, True, False), (True, False, False), @@ -41,27 +51,29 @@ def test_running_under_virtualenv( ], ) def test_virtualenv_no_global_with_regular_virtualenv( - monkeypatch, - tmpdir, - under_virtualenv, - no_global_file, - expected, -): - monkeypatch.setattr(virtualenv, '_running_under_venv', lambda: False) - - monkeypatch.setattr(site, '__file__', tmpdir / 'site.py') + monkeypatch: pytest.MonkeyPatch, + tmpdir: Path, + under_virtualenv: bool, + no_global_file: bool, + expected: bool, +) -> None: + monkeypatch.setattr(virtualenv, "_running_under_venv", lambda: False) + + monkeypatch.setattr(site, "__file__", tmpdir / "site.py") monkeypatch.setattr( - virtualenv, '_running_under_regular_virtualenv', + virtualenv, + "_running_under_regular_virtualenv", lambda: under_virtualenv, ) if no_global_file: - (tmpdir / 'no-global-site-packages.txt').touch() + (tmpdir / "no-global-site-packages.txt").touch() assert virtualenv.virtualenv_no_global() == expected @pytest.mark.parametrize( - "pyvenv_cfg_lines, under_venv, expected, expect_warning", [ + "pyvenv_cfg_lines, under_venv, expected, expect_warning", + [ (None, False, False, False), (None, True, True, True), # this has a warning. ( @@ -87,20 +99,16 @@ def test_virtualenv_no_global_with_regular_virtualenv( ], ) def test_virtualenv_no_global_with_pep_405_virtual_environment( - monkeypatch, - caplog, - pyvenv_cfg_lines, - under_venv, - expected, - expect_warning, -): - monkeypatch.setattr( - virtualenv, '_running_under_regular_virtualenv', lambda: False - ) - monkeypatch.setattr( - virtualenv, '_get_pyvenv_cfg_lines', lambda: pyvenv_cfg_lines - ) - monkeypatch.setattr(virtualenv, '_running_under_venv', lambda: under_venv) + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + pyvenv_cfg_lines: Optional[List[str]], + under_venv: bool, + expected: bool, + expect_warning: bool, +) -> None: + monkeypatch.setattr(virtualenv, "_running_under_regular_virtualenv", lambda: False) + monkeypatch.setattr(virtualenv, "_get_pyvenv_cfg_lines", lambda: pyvenv_cfg_lines) + monkeypatch.setattr(virtualenv, "_running_under_venv", lambda: under_venv) with caplog.at_level(logging.WARNING): assert virtualenv.virtualenv_no_global() == expected @@ -115,21 +123,22 @@ def test_virtualenv_no_global_with_pep_405_virtual_environment( @pytest.mark.parametrize( - "contents, expected", [ + "contents, expected", + [ (None, None), ("", []), ("a = b\nc = d\n", ["a = b", "c = d"]), ("a = b\nc = d", ["a = b", "c = d"]), # no trailing newlines - ] + ], ) def test_get_pyvenv_cfg_lines_for_pep_405_virtual_environment( - monkeypatch, - tmpdir, - contents, - expected, -): - monkeypatch.setattr(sys, 'prefix', str(tmpdir)) + monkeypatch: pytest.MonkeyPatch, + tmpdir: Path, + contents: Optional[str], + expected: Optional[List[str]], +) -> None: + monkeypatch.setattr(sys, "prefix", str(tmpdir)) if contents is not None: - tmpdir.joinpath('pyvenv.cfg').write_text(contents) + tmpdir.joinpath("pyvenv.cfg").write_text(contents) assert virtualenv._get_pyvenv_cfg_lines() == expected diff --git a/tests/unit/test_utils_wheel.py b/tests/unit/test_utils_wheel.py index a73ecd6c3d6..53e149f9493 100644 --- a/tests/unit/test_utils_wheel.py +++ b/tests/unit/test_utils_wheel.py @@ -1,23 +1,23 @@ import os +from contextlib import ExitStack from email import message_from_string from io import BytesIO +from typing import Callable, Iterator from zipfile import ZipFile import pytest -from pip._vendor.contextlib2 import ExitStack from pip._internal.exceptions import UnsupportedWheel from pip._internal.utils import wheel -from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from tests.lib import TestData +from tests.lib.path import Path -if MYPY_CHECK_RUNNING: - from tests.lib.path import Path +_ZipDir = Callable[[Path], ZipFile] @pytest.fixture -def zip_dir(): - def make_zip(path): - # type: (Path) -> ZipFile +def zip_dir() -> Iterator[_ZipDir]: + def make_zip(path: Path) -> ZipFile: buf = BytesIO() with ZipFile(buf, "w", allowZip64=True) as z: for dirpath, _, filenames in os.walk(path): @@ -36,7 +36,7 @@ def make_zip(path): yield make_zip -def test_wheel_dist_info_dir_found(tmpdir, zip_dir): +def test_wheel_dist_info_dir_found(tmpdir: Path, zip_dir: _ZipDir) -> None: expected = "simple-0.1.dist-info" dist_info_dir = tmpdir / expected dist_info_dir.mkdir() @@ -44,7 +44,7 @@ def test_wheel_dist_info_dir_found(tmpdir, zip_dir): assert wheel.wheel_dist_info_dir(zip_dir(tmpdir), "simple") == expected -def test_wheel_dist_info_dir_multiple(tmpdir, zip_dir): +def test_wheel_dist_info_dir_multiple(tmpdir: Path, zip_dir: _ZipDir) -> None: dist_info_dir_1 = tmpdir / "simple-0.1.dist-info" dist_info_dir_1.mkdir() dist_info_dir_1.joinpath("WHEEL").touch() @@ -56,13 +56,13 @@ def test_wheel_dist_info_dir_multiple(tmpdir, zip_dir): assert "multiple .dist-info directories found" in str(e.value) -def test_wheel_dist_info_dir_none(tmpdir, zip_dir): +def test_wheel_dist_info_dir_none(tmpdir: Path, zip_dir: _ZipDir) -> None: with pytest.raises(UnsupportedWheel) as e: wheel.wheel_dist_info_dir(zip_dir(tmpdir), "simple") assert "directory not found" in str(e.value) -def test_wheel_dist_info_dir_wrong_name(tmpdir, zip_dir): +def test_wheel_dist_info_dir_wrong_name(tmpdir: Path, zip_dir: _ZipDir) -> None: dist_info_dir = tmpdir / "unrelated-0.1.dist-info" dist_info_dir.mkdir() dist_info_dir.joinpath("WHEEL").touch() @@ -71,13 +71,11 @@ def test_wheel_dist_info_dir_wrong_name(tmpdir, zip_dir): assert "does not start with 'simple'" in str(e.value) -def test_wheel_version_ok(tmpdir, data): - assert wheel.wheel_version( - message_from_string("Wheel-Version: 1.9") - ) == (1, 9) +def test_wheel_version_ok(data: TestData) -> None: + assert wheel.wheel_version(message_from_string("Wheel-Version: 1.9")) == (1, 9) -def test_wheel_metadata_fails_missing_wheel(tmpdir, zip_dir): +def test_wheel_metadata_fails_missing_wheel(tmpdir: Path, zip_dir: _ZipDir) -> None: dist_info_dir = tmpdir / "simple-0.1.0.dist-info" dist_info_dir.mkdir() dist_info_dir.joinpath("METADATA").touch() @@ -87,7 +85,7 @@ def test_wheel_metadata_fails_missing_wheel(tmpdir, zip_dir): assert "could not read" in str(e.value) -def test_wheel_metadata_fails_on_bad_encoding(tmpdir, zip_dir): +def test_wheel_metadata_fails_on_bad_encoding(tmpdir: Path, zip_dir: _ZipDir) -> None: dist_info_dir = tmpdir / "simple-0.1.0.dist-info" dist_info_dir.mkdir() dist_info_dir.joinpath("METADATA").touch() @@ -98,27 +96,28 @@ def test_wheel_metadata_fails_on_bad_encoding(tmpdir, zip_dir): assert "error decoding" in str(e.value) -def test_wheel_version_fails_on_no_wheel_version(): +def test_wheel_version_fails_on_no_wheel_version() -> None: with pytest.raises(UnsupportedWheel) as e: wheel.wheel_version(message_from_string("")) assert "missing Wheel-Version" in str(e.value) -@pytest.mark.parametrize("version", [ - ("",), - ("1.b",), - ("1.",), -]) -def test_wheel_version_fails_on_bad_wheel_version(version): +@pytest.mark.parametrize( + "version", + [ + ("",), + ("1.b",), + ("1.",), + ], +) +def test_wheel_version_fails_on_bad_wheel_version(version: str) -> None: with pytest.raises(UnsupportedWheel) as e: - wheel.wheel_version( - message_from_string(f"Wheel-Version: {version}") - ) + wheel.wheel_version(message_from_string(f"Wheel-Version: {version}")) assert "invalid Wheel-Version" in str(e.value) -def test_check_compatibility(): - name = 'test' +def test_check_compatibility() -> None: + name = "test" vc = wheel.VERSION_COMPATIBLE # Major version is higher - should be incompatible @@ -127,7 +126,7 @@ def test_check_compatibility(): # test raises with correct error with pytest.raises(UnsupportedWheel) as e: wheel.check_compatibility(higher_v, name) - assert 'is not compatible' in str(e) + assert "is not compatible" in str(e) # Should only log.warning - minor version is greater higher_v = (vc[0], vc[1] + 1) diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index d36f9f01deb..64f60bc407a 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -1,171 +1,277 @@ import os -from unittest import TestCase +import pathlib +from typing import Any, Dict, List, Optional, Tuple, Type +from unittest import TestCase, mock import pytest -from mock import patch -from pip._vendor.packaging.version import parse as parse_version from pip._internal.exceptions import BadCommand, InstallationError -from pip._internal.utils.misc import hide_url, hide_value +from pip._internal.utils.misc import HiddenText, hide_url, hide_value +from pip._internal.utils.subprocess import CommandArgs from pip._internal.vcs import make_vcs_requirement_url from pip._internal.vcs.bazaar import Bazaar -from pip._internal.vcs.git import Git, looks_like_hash +from pip._internal.vcs.git import Git, RemoteNotValidError, looks_like_hash from pip._internal.vcs.mercurial import Mercurial from pip._internal.vcs.subversion import Subversion from pip._internal.vcs.versioncontrol import RevOptions, VersionControl from tests.lib import is_svn_installed, need_svn +from tests.lib.path import Path @pytest.mark.skipif( - 'TRAVIS' not in os.environ, - reason='Subversion is only required under Travis') -def test_ensure_svn_available(): - """Make sure that svn is available when running in Travis.""" + "CI" not in os.environ, reason="Subversion is only required under CI" +) +def test_ensure_svn_available() -> None: + """Make sure that svn is available when running in CI.""" assert is_svn_installed() -@pytest.mark.parametrize('args, expected', [ - # Test without subdir. - (('git+https://example.com/pkg', 'dev', 'myproj'), - 'git+https://example.com/pkg@dev#egg=myproj'), - # Test with subdir. - (('git+https://example.com/pkg', 'dev', 'myproj', 'sub/dir'), - 'git+https://example.com/pkg@dev#egg=myproj&subdirectory=sub/dir'), - # Test with None subdir. - (('git+https://example.com/pkg', 'dev', 'myproj', None), - 'git+https://example.com/pkg@dev#egg=myproj'), - # Test an unescaped project name. - (('git+https://example.com/pkg', 'dev', 'zope-interface'), - 'git+https://example.com/pkg@dev#egg=zope_interface'), -]) -def test_make_vcs_requirement_url(args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + # Test without subdir. + ( + ("git+https://example.com/pkg", "dev", "myproj"), + "git+https://example.com/pkg@dev#egg=myproj", + ), + # Test with subdir. + ( + ("git+https://example.com/pkg", "dev", "myproj", "sub/dir"), + "git+https://example.com/pkg@dev#egg=myproj&subdirectory=sub/dir", + ), + # Test with None subdir. + ( + ("git+https://example.com/pkg", "dev", "myproj", None), + "git+https://example.com/pkg@dev#egg=myproj", + ), + # Test an unescaped project name. + ( + ("git+https://example.com/pkg", "dev", "zope-interface"), + "git+https://example.com/pkg@dev#egg=zope_interface", + ), + ], +) +def test_make_vcs_requirement_url(args: Tuple[Any, ...], expected: str) -> None: actual = make_vcs_requirement_url(*args) assert actual == expected -def test_rev_options_repr(): - rev_options = RevOptions(Git, 'develop') +def test_rev_options_repr() -> None: + rev_options = RevOptions(Git, "develop") assert repr(rev_options) == "" -@pytest.mark.parametrize(('vc_class', 'expected1', 'expected2', 'kwargs'), [ - # First check VCS-specific RevOptions behavior. - (Bazaar, [], ['-r', '123'], {}), - (Git, ['HEAD'], ['123'], {}), - (Mercurial, [], ['123'], {}), - (Subversion, [], ['-r', '123'], {}), - # Test extra_args. For this, test using a single VersionControl class. - (Git, ['HEAD', 'opt1', 'opt2'], ['123', 'opt1', 'opt2'], - dict(extra_args=['opt1', 'opt2'])), -]) -def test_rev_options_to_args(vc_class, expected1, expected2, kwargs): +@pytest.mark.parametrize( + ("vc_class", "expected1", "expected2", "kwargs"), + [ + # First check VCS-specific RevOptions behavior. + (Bazaar, [], ["-r", "123"], {}), + (Git, ["HEAD"], ["123"], {}), + (Mercurial, [], ["123"], {}), + (Subversion, [], ["-r", "123"], {}), + # Test extra_args. For this, test using a single VersionControl class. + ( + Git, + ["HEAD", "opt1", "opt2"], + ["123", "opt1", "opt2"], + dict(extra_args=["opt1", "opt2"]), + ), + ], +) +def test_rev_options_to_args( + vc_class: Type[VersionControl], + expected1: List[str], + expected2: List[str], + kwargs: Dict[str, Any], +) -> None: """ Test RevOptions.to_args(). """ assert RevOptions(vc_class, **kwargs).to_args() == expected1 - assert RevOptions(vc_class, '123', **kwargs).to_args() == expected2 + assert RevOptions(vc_class, "123", **kwargs).to_args() == expected2 -def test_rev_options_to_display(): +def test_rev_options_to_display() -> None: """ Test RevOptions.to_display(). """ # The choice of VersionControl class doesn't matter here since # the implementation is the same for all of them. rev_options = RevOptions(Git) - assert rev_options.to_display() == '' + assert rev_options.to_display() == "" - rev_options = RevOptions(Git, 'master') - assert rev_options.to_display() == ' (to revision master)' + rev_options = RevOptions(Git, "master") + assert rev_options.to_display() == " (to revision master)" -def test_rev_options_make_new(): +def test_rev_options_make_new() -> None: """ Test RevOptions.make_new(). """ # The choice of VersionControl class doesn't matter here since # the implementation is the same for all of them. - rev_options = RevOptions(Git, 'master', extra_args=['foo', 'bar']) - new_options = rev_options.make_new('develop') + rev_options = RevOptions(Git, "master", extra_args=["foo", "bar"]) + new_options = rev_options.make_new("develop") assert new_options is not rev_options - assert new_options.extra_args == ['foo', 'bar'] - assert new_options.rev == 'develop' + assert new_options.extra_args == ["foo", "bar"] + assert new_options.rev == "develop" assert new_options.vc_class is Git -@pytest.mark.parametrize('sha, expected', [ - ((40 * 'a'), True), - ((40 * 'A'), True), - # Test a string containing all valid characters. - ((18 * 'a' + '0123456789abcdefABCDEF'), True), - ((40 * 'g'), False), - ((39 * 'a'), False), - ((41 * 'a'), False) -]) -def test_looks_like_hash(sha, expected): +@pytest.mark.parametrize( + "sha, expected", + [ + ((40 * "a"), True), + ((40 * "A"), True), + # Test a string containing all valid characters. + ((18 * "a" + "0123456789abcdefABCDEF"), True), + ((40 * "g"), False), + ((39 * "a"), False), + ((41 * "a"), False), + ], +) +def test_looks_like_hash(sha: str, expected: bool) -> None: assert looks_like_hash(sha) == expected -@pytest.mark.parametrize('vcs_cls, remote_url, expected', [ - # Git is one of the subclasses using the base class implementation. - (Git, 'git://example.com/MyProject', False), - (Git, 'http://example.com/MyProject', True), - # Subversion is the only subclass overriding the base class implementation. - (Subversion, 'svn://example.com/MyProject', True), -]) -def test_should_add_vcs_url_prefix(vcs_cls, remote_url, expected): +@pytest.mark.parametrize( + "vcs_cls, remote_url, expected", + [ + # Mercurial is one of the subclasses using the base class implementation. + # `hg://` isn't a real prefix but it tests the default behaviour. + (Mercurial, "hg://user@example.com/MyProject", False), + (Mercurial, "http://example.com/MyProject", True), + # The Git subclasses should return true in all cases. + (Git, "git://example.com/MyProject", True), + (Git, "http://example.com/MyProject", True), + # Subversion also overrides the base class implementation. + (Subversion, "svn://example.com/MyProject", True), + ], +) +def test_should_add_vcs_url_prefix( + vcs_cls: Type[VersionControl], remote_url: str, expected: bool +) -> None: actual = vcs_cls.should_add_vcs_url_prefix(remote_url) assert actual == expected -@patch('pip._internal.vcs.git.Git.get_remote_url') -@patch('pip._internal.vcs.git.Git.get_revision') -@patch('pip._internal.vcs.git.Git.get_subdirectory') +@pytest.mark.parametrize( + "url, target", + [ + # A fully qualified remote url. No changes needed. + ("ssh://bob@server/foo/bar.git", "ssh://bob@server/foo/bar.git"), + ("git://bob@server/foo/bar.git", "git://bob@server/foo/bar.git"), + # User is optional and does not need a default. + ("ssh://server/foo/bar.git", "ssh://server/foo/bar.git"), + # The common scp shorthand for ssh remotes. Pip won't recognise these as + # git remotes until they have a 'ssh://' prefix and the ':' in the middle + # is gone. + ("git@example.com:foo/bar.git", "ssh://git@example.com/foo/bar.git"), + ("example.com:foo.git", "ssh://example.com/foo.git"), + # Http(s) remote names are already complete and should remain unchanged. + ("https://example.com/foo", "https://example.com/foo"), + ("http://example.com/foo/bar.git", "http://example.com/foo/bar.git"), + ("https://bob@example.com/foo", "https://bob@example.com/foo"), + ], +) +def test_git_remote_url_to_pip(url: str, target: str) -> None: + assert Git._git_remote_to_pip_url(url) == target + + +@pytest.mark.parametrize( + "url, platform", + [ + # Windows paths with the ':' drive prefix look dangerously close to SCP. + ("c:/piffle/wiffle/waffle/poffle.git", "nt"), + (r"c:\faffle\waffle\woffle\piffle.git", "nt"), + # Unix paths less so but test them anyway. + ("/muffle/fuffle/pufffle/fluffle.git", "posix"), + ], +) +def test_paths_are_not_mistaken_for_scp_shorthand(url: str, platform: str) -> None: + # File paths should not be mistaken for SCP shorthand. If they do then + # 'c:/piffle/wiffle' would end up as 'ssh://c/piffle/wiffle'. + from pip._internal.vcs.git import SCP_REGEX + + assert not SCP_REGEX.match(url) + + if platform == os.name: + with pytest.raises(RemoteNotValidError): + Git._git_remote_to_pip_url(url) + + +def test_git_remote_local_path(tmpdir: Path) -> None: + path = pathlib.Path(tmpdir, "project.git") + path.mkdir() + # Path must exist to be recognised as a local git remote. + assert Git._git_remote_to_pip_url(str(path)) == path.as_uri() + + +@mock.patch("pip._internal.vcs.git.Git.get_remote_url") +@mock.patch("pip._internal.vcs.git.Git.get_revision") +@mock.patch("pip._internal.vcs.git.Git.get_subdirectory") +@pytest.mark.parametrize( + "git_url, target_url_prefix", + [ + ( + "https://github.com/pypa/pip-test-package", + "git+https://github.com/pypa/pip-test-package", + ), + ( + "git@github.com:pypa/pip-test-package", + "git+ssh://git@github.com/pypa/pip-test-package", + ), + ], + ids=["https", "ssh"], +) @pytest.mark.network def test_git_get_src_requirements( - mock_get_subdirectory, mock_get_revision, mock_get_remote_url -): - git_url = 'https://github.com/pypa/pip-test-package' - sha = '5547fa909e83df8bd743d3978d6667497983a4b7' - - mock_get_remote_url.return_value = git_url + mock_get_subdirectory: mock.Mock, + mock_get_revision: mock.Mock, + mock_get_remote_url: mock.Mock, + git_url: str, + target_url_prefix: str, +) -> None: + sha = "5547fa909e83df8bd743d3978d6667497983a4b7" + + mock_get_remote_url.return_value = Git._git_remote_to_pip_url(git_url) mock_get_revision.return_value = sha mock_get_subdirectory.return_value = None - ret = Git.get_src_requirement('.', 'pip-test-package') + ret = Git.get_src_requirement(".", "pip-test-package") - assert ret == ( - 'git+https://github.com/pypa/pip-test-package' - '@5547fa909e83df8bd743d3978d6667497983a4b7#egg=pip_test_package' - ) + target = f"{target_url_prefix}@{sha}#egg=pip_test_package" + assert ret == target -@patch('pip._internal.vcs.git.Git.get_revision_sha') -def test_git_resolve_revision_rev_exists(get_sha_mock): - get_sha_mock.return_value = ('123456', False) - url = 'git+https://git.example.com' - rev_options = Git.make_rev_options('develop') +@mock.patch("pip._internal.vcs.git.Git.get_revision_sha") +def test_git_resolve_revision_rev_exists(get_sha_mock: mock.Mock) -> None: + get_sha_mock.return_value = ("123456", False) + url = HiddenText("git+https://git.example.com", redacted="*") + rev_options = Git.make_rev_options("develop") - new_options = Git.resolve_revision('.', url, rev_options) - assert new_options.rev == '123456' + new_options = Git.resolve_revision(".", url, rev_options) + assert new_options.rev == "123456" -@patch('pip._internal.vcs.git.Git.get_revision_sha') -def test_git_resolve_revision_rev_not_found(get_sha_mock): +@mock.patch("pip._internal.vcs.git.Git.get_revision_sha") +def test_git_resolve_revision_rev_not_found(get_sha_mock: mock.Mock) -> None: get_sha_mock.return_value = (None, False) - url = 'git+https://git.example.com' - rev_options = Git.make_rev_options('develop') + url = HiddenText("git+https://git.example.com", redacted="*") + rev_options = Git.make_rev_options("develop") - new_options = Git.resolve_revision('.', url, rev_options) - assert new_options.rev == 'develop' + new_options = Git.resolve_revision(".", url, rev_options) + assert new_options.rev == "develop" -@patch('pip._internal.vcs.git.Git.get_revision_sha') -def test_git_resolve_revision_not_found_warning(get_sha_mock, caplog): +@mock.patch("pip._internal.vcs.git.Git.get_revision_sha") +def test_git_resolve_revision_not_found_warning( + get_sha_mock: mock.Mock, caplog: pytest.LogCaptureFixture +) -> None: get_sha_mock.return_value = (None, False) - url = 'git+https://git.example.com' - sha = 40 * 'a' + url = HiddenText("git+https://git.example.com", redacted="*") + sha = 40 * "a" rev_options = Git.make_rev_options(sha) # resolve_revision with a full sha would fail here because @@ -173,44 +279,53 @@ def test_git_resolve_revision_not_found_warning(get_sha_mock, caplog): # test_resolve_commit_not_on_branch. rev_options = Git.make_rev_options(sha[:6]) - new_options = Git.resolve_revision('.', url, rev_options) - assert new_options.rev == 'aaaaaa' + new_options = Git.resolve_revision(".", url, rev_options) + assert new_options.rev == "aaaaaa" # Check that a warning got logged only for the abbreviated hash. messages = [r.getMessage() for r in caplog.records] - messages = [msg for msg in messages if msg.startswith('Did not find ')] + messages = [msg for msg in messages if msg.startswith("Did not find ")] assert messages == [ "Did not find branch or tag 'aaaaaa', assuming revision or ref." ] -@pytest.mark.parametrize('rev_name,result', ( - ('5547fa909e83df8bd743d3978d6667497983a4b7', True), - ('5547fa909', False), - ('5678', False), - ('abc123', False), - ('foo', False), - (None, False), -)) -@patch('pip._internal.vcs.git.Git.get_revision') -def test_git_is_commit_id_equal(mock_get_revision, rev_name, result): +@pytest.mark.parametrize( + "rev_name,result", + ( + ("5547fa909e83df8bd743d3978d6667497983a4b7", True), + ("5547fa909", False), + ("5678", False), + ("abc123", False), + ("foo", False), + (None, False), + ), +) +@mock.patch("pip._internal.vcs.git.Git.get_revision") +def test_git_is_commit_id_equal( + mock_get_revision: mock.Mock, rev_name: Optional[str], result: bool +) -> None: """ Test Git.is_commit_id_equal(). """ - mock_get_revision.return_value = '5547fa909e83df8bd743d3978d6667497983a4b7' - assert Git.is_commit_id_equal('/path', rev_name) is result + mock_get_revision.return_value = "5547fa909e83df8bd743d3978d6667497983a4b7" + assert Git.is_commit_id_equal("/path", rev_name) is result # The non-SVN backends all use the same get_netloc_and_auth(), so only test # Git as a representative. -@pytest.mark.parametrize('args, expected', [ - # Test a basic case. - (('example.com', 'https'), ('example.com', (None, None))), - # Test with username and password. - (('user:pass@example.com', 'https'), - ('user:pass@example.com', (None, None))), -]) -def test_git__get_netloc_and_auth(args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + # Test a basic case. + (("example.com", "https"), ("example.com", (None, None))), + # Test with username and password. + (("user:pass@example.com", "https"), ("user:pass@example.com", (None, None))), + ], +) +def test_git__get_netloc_and_auth( + args: Tuple[str, str], expected: Tuple[str, Tuple[None, None]] +) -> None: """ Test VersionControl.get_netloc_and_auth(). """ @@ -219,21 +334,27 @@ def test_git__get_netloc_and_auth(args, expected): assert actual == expected -@pytest.mark.parametrize('args, expected', [ - # Test https. - (('example.com', 'https'), ('example.com', (None, None))), - # Test https with username and no password. - (('user@example.com', 'https'), ('example.com', ('user', None))), - # Test https with username and password. - (('user:pass@example.com', 'https'), ('example.com', ('user', 'pass'))), - # Test https with URL-encoded reserved characters. - (('user%3Aname:%23%40%5E@example.com', 'https'), - ('example.com', ('user:name', '#@^'))), - # Test ssh with username and password. - (('user:pass@example.com', 'ssh'), - ('user:pass@example.com', (None, None))), -]) -def test_subversion__get_netloc_and_auth(args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + # Test https. + (("example.com", "https"), ("example.com", (None, None))), + # Test https with username and no password. + (("user@example.com", "https"), ("example.com", ("user", None))), + # Test https with username and password. + (("user:pass@example.com", "https"), ("example.com", ("user", "pass"))), + # Test https with URL-encoded reserved characters. + ( + ("user%3Aname:%23%40%5E@example.com", "https"), + ("example.com", ("user:name", "#@^")), + ), + # Test ssh with username and password. + (("user:pass@example.com", "ssh"), ("user:pass@example.com", (None, None))), + ], +) +def test_subversion__get_netloc_and_auth( + args: Tuple[str, str], expected: Tuple[str, Tuple[Optional[str], Optional[str]]] +) -> None: """ Test Subversion.get_netloc_and_auth(). """ @@ -242,29 +363,38 @@ def test_subversion__get_netloc_and_auth(args, expected): assert actual == expected -def test_git__get_url_rev__idempotent(): +def test_git__get_url_rev__idempotent() -> None: """ Check that Git.get_url_rev_and_auth() is idempotent for what the code calls "stub URLs" (i.e. URLs that don't contain "://"). Also check that it doesn't change self.url. """ - url = 'git+git@git.example.com:MyProject#egg=MyProject' + url = "git+git@git.example.com:MyProject#egg=MyProject" result1 = Git.get_url_rev_and_auth(url) result2 = Git.get_url_rev_and_auth(url) - expected = ('git@git.example.com:MyProject', None, (None, None)) + expected = ("git@git.example.com:MyProject", None, (None, None)) assert result1 == expected assert result2 == expected -@pytest.mark.parametrize('url, expected', [ - ('svn+https://svn.example.com/MyProject', - ('https://svn.example.com/MyProject', None, (None, None))), - # Test a "+" in the path portion. - ('svn+https://svn.example.com/My+Project', - ('https://svn.example.com/My+Project', None, (None, None))), -]) -def test_version_control__get_url_rev_and_auth(url, expected): +@pytest.mark.parametrize( + "url, expected", + [ + ( + "svn+https://svn.example.com/MyProject", + ("https://svn.example.com/MyProject", None, (None, None)), + ), + # Test a "+" in the path portion. + ( + "svn+https://svn.example.com/My+Project", + ("https://svn.example.com/My+Project", None, (None, None)), + ), + ], +) +def test_version_control__get_url_rev_and_auth( + url: str, expected: Tuple[str, None, Tuple[None, None]] +) -> None: """ Test the basic case of VersionControl.get_url_rev_and_auth(). """ @@ -272,12 +402,15 @@ def test_version_control__get_url_rev_and_auth(url, expected): assert actual == expected -@pytest.mark.parametrize('url', [ - 'https://svn.example.com/MyProject', - # Test a URL containing a "+" (but not in the scheme). - 'https://svn.example.com/My+Project', -]) -def test_version_control__get_url_rev_and_auth__missing_plus(url): +@pytest.mark.parametrize( + "url", + [ + "https://svn.example.com/MyProject", + # Test a URL containing a "+" (but not in the scheme). + "https://svn.example.com/My+Project", + ], +) +def test_version_control__get_url_rev_and_auth__missing_plus(url: str) -> None: """ Test passing a URL to VersionControl.get_url_rev_and_auth() with a "+" missing from the scheme. @@ -285,14 +418,17 @@ def test_version_control__get_url_rev_and_auth__missing_plus(url): with pytest.raises(ValueError) as excinfo: VersionControl.get_url_rev_and_auth(url) - assert 'malformed VCS url' in str(excinfo.value) + assert "malformed VCS url" in str(excinfo.value) -@pytest.mark.parametrize('url', [ - # Test a URL with revision part as empty. - 'git+https://github.com/MyUser/myProject.git@#egg=py_pkg', -]) -def test_version_control__get_url_rev_and_auth__no_revision(url): +@pytest.mark.parametrize( + "url", + [ + # Test a URL with revision part as empty. + "git+https://github.com/MyUser/myProject.git@#egg=py_pkg", + ], +) +def test_version_control__get_url_rev_and_auth__no_revision(url: str) -> None: """ Test passing a URL to VersionControl.get_url_rev_and_auth() with empty revision @@ -300,30 +436,66 @@ def test_version_control__get_url_rev_and_auth__no_revision(url): with pytest.raises(InstallationError) as excinfo: VersionControl.get_url_rev_and_auth(url) - assert 'an empty revision (after @)' in str(excinfo.value) - - -@pytest.mark.parametrize('url, expected', [ - # Test http. - ('bzr+http://bzr.myproject.org/MyProject/trunk/#egg=MyProject', - 'http://bzr.myproject.org/MyProject/trunk/'), - # Test https. - ('bzr+https://bzr.myproject.org/MyProject/trunk/#egg=MyProject', - 'https://bzr.myproject.org/MyProject/trunk/'), - # Test ftp. - ('bzr+ftp://bzr.myproject.org/MyProject/trunk/#egg=MyProject', - 'ftp://bzr.myproject.org/MyProject/trunk/'), - # Test sftp. - ('bzr+sftp://bzr.myproject.org/MyProject/trunk/#egg=MyProject', - 'sftp://bzr.myproject.org/MyProject/trunk/'), - # Test launchpad. - ('bzr+lp:MyLaunchpadProject#egg=MyLaunchpadProject', - 'lp:MyLaunchpadProject'), - # Test ssh (special handling). - ('bzr+ssh://bzr.myproject.org/MyProject/trunk/#egg=MyProject', - 'bzr+ssh://bzr.myproject.org/MyProject/trunk/'), -]) -def test_bazaar__get_url_rev_and_auth(url, expected): + assert "an empty revision (after @)" in str(excinfo.value) + + +@pytest.mark.parametrize("vcs_cls", [Bazaar, Git, Mercurial, Subversion]) +@pytest.mark.parametrize( + "exc_cls, msg_re", + [ + (FileNotFoundError, r"Cannot find command '{name}'"), + (PermissionError, r"No permission to execute '{name}'"), + ], + ids=["FileNotFoundError", "PermissionError"], +) +def test_version_control__run_command__fails( + vcs_cls: Type[VersionControl], exc_cls: Type[Exception], msg_re: str +) -> None: + """ + Test that ``VersionControl.run_command()`` raises ``BadCommand`` + when the command is not found or when the user have no permission + to execute it. The error message must contains the command name. + """ + with mock.patch("pip._internal.vcs.versioncontrol.call_subprocess") as call: + call.side_effect = exc_cls + with pytest.raises(BadCommand, match=msg_re.format(name=vcs_cls.name)): + # https://github.com/python/mypy/issues/3283 + vcs_cls.run_command([]) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "url, expected", + [ + # Test http. + ( + "bzr+http://bzr.myproject.org/MyProject/trunk/#egg=MyProject", + "http://bzr.myproject.org/MyProject/trunk/", + ), + # Test https. + ( + "bzr+https://bzr.myproject.org/MyProject/trunk/#egg=MyProject", + "https://bzr.myproject.org/MyProject/trunk/", + ), + # Test ftp. + ( + "bzr+ftp://bzr.myproject.org/MyProject/trunk/#egg=MyProject", + "ftp://bzr.myproject.org/MyProject/trunk/", + ), + # Test sftp. + ( + "bzr+sftp://bzr.myproject.org/MyProject/trunk/#egg=MyProject", + "sftp://bzr.myproject.org/MyProject/trunk/", + ), + # Test launchpad. + ("bzr+lp:MyLaunchpadProject#egg=MyLaunchpadProject", "lp:MyLaunchpadProject"), + # Test ssh (special handling). + ( + "bzr+ssh://bzr.myproject.org/MyProject/trunk/#egg=MyProject", + "bzr+ssh://bzr.myproject.org/MyProject/trunk/", + ), + ], +) +def test_bazaar__get_url_rev_and_auth(url: str, expected: str) -> None: """ Test Bazaar.get_url_rev_and_auth(). """ @@ -331,21 +503,34 @@ def test_bazaar__get_url_rev_and_auth(url, expected): assert actual == (expected, None, (None, None)) -@pytest.mark.parametrize('url, expected', [ - # Test an https URL. - ('svn+https://svn.example.com/MyProject#egg=MyProject', - ('https://svn.example.com/MyProject', None, (None, None))), - # Test an https URL with a username and password. - ('svn+https://user:pass@svn.example.com/MyProject#egg=MyProject', - ('https://svn.example.com/MyProject', None, ('user', 'pass'))), - # Test an ssh URL. - ('svn+ssh://svn.example.com/MyProject#egg=MyProject', - ('svn+ssh://svn.example.com/MyProject', None, (None, None))), - # Test an ssh URL with a username. - ('svn+ssh://user@svn.example.com/MyProject#egg=MyProject', - ('svn+ssh://user@svn.example.com/MyProject', None, (None, None))), -]) -def test_subversion__get_url_rev_and_auth(url, expected): +@pytest.mark.parametrize( + "url, expected", + [ + # Test an https URL. + ( + "svn+https://svn.example.com/MyProject#egg=MyProject", + ("https://svn.example.com/MyProject", None, (None, None)), + ), + # Test an https URL with a username and password. + ( + "svn+https://user:pass@svn.example.com/MyProject#egg=MyProject", + ("https://svn.example.com/MyProject", None, ("user", "pass")), + ), + # Test an ssh URL. + ( + "svn+ssh://svn.example.com/MyProject#egg=MyProject", + ("svn+ssh://svn.example.com/MyProject", None, (None, None)), + ), + # Test an ssh URL with a username. + ( + "svn+ssh://user@svn.example.com/MyProject#egg=MyProject", + ("svn+ssh://user@svn.example.com/MyProject", None, (None, None)), + ), + ], +) +def test_subversion__get_url_rev_and_auth( + url: str, expected: Tuple[str, None, Tuple[Optional[str], Optional[str]]] +) -> None: """ Test Subversion.get_url_rev_and_auth(). """ @@ -355,12 +540,17 @@ def test_subversion__get_url_rev_and_auth(url, expected): # The non-SVN backends all use the same make_rev_args(), so only test # Git as a representative. -@pytest.mark.parametrize('username, password, expected', [ - (None, None, []), - ('user', None, []), - ('user', hide_value('pass'), []), -]) -def test_git__make_rev_args(username, password, expected): +@pytest.mark.parametrize( + "username, password, expected", + [ + (None, None, []), + ("user", None, []), + ("user", hide_value("pass"), []), + ], +) +def test_git__make_rev_args( + username: Optional[str], password: Optional[HiddenText], expected: CommandArgs +) -> None: """ Test VersionControl.make_rev_args(). """ @@ -368,13 +558,21 @@ def test_git__make_rev_args(username, password, expected): assert actual == expected -@pytest.mark.parametrize('username, password, expected', [ - (None, None, []), - ('user', None, ['--username', 'user']), - ('user', hide_value('pass'), - ['--username', 'user', '--password', hide_value('pass')]), -]) -def test_subversion__make_rev_args(username, password, expected): +@pytest.mark.parametrize( + "username, password, expected", + [ + (None, None, []), + ("user", None, ["--username", "user"]), + ( + "user", + hide_value("pass"), + ["--username", "user", "--password", hide_value("pass")], + ), + ], +) +def test_subversion__make_rev_args( + username: Optional[str], password: Optional[HiddenText], expected: CommandArgs +) -> None: """ Test Subversion.make_rev_args(). """ @@ -382,38 +580,40 @@ def test_subversion__make_rev_args(username, password, expected): assert actual == expected -def test_subversion__get_url_rev_options(): +def test_subversion__get_url_rev_options() -> None: """ Test Subversion.get_url_rev_options(). """ - secret_url = ( - 'svn+https://user:pass@svn.example.com/MyProject@v1.0#egg=MyProject' - ) + secret_url = "svn+https://user:pass@svn.example.com/MyProject@v1.0#egg=MyProject" hidden_url = hide_url(secret_url) url, rev_options = Subversion().get_url_rev_options(hidden_url) - assert url == hide_url('https://svn.example.com/MyProject') - assert rev_options.rev == 'v1.0' + assert url == hide_url("https://svn.example.com/MyProject") + assert rev_options.rev == "v1.0" assert rev_options.extra_args == ( - ['--username', 'user', '--password', hide_value('pass')] + ["--username", "user", "--password", hide_value("pass")] ) -def test_get_git_version(): +def test_get_git_version() -> None: git_version = Git().get_git_version() - assert git_version >= parse_version('1.0.0') - - -@pytest.mark.parametrize('use_interactive,is_atty,expected', [ - (None, False, False), - (None, True, True), - (False, False, False), - (False, True, False), - (True, False, True), - (True, True, True), -]) -@patch('sys.stdin.isatty') + assert git_version >= (1, 0, 0) + + +@pytest.mark.parametrize( + "use_interactive,is_atty,expected", + [ + (None, False, False), + (None, True, True), + (False, False, False), + (False, True, False), + (True, False, True), + (True, True, True), + ], +) +@mock.patch("sys.stdin.isatty") def test_subversion__init_use_interactive( - mock_isatty, use_interactive, is_atty, expected): + mock_isatty: mock.Mock, use_interactive: bool, is_atty: bool, expected: bool +) -> None: """ Test Subversion.__init__() with mocked sys.stdin.isatty() output. """ @@ -423,7 +623,7 @@ def test_subversion__init_use_interactive( @need_svn -def test_subversion__call_vcs_version(): +def test_subversion__call_vcs_version() -> None: """ Test Subversion.call_vcs_version() against local ``svn``. """ @@ -435,25 +635,33 @@ def test_subversion__call_vcs_version(): assert version[0] >= 1 -@pytest.mark.parametrize('svn_output, expected_version', [ - ('svn, version 1.10.3 (r1842928)\n' - ' compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0', - (1, 10, 3)), - ('svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)\n' - ' compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2', - (1, 12, 0)), - ('svn, version 1.9.7 (r1800392)', (1, 9, 7)), - ('svn, version 1.9.7a1 (r1800392)', ()), - ('svn, version 1.9 (r1800392)', (1, 9)), - ('svn, version .9.7 (r1800392)', ()), - ('svn version 1.9.7 (r1800392)', ()), - ('svn 1.9.7', ()), - ('svn, version . .', ()), - ('', ()), -]) -@patch('pip._internal.vcs.subversion.Subversion.run_command') +@pytest.mark.parametrize( + "svn_output, expected_version", + [ + ( + "svn, version 1.10.3 (r1842928)\n" + " compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0", + (1, 10, 3), + ), + ( + "svn, version 1.12.0-SlikSvn (SlikSvn/1.12.0)\n" + " compiled May 28 2019, 13:44:56 on x86_64-microsoft-windows6.2", + (1, 12, 0), + ), + ("svn, version 1.9.7 (r1800392)", (1, 9, 7)), + ("svn, version 1.9.7a1 (r1800392)", ()), + ("svn, version 1.9 (r1800392)", (1, 9)), + ("svn, version .9.7 (r1800392)", ()), + ("svn version 1.9.7 (r1800392)", ()), + ("svn 1.9.7", ()), + ("svn, version . .", ()), + ("", ()), + ], +) +@mock.patch("pip._internal.vcs.subversion.Subversion.run_command") def test_subversion__call_vcs_version_patched( - mock_run_command, svn_output, expected_version): + mock_run_command: mock.Mock, svn_output: str, expected_version: Tuple[int, ...] +) -> None: """ Test Subversion.call_vcs_version() against patched output. """ @@ -462,8 +670,10 @@ def test_subversion__call_vcs_version_patched( assert version == expected_version -@patch('pip._internal.vcs.subversion.Subversion.run_command') -def test_subversion__call_vcs_version_svn_not_installed(mock_run_command): +@mock.patch("pip._internal.vcs.subversion.Subversion.run_command") +def test_subversion__call_vcs_version_svn_not_installed( + mock_run_command: mock.Mock, +) -> None: """ Test Subversion.call_vcs_version() when svn is not installed. """ @@ -472,13 +682,16 @@ def test_subversion__call_vcs_version_svn_not_installed(mock_run_command): Subversion().call_vcs_version() -@pytest.mark.parametrize('version', [ - (), - (1,), - (1, 8), - (1, 8, 0), -]) -def test_subversion__get_vcs_version_cached(version): +@pytest.mark.parametrize( + "version", + [ + (), + (1,), + (1, 8), + (1, 8, 0), + ], +) +def test_subversion__get_vcs_version_cached(version: Tuple[int, ...]) -> None: """ Test Subversion.get_vcs_version() with previously cached result. """ @@ -487,13 +700,18 @@ def test_subversion__get_vcs_version_cached(version): assert svn.get_vcs_version() == version -@pytest.mark.parametrize('vcs_version', [ - (), - (1, 7), - (1, 8, 0), -]) -@patch('pip._internal.vcs.subversion.Subversion.call_vcs_version') -def test_subversion__get_vcs_version_call_vcs(mock_call_vcs, vcs_version): +@pytest.mark.parametrize( + "vcs_version", + [ + (), + (1, 7), + (1, 8, 0), + ], +) +@mock.patch("pip._internal.vcs.subversion.Subversion.call_vcs_version") +def test_subversion__get_vcs_version_call_vcs( + mock_call_vcs: mock.Mock, vcs_version: Tuple[int, ...] +) -> None: """ Test Subversion.get_vcs_version() with mocked output from call_vcs_version(). @@ -506,16 +724,20 @@ def test_subversion__get_vcs_version_call_vcs(mock_call_vcs, vcs_version): assert svn._vcs_version == vcs_version -@pytest.mark.parametrize('use_interactive,vcs_version,expected_options', [ - (False, (), ['--non-interactive']), - (False, (1, 7, 0), ['--non-interactive']), - (False, (1, 8, 0), ['--non-interactive']), - (True, (), []), - (True, (1, 7, 0), []), - (True, (1, 8, 0), ['--force-interactive']), -]) +@pytest.mark.parametrize( + "use_interactive,vcs_version,expected_options", + [ + (False, (), ["--non-interactive"]), + (False, (1, 7, 0), ["--non-interactive"]), + (False, (1, 8, 0), ["--non-interactive"]), + (True, (), []), + (True, (1, 7, 0), []), + (True, (1, 8, 0), ["--force-interactive"]), + ], +) def test_subversion__get_remote_call_options( - use_interactive, vcs_version, expected_options): + use_interactive: bool, vcs_version: Tuple[int, ...], expected_options: List[str] +) -> None: """ Test Subversion.get_remote_call_options(). """ @@ -525,65 +747,87 @@ def test_subversion__get_remote_call_options( class TestSubversionArgs(TestCase): - def setUp(self): - patcher = patch('pip._internal.vcs.versioncontrol.call_subprocess') + def setUp(self) -> None: + patcher = mock.patch("pip._internal.vcs.versioncontrol.call_subprocess") self.addCleanup(patcher.stop) self.call_subprocess_mock = patcher.start() # Test Data. - self.url = 'svn+http://username:password@svn.example.com/' + self.url = "svn+http://username:password@svn.example.com/" # use_interactive is set to False to test that remote call options are # properly added. self.svn = Subversion(use_interactive=False) self.rev_options = RevOptions(Subversion) - self.dest = '/tmp/test' + self.dest = "/tmp/test" - def assert_call_args(self, args): + def assert_call_args(self, args: CommandArgs) -> None: assert self.call_subprocess_mock.call_args[0][0] == args - def test_obtain(self): - self.svn.obtain(self.dest, hide_url(self.url)) - self.assert_call_args([ - 'svn', 'checkout', '-q', '--non-interactive', '--username', - 'username', '--password', hide_value('password'), - hide_url('http://svn.example.com/'), '/tmp/test', - ]) - - def test_export(self): - self.svn.export(self.dest, hide_url(self.url)) - self.assert_call_args([ - 'svn', 'export', '--non-interactive', '--username', 'username', - '--password', hide_value('password'), - hide_url('http://svn.example.com/'), '/tmp/test', - ]) - - def test_fetch_new(self): - self.svn.fetch_new(self.dest, hide_url(self.url), self.rev_options) - self.assert_call_args([ - 'svn', 'checkout', '-q', '--non-interactive', - hide_url('svn+http://username:password@svn.example.com/'), - '/tmp/test', - ]) - - def test_fetch_new_revision(self): - rev_options = RevOptions(Subversion, '123') - self.svn.fetch_new(self.dest, hide_url(self.url), rev_options) - self.assert_call_args([ - 'svn', 'checkout', '-q', '--non-interactive', '-r', '123', - hide_url('svn+http://username:password@svn.example.com/'), - '/tmp/test', - ]) - - def test_switch(self): + def test_obtain(self) -> None: + self.svn.obtain(self.dest, hide_url(self.url), verbosity=0) + self.assert_call_args( + [ + "svn", + "checkout", + "--quiet", + "--non-interactive", + "--username", + "username", + "--password", + hide_value("password"), + hide_url("http://svn.example.com/"), + "/tmp/test", + ] + ) + + def test_fetch_new(self) -> None: + self.svn.fetch_new(self.dest, hide_url(self.url), self.rev_options, verbosity=0) + self.assert_call_args( + [ + "svn", + "checkout", + "--quiet", + "--non-interactive", + hide_url("svn+http://username:password@svn.example.com/"), + "/tmp/test", + ] + ) + + def test_fetch_new_revision(self) -> None: + rev_options = RevOptions(Subversion, "123") + self.svn.fetch_new(self.dest, hide_url(self.url), rev_options, verbosity=0) + self.assert_call_args( + [ + "svn", + "checkout", + "--quiet", + "--non-interactive", + "-r", + "123", + hide_url("svn+http://username:password@svn.example.com/"), + "/tmp/test", + ] + ) + + def test_switch(self) -> None: self.svn.switch(self.dest, hide_url(self.url), self.rev_options) - self.assert_call_args([ - 'svn', 'switch', '--non-interactive', - hide_url('svn+http://username:password@svn.example.com/'), - '/tmp/test', - ]) - - def test_update(self): + self.assert_call_args( + [ + "svn", + "switch", + "--non-interactive", + hide_url("svn+http://username:password@svn.example.com/"), + "/tmp/test", + ] + ) + + def test_update(self) -> None: self.svn.update(self.dest, hide_url(self.url), self.rev_options) - self.assert_call_args([ - 'svn', 'update', '--non-interactive', '/tmp/test', - ]) + self.assert_call_args( + [ + "svn", + "update", + "--non-interactive", + "/tmp/test", + ] + ) diff --git a/tests/unit/test_vcs_mercurial.py b/tests/unit/test_vcs_mercurial.py index 07224c0a4d6..22ec2b60ed4 100644 --- a/tests/unit/test_vcs_mercurial.py +++ b/tests/unit/test_vcs_mercurial.py @@ -8,25 +8,26 @@ from pip._internal.utils.misc import hide_url from pip._internal.vcs.mercurial import Mercurial from tests.lib import need_mercurial +from tests.lib.path import Path @need_mercurial -def test_mercurial_switch_updates_config_file_when_found(tmpdir): +def test_mercurial_switch_updates_config_file_when_found(tmpdir: Path) -> None: hg = Mercurial() options = hg.make_rev_options() - hg_dir = os.path.join(tmpdir, '.hg') + hg_dir = os.path.join(tmpdir, ".hg") os.mkdir(hg_dir) config = configparser.RawConfigParser() - config.add_section('paths') - config.set('paths', 'default', 'old_url') + config.add_section("paths") + config.set("paths", "default", "old_url") - hgrc_path = os.path.join(hg_dir, 'hgrc') - with open(hgrc_path, 'w') as f: + hgrc_path = os.path.join(hg_dir, "hgrc") + with open(hgrc_path, "w") as f: config.write(f) - hg.switch(tmpdir, hide_url('new_url'), options) + hg.switch(tmpdir, hide_url("new_url"), options) config.read(hgrc_path) - default_path = config.get('paths', 'default') - assert default_path == 'new_url' + default_path = config.get("paths", "default") + assert default_path == "new_url" diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 52a5fe0436a..a698656b2df 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -2,11 +2,13 @@ import csv import logging import os +import pathlib import textwrap from email import message_from_string +from typing import Dict, List, Optional, Tuple, cast +from unittest.mock import patch import pytest -from mock import patch from pip._vendor.packaging.requirements import Requirement from pip._internal.exceptions import InstallationError @@ -19,60 +21,70 @@ from pip._internal.models.scheme import Scheme from pip._internal.operations.build.wheel_legacy import get_legacy_build_wheel_path from pip._internal.operations.install import wheel +from pip._internal.operations.install.wheel import InstalledCSVRow, RecordPath from pip._internal.utils.compat import WINDOWS from pip._internal.utils.misc import hash_file from pip._internal.utils.unpacking import unpack_file -from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel -from tests.lib import DATA_DIR, assert_paths_equal +from tests.lib import DATA_DIR, TestData, assert_paths_equal +from tests.lib.path import Path from tests.lib.wheel import make_wheel -def call_get_legacy_build_wheel_path(caplog, names): +def call_get_legacy_build_wheel_path( + caplog: pytest.LogCaptureFixture, names: List[str] +) -> Optional[str]: wheel_path = get_legacy_build_wheel_path( names=names, - temp_dir='/tmp/abcd', - name='pendulum', - command_args=['arg1', 'arg2'], - command_output='output line 1\noutput line 2\n', + temp_dir="/tmp/abcd", + name="pendulum", + command_args=["arg1", "arg2"], + command_output="output line 1\noutput line 2\n", ) return wheel_path -def test_get_legacy_build_wheel_path(caplog): - actual = call_get_legacy_build_wheel_path(caplog, names=['name']) - assert_paths_equal(actual, '/tmp/abcd/name') +def test_get_legacy_build_wheel_path(caplog: pytest.LogCaptureFixture) -> None: + actual = call_get_legacy_build_wheel_path(caplog, names=["name"]) + assert actual is not None + assert_paths_equal(actual, "/tmp/abcd/name") assert not caplog.records -def test_get_legacy_build_wheel_path__no_names(caplog): +def test_get_legacy_build_wheel_path__no_names( + caplog: pytest.LogCaptureFixture, +) -> None: caplog.set_level(logging.INFO) actual = call_get_legacy_build_wheel_path(caplog, names=[]) assert actual is None assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'WARNING' + assert record.levelname == "WARNING" assert record.message.splitlines() == [ "Legacy build of wheel for 'pendulum' created no files.", "Command arguments: arg1 arg2", - 'Command output: [use --verbose to show]', + "Command output: [use --verbose to show]", ] -def test_get_legacy_build_wheel_path__multiple_names(caplog): +def test_get_legacy_build_wheel_path__multiple_names( + caplog: pytest.LogCaptureFixture, +) -> None: caplog.set_level(logging.INFO) # Deliberately pass the names in non-sorted order. actual = call_get_legacy_build_wheel_path( - caplog, names=['name2', 'name1'], + caplog, + names=["name2", "name1"], ) - assert_paths_equal(actual, '/tmp/abcd/name1') + assert actual is not None + assert_paths_equal(actual, "/tmp/abcd/name1") assert len(caplog.records) == 1 record = caplog.records[0] - assert record.levelname == 'WARNING' + assert record.levelname == "WARNING" assert record.message.splitlines() == [ "Legacy build of wheel for 'pendulum' created more than one file.", "Filenames (choosing first): ['name1', 'name2']", "Command arguments: arg1 arg2", - 'Command output: [use --verbose to show]', + "Command output: [use --verbose to show]", ] @@ -84,152 +96,176 @@ def test_get_legacy_build_wheel_path__multiple_names(caplog): "進入點 = 套件.模組:函式", ], ) -def test_get_entrypoints(console_scripts): +def test_get_entrypoints(tmp_path: pathlib.Path, console_scripts: str) -> None: entry_points_text = """ [console_scripts] {} [section] common:one = module:func common:two = module:other_func - """.format(console_scripts) + """.format( + console_scripts + ) - wheel_zip = make_wheel( + distribution = make_wheel( "simple", "0.1.0", extra_metadata_files={ "entry_points.txt": entry_points_text, }, - ).as_zipfile() - distribution = pkg_resources_distribution_for_wheel( - wheel_zip, "simple", "" - ) + ).as_distribution("simple") - assert wheel.get_entrypoints(distribution) == ( - dict([console_scripts.split(' = ')]), - {}, - ) + entry_point, entry_point_value = console_scripts.split(" = ") + assert wheel.get_entrypoints(distribution) == ({entry_point: entry_point_value}, {}) -def test_get_entrypoints_no_entrypoints(): - wheel_zip = make_wheel("simple", "0.1.0").as_zipfile() - distribution = pkg_resources_distribution_for_wheel( - wheel_zip, "simple", "" - ) +def test_get_entrypoints_no_entrypoints(tmp_path: pathlib.Path) -> None: + distribution = make_wheel("simple", "0.1.0").as_distribution("simple") console, gui = wheel.get_entrypoints(distribution) assert console == {} assert gui == {} -@pytest.mark.parametrize("outrows, expected", [ - ([ - ('', '', 'a'), - ('', '', ''), - ], [ - ('', '', ''), - ('', '', 'a'), - ]), - ([ - # Include an int to check avoiding the following error: - # > TypeError: '<' not supported between instances of 'str' and 'int' - ('', '', 1), - ('', '', ''), - ], [ - ('', '', ''), - ('', '', '1'), - ]), - ([ - # Test the normalization correctly encode everything for csv.writer(). - ('😉', '', 1), - ('', '', ''), - ], [ - ('', '', ''), - ('😉', '', '1'), - ]), -]) -def test_normalized_outrows(outrows, expected): +@pytest.mark.parametrize( + "outrows, expected", + [ + ( + [ + ("", "", "a"), + ("", "", ""), + ], + [ + ("", "", ""), + ("", "", "a"), + ], + ), + ( + [ + # Include an int to check avoiding the following error: + # > TypeError: '<' not supported between instances of 'str' and 'int' + ("", "", 1), + ("", "", ""), + ], + [ + ("", "", ""), + ("", "", "1"), + ], + ), + ( + [ + # Test the normalization correctly encode everything for csv.writer(). + ("😉", "", 1), + ("", "", ""), + ], + [ + ("", "", ""), + ("😉", "", "1"), + ], + ), + ], +) +def test_normalized_outrows( + outrows: List[Tuple[RecordPath, str, str]], expected: List[Tuple[str, str, str]] +) -> None: actual = wheel._normalized_outrows(outrows) assert actual == expected -def call_get_csv_rows_for_installed(tmpdir, text): - path = tmpdir.joinpath('temp.txt') +def call_get_csv_rows_for_installed(tmpdir: Path, text: str) -> List[InstalledCSVRow]: + path = tmpdir.joinpath("temp.txt") path.write_text(text) # Test that an installed file appearing in RECORD has its filename # updated in the new RECORD file. - installed = {'a': 'z'} - changed = set() - generated = [] - lib_dir = '/lib/dir' + installed = cast(Dict[RecordPath, RecordPath], {"a": "z"}) + lib_dir = "/lib/dir" - with open(path, **wheel.csv_io_kwargs('r')) as f: + with open(path, **wheel.csv_io_kwargs("r")) as f: record_rows = list(csv.reader(f)) outrows = wheel.get_csv_rows_for_installed( - record_rows, installed=installed, changed=changed, - generated=generated, lib_dir=lib_dir, + record_rows, + installed=installed, + changed=set(), + generated=[], + lib_dir=lib_dir, ) return outrows -def test_get_csv_rows_for_installed(tmpdir, caplog): - text = textwrap.dedent("""\ +def test_get_csv_rows_for_installed( + tmpdir: Path, caplog: pytest.LogCaptureFixture +) -> None: + text = textwrap.dedent( + """\ a,b,c d,e,f - """) + """ + ) outrows = call_get_csv_rows_for_installed(tmpdir, text) expected = [ - ('z', 'b', 'c'), - ('d', 'e', 'f'), + ("z", "b", "c"), + ("d", "e", "f"), ] assert outrows == expected # Check there were no warnings. assert len(caplog.records) == 0 -def test_get_csv_rows_for_installed__long_lines(tmpdir, caplog): - text = textwrap.dedent("""\ +def test_get_csv_rows_for_installed__long_lines( + tmpdir: Path, caplog: pytest.LogCaptureFixture +) -> None: + text = textwrap.dedent( + """\ a,b,c,d e,f,g h,i,j,k - """) + """ + ) outrows = call_get_csv_rows_for_installed(tmpdir, text) - - expected = [ - ('z', 'b', 'c'), - ('e', 'f', 'g'), - ('h', 'i', 'j'), + assert outrows == [ + ("z", "b", "c"), + ("e", "f", "g"), + ("h", "i", "j"), ] - assert outrows == expected messages = [rec.message for rec in caplog.records] - expected = [ + assert messages == [ "RECORD line has more than three elements: ['a', 'b', 'c', 'd']", - "RECORD line has more than three elements: ['h', 'i', 'j', 'k']" + "RECORD line has more than three elements: ['h', 'i', 'j', 'k']", ] - assert messages == expected - - -@pytest.mark.parametrize("text,expected", [ - ("Root-Is-Purelib: true", True), - ("Root-Is-Purelib: false", False), - ("Root-Is-Purelib: hello", False), - ("", False), - ("root-is-purelib: true", True), - ("root-is-purelib: True", True), -]) -def test_wheel_root_is_purelib(text, expected): + + +@pytest.mark.parametrize( + "text,expected", + [ + ("Root-Is-Purelib: true", True), + ("Root-Is-Purelib: false", False), + ("Root-Is-Purelib: hello", False), + ("", False), + ("root-is-purelib: true", True), + ("root-is-purelib: True", True), + ], +) +def test_wheel_root_is_purelib(text: str, expected: bool) -> None: assert wheel.wheel_root_is_purelib(message_from_string(text)) == expected -class TestWheelFile: +def test_dist_from_broken_wheel_fails(data: TestData) -> None: + from pip._internal.exceptions import InvalidWheel + from pip._internal.metadata import FilesystemWheel, get_wheel_distribution - def test_unpack_wheel_no_flatten(self, tmpdir): - filepath = os.path.join(DATA_DIR, 'packages', - 'meta-1.0-py2.py3-none-any.whl') + package = data.packages.joinpath("corruptwheel-1.0-py2.py3-none-any.whl") + with pytest.raises(InvalidWheel): + get_wheel_distribution(FilesystemWheel(package), "brokenwheel") + + +class TestWheelFile: + def test_unpack_wheel_no_flatten(self, tmpdir: Path) -> None: + filepath = os.path.join(DATA_DIR, "packages", "meta-1.0-py2.py3-none-any.whl") unpack_file(filepath, tmpdir) - assert os.path.isdir(os.path.join(tmpdir, 'meta-1.0.dist-info')) + assert os.path.isdir(os.path.join(tmpdir, "meta-1.0.dist-info")) class TestInstallUnpackedWheel: @@ -237,13 +273,13 @@ class TestInstallUnpackedWheel: Tests for moving files from wheel src to scheme paths """ - def prep(self, data, tmpdir): + def prep(self, data: TestData, tmpdir: str) -> None: # Since Path implements __add__, os.path.join returns a Path object. # Passing Path objects to interfaces expecting str (like # `compileall.compile_file`) can cause failures, so we normalize it # to a string here. tmpdir = str(tmpdir) - self.name = 'sample' + self.name = "sample" self.wheelpath = make_wheel( "sample", "1.2.0", @@ -290,43 +326,41 @@ def main(): "gui_scripts": ["sample2 = sample:main"], }, ).save_to_dir(tmpdir) - self.req = Requirement('sample') - self.src = os.path.join(tmpdir, 'src') - self.dest = os.path.join(tmpdir, 'dest') + self.req = Requirement("sample") + self.src = os.path.join(tmpdir, "src") + self.dest = os.path.join(tmpdir, "dest") self.scheme = Scheme( - purelib=os.path.join(self.dest, 'lib'), - platlib=os.path.join(self.dest, 'lib'), - headers=os.path.join(self.dest, 'headers'), - scripts=os.path.join(self.dest, 'bin'), - data=os.path.join(self.dest, 'data'), + purelib=os.path.join(self.dest, "lib"), + platlib=os.path.join(self.dest, "lib"), + headers=os.path.join(self.dest, "headers"), + scripts=os.path.join(self.dest, "bin"), + data=os.path.join(self.dest, "data"), ) - self.src_dist_info = os.path.join( - self.src, 'sample-1.2.0.dist-info') + self.src_dist_info = os.path.join(self.src, "sample-1.2.0.dist-info") self.dest_dist_info = os.path.join( - self.scheme.purelib, 'sample-1.2.0.dist-info') + self.scheme.purelib, "sample-1.2.0.dist-info" + ) - def assert_permission(self, path, mode): + def assert_permission(self, path: str, mode: int) -> None: target_mode = os.stat(path).st_mode & 0o777 assert (target_mode & mode) == mode, oct(target_mode) - def assert_installed(self, expected_permission): + def assert_installed(self, expected_permission: int) -> None: # lib - assert os.path.isdir( - os.path.join(self.scheme.purelib, 'sample')) + assert os.path.isdir(os.path.join(self.scheme.purelib, "sample")) # dist-info - metadata = os.path.join(self.dest_dist_info, 'METADATA') + metadata = os.path.join(self.dest_dist_info, "METADATA") self.assert_permission(metadata, expected_permission) - record = os.path.join(self.dest_dist_info, 'RECORD') + record = os.path.join(self.dest_dist_info, "RECORD") self.assert_permission(record, expected_permission) # data files - data_file = os.path.join(self.scheme.data, 'my_data', 'data_file') + data_file = os.path.join(self.scheme.data, "my_data", "data_file") assert os.path.isfile(data_file) # package data - pkg_data = os.path.join( - self.scheme.purelib, 'sample', 'package_data.dat') + pkg_data = os.path.join(self.scheme.purelib, "sample", "package_data.dat") assert os.path.isfile(pkg_data) - def test_std_install(self, data, tmpdir): + def test_std_install(self, data: TestData, tmpdir: Path) -> None: self.prep(data, tmpdir) wheel.install_wheel( self.name, @@ -336,11 +370,10 @@ def test_std_install(self, data, tmpdir): ) self.assert_installed(0o644) - @pytest.mark.parametrize("user_mask, expected_permission", [ - (0o27, 0o640) - ]) - def test_std_install_with_custom_umask(self, data, tmpdir, - user_mask, expected_permission): + @pytest.mark.parametrize("user_mask, expected_permission", [(0o27, 0o640)]) + def test_std_install_with_custom_umask( + self, data: TestData, tmpdir: Path, user_mask: int, expected_permission: int + ) -> None: """Test that the files created after install honor the permissions set when the user sets a custom umask""" @@ -357,7 +390,7 @@ def test_std_install_with_custom_umask(self, data, tmpdir, finally: os.umask(prev_umask) - def test_std_install_requested(self, data, tmpdir): + def test_std_install_requested(self, data: TestData, tmpdir: Path) -> None: self.prep(data, tmpdir) wheel.install_wheel( self.name, @@ -367,10 +400,10 @@ def test_std_install_requested(self, data, tmpdir): requested=True, ) self.assert_installed(0o644) - requested_path = os.path.join(self.dest_dist_info, 'REQUESTED') + requested_path = os.path.join(self.dest_dist_info, "REQUESTED") assert os.path.isfile(requested_path) - def test_std_install_with_direct_url(self, data, tmpdir): + def test_std_install_with_direct_url(self, data: TestData, tmpdir: Path) -> None: """Test that install_wheel creates direct_url.json metadata when provided with a direct_url argument. Also test that the RECORDS file contains an entry for direct_url.json in that case. @@ -389,26 +422,24 @@ def test_std_install_with_direct_url(self, data, tmpdir): req_description=str(self.req), direct_url=direct_url, ) - direct_url_path = os.path.join( - self.dest_dist_info, DIRECT_URL_METADATA_NAME - ) + direct_url_path = os.path.join(self.dest_dist_info, DIRECT_URL_METADATA_NAME) self.assert_permission(direct_url_path, 0o644) - with open(direct_url_path, 'rb') as f: + with open(direct_url_path, "rb") as f1: expected_direct_url_json = direct_url.to_json() - direct_url_json = f.read().decode("utf-8") + direct_url_json = f1.read().decode("utf-8") assert direct_url_json == expected_direct_url_json # check that the direc_url file is part of RECORDS - with open(os.path.join(self.dest_dist_info, "RECORD")) as f: - assert DIRECT_URL_METADATA_NAME in f.read() + with open(os.path.join(self.dest_dist_info, "RECORD")) as f2: + assert DIRECT_URL_METADATA_NAME in f2.read() - def test_install_prefix(self, data, tmpdir): - prefix = os.path.join(os.path.sep, 'some', 'path') + def test_install_prefix(self, data: TestData, tmpdir: Path) -> None: + prefix = os.path.join(os.path.sep, "some", "path") self.prep(data, tmpdir) scheme = get_scheme( self.name, user=False, home=None, - root=tmpdir, + root=str(tmpdir), # Casting needed for CPython 3.10+. See GH-10358. isolated=False, prefix=prefix, ) @@ -419,11 +450,11 @@ def test_install_prefix(self, data, tmpdir): req_description=str(self.req), ) - bin_dir = 'Scripts' if WINDOWS else 'bin' - assert os.path.exists(os.path.join(tmpdir, 'some', 'path', bin_dir)) - assert os.path.exists(os.path.join(tmpdir, 'some', 'path', 'my_data')) + bin_dir = "Scripts" if WINDOWS else "bin" + assert os.path.exists(os.path.join(tmpdir, "some", "path", bin_dir)) + assert os.path.exists(os.path.join(tmpdir, "some", "path", "my_data")) - def test_dist_info_contains_empty_dir(self, data, tmpdir): + def test_dist_info_contains_empty_dir(self, data: TestData, tmpdir: Path) -> None: """ Test that empty dirs are not installed """ @@ -436,14 +467,12 @@ def test_dist_info_contains_empty_dir(self, data, tmpdir): req_description=str(self.req), ) self.assert_installed(0o644) - assert not os.path.isdir( - os.path.join(self.dest_dist_info, 'empty_dir')) + assert not os.path.isdir(os.path.join(self.dest_dist_info, "empty_dir")) - @pytest.mark.parametrize( - "path", - ["/tmp/example", "../example", "./../example"] - ) - def test_wheel_install_rejects_bad_paths(self, data, tmpdir, path): + @pytest.mark.parametrize("path", ["/tmp/example", "../example", "./../example"]) + def test_wheel_install_rejects_bad_paths( + self, data: TestData, tmpdir: Path, path: str + ) -> None: self.prep(data, tmpdir) wheel_path = make_wheel( "simple", "0.1.0", extra_files={path: "example contents\n"} @@ -461,15 +490,11 @@ def test_wheel_install_rejects_bad_paths(self, data, tmpdir, path): assert "example" in exc_text @pytest.mark.xfail(strict=True) - @pytest.mark.parametrize( - "entrypoint", ["hello = hello", "hello = hello:"] - ) - @pytest.mark.parametrize( - "entrypoint_type", ["console_scripts", "gui_scripts"] - ) + @pytest.mark.parametrize("entrypoint", ["hello = hello", "hello = hello:"]) + @pytest.mark.parametrize("entrypoint_type", ["console_scripts", "gui_scripts"]) def test_invalid_entrypoints_fail( - self, data, tmpdir, entrypoint, entrypoint_type - ): + self, data: TestData, tmpdir: Path, entrypoint: str, entrypoint_type: str + ) -> None: self.prep(data, tmpdir) wheel_path = make_wheel( "simple", "0.1.0", entry_points={entrypoint_type: [entrypoint]} @@ -494,41 +519,34 @@ class TestMessageAboutScriptsNotOnPATH: "which may not be expanded by all applications." ) - def _template(self, paths, scripts): - with patch.dict('os.environ', {'PATH': os.pathsep.join(paths)}): + def _template(self, paths: List[str], scripts: List[str]) -> Optional[str]: + with patch.dict("os.environ", {"PATH": os.pathsep.join(paths)}): return wheel.message_about_scripts_not_on_PATH(scripts) - def test_no_script(self): - retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=[] - ) + def test_no_script(self) -> None: + retval = self._template(paths=["/a/b", "/c/d/bin"], scripts=[]) assert retval is None - def test_single_script__single_dir_not_on_PATH(self): - retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/c/d/foo'] - ) + def test_single_script__single_dir_not_on_PATH(self) -> None: + retval = self._template(paths=["/a/b", "/c/d/bin"], scripts=["/c/d/foo"]) assert retval is not None assert "--no-warn-script-location" in retval assert "foo is installed in '/c/d'" in retval assert self.tilde_warning_msg not in retval - def test_two_script__single_dir_not_on_PATH(self): + def test_two_script__single_dir_not_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/c/d/foo', '/c/d/baz'] + paths=["/a/b", "/c/d/bin"], scripts=["/c/d/foo", "/c/d/baz"] ) assert retval is not None assert "--no-warn-script-location" in retval assert "baz and foo are installed in '/c/d'" in retval assert self.tilde_warning_msg not in retval - def test_multi_script__multi_dir_not_on_PATH(self): + def test_multi_script__multi_dir_not_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/c/d/foo', '/c/d/bar', '/c/d/baz', '/a/b/c/spam'] + paths=["/a/b", "/c/d/bin"], + scripts=["/c/d/foo", "/c/d/bar", "/c/d/baz", "/a/b/c/spam"], ) assert retval is not None assert "--no-warn-script-location" in retval @@ -536,13 +554,10 @@ def test_multi_script__multi_dir_not_on_PATH(self): assert "spam is installed in '/a/b/c'" in retval assert self.tilde_warning_msg not in retval - def test_multi_script_all__multi_dir_not_on_PATH(self): + def test_multi_script_all__multi_dir_not_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=[ - '/c/d/foo', '/c/d/bar', '/c/d/baz', - '/a/b/c/spam', '/a/b/c/eggs' - ] + paths=["/a/b", "/c/d/bin"], + scripts=["/c/d/foo", "/c/d/bar", "/c/d/baz", "/a/b/c/spam", "/a/b/c/eggs"], ) assert retval is not None assert "--no-warn-script-location" in retval @@ -550,77 +565,71 @@ def test_multi_script_all__multi_dir_not_on_PATH(self): assert "eggs and spam are installed in '/a/b/c'" in retval assert self.tilde_warning_msg not in retval - def test_two_script__single_dir_on_PATH(self): + def test_two_script__single_dir_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/a/b/foo', '/a/b/baz'] + paths=["/a/b", "/c/d/bin"], scripts=["/a/b/foo", "/a/b/baz"] ) assert retval is None - def test_multi_script__multi_dir_on_PATH(self): + def test_multi_script__multi_dir_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/a/b/foo', '/a/b/bar', '/a/b/baz', '/c/d/bin/spam'] + paths=["/a/b", "/c/d/bin"], + scripts=["/a/b/foo", "/a/b/bar", "/a/b/baz", "/c/d/bin/spam"], ) assert retval is None - def test_multi_script__single_dir_on_PATH(self): + def test_multi_script__single_dir_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/a/b/foo', '/a/b/bar', '/a/b/baz'] + paths=["/a/b", "/c/d/bin"], scripts=["/a/b/foo", "/a/b/bar", "/a/b/baz"] ) assert retval is None - def test_single_script__single_dir_on_PATH(self): - retval = self._template( - paths=['/a/b', '/c/d/bin'], - scripts=['/a/b/foo'] - ) + def test_single_script__single_dir_on_PATH(self) -> None: + retval = self._template(paths=["/a/b", "/c/d/bin"], scripts=["/a/b/foo"]) assert retval is None - def test_PATH_check_case_insensitive_on_windows(self): - retval = self._template( - paths=['C:\\A\\b'], - scripts=['c:\\a\\b\\c', 'C:/A/b/d'] - ) + def test_PATH_check_case_insensitive_on_windows(self) -> None: + retval = self._template(paths=["C:\\A\\b"], scripts=["c:\\a\\b\\c", "C:/A/b/d"]) if WINDOWS: assert retval is None else: assert retval is not None assert self.tilde_warning_msg not in retval - def test_trailing_ossep_removal(self): + def test_trailing_ossep_removal(self) -> None: retval = self._template( - paths=[os.path.join('a', 'b', '')], - scripts=[os.path.join('a', 'b', 'c')] + paths=[os.path.join("a", "b", "")], scripts=[os.path.join("a", "b", "c")] ) assert retval is None - def test_missing_PATH_env_treated_as_empty_PATH_env(self, monkeypatch): - scripts = ['a/b/foo'] + def test_missing_PATH_env_treated_as_empty_PATH_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + scripts = ["a/b/foo"] - monkeypatch.delenv('PATH') + monkeypatch.delenv("PATH") retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) - monkeypatch.setenv('PATH', '') + monkeypatch.setenv("PATH", "") retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) assert retval_missing == retval_empty - def test_no_script_tilde_in_path(self): - retval = self._template( - paths=['/a/b', '/c/d/bin', '~/e', '/f/g~g'], - scripts=[] - ) + def test_no_script_tilde_in_path(self) -> None: + retval = self._template(paths=["/a/b", "/c/d/bin", "~/e", "/f/g~g"], scripts=[]) assert retval is None - def test_multi_script_all_tilde__multi_dir_not_on_PATH(self): + def test_multi_script_all_tilde__multi_dir_not_on_PATH(self) -> None: retval = self._template( - paths=['/a/b', '/c/d/bin', '~e/f'], + paths=["/a/b", "/c/d/bin", "~e/f"], scripts=[ - '/c/d/foo', '/c/d/bar', '/c/d/baz', - '/a/b/c/spam', '/a/b/c/eggs', '/e/f/tilde' - ] + "/c/d/foo", + "/c/d/bar", + "/c/d/baz", + "/a/b/c/spam", + "/a/b/c/eggs", + "/e/f/tilde", + ], ) assert retval is not None assert "--no-warn-script-location" in retval @@ -629,13 +638,16 @@ def test_multi_script_all_tilde__multi_dir_not_on_PATH(self): assert "tilde is installed in '/e/f'" in retval assert self.tilde_warning_msg in retval - def test_multi_script_all_tilde_not_at_start__multi_dir_not_on_PATH(self): + def test_multi_script_all_tilde_not_at_start__multi_dir_not_on_PATH(self) -> None: retval = self._template( - paths=['/e/f~f', '/c/d/bin'], + paths=["/e/f~f", "/c/d/bin"], scripts=[ - '/c/d/foo', '/c/d/bar', '/c/d/baz', - '/e/f~f/c/spam', '/e/f~f/c/eggs' - ] + "/c/d/foo", + "/c/d/bar", + "/c/d/baz", + "/e/f~f/c/spam", + "/e/f~f/c/eggs", + ], ) assert retval is not None assert "--no-warn-script-location" in retval @@ -645,25 +657,26 @@ def test_multi_script_all_tilde_not_at_start__multi_dir_not_on_PATH(self): class TestWheelHashCalculators: - - def prep(self, tmpdir): + def prep(self, tmpdir: Path) -> None: self.test_file = tmpdir.joinpath("hash.file") # Want this big enough to trigger the internal read loops. self.test_file_len = 2 * 1024 * 1024 with open(str(self.test_file), "w") as fp: fp.truncate(self.test_file_len) - self.test_file_hash = \ - '5647f05ec18958947d32874eeb788fa396a05d0bab7c1b71f112ceb7e9b31eee' - self.test_file_hash_encoded = \ - 'sha256=VkfwXsGJWJR9ModO63iPo5agXQurfBtx8RLOt-mzHu4' + self.test_file_hash = ( + "5647f05ec18958947d32874eeb788fa396a05d0bab7c1b71f112ceb7e9b31eee" + ) + self.test_file_hash_encoded = ( + "sha256=VkfwXsGJWJR9ModO63iPo5agXQurfBtx8RLOt-mzHu4" + ) - def test_hash_file(self, tmpdir): + def test_hash_file(self, tmpdir: Path) -> None: self.prep(tmpdir) h, length = hash_file(self.test_file) assert length == self.test_file_len assert h.hexdigest() == self.test_file_hash - def test_rehash(self, tmpdir): + def test_rehash(self, tmpdir: Path) -> None: self.prep(tmpdir) h, length = wheel.rehash(self.test_file) assert length == str(self.test_file_len) diff --git a/tests/unit/test_wheel_builder.py b/tests/unit/test_wheel_builder.py index bd42f305956..0468273d66a 100644 --- a/tests/unit/test_wheel_builder.py +++ b/tests/unit/test_wheel_builder.py @@ -1,12 +1,14 @@ import logging +from typing import Optional, cast +from unittest import mock import pytest -from mock import patch from pip._internal import wheel_builder from pip._internal.models.link import Link from pip._internal.operations.build.wheel_legacy import format_command_result -from tests.lib import _create_test_package +from pip._internal.req.req_install import InstallRequirement +from tests.lib import PipTestEnvironment, _create_test_package @pytest.mark.parametrize( @@ -14,33 +16,31 @@ [ # Trivial. ("pip-18.0", True), - # Ambiguous. ("foo-2-2", True), ("im-valid", True), - # Invalid. ("invalid", False), ("im_invalid", False), ], ) -def test_contains_egg_info(s, expected): +def test_contains_egg_info(s: str, expected: bool) -> None: result = wheel_builder._contains_egg_info(s) assert result == expected class ReqMock: - def __init__( self, - name="pendulum", - is_wheel=False, - editable=False, - link=None, - constraint=False, - source_dir="/tmp/pip-install-123/pendulum", - use_pep517=True, - ): + name: str = "pendulum", + is_wheel: bool = False, + editable: bool = False, + link: Optional[Link] = None, + constraint: bool = False, + source_dir: Optional[str] = "/tmp/pip-install-123/pendulum", + use_pep517: bool = True, + supports_pyproject_editable: bool = False, + ) -> None: self.name = name self.is_wheel = is_wheel self.editable = editable @@ -48,6 +48,10 @@ def __init__( self.constraint = constraint self.source_dir = source_dir self.use_pep517 = use_pep517 + self._supports_pyproject_editable = supports_pyproject_editable + + def supports_pyproject_editable(self) -> bool: + return self._supports_pyproject_editable @pytest.mark.parametrize( @@ -64,8 +68,17 @@ def __init__( (ReqMock(constraint=True), False, False), # We don't build reqs that are already wheels. (ReqMock(is_wheel=True), False, False), - # We don't build editables. - (ReqMock(editable=True), False, False), + (ReqMock(editable=True, use_pep517=False), False, False), + ( + ReqMock(editable=True, use_pep517=True, supports_pyproject_editable=True), + False, + True, + ), + ( + ReqMock(editable=True, use_pep517=True, supports_pyproject_editable=False), + False, + False, + ), (ReqMock(source_dir=None), False, False), # By default (i.e. when binaries are allowed), VCS requirements # should be built in install mode. @@ -93,9 +106,11 @@ def __init__( ), ], ) -def test_should_build_for_install_command(req, disallow_binaries, expected): +def test_should_build_for_install_command( + req: ReqMock, disallow_binaries: bool, expected: bool +) -> None: should_build = wheel_builder.should_build_for_install_command( - req, + cast(InstallRequirement, req), check_binary_allowed=lambda req: not disallow_binaries, ) assert should_build is expected @@ -107,33 +122,36 @@ def test_should_build_for_install_command(req, disallow_binaries, expected): (ReqMock(), True), (ReqMock(constraint=True), False), (ReqMock(is_wheel=True), False), - (ReqMock(editable=True), True), + (ReqMock(editable=True, use_pep517=False), True), + (ReqMock(editable=True, use_pep517=True), True), (ReqMock(source_dir=None), True), (ReqMock(link=Link("git+https://g.c/org/repo")), True), ], ) -def test_should_build_for_wheel_command(req, expected): - should_build = wheel_builder.should_build_for_wheel_command(req) +def test_should_build_for_wheel_command(req: ReqMock, expected: bool) -> None: + should_build = wheel_builder.should_build_for_wheel_command( + cast(InstallRequirement, req) + ) assert should_build is expected -@patch("pip._internal.wheel_builder.is_wheel_installed") -def test_should_build_legacy_wheel_not_installed(is_wheel_installed): +@mock.patch("pip._internal.wheel_builder.is_wheel_installed") +def test_should_build_legacy_wheel_not_installed(is_wheel_installed: mock.Mock) -> None: is_wheel_installed.return_value = False legacy_req = ReqMock(use_pep517=False) should_build = wheel_builder.should_build_for_install_command( - legacy_req, + cast(InstallRequirement, legacy_req), check_binary_allowed=lambda req: True, ) assert not should_build -@patch("pip._internal.wheel_builder.is_wheel_installed") -def test_should_build_legacy_wheel_installed(is_wheel_installed): +@mock.patch("pip._internal.wheel_builder.is_wheel_installed") +def test_should_build_legacy_wheel_installed(is_wheel_installed: mock.Mock) -> None: is_wheel_installed.return_value = True legacy_req = ReqMock(use_pep517=False) should_build = wheel_builder.should_build_for_install_command( - legacy_req, + cast(InstallRequirement, legacy_req), check_binary_allowed=lambda req: True, ) assert should_build @@ -142,76 +160,81 @@ def test_should_build_legacy_wheel_installed(is_wheel_installed): @pytest.mark.parametrize( "req, expected", [ - (ReqMock(editable=True), False), + (ReqMock(editable=True, use_pep517=False), False), + (ReqMock(editable=True, use_pep517=True), False), (ReqMock(source_dir=None), False), (ReqMock(link=Link("git+https://g.c/org/repo")), False), (ReqMock(link=Link("https://g.c/dist.tgz")), False), (ReqMock(link=Link("https://g.c/dist-2.0.4.tgz")), True), ], ) -def test_should_cache(req, expected): - assert wheel_builder._should_cache(req) is expected +def test_should_cache(req: ReqMock, expected: bool) -> None: + assert wheel_builder._should_cache(cast(InstallRequirement, req)) is expected -def test_should_cache_git_sha(script, tmpdir): +def test_should_cache_git_sha(script: PipTestEnvironment) -> None: repo_path = _create_test_package(script, name="mypkg") - commit = script.run( - "git", "rev-parse", "HEAD", cwd=repo_path - ).stdout.strip() + commit = script.run("git", "rev-parse", "HEAD", cwd=repo_path).stdout.strip() # a link referencing a sha should be cached url = "git+https://g.c/o/r@" + commit + "#egg=mypkg" req = ReqMock(link=Link(url), source_dir=repo_path) - assert wheel_builder._should_cache(req) + assert wheel_builder._should_cache(cast(InstallRequirement, req)) # a link not referencing a sha should not be cached url = "git+https://g.c/o/r@master#egg=mypkg" req = ReqMock(link=Link(url), source_dir=repo_path) - assert not wheel_builder._should_cache(req) + assert not wheel_builder._should_cache(cast(InstallRequirement, req)) -def test_format_command_result__INFO(caplog): +def test_format_command_result__INFO(caplog: pytest.LogCaptureFixture) -> None: caplog.set_level(logging.INFO) actual = format_command_result( # Include an argument with a space to test argument quoting. - command_args=['arg1', 'second arg'], - command_output='output line 1\noutput line 2\n', + command_args=["arg1", "second arg"], + command_output="output line 1\noutput line 2\n", ) assert actual.splitlines() == [ "Command arguments: arg1 'second arg'", - 'Command output: [use --verbose to show]', + "Command output: [use --verbose to show]", ] -@pytest.mark.parametrize('command_output', [ - # Test trailing newline. - 'output line 1\noutput line 2\n', - # Test no trailing newline. - 'output line 1\noutput line 2', -]) -def test_format_command_result__DEBUG(caplog, command_output): +@pytest.mark.parametrize( + "command_output", + [ + # Test trailing newline. + "output line 1\noutput line 2\n", + # Test no trailing newline. + "output line 1\noutput line 2", + ], +) +def test_format_command_result__DEBUG( + caplog: pytest.LogCaptureFixture, command_output: str +) -> None: caplog.set_level(logging.DEBUG) actual = format_command_result( - command_args=['arg1', 'arg2'], + command_args=["arg1", "arg2"], command_output=command_output, ) assert actual.splitlines() == [ "Command arguments: arg1 arg2", - 'Command output:', - 'output line 1', - 'output line 2', - '----------------------------------------', + "Command output:", + "output line 1", + "output line 2", ] -@pytest.mark.parametrize('log_level', ['DEBUG', 'INFO']) -def test_format_command_result__empty_output(caplog, log_level): +@pytest.mark.parametrize("log_level", ["DEBUG", "INFO"]) +def test_format_command_result__empty_output( + caplog: pytest.LogCaptureFixture, log_level: str +) -> None: caplog.set_level(log_level) actual = format_command_result( - command_args=['arg1', 'arg2'], - command_output='', + command_args=["arg1", "arg2"], + command_output="", ) assert actual.splitlines() == [ "Command arguments: arg1 arg2", - 'Command output: None', + "Command output: None", ] diff --git a/tests/yaml/ERRORS.md b/tests/yaml/ERRORS.md deleted file mode 100644 index 700e3d4ea5d..00000000000 --- a/tests/yaml/ERRORS.md +++ /dev/null @@ -1,60 +0,0 @@ -# New resolver error messages - - -## Incompatible requirements - -Most resolver error messages are due to incompatible requirements. -That is, the dependency tree contains conflicting versions of the same -package. Take the example: - - base: - available: - - A 1.0.0; depends B == 1.0.0, C == 2.0.0 - - B 1.0.0; depends C == 1.0.0 - - C 1.0.0 - - C 2.0.0 - -Here, `A` cannot be installed because it depends on `B` (which depends on -a different version of `C` than `A` itself. In real world examples, the -conflicting version are not so easy to spot. I'm suggesting an error -message which looks something like this: - - A 1.0.0 -> B 1.0.0 -> C 1.0.0 - A 1.0.0 -> C 2.0.0 - -That is, for the conflicting package, we show the user where exactly the -requirement came from. - - -## Double requirement - -I've noticed that in many cases the old resolver messages are more -informative. For example, in the simple example: - - base: - available: - - B 1.0.0 - - B 2.0.0 - -Now if we want to install both version of `B` at the same time, -i.e. the requirement `B==1.0.0 B==2.0.0`, we get: - - ERROR: Could not find a version that satisfies the requirement B==1.0.0 - ERROR: Could not find a version that satisfies the requirement B==2.0.0 - No matching distribution found for b, b - -Even though both version are actually available and satisfy each requirement, -just not at once. When trying to install a version of `B` which does not -exist, say requirement `B==1.5.0`, you get the same type of error message: - - Could not find a version that satisfies the requirement B==1.5.0 - No matching distribution found for b - -For this case, the old error message was: - - Could not find a version that satisfies the requirement B==1.5.0 (from versions: 1.0.0, 2.0.0) - No matching distribution found for B==1.5.0 - -And the old error message for the requirement `B==1.0.0 B==2.0.0`: - - Double requirement given: B==2.0.0 (already in B==1.0.0, name='B') diff --git a/tests/yaml/README.md b/tests/yaml/README.md deleted file mode 100644 index 1a379fdcbb0..00000000000 --- a/tests/yaml/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# YAML tests for pip's resolver - -This directory contains fixtures for testing pip's resolver. -The fixtures are written as `.yml` files, with a convenient format -that allows for specifying a custom index for temporary use. - -The `.yml` files are typically organized in the following way. Here, we are -going to take a closer look at the `simple.yml` file and step through the -test cases. A `base` section defines which packages are available upstream: - - base: - available: - - simple 0.1.0 - - simple 0.2.0 - - base 0.1.0; depends dep - - dep 0.1.0 - -Each package has a name and version number. Here, there are two -packages `simple` (with versoin `0.1.0` and `0.2.0`). The package -`base 0.1.0` depends on the requirement `dep` (which simply means it -depends on any version of `dep`. More generally, a package can also -depend on a specific version of another package, or a range of versions. - -Next, in our yaml file, we have the `cases:` section which is a list of -test cases. Each test case has a request and a response. The request -is what the user would want to do: - - cases: - - - request: - - install: simple - - uninstall: simple - response: - - state: - - simple 0.2.0 - - state: null - -Here the first request is to install the package simple, this would -basically be equivalent to typing `pip install simple`, and the corresponding -first response is that the state of installed packages is `simple 0.2.0`. -Note that by default the highest version of an available package will be -installed. - -The second request is to uninstall simple again, which will result in the -state `null` (basically an empty list of installed packages). - -When the yaml tests are run, each response is verified by checking which -packages got actually installed. Note that this is check is done in -alphabetical order. - - - -The linter is very useful for initally checking `.yml` files, e.g.: - - $ python linter.py -v simple.yml - -To run only the yaml tests, use (from the root of the source tree): - - $ tox -e py38 -- -m yaml -vv - -Or, in order to avoid collecting all the test cases: - - $ tox -e py38 -- tests/functional/test_yaml.py - -Or, only a specific test: - - $ tox -e py38 -- tests/functional/test_yaml.py -k simple - -Or, just a specific test case: - - $ tox -e py38 -- tests/functional/test_yaml.py -k simple-0 - - - diff --git a/tests/yaml/backtrack.yml b/tests/yaml/backtrack.yml deleted file mode 100644 index ffcb722b88c..00000000000 --- a/tests/yaml/backtrack.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Pradyun's backtracking example -base: - available: - - A 1.0.0; depends B == 1.0.0 - - A 2.0.0; depends B == 2.0.0, C == 1.0.0 - - A 3.0.0; depends B == 3.0.0, C == 2.0.0 - - A 4.0.0; depends B == 4.0.0, C == 3.0.0 - - A 5.0.0; depends B == 5.0.0, C == 4.0.0 - - A 6.0.0; depends B == 6.0.0, C == 5.0.0 - - A 7.0.0; depends B == 7.0.0, C == 6.0.0 - - A 8.0.0; depends B == 8.0.0, C == 7.0.0 - - - B 1.0.0; depends C == 1.0.0 - - B 2.0.0; depends C == 2.0.0 - - B 3.0.0; depends C == 3.0.0 - - B 4.0.0; depends C == 4.0.0 - - B 5.0.0; depends C == 5.0.0 - - B 6.0.0; depends C == 6.0.0 - - B 7.0.0; depends C == 7.0.0 - - B 8.0.0; depends C == 8.0.0 - - - C 1.0.0 - - C 2.0.0 - - C 3.0.0 - - C 4.0.0 - - C 5.0.0 - - C 6.0.0 - - C 7.0.0 - - C 8.0.0 - -cases: -- - request: - - install: A - response: - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - skip: legacy diff --git a/tests/yaml/circular.yml b/tests/yaml/circular.yml deleted file mode 100644 index 95c535454fa..00000000000 --- a/tests/yaml/circular.yml +++ /dev/null @@ -1,45 +0,0 @@ -base: - available: - - A 1.0.0; depends B == 1.0.0 - - B 1.0.0; depends C == 1.0.0 - - C 1.0.0; depends D == 1.0.0 - - D 1.0.0; depends A == 1.0.0 - -cases: -# NOTE: Do we want to check the order? -- - request: - - install: A - response: - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - - D 1.0.0 -- - request: - - install: B - response: - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - - D 1.0.0 -- - request: - - install: C - response: - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - - D 1.0.0 -- - request: - - install: D - response: - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - - D 1.0.0 diff --git a/tests/yaml/conflict_1.yml b/tests/yaml/conflict_1.yml deleted file mode 100644 index dc18be32a1f..00000000000 --- a/tests/yaml/conflict_1.yml +++ /dev/null @@ -1,77 +0,0 @@ -base: - available: - - A 1.0.0; depends B == 1.0.0, B == 2.0.0 - - B 1.0.0 - - B 2.0.0 - -cases: -- - request: - - install: A - response: - - error: - code: 0 - stderr: ['incompatible'] - skip: legacy - # -- a good error message would be: - # A 1.0.0 has incompatible requirements B==1.0.0, B==2.0.0 - -- - request: - - install: ['B==1.0.0', 'B'] - response: - - state: - - B 1.0.0 - skip: legacy - # -- old error: - # Double requirement given: B (already in B==1.0.0, name='B') - -- - request: - - install: ['B==1.0.0', 'B==2.0.0'] - response: - - state: null - error: - code: 1 - stderr: >- - Cannot install B==1.0.0 and B==2.0.0 because these - package versions have conflicting dependencies. - skip: legacy - # -- currently the (new resolver) error message is: - # Could not find a version that satisfies the requirement B==1.0.0 - # Could not find a version that satisfies the requirement B==2.0.0 - # No matching distribution found for b, b - # -- better would be: - # cannot install different version (1.0.0, 2.0.0) of package B at the - # same time. - # -- the old error message was actually better here: - # Double requirement given: B==2.0.0 (already in B==1.0.0, name='B') - -- - request: - - install: B==1.5.0 - response: - - state: null - error: - code: 1 - stderr: 'no\s+matching\s+distribution' - skip: legacy - # -- currently (new resolver) error message is: - # Could not find a version that satisfies the requirement B==1.5.0 - # No matching distribution found for b - # -- the old error message was actually better here: - # Could not find a version that satisfies the requirement B==1.5.0 (from versions: 1.0.0, 2.0.0) - # No matching distribution found for B==1.5.0 - -- - request: - - install: A==2.0 - response: - - state: null - error: - code: 1 - stderr: 'no\s+matching\s+distribution' - skip: legacy - # -- currently the error message is: - # Could not find a version that satisfies the requirement A==2.0 - # No matching distribution found for a diff --git a/tests/yaml/conflict_2.yml b/tests/yaml/conflict_2.yml deleted file mode 100644 index 7ec5848ed8f..00000000000 --- a/tests/yaml/conflict_2.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Tzu-ping mentioned this example -base: - available: - - name: virtualenv - version: 20.0.2 - depends: ['six>=1.12.0,<2'] - - six 1.11 - - six 1.12 - - six 1.13 - -cases: -- - request: - - install: virtualenv - response: - - state: - - six 1.13 - - virtualenv 20.0.2 -- - request: - - install: ['six<1.12', 'virtualenv==20.0.2'] - response: - - state: null - error: - stderr: >- - Cannot install six<1.12 and virtualenv 20.0.2 because these - package versions have conflicting dependencies. - skip: legacy diff --git a/tests/yaml/conflict_3.yml b/tests/yaml/conflict_3.yml deleted file mode 100644 index 53f2b4a981f..00000000000 --- a/tests/yaml/conflict_3.yml +++ /dev/null @@ -1,22 +0,0 @@ -base: - available: - - A 1.0.0; depends B == 1.0.0, C == 2.0.0 - - B 1.0.0; depends C == 1.0.0 - - C 1.0.0 - - C 2.0.0 - -cases: -- - request: - - install: A - response: - - state: null - skip: legacy - # -- currently the error message is: - # Could not find a version that satisfies the requirement C==2.0.0 (from a) - # Could not find a version that satisfies the requirement C==1.0.0 (from b) - # No matching distribution found for c, c - # -- This is a bit confusing, as both versions of C are available. - # -- better would be something like: - # A 1.0.0 -> B 1.0.0 -> C 1.0.0 - # A 1.0.0 -> C 2.0.0 diff --git a/tests/yaml/conflicting_diamond.yml b/tests/yaml/conflicting_diamond.yml deleted file mode 100644 index c28b667ac6b..00000000000 --- a/tests/yaml/conflicting_diamond.yml +++ /dev/null @@ -1,19 +0,0 @@ -cases: -- - available: - - A 1.0.0; depends B == 1.0.0, C == 1.0.0 - - B 1.0.0; depends D == 1.0.0 - - C 1.0.0; depends D == 2.0.0 - - D 1.0.0 - - D 2.0.0 - request: - - install: A - response: - - error: - code: 1 - stderr: >- - Cannot install A and A because these package - versions have conflicting dependencies. - # TODO: Tweak this error message to make sense. - # https://github.com/pypa/pip/issues/8495 - skip: legacy diff --git a/tests/yaml/conflicting_triangle.yml b/tests/yaml/conflicting_triangle.yml deleted file mode 100644 index 02b348ca2f2..00000000000 --- a/tests/yaml/conflicting_triangle.yml +++ /dev/null @@ -1,18 +0,0 @@ -cases: -- - available: - - A 1.0.0; depends C == 1.0.0 - - B 1.0.0; depends C == 2.0.0 - - C 1.0.0 - - C 2.0.0 - request: - - install: A - - install: B - response: - - state: - - A 1.0.0 - - C 1.0.0 - - error: - code: 0 - stderr: ['c==1\.0\.0', 'incompatible'] - skip: legacy diff --git a/tests/yaml/extras.yml b/tests/yaml/extras.yml deleted file mode 100644 index b0f4e992c9c..00000000000 --- a/tests/yaml/extras.yml +++ /dev/null @@ -1,49 +0,0 @@ -base: - available: - - A 1.0.0; depends B == 1.0.0, C == 1.0.0, D == 1.0.0 - - B 1.0.0; depends D[extra_1] == 1.0.0 - - C 1.0.0; depends D[extra_2] == 1.0.0 - - name: D - version: 1.0.0 - depends: [] - extras: - extra_1: [E == 1.0.0] - extra_2: [F == 1.0.0] - - E 1.0.0 - - F 1.0.0 -cases: -- - request: - - install: B - response: - - state: - - B 1.0.0 - - D 1.0.0 - - E 1.0.0 -- - request: - - install: C - response: - - state: - - C 1.0.0 - - D 1.0.0 - - F 1.0.0 -- - request: - - install: A - response: - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - - D 1.0.0 - - E 1.0.0 - - F 1.0.0 - skip: legacy -- - request: - - install: D[extra_1] - options: --no-deps - response: - - state: - - D 1.0.0 diff --git a/tests/yaml/fallback.yml b/tests/yaml/fallback.yml deleted file mode 100644 index 86925398a56..00000000000 --- a/tests/yaml/fallback.yml +++ /dev/null @@ -1,20 +0,0 @@ -base: - available: - - A 1.0.0; depends B == 1.0.0, C == 1.0.0 - - A 0.8.0 - - B 1.0.0; depends D == 1.0.0 - - C 1.0.0; depends D == 2.0.0 - - D 1.0.0 - - D 2.0.0 - -cases: -- - request: - - install: A - response: - - state: - - A 0.8.0 - # the old resolver tries to install A 1.0.0 (which fails), but the new - # resolver realises that A 1.0.0 cannot be installed and falls back to - # installing the older version A 0.8.0 instead. - skip: legacy diff --git a/tests/yaml/huge.yml b/tests/yaml/huge.yml deleted file mode 100644 index 01bfdf26f3c..00000000000 --- a/tests/yaml/huge.yml +++ /dev/null @@ -1,1260 +0,0 @@ -base: - available: - - alabaster 0.7.10 - - alabaster 0.7.11 - - appdirs 1.4.3 - - asn1crypto 0.22.0 - - asn1crypto 0.23.0 - - asn1crypto 0.24.0 - - name: astroid - version: 1.5.3 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 1.6.0 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 1.6.1 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 1.6.2 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 1.6.3 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 1.6.4 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 1.6.5 - depends: ['lazy-object-proxy', 'setuptools', 'six', 'wrapt'] - - name: astroid - version: 2.0.2 - depends: ['lazy-object-proxy', 'six', 'wrapt'] - - name: astroid - version: 2.0.4 - depends: ['lazy-object-proxy', 'six', 'wrapt'] - - name: attrs - version: 17.2.0 - depends: ['hypothesis', 'pympler', 'zope', 'zope.interface'] - - name: attrs - version: 17.3.0 - depends: ['hypothesis', 'pympler', 'zope', 'zope.interface'] - - attrs 17.4.0 - - attrs 18.1.0 - - name: automat - version: 0.6.0 - depends: ['attrs', 'six'] - - name: automat - version: 0.7.0 - depends: ['attrs', 'six'] - - name: babel - version: 2.5.0 - depends: ['pytz'] - - name: babel - version: 2.5.1 - depends: ['pytz'] - - name: babel - version: 2.5.3 - depends: ['pytz'] - - name: babel - version: 2.6.0 - depends: ['pytz'] - - backcall 0.1.0 - - backports 1.0 - - name: backports.functools_lru_cache - version: '1.4' - depends: ['backports', 'setuptools'] - - name: backports.functools_lru_cache - version: '1.5' - depends: ['backports', 'setuptools'] - - name: backports.shutil_get_terminal_size - version: 1.0.0 - depends: ['backports'] - - backports_abc 0.5 - - beautifulsoup4 4.6.0 - - beautifulsoup4 4.6.1 - - beautifulsoup4 4.6.3 - - bitarray 0.8.1 - - bitarray 0.8.2 - - bitarray 0.8.3 - - name: bkcharts - version: '0.2' - depends: ['numpy >=1.7.1', 'pandas', 'six >=1.5.2'] - - name: bleach - version: 2.0.0 - depends: ['html5lib >=0.99999999', 'six'] - - name: bleach - version: 2.1.1 - depends: ['html5lib >=0.99999999', 'setuptools', 'six'] - - name: bleach - version: 2.1.2 - depends: ['html5lib >=0.99999999', 'setuptools', 'six'] - - name: bleach - version: 2.1.3 - depends: ['html5lib >=0.99999999', 'setuptools', 'six'] - - name: bokeh - version: 0.12.10 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.11 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.13 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.14 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'packaging >=16.8', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.15 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'packaging >=16.8', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.16 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'packaging >=16.8', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.7 - depends: ['bkcharts >=0.2', 'jinja2 >=2.7', 'matplotlib', 'numpy >=1.7.1', 'pandas', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'requests >=1.2.3', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.12.9 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: bokeh - version: 0.13.0 - depends: ['jinja2 >=2.7', 'numpy >=1.7.1', 'packaging >=16.8', 'python-dateutil >=2.1', 'pyyaml >=3.10', 'six >=1.5.2', 'tornado >=4.3'] - - name: boto3 - version: 1.4.7 - depends: ['botocore >=1.7.0,<1.8.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.4.8 - depends: ['botocore >=1.8.0,<1.9.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.5.32 - depends: ['botocore >=1.8.46,<1.9.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.6.18 - depends: ['botocore >=1.9.18,<1.10.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.7.24 - depends: ['botocore >=1.10.24,<1.11.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.7.32 - depends: ['botocore >=1.10.32,<1.11.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.7.4 - depends: ['botocore >=1.10.4,<1.11.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.7.45 - depends: ['botocore >=1.10.45,<1.11.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: boto3 - version: 1.7.62 - depends: ['botocore >=1.10.62,<1.11.0', 'jmespath >=0.7.1,<1.0.0', 's3transfer >=0.1.10,<0.2.0'] - - name: botocore - version: 1.10.12 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.10.24 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.10.32 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.10.4 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<2.7.0'] - - name: botocore - version: 1.10.45 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.10.62 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.5.78 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.7.14 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.7.20 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.7.40 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.7.5 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.8.21 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.8.46 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.8.5 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<3.0.0'] - - name: botocore - version: 1.9.18 - depends: ['docutils >=0.10', 'jmespath >=0.7.1,<1.0.0', 'python-dateutil >=2.1,<2.7.0'] - - certifi 2017.11.5 - - certifi 2017.7.27.1 - - certifi 2018.1.18 - - certifi 2018.4.16 - - certifi 2018.8.13 - # cffi is a bundled module in PyPy and causes resolution errors if pip - # tries to installed it. Give it a different name since we are simply - # checking the graph anyway and the identifier doesn't really matter. - - name: cffi_not_really - version: 1.10.0 - depends: ['pycparser'] - - name: cffi_not_really - version: 1.11.2 - depends: ['pycparser'] - - name: cffi_not_really - version: 1.11.4 - depends: ['pycparser'] - - name: cffi_not_really - version: 1.11.5 - depends: ['pycparser'] - - chardet 3.0.4 - - click 6.7 - - cloudpickle 0.4.0 - - cloudpickle 0.4.2 - - cloudpickle 0.5.2 - - cloudpickle 0.5.3 - - colorama 0.3.9 - - configparser 3.5.0 - - constantly 15.1.0 - - contextlib2 0.5.5 - - coverage 4.4.2 - - coverage 4.5.1 - - name: cryptography - version: 2.0.3 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'openssl 1.0.*', 'six >=1.4.1'] - - name: cryptography - version: 2.1.3 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'openssl 1.0.*', 'openssl >=1.0.2m,<1.0.3a', 'six >=1.4.1'] - - name: cryptography - version: 2.1.4 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'openssl 1.0.*', 'openssl >=1.0.2m,<1.0.3a', 'six >=1.4.1'] - - name: cryptography - version: 2.2.1 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'openssl 1.0.*', 'openssl >=1.0.2n,<1.0.3a', 'six >=1.4.1'] - - name: cryptography - version: 2.2.2 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'openssl 1.0.*', 'openssl >=1.0.2o,<1.0.3a', 'six >=1.4.1'] - - name: cryptography - version: '2.3' - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'cryptography-vectors 2.3.*', 'idna >=2.1', 'openssl >=1.0.2o,<1.0.3a', 'six >=1.4.1'] - - cryptography-vectors 2.0.3 - - cryptography-vectors 2.1.3 - - cryptography-vectors 2.1.4 - - cryptography-vectors 2.2.1 - - cryptography-vectors 2.2.2 - - cryptography-vectors 2.3 - - name: cycler - version: 0.10.0 - depends: ['six'] - - name: cytoolz - version: 0.8.2 - depends: ['toolz >=0.8.0'] - - name: cytoolz - version: 0.9.0 - depends: ['toolz >=0.8.0'] - - name: cytoolz - version: 0.9.0.1 - depends: ['toolz >=0.8.0'] - - name: dask - version: 0.15.2 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.15.2.*', 'distributed >=1.16.0', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.15.3 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.15.3.*', 'distributed >=1.19.0', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.15.4 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.15.4.*', 'distributed >=1.19.0', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.16.0 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.16.0.*', 'distributed >=1.20.0', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.16.1 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.16.1.*', 'distributed >=1.20.0', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.17.0 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.17.0.*', 'distributed >=1.21.0', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.17.1 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'dask-core 0.17.1.*', 'distributed >=1.21.1', 'numpy >=1.10', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.17.2 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.17.2.*', 'distributed >=1.21.0', 'numpy >=1.10.4', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.17.3 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.17.3.*', 'distributed >=1.21.0', 'numpy >=1.11.0', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.17.4 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.17.4.*', 'distributed >=1.21.0', 'numpy >=1.11.0', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.17.5 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.17.5.*', 'distributed >=1.21.0', 'numpy >=1.11.0', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.18.0 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.18.0.*', 'distributed >=1.22.0', 'numpy >=1.11.0', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.18.1 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.18.1.*', 'distributed >=1.22.0', 'numpy >=1.11.0', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - name: dask - version: 0.18.2 - depends: ['bokeh', 'cloudpickle >=0.2.1', 'cytoolz >=0.7.3', 'dask-core 0.18.2.*', 'distributed >=1.22.0', 'numpy >=1.11.0', 'pandas >=0.19.0', 'partd >=0.3.8', 'toolz >=0.7.3'] - - dask-core 0.15.2 - - dask-core 0.15.3 - - dask-core 0.15.4 - - dask-core 0.16.0 - - dask-core 0.16.1 - - dask-core 0.17.0 - - dask-core 0.17.1 - - dask-core 0.17.2 - - dask-core 0.17.3 - - dask-core 0.17.4 - - dask-core 0.17.5 - - dask-core 0.18.0 - - dask-core 0.18.1 - - dask-core 0.18.2 - - decorator 4.1.2 - - decorator 4.2.1 - - decorator 4.3.0 - - dill 0.2.7.1 - - dill 0.2.8.2 - - name: distributed - version: 1.18.3 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.15.2', 'msgpack-python', 'psutil', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.2'] - - name: distributed - version: 1.19.1 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.15.2', 'msgpack-python', 'psutil', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.20.0 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.16.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.20.1 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.16.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.20.2 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.16.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.0 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.1 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.2 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.3 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.4 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.5 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.6 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.21.8 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.17.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.22.0 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.18.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - name: distributed - version: 1.22.1 - depends: ['click >=6.6', 'cloudpickle >=0.2.2', 'cytoolz >=0.7.4', 'dask-core >=0.18.0', 'msgpack-python', 'psutil', 'pyyaml', 'six', 'sortedcontainers', 'tblib', 'toolz >=0.7.4', 'tornado >=4.5.1', 'zict >=0.1.3'] - - docutils 0.14 - - entrypoints 0.2.3 - - enum34 1.1.6 - - expat 2.2.4 - - expat 2.2.5 - - filelock 2.0.12 - - filelock 2.0.13 - - filelock 3.0.4 - - name: flask - version: 0.12.2 - depends: ['click >=2.0', 'itsdangerous >=0.21', 'jinja2 >=2.4', 'werkzeug >=0.7'] - - name: flask - version: 1.0.2 - depends: ['click >=5.1', 'itsdangerous >=0.24', 'jinja2 >=2.10', 'werkzeug >=0.14'] - - fribidi 1.0.2 - - fribidi 1.0.4 - - funcsigs 1.0.2 - - functools32 3.2.3.2 - - future 0.16.0 - - futures 3.1.1 - - futures 3.2.0 - - name: gevent - version: 1.2.2 - depends: ['cffi_not_really >=1.3.0', 'greenlet >=0.4.10'] - - name: gevent - version: 1.3.0 - depends: ['cffi_not_really >=1.11.5', 'greenlet >=0.4.10'] - - name: gevent - version: 1.3.2.post0 - depends: ['cffi_not_really >=1.11.5', 'greenlet >=0.4.13'] - - name: gevent - version: 1.3.3 - depends: ['cffi_not_really >=1.11.5', 'greenlet >=0.4.13'] - - name: gevent - version: 1.3.4 - depends: ['cffi_not_really >=1.11.5', 'greenlet >=0.4.13'] - - name: gevent - version: 1.3.5 - depends: ['cffi_not_really >=1.11.5', 'greenlet >=0.4.13'] - - glob2 0.5 - - glob2 0.6 - - gmp 6.1.2 - - graphite2 1.3.10 - - graphite2 1.3.11 - - greenlet 0.4.12 - - greenlet 0.4.13 - - greenlet 0.4.14 - - name: html5lib - version: '0.999999999' - depends: ['six >=1.9', 'webencodings'] - - name: html5lib - version: 1.0.1 - depends: ['six >=1.9', 'webencodings'] - - name: hyperlink - version: 18.0.0 - depends: ['idna >=2.5'] - - hypothesis 3.23.0 - - name: hypothesis - version: 3.37.0 - depends: ['attrs', 'coverage'] - - name: hypothesis - version: 3.38.5 - depends: ['attrs', 'coverage'] - - name: hypothesis - version: 3.46.0 - depends: ['attrs', 'coverage'] - - name: hypothesis - version: 3.52.0 - depends: ['attrs >=16.0.0', 'coverage'] - - name: hypothesis - version: 3.53.0 - depends: ['attrs >=16.0.0', 'coverage'] - - name: hypothesis - version: 3.56.0 - depends: ['attrs >=16.0.0', 'coverage'] - - name: hypothesis - version: 3.57.0 - depends: ['attrs >=16.0.0', 'coverage'] - - name: hypothesis - version: 3.59.1 - depends: ['attrs >=16.0.0', 'coverage'] - - name: ibis-framework - version: 0.12.0 - depends: ['impyla >=0.14.0', 'multipledispatch', 'numpy >=1.10.0', 'pandas >=0.18.1', 'psycopg2', 'python-graphviz', 'setuptools', 'six', 'sqlalchemy >=1.0.0', 'thrift', 'thriftpy <=0.3.9', 'toolz'] - - name: ibis-framework - version: 0.13.0 - depends: ['impyla >=0.14.0', 'multipledispatch', 'numpy >=1.10.0', 'pandas >=0.18.1', 'psycopg2', 'python-graphviz', 'setuptools', 'six', 'sqlalchemy >=1.0.0', 'thrift', 'thriftpy <=0.3.9', 'toolz'] - - icu 58.2 - - idna 2.6 - - idna 2.7 - - imagesize 0.7.1 - - imagesize 1.0.0 - - name: impyla - version: 0.14.0 - depends: ['bitarray', 'setuptools', 'six', 'thriftpy >=0.3.5'] - - name: impyla - version: 0.14.1 - depends: ['bitarray', 'setuptools', 'six', 'thriftpy >=0.3.5'] - - incremental 17.5.0 - - ipaddress 1.0.18 - - ipaddress 1.0.19 - - ipaddress 1.0.22 - - name: ipykernel - version: 4.6.1 - depends: ['ipython', 'jupyter_client', 'tornado >=4.0', 'traitlets >=4.1'] - - name: ipykernel - version: 4.7.0 - depends: ['ipython', 'jupyter_client', 'tornado >=4.0', 'traitlets >=4.1'] - - name: ipykernel - version: 4.8.0 - depends: ['ipython >=4.0.0', 'jupyter_client', 'tornado >=4.0', 'traitlets >=4.1'] - - name: ipykernel - version: 4.8.2 - depends: ['ipython >=4.0.0', 'jupyter_client', 'tornado >=4.0', 'traitlets >=4.1'] - - name: ipython - version: 5.4.1 - depends: ['decorator', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 5.5.0 - depends: ['decorator', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 5.6.0 - depends: ['decorator', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 5.7.0 - depends: ['backports.shutil_get_terminal_size', 'decorator', 'pathlib2', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 5.8.0 - depends: ['decorator', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 6.1.0 - depends: ['decorator', 'jedi >=0.10', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 6.2.1 - depends: ['decorator', 'jedi >=0.10', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets'] - - name: ipython - version: 6.3.0 - depends: ['backcall', 'decorator', 'jedi >=0.10', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets >=4.2'] - - name: ipython - version: 6.3.1 - depends: ['backcall', 'decorator', 'jedi >=0.10', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets >=4.2'] - - name: ipython - version: 6.4.0 - depends: ['backcall', 'decorator', 'jedi >=0.10', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets >=4.2'] - - name: ipython - version: 6.5.0 - depends: ['backcall', 'decorator', 'jedi >=0.10', 'pexpect', 'pickleshare', 'prompt_toolkit >=1.0.4,<2.0.0', 'pygments', 'simplegeneric >0.8', 'traitlets >=4.2'] - - name: ipython-notebook - version: 0.13.2 - depends: ['ipython 0.13.2', 'pyzmq 2.2.0.1', 'tornado'] - - name: ipython-notebook - version: 1.0.0 - depends: ['ipython 1.0.0', 'pyzmq 2.2.0.1', 'tornado'] - - name: ipython-notebook - version: 1.1.0 - depends: ['ipython 1.1.0', 'jinja2', 'pyzmq 2.2.0.1', 'tornado'] - - name: ipython-notebook - version: 2.0.0 - depends: ['ipython 2.0.0', 'jinja2', 'pyzmq 14.*', 'tornado'] - - name: ipython-notebook - version: 2.1.0 - depends: ['ipython 2.1.0', 'jinja2', 'pyzmq 14.*', 'tornado'] - - name: ipython-notebook - version: 2.2.0 - depends: ['ipython 2.2.0', 'jinja2', 'pyzmq 14.*', 'tornado'] - - name: ipython-notebook - version: 2.3.0 - depends: ['ipython 2.3.0', 'jinja2', 'pyzmq 14.*', 'tornado'] - - name: ipython-notebook - version: 2.3.1 - depends: ['ipython 2.3.1', 'jinja2', 'pyzmq 14.*', 'tornado'] - - name: ipython-notebook - version: 2.4.1 - depends: ['ipython 2.4.1', 'jinja2', 'pyzmq 14.*', 'tornado'] - - name: ipython-notebook - version: 3.0.0 - depends: ['ipython 3.0.0', 'jinja2', 'jsonschema 2.4.0', 'mistune', 'pygments', 'pyzmq 14.*', 'terminado 0.5', 'tornado'] - - name: ipython-notebook - version: 3.1.0 - depends: ['ipython 3.1.0', 'jinja2', 'jsonschema 2.4.0', 'mistune', 'pygments', 'pyzmq 14.*', 'terminado 0.5', 'tornado'] - - name: ipython-notebook - version: 3.2.0 - depends: ['ipython 3.2.0', 'jinja2', 'jsonschema 2.4.0', 'mistune', 'pygments', 'pyzmq 14.*', 'terminado 0.5', 'tornado'] - - name: ipython-notebook - version: 3.2.1 - depends: ['ipython 3.2.1', 'jinja2', 'jsonschema 2.4.0', 'mistune', 'pygments', 'pyzmq 14.*', 'terminado 0.5', 'tornado'] - - name: ipython-notebook - version: 4.0.4 - depends: ['notebook'] - - ipython_genutils 0.2.0 - - name: ipywidgets - version: 7.0.0 - depends: ['ipykernel >=4.5.1', 'ipython', 'nbformat >=4.2.0', 'traitlets >=4.3.1', 'widgetsnbextension >=3.0.0'] - - name: ipywidgets - version: 7.0.5 - depends: ['ipykernel >=4.5.1', 'ipython', 'nbformat >=4.2.0', 'traitlets >=4.3.1', 'widgetsnbextension >=3.0.0'] - - name: ipywidgets - version: 7.1.0 - depends: ['ipykernel >=4.5.1', 'ipython', 'nbformat >=4.2.0', 'traitlets >=4.3.1', 'widgetsnbextension >=3.0.0'] - - name: ipywidgets - version: 7.1.1 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.1.0,<4.0'] - - name: ipywidgets - version: 7.1.2 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.1.0,<4.0'] - - name: ipywidgets - version: 7.2.0 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.2.0,<4.0.0'] - - name: ipywidgets - version: 7.2.1 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.2.0,<4.0.0'] - - name: ipywidgets - version: 7.3.0 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.3.0,<3.4.0'] - - name: ipywidgets - version: 7.3.1 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.3.0,<3.4.0'] - - name: ipywidgets - version: 7.4.0 - depends: ['ipykernel >=4.5.1', 'ipython >=4.0.0', 'nbformat >=4.2.0', 'traitlets >=4.3.1,<5.0.0', 'widgetsnbextension >=3.4.0,<3.5.0'] - - itsdangerous 0.24 - - jedi 0.10.2 - - name: jedi - version: 0.11.0 - depends: ['parso ==0.1.0'] - - name: jedi - version: 0.11.1 - depends: ['numpydoc', 'parso >=0.1.0,<0.2'] - - name: jedi - version: 0.12.0 - depends: ['parso >=0.2.0'] - - name: jedi - version: 0.12.1 - depends: ['parso >=0.3.0'] - - name: jinja2 - version: '2.10' - depends: ['markupsafe >=0.23', 'setuptools'] - - name: jinja2 - version: 2.9.6 - depends: ['markupsafe >=0.23', 'setuptools'] - - jmespath 0.9.3 - - jpeg 9b - - name: jsonschema - version: 2.6.0 - depends: ['setuptools'] - - name: jupyter - version: 1.0.0 - depends: ['ipykernel', 'ipywidgets', 'jupyter_console', 'nbconvert', 'notebook', 'qtconsole'] - - name: jupyter_client - version: 5.1.0 - depends: ['jupyter_core', 'python-dateutil >=2.1', 'pyzmq >=13', 'traitlets'] - - name: jupyter_client - version: 5.2.1 - depends: ['jupyter_core', 'python-dateutil >=2.1', 'pyzmq >=13', 'traitlets'] - - name: jupyter_client - version: 5.2.2 - depends: ['jupyter_core', 'python-dateutil >=2.1', 'pyzmq >=13', 'tornado', 'traitlets'] - - name: jupyter_client - version: 5.2.3 - depends: ['jupyter_core', 'python-dateutil >=2.1', 'pyzmq >=13', 'tornado', 'traitlets'] - - name: jupyter_console - version: 5.2.0 - depends: ['ipykernel', 'ipython', 'jupyter_client', 'pexpect', 'prompt_toolkit', 'pygments'] - - name: jupyter_core - version: 4.3.0 - depends: ['traitlets'] - - name: jupyter_core - version: 4.4.0 - depends: ['traitlets'] - - kiwisolver 1.0.0 - - kiwisolver 1.0.1 - - lazy-object-proxy 1.3.1 - - llvmlite 0.20.0 - - llvmlite 0.21.0 - - llvmlite 0.22.0 - - locket 0.2.0 - - name: logilab-common - version: 1.4.1 - depends: ['setuptools', 'six >=1.4.0'] - - make 4.2.1 - - markupsafe 1.0 - - name: matplotlib - version: 2.0.2 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt 5.6.*', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - name: matplotlib - version: 2.1.0 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt 5.6.*', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - name: matplotlib - version: 2.1.1 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt 5.6.*', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - name: matplotlib - version: 2.1.2 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt 5.6.*', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - name: matplotlib - version: 2.2.0 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt 5.6.*', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - name: matplotlib - version: 2.2.2 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt >=5.6,<6.0a0', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - name: matplotlib - version: 2.2.3 - depends: ['cycler >=0.10', 'numpy', 'pyparsing', 'pyqt 5.9.*', 'python-dateutil', 'pytz', 'setuptools', 'tornado'] - - mistune 0.7.4 - - mistune 0.8.1 - - mistune 0.8.3 - - msgpack-python 0.4.8 - - msgpack-python 0.5.1 - - msgpack-python 0.5.5 - - msgpack-python 0.5.6 - - multipledispatch 0.4.9 - - multipledispatch 0.5.0 - - name: multipledispatch - version: 0.6.0 - depends: ['six'] - - name: nbconvert - version: 5.3.1 - depends: ['bleach', 'entrypoints >=0.2.2', 'jinja2', 'jupyter_client >=4.2', 'jupyter_core', 'mistune >0.6', 'nbformat', 'pandoc', 'pandocfilters >=1.4.1', 'pygments', 'testpath', 'traitlets'] - - name: nbformat - version: 4.4.0 - depends: ['ipython_genutils', 'jsonschema >=2.4,!=2.5.0', 'jupyter_core', 'traitlets >=4.1'] - - ncurses 6.0 - - ncurses 6.1 - - name: nose - version: 1.3.7 - depends: ['setuptools'] - - name: notebook - version: 5.0.0 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client', 'jupyter_core', 'nbconvert', 'nbformat', 'terminado >=0.3.3', 'tornado >=4', 'traitlets >=4.3'] - - name: notebook - version: 5.1.0 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client', 'jupyter_core', 'nbconvert', 'nbformat', 'terminado >=0.3.3', 'tornado >=4', 'traitlets >=4.3'] - - name: notebook - version: 5.2.0 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client', 'jupyter_core', 'nbconvert', 'nbformat', 'terminado >=0.3.3', 'tornado >=4', 'traitlets >=4.3'] - - name: notebook - version: 5.2.1 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client', 'jupyter_core', 'nbconvert', 'nbformat', 'terminado >=0.3.3', 'tornado >=4', 'traitlets >=4.3'] - - name: notebook - version: 5.2.2 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client', 'jupyter_core', 'nbconvert', 'nbformat', 'terminado >=0.3.3', 'tornado >=4', 'traitlets >=4.3'] - - name: notebook - version: 5.3.1 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client >=5.2.0', 'jupyter_core >=4.4.0', 'nbconvert', 'nbformat', 'send2trash', 'terminado >=0.8.1', 'tornado >=4', 'traitlets >=4.2.1'] - - name: notebook - version: 5.4.0 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client >=5.2.0', 'jupyter_core >=4.4.0', 'nbconvert', 'nbformat', 'send2trash', 'terminado >=0.8.1', 'tornado >=4', 'traitlets >=4.2.1'] - - name: notebook - version: 5.4.1 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client >=5.2.0', 'jupyter_core >=4.4.0', 'nbconvert', 'nbformat', 'send2trash', 'terminado >=0.8.1', 'tornado >=4', 'traitlets >=4.2.1'] - - name: notebook - version: 5.5.0 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client >=5.2.0', 'jupyter_core >=4.4.0', 'nbconvert', 'nbformat', 'pyzmq >=17', 'send2trash', 'terminado >=0.8.1', 'tornado >=4', 'traitlets >=4.2.1'] - - name: notebook - version: 5.6.0 - depends: ['ipykernel', 'ipython_genutils', 'jinja2', 'jupyter_client >=5.2.0', 'jupyter_core >=4.4.0', 'nbconvert', 'nbformat', 'prometheus_client', 'pyzmq >=17', 'send2trash', 'terminado >=0.8.1', 'tornado >=4', 'traitlets >=4.2.1'] - - numpy 1.11.3 - - numpy 1.12.1 - - numpy 1.13.1 - - numpy 1.13.3 - - numpy 1.14.0 - - numpy 1.14.1 - - numpy 1.14.2 - - numpy 1.14.3 - - numpy 1.14.4 - - numpy 1.14.5 - - numpy 1.15.0 - - numpy 1.9.3 - - name: numpydoc - version: 0.7.0 - depends: ['sphinx'] - - name: numpydoc - version: 0.8.0 - depends: ['sphinx'] - - name: openssl - version: 1.0.2l - depends: ['ca-certificates'] - - name: openssl - version: 1.0.2m - depends: ['ca-certificates'] - - name: openssl - version: 1.0.2n - depends: ['ca-certificates'] - - name: openssl - version: 1.0.2o - depends: ['ca-certificates'] - - name: openssl - version: 1.0.2p - depends: ['ca-certificates'] - - name: packaging - version: '16.8' - depends: ['pyparsing', 'six'] - - name: packaging - version: '17.1' - depends: ['pyparsing', 'six'] - - name: pandas - version: 0.20.3 - depends: ['numpy >=1.9', 'python-dateutil', 'pytz'] - - name: pandas - version: 0.21.0 - depends: ['numpy >=1.9.3,<2.0a0', 'python-dateutil', 'pytz'] - - name: pandas - version: 0.21.1 - depends: ['numpy >=1.9.3,<2.0a0', 'python-dateutil', 'pytz'] - - name: pandas - version: 0.22.0 - depends: ['numpy >=1.9.3,<2.0a0', 'python-dateutil', 'pytz'] - - name: pandas - version: 0.23.0 - depends: ['numpy >=1.9.3,<2.0a0', 'python-dateutil', 'pytz'] - - name: pandas - version: 0.23.1 - depends: ['numpy >=1.9.3,<2.0a0', 'python-dateutil >=2.5.*', 'pytz'] - - name: pandas - version: 0.23.2 - depends: ['numpy >=1.11.3,<2.0a0', 'python-dateutil >=2.5.*', 'pytz'] - - name: pandas - version: 0.23.3 - depends: ['numpy >=1.11.3,<2.0a0', 'python-dateutil >=2.5.*', 'pytz'] - - name: pandas - version: 0.23.4 - depends: ['numpy >=1.11.3,<2.0a0', 'python-dateutil >=2.5.*', 'pytz'] - - pandocfilters 1.4.2 - - parso 0.1.0 - - parso 0.1.1 - - parso 0.2.0 - - parso 0.2.1 - - parso 0.3.0 - - parso 0.3.1 - - name: partd - version: 0.3.8 - depends: ['locket', 'toolz'] - - patchelf 0.9 - - path.py 10.3.1 - - path.py 10.5 - - path.py 11.0 - - path.py 11.0.1 - - name: pathlib2 - version: 2.3.0 - depends: ['six'] - - name: pathlib2 - version: 2.3.2 - depends: ['six'] - - pcre 8.41 - - pcre 8.42 - - perl 5.26.2 - - name: perl-app-cpanminus - version: '1.7039' - depends: ['perl 5.22.0*'] - - name: perl-encode-locale - version: '1.05' - depends: ['perl >=5.26.2,<5.27.0a0'] - - name: pexpect - version: 4.2.1 - depends: ['ptyprocess >=0.5'] - - name: pexpect - version: 4.3.0 - depends: ['ptyprocess >=0.5'] - - name: pexpect - version: 4.3.1 - depends: ['ptyprocess >=0.5'] - - name: pexpect - version: 4.4.0 - depends: ['ptyprocess >=0.5'] - - name: pexpect - version: 4.5.0 - depends: ['ptyprocess >=0.5'] - - name: pexpect - version: 4.6.0 - depends: ['ptyprocess >=0.5'] - - pickleshare 0.7.4 - - name: pip - version: 10.0.1 - depends: ['setuptools', 'wheel'] - - name: pip - version: 9.0.1 - depends: ['setuptools', 'wheel'] - - name: pip - version: 9.0.3 - depends: ['setuptools', 'wheel'] - - pixman 0.34.0 - - pkginfo 1.4.1 - - pkginfo 1.4.2 - - ply 3.10 - - ply 3.11 - - name: prometheus_client - version: 0.2.0 - depends: ['twisted'] - - name: prometheus_client - version: 0.3.0 - depends: ['twisted'] - - name: prometheus_client - version: 0.3.1 - depends: ['twisted'] - - name: prompt_toolkit - version: 1.0.15 - depends: ['pygments', 'six >=1.9.0', 'wcwidth'] - - name: prompt_toolkit - version: 2.0.2 - depends: ['pygments', 'six >=1.9.0', 'wcwidth'] - - name: prompt_toolkit - version: 2.0.3 - depends: ['pygments', 'six >=1.9.0', 'wcwidth'] - - name: prompt_toolkit - version: 2.0.4 - depends: ['pygments', 'six >=1.9.0', 'wcwidth'] - - psutil 5.2.2 - - psutil 5.3.1 - - psutil 5.4.0 - - psutil 5.4.1 - - psutil 5.4.3 - - psutil 5.4.5 - - psutil 5.4.6 - - psycopg2 2.7.3.1 - - psycopg2 2.7.3.2 - - psycopg2 2.7.4 - - psycopg2 2.7.5 - - ptyprocess 0.5.2 - - ptyprocess 0.6.0 - - pyasn1 0.3.7 - - pyasn1 0.4.2 - - pyasn1 0.4.3 - - pyasn1 0.4.4 - - name: pyasn1-modules - version: 0.2.1 - depends: ['pyasn1 >=0.4.1,<0.5.0'] - - name: pyasn1-modules - version: 0.2.2 - depends: ['pyasn1 >=0.4.1,<0.5.0'] - - pycosat 0.6.2 - - pycosat 0.6.3 - - pycparser 2.18 - - name: pygments - version: 2.2.0 - depends: ['setuptools'] - - pympler 0.5 - - name: pyopenssl - version: 17.2.0 - depends: ['cryptography >=1.9', 'six >=1.5.2'] - - name: pyopenssl - version: 17.4.0 - depends: ['cryptography >=1.9', 'six >=1.5.2'] - - name: pyopenssl - version: 17.5.0 - depends: ['cryptography >=2.1.4', 'six >=1.5.2'] - - name: pyopenssl - version: 18.0.0 - depends: ['cryptography >=2.2.1', 'six >=1.5.2'] - - pyparsing 2.2.0 - - name: pyqt - version: 5.6.0 - depends: ['qt 5.6.*', 'sip 4.18.*'] - - name: pyqt - version: 5.9.2 - depends: ['dbus >=1.13.2,<2.0a0', 'qt 5.9.*', 'qt >=5.9.6,<5.10.0a0', 'sip >=4.19.4'] - - pysocks 1.6.7 - - pysocks 1.6.8 - - name: python-dateutil - version: 2.6.1 - depends: ['six'] - - name: python-dateutil - version: 2.7.0 - depends: ['six >=1.5'] - - name: python-dateutil - version: 2.7.2 - depends: ['six >=1.5'] - - name: python-dateutil - version: 2.7.3 - depends: ['six >=1.5'] - - name: python-digest - version: 1.1.1 - depends: ['cryptography <2.2'] - - python-graphviz 0.8.2 - - python-graphviz 0.8.3 - - python-graphviz 0.8.4 - - pytz 2017.2 - - pytz 2017.3 - - pytz 2018.3 - - pytz 2018.4 - - pytz 2018.5 - - pyyaml 3.12 - - pyyaml 3.13 - - pyzmq 16.0.2 - - pyzmq 16.0.3 - - pyzmq 17.0.0 - - pyzmq 17.1.0 - - pyzmq 17.1.2 - - name: qtconsole - version: 4.3.1 - depends: ['ipykernel >=4.1', 'jupyter_client >=4.1', 'jupyter_core', 'pygments', 'pyqt', 'traitlets'] - - name: qtconsole - version: 4.4.0 - depends: ['ipykernel >=4.1', 'jupyter_client >=4.1', 'jupyter_core', 'pygments', 'pyqt >=5.9.2,<5.10.0a0', 'traitlets'] - - redis 4.0.10 - - redis 4.0.2 - - redis 4.0.8 - - redis 4.0.9 - - redis-py 2.10.6 - - name: requests - version: 2.18.4 - depends: ['certifi >=2017.4.17', 'chardet >=3.0.2,<3.1.0', 'idna >=2.5,<2.7', 'urllib3 >=1.21.1,<1.23'] - - name: requests - version: 2.19.1 - depends: ['certifi >=2017.4.17', 'chardet >=3.0.2,<3.1.0', 'idna >=2.5,<2.8', 'urllib3 >=1.21.1,<1.24'] - - name: ruamel_yaml - version: 0.11.14 - depends: ['yaml'] - - name: ruamel_yaml - version: 0.15.35 - depends: ['yaml', 'yaml >=0.1.7,<0.2.0a0'] - - name: ruamel_yaml - version: 0.15.37 - depends: ['yaml >=0.1.7,<0.2.0a0'] - - name: ruamel_yaml - version: 0.15.40 - depends: ['yaml >=0.1.7,<0.2.0a0'] - - name: ruamel_yaml - version: 0.15.42 - depends: ['yaml >=0.1.7,<0.2.0a0'] - - name: ruamel_yaml - version: 0.15.46 - depends: ['yaml >=0.1.7,<0.2.0a0'] - - name: s3fs - version: 0.1.3 - depends: ['boto3'] - - name: s3fs - version: 0.1.4 - depends: ['boto3'] - - name: s3fs - version: 0.1.5 - depends: ['boto3'] - - name: s3transfer - version: 0.1.10 - depends: ['botocore >=1.3.0,<2.0.0'] - - name: s3transfer - version: 0.1.11 - depends: ['botocore >=1.3.0,<2.0.0'] - - name: s3transfer - version: 0.1.13 - depends: ['botocore >=1.3.0,<2.0.0'] - - scandir 1.5 - - scandir 1.6 - - scandir 1.7 - - scandir 1.8 - - scandir 1.9.0 - - name: scipy - version: 0.19.1 - depends: ['numpy >=1.9.3,<2.0a0'] - - name: scipy - version: 1.0.0 - depends: ['numpy >=1.9.3,<2.0a0'] - - name: scipy - version: 1.0.1 - depends: ['numpy >=1.9.3,<2.0a0'] - - name: scipy - version: 1.1.0 - depends: ['numpy >=1.11.3,<2.0a0'] - - send2trash 1.4.2 - - send2trash 1.5.0 - - name: service_identity - version: 17.0.0 - depends: ['attrs >=16.0.0', 'pyasn1', 'pyasn1-modules', 'pyopenssl >=0.12'] - - name: setuptools - version: 36.5.0 - depends: ['certifi'] - - name: setuptools - version: 38.4.0 - depends: ['certifi >=2016.09'] - - name: setuptools - version: 38.5.1 - depends: ['certifi >=2016.09'] - - name: setuptools - version: 39.0.1 - depends: ['certifi >=2016.09'] - - name: setuptools - version: 39.1.0 - depends: ['certifi >=2016.09'] - - name: setuptools - version: 39.2.0 - depends: ['certifi >=2016.09'] - - name: setuptools - version: 40.0.0 - depends: ['certifi >=2016.09'] - - simplegeneric 0.8.1 - - name: singledispatch - version: 3.4.0.3 - depends: ['six'] - - sip 4.18.1 - - sip 4.19.8 - - six 1.10.0 - - six 1.11.0 - - snowballstemmer 1.2.1 - - name: sortedcollections - version: 0.5.3 - depends: ['sortedcontainers'] - - name: sortedcollections - version: 0.6.1 - depends: ['sortedcontainers'] - - name: sortedcollections - version: 1.0.1 - depends: ['sortedcontainers >=2.0'] - - sortedcontainers 1.5.10 - - sortedcontainers 1.5.7 - - sortedcontainers 1.5.9 - - sortedcontainers 2.0.2 - - sortedcontainers 2.0.3 - - sortedcontainers 2.0.4 - - name: sphinx - version: 1.6.3 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.6.6 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.7.0 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'packaging', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.7.1 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'packaging', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.7.2 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'packaging', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.7.3 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'packaging', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.7.4 - depends: ['alabaster', 'babel', 'docutils', 'imagesize', 'jinja2', 'packaging', 'pygments', 'requests', 'six', 'snowballstemmer', 'sphinxcontrib-websupport', 'typing'] - - name: sphinx - version: 1.7.5 - depends: ['alabaster >=0.7,<0.8', 'babel >=1.3,!=2.0', 'docutils >=0.11', 'imagesize', 'jinja2 >=2.3', 'packaging', 'pygments >2.0', 'requests >2.0.0', 'six >=1.5', 'snowballstemmer >=1.1', 'sphinxcontrib-websupport'] - - name: sphinx - version: 1.7.6 - depends: ['alabaster >=0.7,<0.8', 'babel >=1.3,!=2.0', 'docutils >=0.11', 'imagesize', 'jinja2 >=2.3', 'packaging', 'pygments >2.0', 'requests >2.0.0', 'six >=1.5', 'snowballstemmer >=1.1', 'sphinxcontrib-websupport'] - - sphinxcontrib 1.0 - - name: sphinxcontrib-websupport - version: 1.0.1 - depends: ['sphinxcontrib'] - - name: sphinxcontrib-websupport - version: 1.1.0 - depends: ['sphinxcontrib'] - - sqlalchemy 1.1.13 - - sqlalchemy 1.2.0 - - sqlalchemy 1.2.1 - - sqlalchemy 1.2.10 - - sqlalchemy 1.2.3 - - sqlalchemy 1.2.4 - - sqlalchemy 1.2.5 - - sqlalchemy 1.2.6 - - sqlalchemy 1.2.7 - - sqlalchemy 1.2.8 - - name: ssl_match_hostname - version: 3.5.0.1 - depends: ['backports'] - - subprocess32 3.2.7 - - subprocess32 3.5.0 - - subprocess32 3.5.1 - - subprocess32 3.5.2 - - tblib 1.3.2 - - name: terminado - version: '0.6' - depends: ['ptyprocess', 'tornado >=4'] - - name: terminado - version: 0.8.1 - depends: ['ptyprocess', 'tornado >=4'] - - testpath 0.3.1 - - name: thrift - version: 0.11.0 - depends: ['six >=1.7.2'] - - thrift 0.9.3 - - name: thriftpy - version: 0.3.9 - depends: ['ply >=3.4,<4.0'] - - toolz 0.8.2 - - toolz 0.9.0 - - tornado 4.5.2 - - tornado 4.5.3 - - tornado 5.0 - - tornado 5.0.1 - - tornado 5.0.2 - - tornado 5.1 - - name: traitlets - version: 4.3.2 - depends: ['decorator', 'ipython_genutils', 'six'] - - name: twisted - version: 17.9.0 - depends: ['appdirs >=1.4.0', 'automat >=0.3.0', 'constantly >=15.1', 'cryptography >=1.5', 'hyperlink >=17.1.1', 'idna >=0.6,!=2.3', 'incremental >=16.10.1', 'pyasn1', 'pyopenssl >=16.0.0', 'service_identity', 'zope.interface >=4.0.2'] - - name: twisted - version: 18.4.0 - depends: ['appdirs >=1.4.0', 'automat >=0.3.0', 'constantly >=15.1', 'cryptography >=1.5', 'hyperlink >=17.1.1', 'idna >=0.6,!=2.3', 'incremental >=16.10.1', 'pyasn1', 'pyopenssl >=16.0.0', 'service_identity', 'zope.interface >=4.0.2'] - - name: twisted - version: 18.7.0 - depends: ['appdirs >=1.4.0', 'automat >=0.3.0', 'constantly >=15.1', 'cryptography >=1.5', 'hyperlink >=17.1.1', 'idna >=0.6,!=2.3', 'incremental >=16.10.1', 'pyasn1', 'pyopenssl >=16.0.0', 'service_identity', 'zope.interface >=4.0.2'] - - typed-ast 1.1.0 - - typing 3.6.2 - - typing 3.6.4 - - ujson 1.35 - - name: urllib3 - version: '1.22' - depends: ['certifi', 'cryptography >=1.3.4', 'idna >=2.0.0', 'pyopenssl >=0.14', 'pysocks >=1.5.6,<2.0,!=1.5.7'] - - name: urllib3 - version: '1.23' - depends: ['certifi', 'cryptography >=1.3.4', 'idna >=2.0.0', 'pyopenssl >=0.14', 'pysocks >=1.5.6,<2.0,!=1.5.7'] - - wcwidth 0.1.7 - - webencodings 0.5.1 - - werkzeug 0.12.2 - - werkzeug 0.14.1 - - name: wheel - version: 0.29.0 - depends: ['setuptools'] - - name: wheel - version: 0.30.0 - depends: ['setuptools'] - - name: wheel - version: 0.31.0 - depends: ['setuptools'] - - name: wheel - version: 0.31.1 - depends: ['setuptools'] - - name: widgetsnbextension - version: 3.0.2 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.0.8 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.1.0 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.1.4 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.2.0 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.2.1 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.3.0 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.3.1 - depends: ['notebook >=4.4.1'] - - name: widgetsnbextension - version: 3.4.0 - depends: ['notebook >=4.4.1'] - - wrapt 1.10.11 - - xz 5.2.3 - - xz 5.2.4 - - yaml 0.1.7 - - zeromq 4.2.2 - - zeromq 4.2.3 - - zeromq 4.2.5 - - name: zict - version: 0.1.2 - depends: ['heapdict'] - - name: zict - version: 0.1.3 - depends: ['heapdict'] - - zope 1.0 - - name: zope.interface - version: 4.4.3 - depends: ['zope'] - - name: zope.interface - version: 4.5.0 - depends: ['zope'] - -cases: -- - request: - - install: alabaster - response: - - state: - - alabaster 0.7.11 -- - request: - - install: ipython==6.3.1 - response: - - state: - - backcall 0.1.0 - - decorator 4.3.0 - - ipython 6.3.1 - - ipython_genutils 0.2.0 - - jedi 0.12.1 - - parso 0.3.1 - - pexpect 4.6.0 - - pickleshare 0.7.4 - - prompt_toolkit 1.0.15 - - ptyprocess 0.6.0 - - pygments 2.2.0 - - simplegeneric 0.8.1 - - six 1.11.0 - - traitlets 4.3.2 - - wcwidth 0.1.7 diff --git a/tests/yaml/large.yml b/tests/yaml/large.yml deleted file mode 100644 index fbb1c737eca..00000000000 --- a/tests/yaml/large.yml +++ /dev/null @@ -1,295 +0,0 @@ -# The 129 available packages have been obtained by transforming a -# conda repodata.json, and doing some manual fixes. -base: - available: - - affine 2.2.0 - - affine 2.2.1 - - asn1crypto 0.22.0 - - asn1crypto 0.23.0 - - asn1crypto 0.24.0 - - backports 1.0 - - name: backports.functools_lru_cache - version: '1.4' - depends: ['backports', 'setuptools'] - - name: backports.functools_lru_cache - version: '1.5' - depends: ['backports', 'setuptools'] - - beautifulsoup4 4.6.0 - - beautifulsoup4 4.6.1 - - beautifulsoup4 4.6.3 - - name: cachecontrol - version: 0.12.3 - depends: ['msgpack_python', 'requests'] - - name: cachecontrol - version: 0.12.4 - depends: ['msgpack_python', 'requests'] - - name: cachecontrol - version: 0.12.5 - depends: ['msgpack_python', 'requests'] - - certifi 2017.11.5 - - certifi 2017.7.27.1 - - certifi 2018.1.18 - - certifi 2018.4.16 - - certifi 2018.8.13 - # cffi is a bundled module in PyPy and causes resolution errors if pip - # tries to installed it. Give it a different name since we are simply - # checking the graph anyway and the identifier doesn't really matter. - - name: cffi_not_really - version: 1.10.0 - depends: ['pycparser'] - - name: cffi_not_really - version: 1.11.2 - depends: ['pycparser'] - - name: cffi_not_really - version: 1.11.4 - depends: ['pycparser'] - - name: cffi_not_really - version: 1.11.5 - depends: ['pycparser'] - - chardet 3.0.4 - - click 6.7 - - colorama 0.3.9 - - colour 0.1.4 - - colour 0.1.5 - - contextlib2 0.5.5 - - name: cryptography - version: 2.0.3 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'six >=1.4.1'] - - name: cryptography - version: 2.1.3 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'six >=1.4.1'] - - name: cryptography - version: 2.1.4 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'six >=1.4.1'] - - name: cryptography - version: 2.2.1 - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'idna >=2.1', 'six >=1.4.1'] - - name: cryptography - version: '2.3' - depends: ['asn1crypto >=0.21.0', 'cffi_not_really >=1.7', 'cryptography_vectors ~=2.3', 'idna >=2.1', 'six >=1.4.1'] - - cryptography_vectors 2.0.3 - - cryptography_vectors 2.1.3 - - cryptography_vectors 2.1.4 - - cryptography_vectors 2.2.1 - - cryptography_vectors 2.2.2 - - cryptography_vectors 2.3.0 - - name: cytoolz - version: 0.8.2 - depends: ['toolz >=0.8.0'] - - name: cytoolz - version: 0.9.0 - depends: ['toolz >=0.8.0'] - - name: cytoolz - version: 0.9.0.1 - depends: ['toolz >=0.8.0'] - - distlib 0.2.5 - - distlib 0.2.6 - - distlib 0.2.7 - - enum34 1.1.6 - - filelock 2.0.12 - - filelock 2.0.13 - - filelock 3.0.4 - - future 0.16.0 - - futures 3.1.1 - - futures 3.2.0 - - glob2 0.5 - - glob2 0.6 - - name: html5lib - version: '0.999999999' - depends: ['six >=1.9', 'webencodings'] - - name: html5lib - version: 1.0.1 - depends: ['six >=1.9', 'webencodings'] - - idna 2.6 - - idna 2.7 - - ipaddress 1.0.18 - - ipaddress 1.0.19 - - ipaddress 1.0.22 - - name: jinja2 - version: '2.10' - depends: ['markupsafe >=0.23', 'setuptools'] - - name: jinja2 - version: 2.9.6 - depends: ['markupsafe >=0.23', 'setuptools'] - - lockfile 0.12.2 - - markupsafe 1.0 - - msgpack_python 0.4.8 - - msgpack_python 0.5.1 - - msgpack_python 0.5.5 - - msgpack_python 0.5.6 - - name: packaging - version: '16.8' - depends: ['pyparsing', 'six'] - - name: packaging - version: '17.1' - depends: ['pyparsing', 'six'] - - name: pip - version: 10.0.1 - depends: ['setuptools', 'wheel'] - - name: pip - version: 9.0.1 - depends: ['cachecontrol', 'colorama', 'distlib', 'html5lib', 'lockfile', 'packaging', 'progress', 'requests', 'setuptools', 'webencodings', 'wheel'] - - name: pip - version: 9.0.3 - depends: ['setuptools', 'wheel'] - - pkginfo 1.4.1 - - pkginfo 1.4.2 - - progress 1.3 - - progress 1.4 - - psutil 5.2.2 - - psutil 5.3.1 - - psutil 5.4.0 - - psutil 5.4.1 - - psutil 5.4.3 - - psutil 5.4.5 - - psutil 5.4.6 - - pycosat 0.6.2 - - pycosat 0.6.3 - - pycparser 2.18 - - name: pyopenssl - version: 17.2.0 - depends: ['cryptography >=1.9', 'six >=1.5.2'] - - name: pyopenssl - version: 17.4.0 - depends: ['cryptography >=1.9', 'six >=1.5.2'] - - name: pyopenssl - version: 17.5.0 - depends: ['cryptography >=2.1.4', 'six >=1.5.2'] - - name: pyopenssl - version: 18.0.0 - depends: ['cryptography >=2.2.1', 'six >=1.5.2'] - - pyparsing 2.2.0 - - name: pysocks - version: 1.6.7 - depends: ['win_inet_pton'] - - name: pysocks - version: 1.6.8 - depends: ['win_inet_pton'] - - pywin32 221 - - pywin32 222 - - pywin32 223 - - pyyaml 3.12 - - pyyaml 3.13 - - name: requests - version: 2.18.4 - depends: ['certifi >=2017.4.17', 'chardet >=3.0.2,<3.1.0', 'idna >=2.5,<2.7', 'urllib3 >=1.21.1,<1.23'] - - name: requests - version: 2.19.1 - depends: ['certifi >=2017.4.17', 'chardet >=3.0.2,<3.1.0', 'idna >=2.5,<2.8', 'urllib3 >=1.21.1,<1.24'] - - scandir 1.5 - - scandir 1.6 - - scandir 1.7 - - scandir 1.8 - - scandir 1.9.0 - - name: setuptools - version: 36.2.2 - depends: ['certifi', 'wincertstore'] - - name: setuptools - version: 36.5.0 - depends: ['certifi', 'wincertstore'] - - name: setuptools - version: 38.4.0 - depends: ['certifi >=2016.09', 'wincertstore >=0.2'] - - name: setuptools - version: 38.5.1 - depends: ['certifi >=2016.09', 'wincertstore >=0.2'] - - name: setuptools - version: 39.0.1 - depends: ['certifi >=2016.09', 'wincertstore >=0.2'] - - name: setuptools - version: 39.1.0 - depends: ['certifi >=2016.09', 'wincertstore >=0.2'] - - name: setuptools - version: 39.2.0 - depends: ['certifi >=2016.09', 'wincertstore >=0.2'] - - name: setuptools - version: 40.0.0 - depends: ['certifi >=2016.09', 'wincertstore >=0.2'] - - six 1.8.2 - - six 1.10.0 - - six 1.11.0 - - toolz 0.8.2 - - toolz 0.9.0 - - name: urllib3 - version: '1.22' - depends: ['certifi', 'cryptography >=1.3.4', 'idna >=2.0.0', 'pyopenssl >=0.14', 'pysocks >=1.5.6,<2.0,!=1.5.7'] - - name: urllib3 - version: '1.23' - depends: ['certifi', 'cryptography >=1.3.4', 'idna >=2.0.0', 'pyopenssl >=0.14', 'pysocks >=1.5.6,<2.0,!=1.5.7'] - - webencodings 0.5.1 - - name: wheel - version: 0.29.0 - depends: ['setuptools'] - - name: wheel - version: 0.30.0 - depends: ['setuptools'] - - name: wheel - version: 0.31.0 - depends: ['setuptools'] - - name: wheel - version: 0.31.1 - depends: ['setuptools'] - - win_inet_pton 1.0.1 - - wincertstore 0.2 - -cases: -- - request: - - install: affine - response: - - state: - - affine 2.2.1 -- - request: - - install: cryptography - response: - - state: - - asn1crypto 0.24.0 - - cffi_not_really 1.11.5 - - cryptography 2.3 - - cryptography_vectors 2.3.0 - - idna 2.7 - - pycparser 2.18 - - six 1.11.0 - skip: legacy -- - request: - - install: cachecontrol - response: - - state: - - asn1crypto 0.24.0 - - cachecontrol 0.12.5 - - certifi 2018.8.13 - - cffi_not_really 1.11.5 - - chardet 3.0.4 - - cryptography 2.3 - - cryptography_vectors 2.3.0 - - idna 2.7 - - msgpack_python 0.5.6 - - pycparser 2.18 - - pyopenssl 18.0.0 - - pysocks 1.6.8 - - requests 2.19.1 - - six 1.11.0 - - urllib3 1.23 - - win_inet_pton 1.0.1 -- - request: - - install: cytoolz - response: - - state: - - cytoolz 0.9.0.1 - - toolz 0.9.0 -- - request: - - install: ['html5lib', 'six ==1.8.2'] - response: - - state: null - error: - code: 1 - stderr: >- - Cannot install six==1.8.2, html5lib 1.0.1, six==1.8.2 and - html5lib 0.999999999 because these package versions have - conflicting dependencies. - - skip: legacy diff --git a/tests/yaml/linter.py b/tests/yaml/linter.py deleted file mode 100644 index ac17bbc41be..00000000000 --- a/tests/yaml/linter.py +++ /dev/null @@ -1,108 +0,0 @@ -import re -import sys -from pprint import pprint - -import yaml - -sys.path.insert(0, '../../src') -sys.path.insert(0, '../..') - - -def check_dict(d, required=None, optional=None): - assert isinstance(d, dict) - if required is None: - required = [] - if optional is None: - optional = [] - for key in required: - if key not in d: - sys.exit("key %r is required" % key) - allowed_keys = set(required) - allowed_keys.update(optional) - for key in d.keys(): - if key not in allowed_keys: - sys.exit("key %r is not allowed. Allowed keys are: %r" % - (key, allowed_keys)) - - -def lint_case(case, verbose=False): - from tests.functional.test_yaml import convert_to_dict - - if verbose: - print("--- linting case ---") - pprint(case) - - check_dict(case, optional=['available', 'request', 'response', 'skip']) - available = case.get("available", []) - requests = case.get("request", []) - responses = case.get("response", []) - assert isinstance(available, list) - assert isinstance(requests, list) - assert isinstance(responses, list) - assert len(requests) == len(responses) - - for package in available: - if isinstance(package, str): - package = convert_to_dict(package) - if verbose: - pprint(package) - check_dict(package, - required=['name', 'version'], - optional=['depends', 'extras']) - version = package['version'] - assert isinstance(version, str), repr(version) - - for request, response in zip(requests, responses): - check_dict(request, optional=['install', 'uninstall', 'options']) - check_dict(response, optional=['state', 'error']) - assert len(response) >= 1 - assert isinstance(response.get('state') or [], list) - error = response.get('error') - if error: - check_dict(error, optional=['code', 'stderr']) - stderr = error.get('stderr') - if stderr: - if isinstance(stderr, str): - patters = [stderr] - elif isinstance(stderr, list): - patters = stderr - else: - raise "string or list expected, found %r" % stderr - for patter in patters: - re.compile(patter, re.I) - - -def lint_yml(yml_file, verbose=False): - if verbose: - print("=== linting: %s ===" % yml_file) - assert yml_file.endswith(".yml") - with open(yml_file) as fi: - data = yaml.safe_load(fi) - if verbose: - pprint(data) - - check_dict(data, required=['cases'], optional=['base']) - base = data.get("base", {}) - cases = data["cases"] - for _, case_template in enumerate(cases): - case = base.copy() - case.update(case_template) - lint_case(case, verbose) - - -if __name__ == '__main__': - from optparse import OptionParser - - p = OptionParser(usage="usage: %prog [options] FILE ...", - description="linter for pip's yaml test FILE(s)") - - p.add_option('-v', '--verbose', - action="store_true") - - opts, args = p.parse_args() - - if len(args) < 1: - p.error('at least one argument required, try -h') - - for yml_file in args: - lint_yml(yml_file, opts.verbose) diff --git a/tests/yaml/non_pinned.yml b/tests/yaml/non_pinned.yml deleted file mode 100644 index 6e9b26c4c10..00000000000 --- a/tests/yaml/non_pinned.yml +++ /dev/null @@ -1,24 +0,0 @@ -base: - available: - - A 1.0.0; depends B < 2.0.0 - - A 2.0.0; depends B < 3.0.0 - - B 1.0.0 - - B 2.0.0 - - B 2.1.0 - - B 3.0.0 - -cases: -- - request: - - install: A >= 2.0.0 - response: - - state: - - A 2.0.0 - - B 2.1.0 -- - request: - - install: A < 2.0.0 - response: - - state: - - A 1.0.0 - - B 1.0.0 diff --git a/tests/yaml/overlap1.yml b/tests/yaml/overlap1.yml deleted file mode 100644 index 9afbb04c379..00000000000 --- a/tests/yaml/overlap1.yml +++ /dev/null @@ -1,44 +0,0 @@ -# https://medium.com/knerd/the-nine-circles-of-python-dependency-hell-481d53e3e025 -# Circle 4: Overlapping transitive dependencies -base: - available: - - myapp 0.2.4; depends fussy, capridous - - name: fussy - version: 3.8.0 - depends: ['requests >=1.2.0,<3'] - - name: capridous - version: 1.1.0 - depends: ['requests >=1.0.3,<2'] - - requests 1.0.1 - - requests 1.0.3 - - requests 1.1.0 - - requests 1.2.0 - - requests 1.3.0 - - requests 2.1.0 - - requests 3.2.0 - -cases: -- - request: - - install: myapp - response: - - state: - - capridous 1.1.0 - - fussy 3.8.0 - - myapp 0.2.4 - - requests 1.3.0 - skip: legacy -- - request: - - install: fussy - response: - - state: - - fussy 3.8.0 - - requests 2.1.0 -- - request: - - install: capridous - response: - - state: - - capridous 1.1.0 - - requests 1.3.0 diff --git a/tests/yaml/pinned.yml b/tests/yaml/pinned.yml deleted file mode 100644 index c8bd3f35dbf..00000000000 --- a/tests/yaml/pinned.yml +++ /dev/null @@ -1,29 +0,0 @@ -base: - available: - - A 1.0.0 - - A 2.0.0 - - B 1.0.0; depends A == 1.0.0 - - B 2.0.0; depends A == 2.0.0 - -cases: -- - request: - - install: B - response: - - state: - - A 2.0.0 - - B 2.0.0 -- - request: - - install: B == 2.0.0 - response: - - state: - - A 2.0.0 - - B 2.0.0 -- - request: - - install: B == 1.0.0 - response: - - state: - - A 1.0.0 - - B 1.0.0 diff --git a/tests/yaml/pip988.yml b/tests/yaml/pip988.yml deleted file mode 100644 index 1190d2a4e07..00000000000 --- a/tests/yaml/pip988.yml +++ /dev/null @@ -1,37 +0,0 @@ -# https://github.com/pypa/pip/issues/988#issuecomment-606967707 -base: - available: - - A 1.0.0; depends B >= 1.0.0, C >= 1.0.0 - - A 2.0.0; depends B >= 2.0.0, C >= 1.0.0 - - B 1.0.0; depends C >= 1.0.0 - - B 2.0.0; depends C >= 2.0.0 - - C 1.0.0 - - C 2.0.0 - -cases: -- - request: - - install: C==1.0.0 - - install: B==1.0.0 - - install: A==1.0.0 - - install: A==2.0.0 - response: - - state: - - C 1.0.0 - - state: - - B 1.0.0 - - C 1.0.0 - - state: - - A 1.0.0 - - B 1.0.0 - - C 1.0.0 - - state: - - A 2.0.0 - - B 2.0.0 - - C 2.0.0 - # for the last install (A==2.0.0) the old resolver gives - # - A 2.0.0 - # - B 2.0.0 - # - C 1.0.0 - # but because B 2.0.0 depends on C >=2.0.0 this is wrong - skip: legacy diff --git a/tests/yaml/poetry2298.yml b/tests/yaml/poetry2298.yml deleted file mode 100644 index 8b0670896ae..00000000000 --- a/tests/yaml/poetry2298.yml +++ /dev/null @@ -1,24 +0,0 @@ -# see: https://github.com/python-poetry/poetry/issues/2298 -base: - available: - - poetry 1.0.5; depends zappa == 0.51.0, sphinx == 3.0.1 - - zappa 0.51.0; depends boto3 - - sphinx 3.0.1; depends docutils - - boto3 1.4.5; depends botocore ~=1.5.0 - - botocore 1.5.92; depends docutils <0.16 - - docutils 0.16.0 - - docutils 0.15.0 - -cases: -- - request: - - install: poetry - response: - - state: - - boto3 1.4.5 - - botocore 1.5.92 - - docutils 0.15.0 - - poetry 1.0.5 - - sphinx 3.0.1 - - zappa 0.51.0 - skip: legacy diff --git a/tests/yaml/simple.yml b/tests/yaml/simple.yml deleted file mode 100644 index 8e90e605d54..00000000000 --- a/tests/yaml/simple.yml +++ /dev/null @@ -1,47 +0,0 @@ -base: - available: - - simple 0.1.0 - - simple 0.2.0 - - base 0.1.0; depends dep - - dep 0.1.0 - -cases: -- - request: - - install: simple - - uninstall: simple - response: - - state: - - simple 0.2.0 - - state: null -- - request: - - install: simple - - install: dep - response: - - state: - - simple 0.2.0 - - state: - - dep 0.1.0 - - simple 0.2.0 -- - request: - - install: base - response: - - state: - - base 0.1.0 - - dep 0.1.0 -- - request: - - install: base - options: --no-deps - response: - - state: - - base 0.1.0 -- - request: - - install: ['dep', 'simple==0.1.0'] - response: - - state: - - dep 0.1.0 - - simple 0.1.0 diff --git a/tests/yaml/trivial.yml b/tests/yaml/trivial.yml deleted file mode 100644 index 418422044e4..00000000000 --- a/tests/yaml/trivial.yml +++ /dev/null @@ -1,24 +0,0 @@ -base: - available: - - a 0.1.0 - - b 0.2.0 - - c 0.3.0 - -cases: -- - request: - - install: ['a', 'b'] - - install: c - - uninstall: ['b', 'c'] - - uninstall: a - response: - - state: - - a 0.1.0 - - b 0.2.0 - - state: - - a 0.1.0 - - b 0.2.0 - - c 0.3.0 - - state: - - a 0.1.0 - - state: null diff --git a/tools/automation/news/template.rst b/tools/automation/news/template.rst deleted file mode 100644 index 7ea90573dc2..00000000000 --- a/tools/automation/news/template.rst +++ /dev/null @@ -1,38 +0,0 @@ -{% for section in sections %} -{% set underline = "-" %} -{% if section %} -{{section}} -{{ underline * section|length }}{% set underline = "~" %} - -{% endif %} -{% if sections[section] %} -{% for category, val in definitions.items() if category in sections[section] and category != 'trivial' %} - -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} - -{% if definitions[category]['showcontent'] %} -{% for text, values in sections[section][category]|dictsort(by='value') %} -- {{ text }}{% if category != 'vendor' and category != 'process' %} ({{ values|sort|join(', ') }}){% endif %} - -{% endfor %} -{% else %} -- {{ sections[section][category]['']|sort|join(', ') }} - - -{% endif %} -{% if sections[section][category]|length == 0 %} - -No significant changes. - - -{% else %} -{% endif %} -{% endfor %} -{% else %} - -No significant changes. - - -{% endif %} -{% endfor %} diff --git a/tools/automation/vendoring/patches/appdirs.patch b/tools/automation/vendoring/patches/appdirs.patch deleted file mode 100644 index 69afd3e8681..00000000000 --- a/tools/automation/vendoring/patches/appdirs.patch +++ /dev/null @@ -1,115 +0,0 @@ -diff --git a/src/pip/_vendor/appdirs.py b/src/pip/_vendor/appdirs.py -index ae67001a..3a52b758 100644 ---- a/src/pip/_vendor/appdirs.py -+++ b/src/pip/_vendor/appdirs.py -@@ -37,6 +37,10 @@ if sys.platform.startswith('java'): - # are actually checked for and the rest of the module expects - # *sys.platform* style strings. - system = 'linux2' -+elif sys.platform == 'cli' and os.name == 'nt': -+ # Detect Windows in IronPython to match pip._internal.utils.compat.WINDOWS -+ # Discussion: -+ system = 'win32' - else: - system = sys.platform - -@@ -64,7 +68,7 @@ def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): - for a discussion of issues. - - Typical user data directories are: -- Mac OS X: ~/Library/Application Support/ -+ Mac OS X: ~/Library/Application Support/ # or ~/.config/, if the other does not exist - Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined - Win XP (not roaming): C:\Documents and Settings\\Application Data\\ - Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ -@@ -150,7 +154,7 @@ def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): - if appname: - if version: - appname = os.path.join(appname, version) -- pathlist = [os.sep.join([x, appname]) for x in pathlist] -+ pathlist = [os.path.join(x, appname) for x in pathlist] - - if multipath: - path = os.pathsep.join(pathlist) -@@ -203,6 +203,8 @@ def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): - return path - - -+# for the discussion regarding site_config_dir locations -+# see - def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): - r"""Return full path to the user-shared data dir for this application. - -@@ -238,14 +244,15 @@ def site_config_dir(appname=None, appauthor=None, version=None, multipath=False) - if appname and version: - path = os.path.join(path, version) - else: -- # XDG default for $XDG_CONFIG_DIRS -+ # XDG default for $XDG_CONFIG_DIRS (missing or empty) -+ # see - # only first, if multipath is False -- path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') -- pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] -+ path = os.getenv('XDG_CONFIG_DIRS') or '/etc/xdg' -+ pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep) if x] - if appname: - if version: - appname = os.path.join(appname, version) -- pathlist = [os.sep.join([x, appname]) for x in pathlist] -+ pathlist = [os.path.join(x, appname) for x in pathlist] - - if multipath: - path = os.pathsep.join(pathlist) -@@ -291,6 +300,10 @@ def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): - if appauthor is None: - appauthor = appname - path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) -+ # When using Python 2, return paths as bytes on Windows like we do on -+ # other operating systems. See helper function docs for more details. -+ if not PY3 and isinstance(path, unicode): -+ path = _win_path_to_bytes(path) - if appname: - if appauthor is not False: - path = os.path.join(path, appauthor, appname) -@@ -557,18 +570,32 @@ def _get_win_folder_with_jna(csidl_name): - - if system == "win32": - try: -- import win32com.shell -- _get_win_folder = _get_win_folder_with_pywin32 -+ from ctypes import windll -+ _get_win_folder = _get_win_folder_with_ctypes - except ImportError: - try: -- from ctypes import windll -- _get_win_folder = _get_win_folder_with_ctypes -+ import com.sun.jna -+ _get_win_folder = _get_win_folder_with_jna - except ImportError: -- try: -- import com.sun.jna -- _get_win_folder = _get_win_folder_with_jna -- except ImportError: -- _get_win_folder = _get_win_folder_from_registry -+ _get_win_folder = _get_win_folder_from_registry -+ -+ -+def _win_path_to_bytes(path): -+ """Encode Windows paths to bytes. Only used on Python 2. -+ -+ Motivation is to be consistent with other operating systems where paths -+ are also returned as bytes. This avoids problems mixing bytes and Unicode -+ elsewhere in the codebase. For more details and discussion see -+ . -+ -+ If encoding using ASCII and MBCS fails, return the original Unicode path. -+ """ -+ for encoding in ('ASCII', 'MBCS'): -+ try: -+ return path.encode(encoding) -+ except (UnicodeEncodeError, LookupError): -+ pass -+ return path - - - #---- self test code diff --git a/tools/automation/vendoring/patches/certifi.patch b/tools/automation/vendoring/patches/certifi.patch deleted file mode 100644 index 9d5395a7b6b..00000000000 --- a/tools/automation/vendoring/patches/certifi.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/pip/_vendor/certifi/core.py b/src/pip/_vendor/certifi/core.py -index 5d2b8cd32..8987449f6 100644 ---- a/src/pip/_vendor/certifi/core.py -+++ b/src/pip/_vendor/certifi/core.py -@@ -33,7 +33,7 @@ try: - # We also have to hold onto the actual context manager, because - # it will do the cleanup whenever it gets garbage collected, so - # we will also store that at the global level as well. -- _CACERT_CTX = get_path("certifi", "cacert.pem") -+ _CACERT_CTX = get_path("pip._vendor.certifi", "cacert.pem") - _CACERT_PATH = str(_CACERT_CTX.__enter__()) - - return _CACERT_PATH diff --git a/tools/automation/vendoring/patches/requests.patch b/tools/automation/vendoring/patches/requests.patch deleted file mode 100644 index 08795ad3a3b..00000000000 --- a/tools/automation/vendoring/patches/requests.patch +++ /dev/null @@ -1,62 +0,0 @@ -diff --git a/src/pip/_vendor/requests/packages.py b/src/pip/_vendor/requests/packages.py -index 6336a07d..9582fa73 100644 ---- a/src/pip/_vendor/requests/packages.py -+++ b/src/pip/_vendor/requests/packages.py -@@ -4,11 +4,13 @@ import sys - # I don't like it either. Just look the other way. :) - - for package in ('urllib3', 'idna', 'chardet'): -- locals()[package] = __import__(package) -+ vendored_package = "pip._vendor." + package -+ locals()[package] = __import__(vendored_package) - # This traversal is apparently necessary such that the identities are - # preserved (requests.packages.urllib3.* is urllib3.*) - for mod in list(sys.modules): -- if mod == package or mod.startswith(package + '.'): -- sys.modules['requests.packages.' + mod] = sys.modules[mod] -+ if mod == vendored_package or mod.startswith(vendored_package + '.'): -+ unprefixed_mod = mod[len("pip._vendor."):] -+ sys.modules['pip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] - - # Kinda cool, though, right? - -diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py -index dc83261a8..517458b5a 100644 ---- a/src/pip/_vendor/requests/__init__.py -+++ b/src/pip/_vendor/requests/__init__.py -@@ -94,6 +94,11 @@ except (AssertionError, ValueError): - # if the standard library doesn't support SNI or the - # 'ssl' library isn't available. - try: -+ # Note: This logic prevents upgrading cryptography on Windows, if imported -+ # as part of pip. -+ from pip._internal.utils.compat import WINDOWS -+ if not WINDOWS: -+ raise ImportError("pip internals: don't import cryptography on Windows") - try: - import ssl - except ImportError: - -diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py -index eb6530d..353ec29 100644 ---- a/src/pip/_vendor/requests/compat.py -+++ b/src/pip/_vendor/requests/compat.py -@@ -25,10 +25,14 @@ - #: Python 3.x? - is_py3 = (_ver[0] == 3) - --try: -- import simplejson as json --except ImportError: -- import json -+# Note: We've patched out simplejson support in pip because it prevents -+# upgrading simplejson on Windows. -+# try: -+# import simplejson as json -+# except (ImportError, SyntaxError): -+# # simplejson does not support Python 3.2, it throws a SyntaxError -+# # because of u'...' Unicode literals. -+import json - - # --------- - # Specifics diff --git a/.azure-pipelines/scripts/New-RAMDisk.ps1 b/tools/ci/New-RAMDisk.ps1 similarity index 100% rename from .azure-pipelines/scripts/New-RAMDisk.ps1 rename to tools/ci/New-RAMDisk.ps1 diff --git a/tools/news/template.rst b/tools/news/template.rst new file mode 100644 index 00000000000..8d0ceb89f3f --- /dev/null +++ b/tools/news/template.rst @@ -0,0 +1,45 @@ +{# This is a heavily customised version of towncrier's default template. #} + +{#- + Only render if there's any changes to show. + + This serves as a compatibility "hack" since we render unreleased news entries + in our changelog with ``sphinxcontrib.towncrier``; which triggers a render even + when there's no entries to be rendered. +#} +{% if sections[''] %} + +{#- Heading for individual version #} +{{ versiondata.version }} ({{ versiondata.date }}) +{{ top_underline * ((versiondata.version + versiondata.date)|length + 3) }} +{# + + The following loop will run exactly once, with ``section_name == ""``. + + This is due to the undocumented "sections" feature in towncrier. + See https://github.com/twisted/towncrier/issues/61. + + We don't use this feature, and this template doesn't render the section + heading for that reason. +#} +{% for section_name, entries_by_type in sections.items() -%} +{# Only show types with entries and ``showcontent = true``, using the order from pyproject.toml #} +{% for type_ in definitions if (sections[section_name][type_] and definitions[type_]['showcontent']) %} + +{# Heading for individual types #} +{{ definitions[type_]['name'] }} +{{ underlines[0] * definitions[type_]['name']|length }} +{# This is the loop that generates individual entries #} +{% for message, issue_reference in sections[section_name][type_]|dictsort(by='value') %} + +- {{ message }} + {%- if type_ not in ["vendor", "process"] %} ({{ issue_reference|sort|join(', ') }}){% endif %} +{% endfor %} + +{% else %} +{# We only have entries where the type has ``showcontent = true``. #} +No significant changes. + +{% endfor -%} +{% endfor -%} +{% endif -%} diff --git a/tools/automation/release/__init__.py b/tools/release/__init__.py similarity index 84% rename from tools/automation/release/__init__.py rename to tools/release/__init__.py index 20775d5e21d..65fa0267632 100644 --- a/tools/automation/release/__init__.py +++ b/tools/release/__init__.py @@ -27,8 +27,8 @@ def get_version_from_arguments(session: Session) -> Optional[str]: session.install("packaging") cmd = [ os.path.join(session.bin, "python"), - "tools/automation/release/check_version.py", - version + "tools/release/check_version.py", + version, ] not_ok = subprocess.run(cmd).returncode if not_ok: @@ -46,8 +46,7 @@ def modified_files_in_git(*args: str) -> int: def get_author_list() -> List[str]: - """Get the list of authors from Git commits. - """ + """Get the list of authors from Git commits.""" # subprocess because session.run doesn't give us stdout # only use names in list of Authors result = subprocess.run( @@ -90,7 +89,7 @@ def generate_news(session: Session, version: str) -> None: def update_version_file(version: str, filepath: str) -> None: - with open(filepath, "r", encoding="utf-8") as f: + with open(filepath, encoding="utf-8") as f: content = list(f) file_modified = False @@ -102,13 +101,16 @@ def update_version_file(version: str, filepath: str) -> None: else: f.write(line) - assert file_modified, \ - f"Version file {filepath} did not get modified" + assert file_modified, f"Version file {filepath} did not get modified" def create_git_tag(session: Session, tag_name: str, *, message: str) -> None: session.run( - "git", "tag", "-m", message, tag_name, external=True, silent=True, + # fmt: off + "git", "tag", "-m", message, tag_name, + # fmt: on + external=True, + silent=True, ) @@ -147,8 +149,8 @@ def have_files_in_folder(folder_name: str) -> bool: @contextlib.contextmanager def workdir( - nox_session: Session, - dir_path: pathlib.Path, + nox_session: Session, + dir_path: pathlib.Path, ) -> Iterator[pathlib.Path]: """Temporarily chdir when entering CM and chdir back on exit.""" orig_dir = pathlib.Path.cwd() @@ -162,35 +164,37 @@ def workdir( @contextlib.contextmanager def isolated_temporary_checkout( - nox_session: Session, - target_ref: str, + nox_session: Session, + target_ref: str, ) -> Iterator[pathlib.Path]: """Make a clean checkout of a given version in tmp dir.""" with tempfile.TemporaryDirectory() as tmp_dir_path: tmp_dir = pathlib.Path(tmp_dir_path) - git_checkout_dir = tmp_dir / f'pip-build-{target_ref}' + git_checkout_dir = tmp_dir / f"pip-build-{target_ref}" nox_session.run( - 'git', 'worktree', 'add', '--force', '--checkout', - str(git_checkout_dir), str(target_ref), - external=True, silent=True, + # fmt: off + "git", "clone", + "--depth", "1", + "--config", "core.autocrlf=false", + "--branch", str(target_ref), + "--", + ".", str(git_checkout_dir), + # fmt: on + external=True, + silent=True, ) - try: - yield git_checkout_dir - finally: - nox_session.run( - 'git', 'worktree', 'remove', '--force', - str(git_checkout_dir), - external=True, silent=True, - ) + yield git_checkout_dir def get_git_untracked_files() -> Iterator[str]: """List all local file paths that aren't tracked by Git.""" git_ls_files_cmd = ( + # fmt: off "git", "ls-files", "--ignored", "--exclude-standard", "--others", "--", ".", + # fmt: on ) # session.run doesn't seem to return any output: ls_files_out = subprocess.check_output(git_ls_files_cmd, text=True) diff --git a/tools/automation/release/check_version.py b/tools/release/check_version.py similarity index 100% rename from tools/automation/release/check_version.py rename to tools/release/check_version.py diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt deleted file mode 100644 index 9b4e9849054..00000000000 --- a/tools/requirements/tests.txt +++ /dev/null @@ -1,16 +0,0 @@ ---use-feature=2020-resolver -cryptography==2.8 -freezegun -mock -pretend -pytest -pytest-cov -pytest-rerunfailures -pytest-timeout -pytest-xdist -pyyaml -scripttest -setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. -https://github.com/pypa/virtualenv/archive/legacy.zip#egg=virtualenv -werkzeug==0.16.0 -wheel diff --git a/tools/tox_pip.py b/tools/tox_pip.py index 5996dade6d2..671518029a5 100644 --- a/tools/tox_pip.py +++ b/tools/tox_pip.py @@ -1,31 +1,37 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - import os import shutil import subprocess import sys from glob import glob +from typing import List -VIRTUAL_ENV = os.environ['VIRTUAL_ENV'] -TOX_PIP_DIR = os.path.join(VIRTUAL_ENV, 'pip') +VIRTUAL_ENV = os.environ["VIRTUAL_ENV"] +TOX_PIP_DIR = os.path.join(VIRTUAL_ENV, "pip") -def pip(args): +def pip(args: List[str]) -> None: # First things first, get a recent (stable) version of pip. if not os.path.exists(TOX_PIP_DIR): - subprocess.check_call([sys.executable, '-m', 'pip', - '--disable-pip-version-check', - 'install', '-t', TOX_PIP_DIR, - 'pip']) - shutil.rmtree(glob(os.path.join(TOX_PIP_DIR, 'pip-*.dist-info'))[0]) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "--disable-pip-version-check", + "install", + "-t", + TOX_PIP_DIR, + "pip", + ] + ) + shutil.rmtree(glob(os.path.join(TOX_PIP_DIR, "pip-*.dist-info"))[0]) # And use that version. - pypath = os.environ.get('PYTHONPATH') - pypath = pypath.split(os.pathsep) if pypath is not None else [] + pypath_env = os.environ.get("PYTHONPATH") + pypath = pypath_env.split(os.pathsep) if pypath_env is not None else [] pypath.insert(0, TOX_PIP_DIR) - os.environ['PYTHONPATH'] = os.pathsep.join(pypath) - subprocess.check_call([sys.executable, '-m', 'pip'] + args) + os.environ["PYTHONPATH"] = os.pathsep.join(pypath) + subprocess.check_call([sys.executable, "-m", "pip"] + args) -if __name__ == '__main__': +if __name__ == "__main__": pip(sys.argv[1:]) diff --git a/tools/travis/install.sh b/tools/travis/install.sh deleted file mode 100755 index 3b12d69a26b..00000000000 --- a/tools/travis/install.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -x - -pip install --upgrade setuptools -pip install --upgrade tox tox-venv -pip freeze --all diff --git a/tools/travis/run.sh b/tools/travis/run.sh deleted file mode 100755 index df8f03e7a57..00000000000 --- a/tools/travis/run.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -set -e - -# Short circuit test runs if there are no code changes involved. -if [[ $TOXENV != docs ]] || [[ $TOXENV != lint ]]; then - if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] - then - echo "This is not a PR -- will do a complete build." - else - # Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE - # may include more changes than desired if the history is convoluted. - # Instead, explicitly fetch the base branch and compare against the - # merge-base commit. - git fetch -q origin +refs/heads/$TRAVIS_BRANCH - changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) - echo "Files changed:" - echo "$changes" - if ! echo "$changes" | grep -qvE '(\.rst$)|(^docs)|(^news)|(^\.github)' - then - echo "Code was not changed -- skipping build." - exit - fi - fi -fi - -# Export the correct TOXENV when not provided. -echo "Determining correct TOXENV..." -if [[ -z "$TOXENV" ]]; then - if [[ ${TRAVIS_PYTHON_VERSION} == pypy* ]]; then - export TOXENV=pypy - else - # We use the syntax ${string:index:length} to make 2.7 -> py27 - _major=${TRAVIS_PYTHON_VERSION:0:1} - _minor=${TRAVIS_PYTHON_VERSION:2:1} - export TOXENV="py${_major}${_minor}" - fi -fi -echo "TOXENV=${TOXENV}" - -if [[ -z "$NEW_RESOLVER" ]]; then - RESOLVER_SWITCH='' -else - RESOLVER_SWITCH='--new-resolver' -fi - -# Print the commands run for this test. -set -x -if [[ "$GROUP" == "1" ]]; then - # Unit tests - tox -- --use-venv -m unit -n auto - # Integration tests (not the ones for 'pip install') - tox -- -m integration -n auto --durations=5 -k "not test_install" \ - --use-venv $RESOLVER_SWITCH -elif [[ "$GROUP" == "2" ]]; then - # Separate Job for running integration tests for 'pip install' - tox -- -m integration -n auto --durations=5 -k "test_install" \ - --use-venv $RESOLVER_SWITCH -elif [[ "$GROUP" == "3" ]]; then - # Separate Job for tests that fail with the new resolver - tox -- -m fails_on_new_resolver -n auto --durations=5 \ - --use-venv $RESOLVER_SWITCH --new-resolver-runtests -else - # Non-Testing Jobs should run once - tox -fi diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh deleted file mode 100755 index c52ce5f167e..00000000000 --- a/tools/travis/setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -echo "Setting Git Credentials..." -git config --global user.email "distutils-sig@python.org" -git config --global user.name "pip" diff --git a/tools/vendoring/patches/certifi.patch b/tools/vendoring/patches/certifi.patch new file mode 100644 index 00000000000..a36a0020ff5 --- /dev/null +++ b/tools/vendoring/patches/certifi.patch @@ -0,0 +1,41 @@ +diff --git a/src/pip/_vendor/certifi/core.py b/src/pip/_vendor/certifi/core.py +index 5d2b8cd32..b8140cf1a 100644 +--- a/src/pip/_vendor/certifi/core.py ++++ b/src/pip/_vendor/certifi/core.py +@@ -8,7 +8,21 @@ This module returns the installation location of cacert.pem or its contents. + """ + import os + ++ ++class _PipPatchedCertificate(Exception): ++ pass ++ ++ + try: ++ # Return a certificate file on disk for a standalone pip zipapp running in ++ # an isolated build environment to use. Passing --cert to the standalone ++ # pip does not work since requests calls where() unconditionally on import. ++ _PIP_STANDALONE_CERT = os.environ.get("_PIP_STANDALONE_CERT") ++ if _PIP_STANDALONE_CERT: ++ def where(): ++ return _PIP_STANDALONE_CERT ++ raise _PipPatchedCertificate() ++ + from importlib.resources import path as get_path, read_text + + _CACERT_CTX = None +@@ -33,11 +47,13 @@ try: + # We also have to hold onto the actual context manager, because + # it will do the cleanup whenever it gets garbage collected, so + # we will also store that at the global level as well. +- _CACERT_CTX = get_path("certifi", "cacert.pem") ++ _CACERT_CTX = get_path("pip._vendor.certifi", "cacert.pem") + _CACERT_PATH = str(_CACERT_CTX.__enter__()) + + return _CACERT_PATH + ++except _PipPatchedCertificate: ++ pass + + except ImportError: + # This fallback will work for Python versions prior to 3.7 that lack the diff --git a/tools/vendoring/patches/pkg_resources.patch b/tools/vendoring/patches/pkg_resources.patch new file mode 100644 index 00000000000..6556a860867 --- /dev/null +++ b/tools/vendoring/patches/pkg_resources.patch @@ -0,0 +1,22 @@ +diff --git a/src/pip/_vendor/pkg_resources/__init__.py b/src/pip/_vendor/pkg_resources/__init__.py +index a457ff27e..4cd562cf9 100644 +--- a/src/pip/_vendor/pkg_resources/__init__.py ++++ b/src/pip/_vendor/pkg_resources/__init__.py +@@ -77,7 +77,7 @@ except ImportError: + importlib_machinery = None + + from . import py31compat +-from pkg_resources.extern import appdirs ++from pkg_resources.extern import platformdirs + from pkg_resources.extern import packaging + __import__('pkg_resources.extern.packaging.version') + __import__('pkg_resources.extern.packaging.specifiers') +@@ -1310,7 +1310,7 @@ def get_default_cache(): + """ + return ( + os.environ.get('PYTHON_EGG_CACHE') +- or appdirs.user_cache_dir(appname='Python-Eggs') ++ or platformdirs.user_cache_dir(appname='Python-Eggs') + ) + + diff --git a/tools/vendoring/patches/platformdirs.patch b/tools/vendoring/patches/platformdirs.patch new file mode 100644 index 00000000000..079187c7a2a --- /dev/null +++ b/tools/vendoring/patches/platformdirs.patch @@ -0,0 +1,22 @@ +diff --git a/src/pip/_vendor/platformdirs/__init__.py b/src/pip/_vendor/platformdirs/__init__.py +index 46b1157d7..2cfaff76c 100644 +--- a/src/pip/_vendor/platformdirs/__init__.py ++++ b/src/pip/_vendor/platformdirs/__init__.py +@@ -19,13 +19,13 @@ + + def _set_platform_dir_class() -> type[PlatformDirsABC]: + if os.getenv("ANDROID_DATA") == "/data" and os.getenv("ANDROID_ROOT") == "/system": +- module, name = "platformdirs.android", "Android" ++ module, name = "pip._vendor.platformdirs.android", "Android" + elif sys.platform == "win32": +- module, name = "platformdirs.windows", "Windows" ++ module, name = "pip._vendor.platformdirs.windows", "Windows" + elif sys.platform == "darwin": +- module, name = "platformdirs.macos", "MacOS" ++ module, name = "pip._vendor.platformdirs.macos", "MacOS" + else: +- module, name = "platformdirs.unix", "Unix" ++ module, name = "pip._vendor.platformdirs.unix", "Unix" + result: type[PlatformDirsABC] = getattr(importlib.import_module(module), name) + return result + diff --git a/tools/vendoring/patches/pygments.patch b/tools/vendoring/patches/pygments.patch new file mode 100644 index 00000000000..3cabf9d6dcc --- /dev/null +++ b/tools/vendoring/patches/pygments.patch @@ -0,0 +1,37 @@ +This patch mainly handles tweaking imports into a form that can be transformed +to import from the vendored namespace. + +diff --git a/src/pip/_vendor/pygments/cmdline.py b/src/pip/_vendor/pygments/cmdline.py +index d9a0fdc8b..db6de0cd3 100644 +--- a/src/pip/_vendor/pygments/cmdline.py ++++ b/src/pip/_vendor/pygments/cmdline.py +@@ -410,11 +410,11 @@ def is_only_option(opt): + outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding) + fmter.encoding = None + try: +- import colorama.initialise ++ import colorama.initialise as colorama_initialise + except ImportError: + pass + else: +- outfile = colorama.initialise.wrap_stream( ++ outfile = colorama_initialise.wrap_stream( + outfile, convert=None, strip=None, autoreset=False, wrap=True) + + # When using the LaTeX formatter and the option `escapeinside` is +diff --git a/src/pip/_vendor/pygments/__main__.py b/src/pip/_vendor/pygments/__main__.py +index c6e2517df..76255b525 100644 +--- a/src/pip/_vendor/pygments/__main__.py ++++ b/src/pip/_vendor/pygments/__main__.py +@@ -9,9 +9,9 @@ + """ + + import sys +-import pygments.cmdline ++from pygments.cmdline import main + + try: +- sys.exit(pygments.cmdline.main(sys.argv)) ++ sys.exit(main(sys.argv)) + except KeyboardInterrupt: + sys.exit(1) diff --git a/tools/vendoring/patches/requests.patch b/tools/vendoring/patches/requests.patch new file mode 100644 index 00000000000..670c1b0cae7 --- /dev/null +++ b/tools/vendoring/patches/requests.patch @@ -0,0 +1,133 @@ +diff --git a/src/pip/_vendor/requests/packages.py b/src/pip/_vendor/requests/packages.py +index 0f8ae0d38..9582fa730 100644 +--- a/src/pip/_vendor/requests/packages.py ++++ b/src/pip/_vendor/requests/packages.py +@@ -1,26 +1,16 @@ + import sys + +-try: +- import chardet +-except ImportError: +- import charset_normalizer as chardet +- import warnings +- +- warnings.filterwarnings('ignore', 'Trying to detect', module='charset_normalizer') +- + # This code exists for backwards compatibility reasons. + # I don't like it either. Just look the other way. :) + +-for package in ('urllib3', 'idna'): +- locals()[package] = __import__(package) ++for package in ('urllib3', 'idna', 'chardet'): ++ vendored_package = "pip._vendor." + package ++ locals()[package] = __import__(vendored_package) + # This traversal is apparently necessary such that the identities are + # preserved (requests.packages.urllib3.* is urllib3.*) + for mod in list(sys.modules): +- if mod == package or mod.startswith(package + '.'): +- sys.modules['requests.packages.' + mod] = sys.modules[mod] ++ if mod == vendored_package or mod.startswith(vendored_package + '.'): ++ unprefixed_mod = mod[len("pip._vendor."):] ++ sys.modules['pip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] + +-target = chardet.__name__ +-for mod in list(sys.modules): +- if mod == target or mod.startswith(target + '.'): +- sys.modules['requests.packages.' + target.replace(target, 'chardet')] = sys.modules[mod] + # Kinda cool, though, right? + +diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py +index 973497f5e..4f80e28fc 100644 +--- a/src/pip/_vendor/requests/__init__.py ++++ b/src/pip/_vendor/requests/__init__.py +@@ -44,10 +44,7 @@ import urllib3 + import warnings + from .exceptions import RequestsDependencyWarning + +-try: +- from charset_normalizer import __version__ as charset_normalizer_version +-except ImportError: +- charset_normalizer_version = None ++charset_normalizer_version = None + + try: + from chardet import __version__ as chardet_version +@@ -107,6 +104,11 @@ except (AssertionError, ValueError): + # if the standard library doesn't support SNI or the + # 'ssl' library isn't available. + try: ++ # Note: This logic prevents upgrading cryptography on Windows, if imported ++ # as part of pip. ++ from pip._internal.utils.compat import WINDOWS ++ if not WINDOWS: ++ raise ImportError("pip internals: don't import cryptography on Windows") + try: + import ssl + except ImportError: + +diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py +index 409b7b028..9e2937167 100644 +--- a/src/pip/_vendor/requests/compat.py ++++ b/src/pip/_vendor/requests/compat.py +@@ -8,10 +8,7 @@ This module handles import compatibility issues between Python 2 and + Python 3. + """ + +-try: +- import chardet +-except ImportError: +- import charset_normalizer as chardet ++import chardet + + import sys + +@@ -28,12 +28,14 @@ is_py2 = (_ver[0] == 2) + #: Python 3.x? + is_py3 = (_ver[0] == 3) + +-has_simplejson = False +-try: +- import simplejson as json +- has_simplejson = True +-except ImportError: +- import json ++# Note: We've patched out simplejson support in pip because it prevents ++# upgrading simplejson on Windows. ++# try: ++# import simplejson as json ++# except (ImportError, SyntaxError): ++# # simplejson does not support Python 3.2, it throws a SyntaxError ++# # because of u'...' Unicode literals. ++import json + + # --------- + # Specifics +@@ -68,10 +70,7 @@ elif is_py3: + # Keep OrderedDict for backwards compatibility. + from collections import OrderedDict + from collections.abc import Callable, Mapping, MutableMapping +- if has_simplejson: +- from simplejson import JSONDecodeError +- else: +- from json import JSONDecodeError ++ from json import JSONDecodeError + + builtin_str = str + str = str + +diff --git a/src/pip/_vendor/requests/help.py b/src/pip/_vendor/requests/help.py +index 3a843404c..745f0d7b3 100644 +--- a/src/pip/_vendor/requests/help.py ++++ b/src/pip/_vendor/requests/help.py +@@ -11,10 +11,7 @@ import urllib3 + + from . import __version__ as requests_version + +-try: +- import charset_normalizer +-except ImportError: +- charset_normalizer = None ++charset_normalizer = None + + try: + import chardet diff --git a/tools/vendoring/patches/tenacity.patch b/tools/vendoring/patches/tenacity.patch new file mode 100644 index 00000000000..85b29c60ca1 --- /dev/null +++ b/tools/vendoring/patches/tenacity.patch @@ -0,0 +1,34 @@ +diff --git a/src/pip/_vendor/tenacity/__init__.py b/src/pip/_vendor/tenacity/__init__.py +index 88c28d2d6..086ad46e1 100644 +--- a/src/pip/_vendor/tenacity/__init__.py ++++ b/src/pip/_vendor/tenacity/__init__.py +@@ -76,10 +76,12 @@ from .after import after_nothing # noqa + from .before_sleep import before_sleep_log # noqa + from .before_sleep import before_sleep_nothing # noqa + +-try: +- import tornado # type: ignore +-except ImportError: +- tornado = None # type: ignore ++# Replace a conditional import with a hard-coded None so that pip does ++# not attempt to use tornado even if it is present in the environment. ++# If tornado is non-None, tenacity will attempt to execute some code ++# that is sensitive to the version of tornado, which could break pip ++# if an old version is found. ++tornado = None # type: ignore + + if t.TYPE_CHECKING: + import types + +--- a/src/pip/_vendor/tenacity/__init__.py ++++ b/src/pip/_vendor/tenacity/__init__.py +@@ -190,7 +190,7 @@ class RetryError(Exception): + self.last_attempt = last_attempt + super().__init__(last_attempt) + +- def reraise(self) -> t.NoReturn: ++ def reraise(self) -> "t.NoReturn": + if self.last_attempt.failed: + raise self.last_attempt.result() + raise self + diff --git a/tools/vendoring/patches/urllib3.patch b/tools/vendoring/patches/urllib3.patch new file mode 100644 index 00000000000..747b81e1d5a --- /dev/null +++ b/tools/vendoring/patches/urllib3.patch @@ -0,0 +1,63 @@ +diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py +index b97555454..189132baa 100644 +--- a/src/pip/_vendor/urllib3/contrib/securetransport.py ++++ b/src/pip/_vendor/urllib3/contrib/securetransport.py +@@ -19,8 +19,8 @@ + + To use this module, simply import and inject it:: + +- import urllib3.contrib.securetransport +- urllib3.contrib.securetransport.inject_into_urllib3() ++ import urllib3.contrib.securetransport as securetransport ++ securetransport.inject_into_urllib3() + + Happy TLSing! + +diff --git a/src/pip/_vendor/urllib3/contrib/pyopenssl.py b/src/pip/_vendor/urllib3/contrib/pyopenssl.py +index c43146279..4cded53f6 100644 +--- a/src/pip/_vendor/urllib3/contrib/pyopenssl.py ++++ b/src/pip/_vendor/urllib3/contrib/pyopenssl.py +@@ -28,7 +28,7 @@ + .. code-block:: python + + try: +- import urllib3.contrib.pyopenssl +- urllib3.contrib.pyopenssl.inject_into_urllib3() ++ import urllib3.contrib.pyopenssl as pyopenssl ++ pyopenssl.inject_into_urllib3() + except ImportError: + pass + +diff --git a/src/pip/_vendor/urllib3/response.py b/src/pip/_vendor/urllib3/response.py +index 38693f4fc..776e49dd2 100644 +--- a/src/pip/_vendor/urllib3/response.py ++++ b/src/pip/_vendor/urllib3/response.py +@@ -7,10 +7,7 @@ from contextlib import contextmanager + from socket import error as SocketError + from socket import timeout as SocketTimeout + +-try: +- import brotli +-except ImportError: +- brotli = None ++brotli = None + + from ._collections import HTTPHeaderDict + from .connection import BaseSSLError, HTTPException +diff --git a/src/pip/_vendor/urllib3/util/request.py b/src/pip/_vendor/urllib3/util/request.py +index 25103383e..330766ef4 100644 +--- a/src/pip/_vendor/urllib3/util/request.py ++++ b/src/pip/_vendor/urllib3/util/request.py +@@ -13,12 +13,6 @@ SKIP_HEADER = "@@@SKIP_HEADER@@@" + SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"]) + + ACCEPT_ENCODING = "gzip,deflate" +-try: +- import brotli as _unused_module_brotli # noqa: F401 +-except ImportError: +- pass +-else: +- ACCEPT_ENCODING += ",br" + + _FAILEDTELL = object() + diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 7fcc2c664a2..00000000000 --- a/tox.ini +++ /dev/null @@ -1,82 +0,0 @@ -[tox] -minversion = 3.4.0 -envlist = - docs, packaging, lint, vendoring, - py36, py37, py38, py39, pypy3 - -[helpers] -# Wrapper for calls to pip that make sure the version being used is the -# original virtualenv (stable) version, and not the code being tested. -pip = python {toxinidir}/tools/tox_pip.py -mkdirp = python -c 'import os, sys; os.path.exists(sys.argv[1]) or os.mkdir(sys.argv[1])' - -[testenv] -# Remove USERNAME once we drop PY2. -passenv = - CI - GIT_SSL_CAINFO - USERNAME - HTTP_PROXY - HTTPS_PROXY - NO_PROXY -setenv = - # This is required in order to get UTF-8 output inside of the subprocesses - # that our tests use. - LC_CTYPE = en_US.UTF-8 -deps = -r{toxinidir}/tools/requirements/tests.txt -commands_pre = - python -c 'import shutil, sys; shutil.rmtree(sys.argv[1], ignore_errors=True)' {toxinidir}/tests/data/common_wheels - {[helpers]pip} wheel -w {toxinidir}/tests/data/common_wheels -r {toxinidir}/tools/requirements/tests-common_wheels.txt -commands = pytest --timeout 300 [] -install_command = {[helpers]pip} install {opts} {packages} -list_dependencies_command = {[helpers]pip} freeze --all - -[testenv:coverage] -basepython = python3 -commands = - {[helpers]mkdirp} {toxinidir}/.coverage-output - pytest --timeout 300 --cov=pip --cov-config={toxinidir}/setup.cfg [] - -setenv = - # Used in coverage configuration in setup.cfg. - COVERAGE_OUTPUT_DIR = {toxinidir}/.coverage-output - # Ensure coverage is enabled in child processes in virtual environments - # since they won't already have been enabled by pytest-cov. - COVERAGE_PROCESS_START = {toxinidir}/setup.cfg - # Used in coverage configuration in setup.cfg. - PIP_CI_COVERAGE_EXCLUDES = if PY2 - -[testenv:docs] -# Don't skip install here since pip_sphinxext uses pip's internals. -deps = -r{toxinidir}/tools/requirements/docs.txt -basepython = python3 -commands = - sphinx-build -W -d {envtmpdir}/doctrees/html -b html docs/html docs/build/html - # Having the conf.py in the docs/html is weird but needed because we - # can not use a different configuration directory vs source directory on RTD - # currently -- https://github.com/rtfd/readthedocs.org/issues/1543. - # That is why we have a "-c docs/html" in the next line. - sphinx-build -W -d {envtmpdir}/doctrees/man -b man docs/man docs/build/man -c docs/html - -[testenv:lint] -skip_install = True -commands_pre = -deps = pre-commit -commands = - pre-commit run [] --all-files --show-diff-on-failure - pre-commit run [] -c .pre-commit-config-slow.yaml --all-files --show-diff-on-failure - -[testenv:vendoring] -basepython = python3 -skip_install = True -commands_pre = -deps = - vendoring~=0.3.3 - # Required, otherwise we interpret --no-binary :all: as - # "do not build wheels", which fails for PEP 517 requirements - pip>=19.3.1 -whitelist_externals = git -commands = - # Check that the vendoring is up-to-date - vendoring sync . -v - git diff --exit-code

' : '\U0001d4ab', + '\\' : '\U0001d4ac', + '\\' : '\U0000211b', + '\\' : '\U0001d4ae', + '\\' : '\U0001d4af', + '\\' : '\U0001d4b0', + '\\' : '\U0001d4b1', + '\\' : '\U0001d4b2', + '\\' : '\U0001d4b3', + '\\' : '\U0001d4b4', + '\\' : '\U0001d4b5', + '\\' : '\U0001d5ba', + '\\' : '\U0001d5bb', + '\\' : '\U0001d5bc', + '\\' : '\U0001d5bd', + '\\' : '\U0001d5be', + '\\' : '\U0001d5bf', + '\\' : '\U0001d5c0', + '\\' : '\U0001d5c1', + '\\' : '\U0001d5c2', + '\\' : '\U0001d5c3', + '\\' : '\U0001d5c4', + '\\' : '\U0001d5c5', + '\\' : '\U0001d5c6', + '\\' : '\U0001d5c7', + '\\' : '\U0001d5c8', + '\\