tests.yml

name: Test

on:
  workflow_dispatch:
  push:
    branches:
      - main
  pull_request:
    paths-ignore:
      - "**/*.md"
      - "scripts/**"

permissions:
  contents: read

env:
  PIP_DISABLE_PIP_VERSION_CHECK: 1
  FORCE_COLOR: 1

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  typeshed-structure:
    name: Check typeshed structure
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - run: uv pip install -r requirements-tests.txt --system
      - run: python ./tests/check_typeshed_structure.py

  pytype:
    name: Run pytype against the stubs
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          # Max supported Python version as of pytype 2024.2.27.
          python-version: "3.11"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - run: uv pip install -r requirements-tests.txt --system
      - name: Install external dependencies for 3rd-party stubs
        run: |
          DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
          if [ -n "$DEPENDENCIES" ]; then
            echo "Installing packages: $DEPENDENCIES"
            uv pip install $DEPENDENCIES --system
          fi
      - run: uv pip freeze
      - run: ./tests/pytype_test.py --print-stderr

  mypy:
    name: Run mypy against the stubs
    runs-on: ubuntu-latest
    strategy:
      matrix:
        platform: ["linux", "win32", "darwin"]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - run: uv pip install -r requirements-tests.txt --system
      - run: python ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }}

  regression-tests:
    name: Run mypy on the test cases
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          # Use py311 for now, as py312 seems to be around 30s slower in CI
          # TODO: figure out why that is (#11590)
          python-version: "3.11"
      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - run: uv pip install -r requirements-tests.txt --system
      - run: python ./tests/regr_test.py --all --verbosity QUIET

  pyright:
    name: Test typeshed with pyright
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-platform: ["Linux", "Windows", "Darwin"]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Install typeshed test-suite requirements
        # Install these so we can run `get_external_stub_requirements.py`
        run: uv pip install -r requirements-tests.txt --system
      - name: Create an isolated venv for testing
        run: uv venv .venv
      - name: Install 3rd-party stub dependencies
        run: |
          DEPENDENCIES=$(python tests/get_external_stub_requirements.py)
          if [ -n "$DEPENDENCIES" ]; then
            source .venv/bin/activate
            echo "Installing packages: $DEPENDENCIES"
            uv pip install $DEPENDENCIES
          fi
      - name: Activate the isolated venv for the rest of the job
        run: echo "$PWD/.venv/bin" >> $GITHUB_PATH
      - name: List 3rd-party stub dependencies installed
        run: uv pip freeze
      - name: Run pyright with basic settings on all the stubs
        uses: jakebailey/pyright-action@v2
        with:
          version: PATH
          python-platform: ${{ matrix.python-platform }}
          python-version: ${{ matrix.python-version }}
          annotate: ${{ matrix.python-version == '3.11' && matrix.python-platform == 'Linux' }} # Having each job create the same comment is too noisy.
      - name: Run pyright with stricter settings on some of the stubs
        uses: jakebailey/pyright-action@v2
        with:
          version: PATH
          python-platform: ${{ matrix.python-platform }}
          python-version: ${{ matrix.python-version }}
          annotate: ${{ matrix.python-version == '3.11' && matrix.python-platform == 'Linux' }} # Having each job create the same comment is too noisy.
          project: ./pyrightconfig.stricter.json
      - name: Run pyright on the test cases
        uses: jakebailey/pyright-action@v2
        with:
          version: PATH
          python-platform: ${{ matrix.python-platform }}
          python-version: ${{ matrix.python-version }}
          annotate: ${{ matrix.python-version == '3.11' && matrix.python-platform == 'Linux' }} # Having each job create the same comment is too noisy.
          project: ./pyrightconfig.testcases.json

  stub-uploader:
    name: Run the stub_uploader tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout typeshed
        uses: actions/checkout@v4
        with:
          path: typeshed
      - name: Checkout stub_uploader
        uses: actions/checkout@v4
        with:
          repository: typeshed-internal/stub_uploader
          path: stub_uploader
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
      - name: Run tests
        run: |
          cd stub_uploader
          uv pip install -r requirements.txt --system
          python -m pytest tests