Calculate linter.config.jobs in cgroupsv2 environments #4406
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Most of this is inspired by the mypy primer | |
| # See: https://github.com/hauntsaninja/mypy_primer | |
| # This is the primer job that runs on every PR | |
| name: Primer / Run | |
| on: | |
| pull_request: | |
| paths: | |
| - "pylint/**" | |
| - "tests/primer/**" | |
| - "requirements*" | |
| - ".github/workflows/**" | |
| # We ignore these specific files because they need to be changed | |
| # on 'main' and will always fail the PR run. | |
| - "!.github/workflows/primer_run_main.yaml" | |
| - "!.github/workflows/primer_comment.yaml" | |
| - "!tests/primer/packages_to_prime.json" | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # This needs to be the SAME as in the Main and comment job | |
| CACHE_VERSION: 4 | |
| KEY_PREFIX: venv-primer | |
| permissions: | |
| contents: read | |
| jobs: | |
| run-primer: | |
| name: Run / ${{ matrix.python-version }} / batch index ${{ matrix.batchIdx }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.13"] | |
| batches: [4] | |
| batchIdx: [0, 1, 2, 3] | |
| steps: | |
| - name: Check out code from GitHub | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| id: python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| check-latest: true | |
| # Restore cached Python environment | |
| - name: Restore Python virtual environment | |
| id: cache-venv | |
| uses: actions/cache/[email protected] | |
| with: | |
| path: venv | |
| key: | |
| ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ | |
| env.KEY_PREFIX }}-${{ env.CACHE_VERSION }}-${{ hashFiles('pyproject.toml', | |
| 'requirements_test.txt', 'requirements_test_min.txt', | |
| 'requirements_test_pre_commit.txt') }} | |
| # Create environment must match step in 'Primer / Main' | |
| - name: Create Python virtual environment | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: | | |
| python -m venv venv | |
| . venv/bin/activate | |
| python -m pip install -U pip setuptools wheel | |
| pip install -U -r requirements_test.txt | |
| # Save cached Python environment (explicit because cancel-in-progress: true) | |
| - name: Save Python virtual environment | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| uses: actions/cache/[email protected] | |
| with: | |
| path: venv | |
| key: | |
| ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ | |
| env.KEY_PREFIX }}-${{ env.CACHE_VERSION }}-${{ hashFiles('pyproject.toml', | |
| 'requirements_test.txt', 'requirements_test_min.txt', | |
| 'requirements_test_pre_commit.txt') }} | |
| # Cache primer packages | |
| - name: Download last 'main' run info | |
| id: download-main-run | |
| uses: actions/[email protected] | |
| env: | |
| COMMIT_STRING_ARTIFACT: primer_commitstring_${{ matrix.python-version }} | |
| OUTPUT_ARTIFACT: | |
| primer_output_main_${{ matrix.python-version }}_batch${{ matrix.batchIdx }} | |
| with: | |
| script: | | |
| const { COMMIT_STRING_ARTIFACT, OUTPUT_ARTIFACT } = process.env | |
| // Download 'main' pylint output | |
| const fs = require('fs'); | |
| const runs = await github.rest.actions.listWorkflowRuns({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: ".github/workflows/primer_run_main.yaml", | |
| status: "success" | |
| }); | |
| const lastRunMain = runs.data.workflow_runs.reduce(function(prev, current) { | |
| return (prev.run_number > current.run_number) ? prev : current | |
| }) | |
| console.log("Last run on main:") | |
| console.log(lastRunMain.html_url) | |
| const artifacts_main = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: lastRunMain.id, | |
| }); | |
| // Get commitstring | |
| const [matchArtifactMain] = artifacts_main.data.artifacts.filter((artifact) => | |
| artifact.name === COMMIT_STRING_ARTIFACT); | |
| const downloadWorkflow = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: matchArtifactMain.id, | |
| archive_format: "zip", | |
| }); | |
| fs.writeFileSync(`${COMMIT_STRING_ARTIFACT}.zip`, Buffer.from(downloadWorkflow.data)); | |
| // Get output | |
| const [matchArtifactMainOutput] = artifacts_main.data.artifacts.filter((artifact) => | |
| artifact.name === OUTPUT_ARTIFACT); | |
| const downloadWorkflowTwo = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: matchArtifactMainOutput.id, | |
| archive_format: "zip", | |
| }); | |
| fs.writeFileSync(`${OUTPUT_ARTIFACT}.zip`, Buffer.from(downloadWorkflowTwo.data)); | |
| return lastRunMain.head_sha; | |
| - name: Copy and unzip the commit string | |
| run: | | |
| unzip primer_commitstring_${{ matrix.python-version }}.zip | |
| cp commit_string_${{ matrix.python-version }}.txt tests/.pylint_primer_tests/commit_string_${{ matrix.python-version }}.txt | |
| - name: Unzip the output of 'main' | |
| run: | | |
| unzip primer_output_main_${{ matrix.python-version }}_batch${{ matrix.batchIdx }}.zip | |
| - name: Get commit string | |
| id: commitstring | |
| run: | | |
| . venv/bin/activate | |
| output=$(python tests/primer/__main__.py prepare --read-commit-string) | |
| echo "commitstring=$output" >> $GITHUB_OUTPUT | |
| - name: Restore projects cache | |
| id: cache-projects | |
| uses: actions/cache/[email protected] | |
| with: | |
| path: tests/.pylint_primer_tests/ | |
| key: >- | |
| ${{ runner.os }}-${{ matrix.python-version }}-${{ | |
| steps.commitstring.outputs.commitstring }}-primer | |
| - name: Regenerate cache | |
| if: steps.cache-projects.outputs.cache-hit != 'true' | |
| run: | | |
| . venv/bin/activate | |
| python tests/primer/__main__.py prepare --clone | |
| - name: Save projects cache | |
| if: steps.cache-projects.outputs.cache-hit != 'true' | |
| uses: actions/cache/[email protected] | |
| with: | |
| path: tests/.pylint_primer_tests/ | |
| key: >- | |
| ${{ runner.os }}-${{ matrix.python-version }}-${{ | |
| steps.commitstring.outputs.commitstring }}-primer | |
| - name: Check cache | |
| run: | | |
| . venv/bin/activate | |
| python tests/primer/__main__.py prepare --check | |
| # Merge the 'main' commit of last successful run | |
| - name: Pull 'main' | |
| shell: bash | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Pylint Primer" | |
| git pull origin ${{ steps.download-main-run.outputs.result }} --no-edit --no-commit --no-rebase | |
| # Run primer | |
| - name: Run pylint primer | |
| run: | | |
| . venv/bin/activate | |
| pip install . | |
| python tests/primer/__main__.py run --type=pr --batches=${{ matrix.batches }} --batchIdx=${{ matrix.batchIdx }} 2>warnings.txt | |
| - name: Echo warnings | |
| if: success() || failure() | |
| run: | | |
| WARNINGS=$(head -c 65000 < warnings.txt) | |
| if [[ $WARNINGS ]] | |
| then echo "::warning ::$WARNINGS" | |
| fi | |
| - name: Upload output of PR | |
| uses: actions/[email protected] | |
| with: | |
| name: | |
| primer_output_pr_${{ matrix.python-version }}_batch${{ matrix.batchIdx }} | |
| path: | |
| tests/.pylint_primer_tests/output_${{ matrix.python-version }}_pr_batch${{ | |
| matrix.batchIdx }}.txt | |
| - name: Upload output of 'main' | |
| uses: actions/[email protected] | |
| with: | |
| name: | |
| primer_output_main_${{ matrix.python-version }}_batch${{ matrix.batchIdx }} | |
| path: output_${{ matrix.python-version }}_main_batch${{ matrix.batchIdx }}.txt | |
| # Save PR number so we know which PR to comment on | |
| - name: Save PR number | |
| run: | | |
| echo ${{ github.event.pull_request.number }} | tee pr_number.txt | |
| - name: Upload PR number | |
| if: | |
| startsWith(steps.python.outputs.python-version, '3.9') && matrix.batchIdx == 0 | |
| uses: actions/[email protected] | |
| with: | |
| name: pr_number | |
| path: pr_number.txt |