From ad2a7ea17b642089653fdb3ad8b5848f5daa6d91 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 19:52:28 -0800 Subject: [PATCH 1/7] Many improvements - Add CI - Only add defaults for the Python version we are running - Support third-party stubs --- .../typeshed_primer_download_errors.js | 19 ++++++++ .../scripts/typeshed_primer_post_comment.js | 30 ++++++++++++ .github/workflows/lint.yml | 23 +++++++++ .github/workflows/pyanalyze.yml | 23 +++++++++ .gitignore | 2 + .pre-commit-config.yaml | 48 +++++++++++++++++++ pyproject.toml | 10 ++++ stubdefaulter.py | 36 ++++++++++---- 8 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/typeshed_primer_download_errors.js create mode 100644 .github/scripts/typeshed_primer_post_comment.js create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pyanalyze.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml diff --git a/.github/scripts/typeshed_primer_download_errors.js b/.github/scripts/typeshed_primer_download_errors.js new file mode 100644 index 0000000..c2717ea --- /dev/null +++ b/.github/scripts/typeshed_primer_download_errors.js @@ -0,0 +1,19 @@ +module.exports = async ({ github, context }) => { + const fs = require('fs') + + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }) + const [matchArtifact] = artifacts.data.artifacts.filter((artifact) => + artifact.name == "typeshed_primer_errors") + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: "zip", + }) + + fs.writeFileSync("errors.zip", Buffer.from(download.data)); +} diff --git a/.github/scripts/typeshed_primer_post_comment.js b/.github/scripts/typeshed_primer_post_comment.js new file mode 100644 index 0000000..0c4a047 --- /dev/null +++ b/.github/scripts/typeshed_primer_post_comment.js @@ -0,0 +1,30 @@ +module.exports = async ({ github, context }) => { + const fs = require('fs') + const DIFF_LINE = { ">": true, "<": true } + + let data = fs.readFileSync('errors_diff.txt', { encoding: 'utf8' }) + // Only keep diff lines + data = data + .split("\n") + .filter(line => line[0] in DIFF_LINE) + .join("\n") + // posting comment fails if too long, so truncate + if (data.length > 30000) { + let truncated_data = data.substring(0, 30000) + let lines_truncated = data.split('\n').length - truncated_data.split('\n').length + data = truncated_data + `\n\n... (truncated ${lines_truncated} lines) ...\n` + } + + const body = data.trim() + ? '⚠ Flake8 diff showing the effect of this PR on typeshed: \n```diff\n' + data + '```' + : 'This change has no effect on typeshed. 🤖🎉' + const issue_number = parseInt(fs.readFileSync("pr_number.txt", { encoding: "utf8" })) + await github.rest.issues.createComment({ + issue_number, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) + + return issue_number +} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7db8095 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,23 @@ +name: Lint + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up latest Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e '.[dev]' + + - name: Run pre-commit hooks + uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/pyanalyze.yml b/.github/workflows/pyanalyze.yml new file mode 100644 index 0000000..6869db4 --- /dev/null +++ b/.github/workflows/pyanalyze.yml @@ -0,0 +1,23 @@ +name: Pyanalyze + +on: [push, pull_request] + +jobs: + main: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install + run: | + python -m pip install -e . + python -m pip install pyanalyze==0.9.0 + - name: Run + # TODO check additional directories + run: PYTHONPATH=. python -m pyanalyze --config-file pyproject.toml taxonomy/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebfe9e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.vscode/ +__pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3e78baa --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +repos: + # Order matters because pyupgrade may make changes that pycln and black have to clean up + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: [--py310-plus] + + - repo: https://github.com/hadialqattan/pycln + rev: v2.1.2 + hooks: + - id: pycln + args: [--config=pyproject.toml] + + - repo: https://github.com/pycqa/isort + rev: 5.11.4 # must match pyproject.toml + hooks: + - id: isort + name: isort (python) + + - repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black + language_version: python3.11 + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear + - flake8-comprehensions + - flake8-simplify + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.991 + hooks: + - id: mypy + additional_dependencies: + - types-requests + - aiohttp diff --git a/pyproject.toml b/pyproject.toml index 05aa027..3388941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,3 +8,13 @@ version = "0.0.0" description = "Autoadd default values to stubs" requires-python = ">=3.10" dependencies = ["libcst", "typeshed_client"] + +[project.optional-dependencies] +dev = [ + "black==22.12.0", # Must match .pre-commit-config.yaml + "flake8-bugbear==23.1.14", + "flake8-noqa==1.3.0", + "isort==5.11.4", # Must match .pre-commit-config.yaml + "mypy==0.991", + "pre-commit-hooks==4.4.0", # Must match .pre-commit-config.yaml +] diff --git a/stubdefaulter.py b/stubdefaulter.py index c549407..601a0cc 100644 --- a/stubdefaulter.py +++ b/stubdefaulter.py @@ -1,3 +1,5 @@ +from __future__ import annotations + """ Tool to add default values to stubs. @@ -6,9 +8,7 @@ TODO: - Support methods, not just top-level functions -- Support third-party stubs - Maybe enable adding more default values (floats?) -- Run on multiple Python versions to pick up the right defaults """ @@ -20,6 +20,7 @@ import libcst import itertools from pathlib import Path +import sys import textwrap import typeshed_client from typing import Any @@ -131,14 +132,33 @@ def add_defaults_to_stub( def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("typeshed_path", help="path to typeshed") + parser.add_argument( + "-s", "--stdlib-path", + help=( + "Path to typeshed's stdlib directory. If given, we will add defaults to" + " stubs in this directory." + ), + ) + parser.add_argument( + "-p", "--packages", + nargs="+", + help=( + "List of packages to add defaults to. We will add defaults to all stubs in" + " these directories. The runtime package must be installed." + ), + ) args = parser.parse_args() - for version in range(11, 6, -1): - context = typeshed_client.finder.get_search_context( - typeshed=Path(args.typeshed_path), version=(3, version) - ) - for module, _ in typeshed_client.get_all_stub_files(context): + stdlib_path = Path(args.stdlib_path) if args.stdlib_path else None + package_paths = [Path(p) for p in args.packages] + + context = typeshed_client.finder.get_search_context( + typeshed=stdlib_path, search_path=package_paths, version=sys.version_info[:2] + ) + for module, path in typeshed_client.get_all_stub_files(context): + if stdlib_path is not None and path.is_relative_to(stdlib_path): + add_defaults_to_stub(module, context) + elif any(path.is_relative_to(p) for p in package_paths): add_defaults_to_stub(module, context) From 99fab971b7fb486bda24077c6aea9df12599ad74 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 20:02:22 -0800 Subject: [PATCH 2/7] fixes --- .flake8 | 4 ++++ .github/workflows/check.yml | 19 +++++++++++++++++++ .github/workflows/pyanalyze.yml | 10 ++++++---- .pre-commit-config.yaml | 3 +-- stubdefaulter.py | 15 +++++++++------ 5 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 .flake8 create mode 100644 .github/workflows/check.yml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..767560f --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +ignore = E203, E266, E501, W503, B028, B950, B011, B006, B008, F821, E711, E712, E731, F401, B901 +max-line-length = 80 +select = B,E,F,W,T4,B9 diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..d4b2634 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,19 @@ +name: tests + +on: [push, pull_request] + +jobs: + tests: + name: test import + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + - run: pip install -e . + - run: python3 stubdefaulter.py --help diff --git a/.github/workflows/pyanalyze.yml b/.github/workflows/pyanalyze.yml index 6869db4..29cc218 100644 --- a/.github/workflows/pyanalyze.yml +++ b/.github/workflows/pyanalyze.yml @@ -5,19 +5,21 @@ on: [push, pull_request] jobs: main: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: ${{ matrix.python-version }} - name: Install run: | python -m pip install -e . python -m pip install pyanalyze==0.9.0 - name: Run - # TODO check additional directories - run: PYTHONPATH=. python -m pyanalyze --config-file pyproject.toml taxonomy/ + run: PYTHONPATH=. python -m pyanalyze --config-file pyproject.toml stubdefaulter.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e78baa..0e4d18f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: rev: v3.3.1 hooks: - id: pyupgrade - args: [--py310-plus] + args: [--py37-plus] - repo: https://github.com/hadialqattan/pycln rev: v2.1.2 @@ -37,7 +37,6 @@ repos: additional_dependencies: - flake8-bugbear - flake8-comprehensions - - flake8-simplify - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.991 diff --git a/stubdefaulter.py b/stubdefaulter.py index 601a0cc..a6724c0 100644 --- a/stubdefaulter.py +++ b/stubdefaulter.py @@ -14,17 +14,18 @@ import argparse import ast -from dataclasses import dataclass import importlib import inspect -import libcst import itertools -from pathlib import Path import sys import textwrap -import typeshed_client +from dataclasses import dataclass +from pathlib import Path from typing import Any +import libcst +import typeshed_client + def contains_ellipses(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: for default in itertools.chain(node.args.defaults, node.args.kw_defaults): @@ -133,14 +134,16 @@ def add_defaults_to_stub( def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( - "-s", "--stdlib-path", + "-s", + "--stdlib-path", help=( "Path to typeshed's stdlib directory. If given, we will add defaults to" " stubs in this directory." ), ) parser.add_argument( - "-p", "--packages", + "-p", + "--packages", nargs="+", help=( "List of packages to add defaults to. We will add defaults to all stubs in" From 4fe39f0af9a426e7381192890211a3aa022df4be Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 20:03:15 -0800 Subject: [PATCH 3/7] must allow 3.7+ --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3388941..52b4476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "stubdefaulter" version = "0.0.0" description = "Autoadd default values to stubs" -requires-python = ">=3.10" +requires-python = ">=3.7" dependencies = ["libcst", "typeshed_client"] [project.optional-dependencies] From 8b56b3d68936566cce574a00b94471680bf91e5e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 20:07:33 -0800 Subject: [PATCH 4/7] fix 3.8 --- .pre-commit-config.yaml | 7 ------- stubdefaulter.py | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e4d18f..4e33f01 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,4 @@ repos: - # Order matters because pyupgrade may make changes that pycln and black have to clean up - - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: [--py37-plus] - - repo: https://github.com/hadialqattan/pycln rev: v2.1.2 hooks: diff --git a/stubdefaulter.py b/stubdefaulter.py index a6724c0..ff21b3f 100644 --- a/stubdefaulter.py +++ b/stubdefaulter.py @@ -131,6 +131,19 @@ def add_defaults_to_stub( f.write(line + "\n") +def is_relative_to(left: Path, right: Path) -> bool: + """Return True if the path is relative to another path or False. + + Redundant with Path.is_relative_to in 3.9+. + + """ + try: + left.relative_to(right) + return True + except ValueError: + return False + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( @@ -159,9 +172,9 @@ def main() -> None: typeshed=stdlib_path, search_path=package_paths, version=sys.version_info[:2] ) for module, path in typeshed_client.get_all_stub_files(context): - if stdlib_path is not None and path.is_relative_to(stdlib_path): + if stdlib_path is not None and is_relative_to(path, stdlib_path): add_defaults_to_stub(module, context) - elif any(path.is_relative_to(p) for p in package_paths): + elif any(is_relative_to(path, p) for p in package_paths): add_defaults_to_stub(module, context) From 987f98b15f6980fdd7cb1ee5918fb140e0aaa144 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 20:09:05 -0800 Subject: [PATCH 5/7] satisfy pyanalyze --- stubdefaulter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stubdefaulter.py b/stubdefaulter.py index ff21b3f..e4f3c0e 100644 --- a/stubdefaulter.py +++ b/stubdefaulter.py @@ -21,7 +21,7 @@ import textwrap from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Dict, List import libcst import typeshed_client @@ -109,7 +109,8 @@ def add_defaults_to_stub( if stub_names is None: raise ValueError(f"Could not find stub for {module_name}") stub_lines = path.read_text().splitlines() - replacement_lines: dict[int, list[str]] = {} + # pyanalyze doesn't let you use dict[] here + replacement_lines: Dict[int, List[str]] = {} for name, info in stub_names.items(): if isinstance( info.ast, (ast.FunctionDef, ast.AsyncFunctionDef) From 3af6ba1fb773cfdbd7eae2fc0bfdd65d77482b97 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 20:12:01 -0800 Subject: [PATCH 6/7] fix 3.7 --- stubdefaulter.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/stubdefaulter.py b/stubdefaulter.py index e4f3c0e..030549b 100644 --- a/stubdefaulter.py +++ b/stubdefaulter.py @@ -69,6 +69,14 @@ def leave_Param( return updated_node +def get_end_lineno(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: + if sys.version_info >= (3, 8): + assert node.end_lineno is not None + return node.end_lineno + else: + return max(child.lineno for child in ast.iter_child_nodes(node)) + + def replace_defaults_in_func( stub_lines: list[str], node: ast.FunctionDef | ast.AsyncFunctionDef, @@ -78,8 +86,8 @@ def replace_defaults_in_func( sig = inspect.signature(runtime_func) except Exception: return {} - assert node.end_lineno is not None - lines = stub_lines[node.lineno - 1 : node.end_lineno] + end_lineno = get_end_lineno(node) + lines = stub_lines[node.lineno - 1 : end_lineno] indentation = len(lines[0]) - len(lines[0].lstrip()) cst = libcst.parse_statement( textwrap.dedent("".join(line + "\n" for line in lines)) @@ -88,7 +96,7 @@ def replace_defaults_in_func( assert isinstance(modified, libcst.FunctionDef) new_code = textwrap.indent(libcst.Module(body=[modified]).code, " " * indentation) output_dict = {node.lineno - 1: new_code.splitlines()} - for i in range(node.lineno, node.end_lineno): + for i in range(node.lineno, end_lineno): output_dict[i] = [] return output_dict From 38eda90aa928eb364eac16af1da30ff96bd4279a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Jan 2023 20:13:10 -0800 Subject: [PATCH 7/7] I should make pyanalyze understand version checks --- stubdefaulter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stubdefaulter.py b/stubdefaulter.py index 030549b..9eedb8e 100644 --- a/stubdefaulter.py +++ b/stubdefaulter.py @@ -71,6 +71,7 @@ def leave_Param( def get_end_lineno(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int: if sys.version_info >= (3, 8): + assert hasattr(node, "end_lineno") assert node.end_lineno is not None return node.end_lineno else: