From 3eb414de57000c25807a0649f52003d7790222e7 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 19 Aug 2022 12:24:55 +0200 Subject: [PATCH] Upgrade pyre, add py.typed marker, make lintable with mypy, fully enable mypy in CI. (#445) --- .github/workflows/pythonlinters.yml | 3 +-- pyproject.toml | 5 +++-- src/antsibull/build_ansible_commands.py | 30 ++++++++++++------------- src/antsibull/build_changelog.py | 18 +++++++-------- src/antsibull/changelog.py | 23 +++++++++---------- src/antsibull/cli/antsibull_build.py | 30 ++++++++++++------------- src/antsibull/new_ansible.py | 2 +- src/antsibull/py.typed | 0 8 files changed, 55 insertions(+), 56 deletions(-) create mode 100644 src/antsibull/py.typed diff --git a/.github/workflows/pythonlinters.yml b/.github/workflows/pythonlinters.yml index 0f10853f..826098ff 100644 --- a/.github/workflows/pythonlinters.yml +++ b/.github/workflows/pythonlinters.yml @@ -79,9 +79,8 @@ jobs: working-directory: antsibull if: always() - # note: mypy is run informationally, never to fail the build - name: Lint with mypy run: | - ./lint-mypy.sh || : + ./lint-mypy.sh working-directory: antsibull if: always() diff --git a/pyproject.toml b/pyproject.toml index e85fed2f..38dde237 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,12 +46,13 @@ cryptography = "*" codecov = "*" flake8 = ">= 3.8.0" mypy = "*" -# https://github.com/facebook/pyre-check/issues/398 -pyre-check = "^0.0.46" +pyre-check = "^0.9.15" pylint = "^2.12.0" pytest = "*" pytest-asyncio = ">= 0.12" pytest-cov = "*" +types-docutils = "*" +types-PyYAML = "*" # For development, we install dependent projects under our control in dev mode: antsibull-changelog = { path = "../antsibull-changelog/", develop = true } antsibull-core = { path = "../antsibull-core/", develop = true } diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index b4aedb78..16863fa9 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -14,7 +14,7 @@ import aiofiles import aiohttp -import asyncio_pool +import asyncio_pool # type: ignore[import] import sh from jinja2 import Template from packaging.version import Version as PypiVer @@ -351,7 +351,7 @@ def prepare_command() -> int: build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file']) build_file = BuildFile(build_filename) build_ansible_version, ansible_core_version, deps = build_file.parse() - ansible_core_version = PypiVer(ansible_core_version) + ansible_core_version_obj = PypiVer(ansible_core_version) # If we're building a feature frozen release (betas and rcs) then we need to # change the upper version limit to not include new features. @@ -359,12 +359,12 @@ def prepare_command() -> int: old_deps, deps = deps, {} # For each collection that's listed... for collection_name, spec in old_deps.items(): - spec = SemVerSpec(spec) + spec_obj = SemVerSpec(spec) new_clauses = [] min_version = None # Look at each clause of the version specification - for clause in spec.clause.clauses: + for clause in spec_obj.clause.clauses: if clause.operator in ('<', '<='): # Omit the upper bound as we're replacing it continue @@ -376,17 +376,17 @@ def prepare_command() -> int: new_clauses.append(str(clause)) if min_version is None: - raise ValueError(f'No minimum version specified for {collection_name}: {spec}') + raise ValueError(f'No minimum version specified for {collection_name}: {spec_obj}') new_clauses.append(f'<{min_version.major}.{min_version.minor + 1}.0') deps[collection_name] = ','.join(new_clauses) included_versions, new_ansible_core_version = asyncio.run( get_collection_and_core_versions( - deps, ansible_core_version, app_ctx.galaxy_url, + deps, ansible_core_version_obj, app_ctx.galaxy_url, ansible_core_allow_prerelease=_is_alpha(app_ctx.extra['ansible_version']))) if new_ansible_core_version: - ansible_core_version = new_ansible_core_version + ansible_core_version_obj = new_ansible_core_version if not str(app_ctx.extra['ansible_version']).startswith(build_ansible_version): print(f'{build_filename} is for version {build_ansible_version} but we need' @@ -396,7 +396,7 @@ def prepare_command() -> int: dependency_data = DependencyFileData( str(app_ctx.extra['ansible_version']), - str(ansible_core_version), + str(ansible_core_version_obj), {collection: str(version) for collection, version in included_versions.items()}) # Get Ansible changelog, add new release @@ -430,7 +430,7 @@ def compile_collection_exclude_paths(collection_names: t.Collection[str], collection_root: str) -> t.Tuple[t.List[str], t.List[str]]: result = set() ignored_files = set() - all_files = [] + all_files: t.List[str] = [] for collection_name in collection_names: namespace, name = collection_name.split('.', 1) prefix = f"{namespace}/{name}/" @@ -650,7 +650,7 @@ def build_multiple_command() -> int: build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file']) build_file = BuildFile(build_filename) build_ansible_version, ansible_core_version, deps = build_file.parse() - ansible_core_version = PypiVer(ansible_core_version) + ansible_core_version_obj = PypiVer(ansible_core_version) # TODO: implement --feature-frozen support @@ -690,10 +690,10 @@ def build_multiple_command() -> int: collection_deps = [] for collection, version in sorted(included_versions.items()): collection_deps.append(f" '{collection}>={version},<{version.next_major()}'") - collection_deps = '\n' + ',\n'.join(collection_deps) - write_build_script(app_ctx.extra['ansible_version'], ansible_core_version, package_dir) - write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version, - [], collection_deps, package_dir) + collection_deps_str = '\n' + ',\n'.join(collection_deps) + write_build_script(app_ctx.extra['ansible_version'], ansible_core_version_obj, package_dir) + write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version_obj, + [], collection_deps_str, package_dir) make_dist(package_dir, app_ctx.extra['sdist_dir']) @@ -702,7 +702,7 @@ def build_multiple_command() -> int: deps_file = DepsFile(deps_filename) deps_file.write( str(app_ctx.extra['ansible_version']), - str(ansible_core_version), + str(ansible_core_version_obj), {collection: str(version) for collection, version in included_versions.items()}) return 0 diff --git a/src/antsibull/build_changelog.py b/src/antsibull/build_changelog.py index 39faf2c8..e38f957c 100644 --- a/src/antsibull/build_changelog.py +++ b/src/antsibull/build_changelog.py @@ -73,9 +73,9 @@ def _add_rst_table_row(builder: RstBuilder, column_widths: t.List[int], row: t.L if j >= len(lines): lines.append([''] * len(column_widths)) lines[j][i] = line - for line in lines: + for cells in lines: parts = ['|'] - for j, cell in enumerate(line): + for j, cell in enumerate(cells): parts.append(' ' + cell + ' ' * (1 + column_widths[j] - len(cell)) + '|') builder.add_raw_rst(''.join(parts)) @@ -90,7 +90,7 @@ def _add_rst_table_line(builder: RstBuilder, column_widths: t.List[int], sep: st def render_rst_table(builder: RstBuilder, headings: t.List[str], cells: t.List[t.List[str]]) -> None: # Determine column widths - column_widths = [] + column_widths: t.List[int] = [] for row in [headings] + cells: while len(row) > len(column_widths): column_widths.append(0) @@ -255,7 +255,7 @@ def common_start(a: t.List[t.Any], b: t.List[t.Any]) -> int: def dump_items(builder: RstBuilder, items: PluginDumpT) -> None: - last_title = [] + last_title: t.List[str] = [] for title, name, description in sorted(items): if title != last_title: if last_title: @@ -375,7 +375,7 @@ def append_changelog(builder: RstBuilder, for section, section_title in DEFAULT_SECTIONS: maybe_add_section_title = create_title_adder(builder, section_title, 1) - for name, dummy, dummy, release_entry in data: + for name, dummy, dummy2, release_entry in data: if not release_entry or release_entry.has_no_changes([section]): continue @@ -557,8 +557,8 @@ def _append_core_porting_guide_bytes(builder: RstBuilder, changelog: Changelog) def _get_porting_guide_bytes(changelog: Changelog) -> bytes: if changelog.ansible_version.major > 2: version = f"{changelog.ansible_version.major}" - core_version = changelog.core_collector.latest - core_version = f"{core_version.major}.{core_version.minor}" + core_version_obj = changelog.core_collector.latest + core_version = f"{core_version_obj.major}.{core_version_obj.minor}" else: version = f"{changelog.ansible_version.major}.{changelog.ansible_version.minor}" core_version = f"{changelog.ansible_version.major}.{changelog.ansible_version.minor}" @@ -621,8 +621,8 @@ def _get_porting_guide_bytes(changelog: Changelog) -> bytes: if any(entry.is_ancestor for entry in changelog.entries): # If there is an ancestor, the earliest ansible-core version will be the # version used in the previous major release. - prev_core_version = changelog.core_collector.earliest - prev_core_version = f"{prev_core_version.major}.{prev_core_version.minor}" + prev_core_version_obj = changelog.core_collector.earliest + prev_core_version = f"{prev_core_version_obj.major}.{prev_core_version_obj.minor}" # Determine whether to include ansible-core porting guide or not if changelog.ansible_version.major == 2 or core_version != prev_core_version: diff --git a/src/antsibull/changelog.py b/src/antsibull/changelog.py index 202fa6d3..ffe21868 100644 --- a/src/antsibull/changelog.py +++ b/src/antsibull/changelog.py @@ -17,8 +17,7 @@ import typing as t import aiohttp -import asyncio_pool -import yaml +import asyncio_pool # type: ignore[import] from packaging.version import Version as PypiVer from semantic_version import Version as SemVer @@ -31,6 +30,7 @@ from antsibull_core.ansible_core import get_ansible_core from antsibull_core.dependency_files import DepsFile, DependencyFileData from antsibull_core.galaxy import CollectionDownloader +from antsibull_core.yaml import load_yaml_bytes, load_yaml_file class ChangelogData: @@ -163,10 +163,10 @@ async def _get_changelog(self, version: SemVer, collection_downloader: CollectionDownloader ) -> t.Optional[ChangelogData]: path = await collection_downloader.download(self.collection, version) - changelog = read_changelog_file(path) - if changelog is None: + changelog_bytes = read_changelog_file(path) + if changelog_bytes is None: return None - changelog_data = yaml.load(changelog, Loader=yaml.SafeLoader) + changelog_data = load_yaml_bytes(changelog_bytes) return ChangelogData.collection(self.collection, str(version), changelog_data) async def _download_changelog_stream(self, start_version: SemVer, @@ -245,15 +245,15 @@ async def _get_changelog_file(version: PypiVer, for root, dummy, files in os.walk(path): if 'changelog.yaml' in files: with open(os.path.join(root, 'changelog.yaml'), 'rb') as f: - changelog = f.read() - changelog_data = yaml.load(changelog, Loader=yaml.SafeLoader) + changelog_bytes = f.read() + changelog_data = load_yaml_bytes(changelog_bytes) changelog = ChangelogData.ansible_core(changelog_data) return changelog if os.path.isfile(path) and path.endswith('.tar.gz'): - changelog = read_changelog_file(path, is_ansible_core=True) - if changelog is None: + maybe_changelog_bytes = read_changelog_file(path, is_ansible_core=True) + if maybe_changelog_bytes is None: return None - changelog_data = yaml.load(changelog, Loader=yaml.SafeLoader) + changelog_data = load_yaml_bytes(maybe_changelog_bytes) return ChangelogData.ansible_core(changelog_data) return None @@ -408,8 +408,7 @@ def __init__(self, deps_dir: t.Optional[str]): if deps_dir is not None: collection_meta_path = os.path.join(deps_dir, 'collection-meta.yaml') if os.path.exists(collection_meta_path): - with open(collection_meta_path, 'rb') as collection_meta_file: - data = yaml.load(collection_meta_file, Loader=yaml.SafeLoader) + data = load_yaml_file(collection_meta_path) if data and 'collections' in data: for collection_name, collection_data in data['collections'].items(): self.data[collection_name] = CollectionMetadata(collection_data) diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index d7bb9b95..1d680d61 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -10,7 +10,7 @@ import sys from typing import List -import twiggy +import twiggy # type: ignore[import] from packaging.version import Version as PypiVer from antsibull_core.logging import log, initialize_app_logging @@ -351,19 +351,19 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: subparsers.add_parser('build-multiple', add_help=False, parents=[build_multiple_parser]) subparsers.add_parser('build-collection', add_help=False, parents=[collection_parser]) - args: argparse.Namespace = parser.parse_args(args) + parsed_args: argparse.Namespace = parser.parse_args(args) # Validation and coercion - normalize_toplevel_options(args) - _normalize_commands(args) - _normalize_build_options(args) - _normalize_build_write_data_options(args) - _normalize_new_release_options(args) - _normalize_release_build_options(args) - _normalize_release_rebuild_options(args) - _normalize_collection_build_options(args) + normalize_toplevel_options(parsed_args) + _normalize_commands(parsed_args) + _normalize_build_options(parsed_args) + _normalize_build_write_data_options(parsed_args) + _normalize_new_release_options(parsed_args) + _normalize_release_build_options(parsed_args) + _normalize_release_rebuild_options(parsed_args) + _normalize_collection_build_options(parsed_args) - return args + return parsed_args def run(args: List[str]) -> int: @@ -379,20 +379,20 @@ def run(args: List[str]) -> int: program_name = os.path.basename(args[0]) try: - args: argparse.Namespace = parse_args(program_name, args[1:]) + parsed_args: argparse.Namespace = parse_args(program_name, args[1:]) except InvalidArgumentError as e: print(e) return 2 - cfg = load_config(args.config_file) + cfg = load_config(parsed_args.config_file) flog.fields(config=cfg).info('Config loaded') - context_data = app_context.create_contexts(args=args, cfg=cfg) + context_data = app_context.create_contexts(args=parsed_args, cfg=cfg) with app_context.app_and_lib_context(context_data) as (app_ctx, dummy_): twiggy.dict_config(app_ctx.logging_cfg.dict()) flog.debug('Set logging config') - return ARGS_MAP[args.command]() + return ARGS_MAP[parsed_args.command]() def main() -> int: diff --git a/src/antsibull/new_ansible.py b/src/antsibull/new_ansible.py index 4008e44f..e156898d 100644 --- a/src/antsibull/new_ansible.py +++ b/src/antsibull/new_ansible.py @@ -9,7 +9,7 @@ import os import aiohttp -import asyncio_pool +import asyncio_pool # type: ignore[import] import semantic_version as semver from antsibull_core import app_context diff --git a/src/antsibull/py.typed b/src/antsibull/py.typed new file mode 100644 index 00000000..e69de29b