Upgrade pyre, add py.typed marker, make lintable with mypy, fully ena…

…ble mypy in CI. (#445)

.github/workflows/pythonlinters.yml

@@ -79,9 +79,8 @@ jobs:
         working-directory: antsibull
         if: always()
 
-      # note: mypy is run informationally, never to fail the build
       - name: Lint with mypy
         run: |
-          ./lint-mypy.sh || :
+          ./lint-mypy.sh
         working-directory: antsibull
         if: always()

pyproject.toml

@@ -46,12 +46,13 @@ cryptography = "*"
 codecov = "*"
 flake8 = ">= 3.8.0"
 mypy = "*"
-# https://github.com/facebook/pyre-check/issues/398
-pyre-check = "^0.0.46"
+pyre-check = "^0.9.15"
 pylint = "^2.12.0"
 pytest = "*"
 pytest-asyncio = ">= 0.12"
 pytest-cov = "*"
+types-docutils = "*"
+types-PyYAML = "*"
 # For development, we install dependent projects under our control in dev mode:
 antsibull-changelog = { path = "../antsibull-changelog/", develop = true }
 antsibull-core = { path = "../antsibull-core/", develop = true }

src/antsibull/build_ansible_commands.py

@@ -14,7 +14,7 @@
 
 import aiofiles
 import aiohttp
-import asyncio_pool
+import asyncio_pool  # type: ignore[import]
 import sh
 from jinja2 import Template
 from packaging.version import Version as PypiVer
@@ -351,20 +351,20 @@ def prepare_command() -> int:
     build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file'])
     build_file = BuildFile(build_filename)
     build_ansible_version, ansible_core_version, deps = build_file.parse()
-    ansible_core_version = PypiVer(ansible_core_version)
+    ansible_core_version_obj = PypiVer(ansible_core_version)
 
     # If we're building a feature frozen release (betas and rcs) then we need to
     # change the upper version limit to not include new features.
     if app_ctx.extra['feature_frozen']:
         old_deps, deps = deps, {}
         # For each collection that's listed...
         for collection_name, spec in old_deps.items():
-            spec = SemVerSpec(spec)
+            spec_obj = SemVerSpec(spec)
             new_clauses = []
             min_version = None
 
             # Look at each clause of the version specification
-            for clause in spec.clause.clauses:
+            for clause in spec_obj.clause.clauses:
                 if clause.operator in ('<', '<='):
                     # Omit the upper bound as we're replacing it
                     continue
@@ -376,17 +376,17 @@ def prepare_command() -> int:
                 new_clauses.append(str(clause))
 
             if min_version is None:
-                raise ValueError(f'No minimum version specified for {collection_name}: {spec}')
+                raise ValueError(f'No minimum version specified for {collection_name}: {spec_obj}')
 
             new_clauses.append(f'<{min_version.major}.{min_version.minor + 1}.0')
             deps[collection_name] = ','.join(new_clauses)
 
     included_versions, new_ansible_core_version = asyncio.run(
         get_collection_and_core_versions(
-            deps, ansible_core_version, app_ctx.galaxy_url,
+            deps, ansible_core_version_obj, app_ctx.galaxy_url,
             ansible_core_allow_prerelease=_is_alpha(app_ctx.extra['ansible_version'])))
     if new_ansible_core_version:
-        ansible_core_version = new_ansible_core_version
+        ansible_core_version_obj = new_ansible_core_version
 
     if not str(app_ctx.extra['ansible_version']).startswith(build_ansible_version):
         print(f'{build_filename} is for version {build_ansible_version} but we need'
@@ -396,7 +396,7 @@ def prepare_command() -> int:
 
     dependency_data = DependencyFileData(
         str(app_ctx.extra['ansible_version']),
-        str(ansible_core_version),
+        str(ansible_core_version_obj),
         {collection: str(version) for collection, version in included_versions.items()})
 
     # Get Ansible changelog, add new release
@@ -430,7 +430,7 @@ def compile_collection_exclude_paths(collection_names: t.Collection[str],
                                      collection_root: str) -> t.Tuple[t.List[str], t.List[str]]:
     result = set()
     ignored_files = set()
-    all_files = []
+    all_files: t.List[str] = []
     for collection_name in collection_names:
         namespace, name = collection_name.split('.', 1)
         prefix = f"{namespace}/{name}/"
@@ -650,7 +650,7 @@ def build_multiple_command() -> int:
     build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file'])
     build_file = BuildFile(build_filename)
     build_ansible_version, ansible_core_version, deps = build_file.parse()
-    ansible_core_version = PypiVer(ansible_core_version)
+    ansible_core_version_obj = PypiVer(ansible_core_version)
 
     # TODO: implement --feature-frozen support
 
@@ -690,10 +690,10 @@ def build_multiple_command() -> int:
         collection_deps = []
         for collection, version in sorted(included_versions.items()):
             collection_deps.append(f"        '{collection}>={version},<{version.next_major()}'")
-        collection_deps = '\n' + ',\n'.join(collection_deps)
-        write_build_script(app_ctx.extra['ansible_version'], ansible_core_version, package_dir)
-        write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version,
-                                 [], collection_deps, package_dir)
+        collection_deps_str = '\n' + ',\n'.join(collection_deps)
+        write_build_script(app_ctx.extra['ansible_version'], ansible_core_version_obj, package_dir)
+        write_python_build_files(app_ctx.extra['ansible_version'], ansible_core_version_obj,
+                                 [], collection_deps_str, package_dir)
 
         make_dist(package_dir, app_ctx.extra['sdist_dir'])
 
@@ -702,7 +702,7 @@ def build_multiple_command() -> int:
     deps_file = DepsFile(deps_filename)
     deps_file.write(
         str(app_ctx.extra['ansible_version']),
-        str(ansible_core_version),
+        str(ansible_core_version_obj),
         {collection: str(version) for collection, version in included_versions.items()})
 
     return 0

src/antsibull/build_changelog.py

@@ -73,9 +73,9 @@ def _add_rst_table_row(builder: RstBuilder, column_widths: t.List[int], row: t.L
             if j >= len(lines):
                 lines.append([''] * len(column_widths))
             lines[j][i] = line
-    for line in lines:
+    for cells in lines:
         parts = ['|']
-        for j, cell in enumerate(line):
+        for j, cell in enumerate(cells):
             parts.append(' ' + cell + ' ' * (1 + column_widths[j] - len(cell)) + '|')
         builder.add_raw_rst(''.join(parts))
 
@@ -90,7 +90,7 @@ def _add_rst_table_line(builder: RstBuilder, column_widths: t.List[int], sep: st
 def render_rst_table(builder: RstBuilder, headings: t.List[str],
                      cells: t.List[t.List[str]]) -> None:
     # Determine column widths
-    column_widths = []
+    column_widths: t.List[int] = []
     for row in [headings] + cells:
         while len(row) > len(column_widths):
             column_widths.append(0)
@@ -255,7 +255,7 @@ def common_start(a: t.List[t.Any], b: t.List[t.Any]) -> int:
 
 
 def dump_items(builder: RstBuilder, items: PluginDumpT) -> None:
-    last_title = []
+    last_title: t.List[str] = []
     for title, name, description in sorted(items):
         if title != last_title:
             if last_title:
@@ -375,7 +375,7 @@ def append_changelog(builder: RstBuilder,
     for section, section_title in DEFAULT_SECTIONS:
         maybe_add_section_title = create_title_adder(builder, section_title, 1)
 
-        for name, dummy, dummy, release_entry in data:
+        for name, dummy, dummy2, release_entry in data:
             if not release_entry or release_entry.has_no_changes([section]):
                 continue
 
@@ -557,8 +557,8 @@ def _append_core_porting_guide_bytes(builder: RstBuilder, changelog: Changelog)
     def _get_porting_guide_bytes(changelog: Changelog) -> bytes:
         if changelog.ansible_version.major > 2:
             version = f"{changelog.ansible_version.major}"
-            core_version = changelog.core_collector.latest
-            core_version = f"{core_version.major}.{core_version.minor}"
+            core_version_obj = changelog.core_collector.latest
+            core_version = f"{core_version_obj.major}.{core_version_obj.minor}"
         else:
             version = f"{changelog.ansible_version.major}.{changelog.ansible_version.minor}"
             core_version = f"{changelog.ansible_version.major}.{changelog.ansible_version.minor}"
@@ -621,8 +621,8 @@ def _get_porting_guide_bytes(changelog: Changelog) -> bytes:
         if any(entry.is_ancestor for entry in changelog.entries):
             # If there is an ancestor, the earliest ansible-core version will be the
             # version used in the previous major release.
-            prev_core_version = changelog.core_collector.earliest
-            prev_core_version = f"{prev_core_version.major}.{prev_core_version.minor}"
+            prev_core_version_obj = changelog.core_collector.earliest
+            prev_core_version = f"{prev_core_version_obj.major}.{prev_core_version_obj.minor}"
 
         # Determine whether to include ansible-core porting guide or not
         if changelog.ansible_version.major == 2 or core_version != prev_core_version:

src/antsibull/changelog.py

@@ -17,8 +17,7 @@
 import typing as t
 
 import aiohttp
-import asyncio_pool
-import yaml
+import asyncio_pool  # type: ignore[import]
 from packaging.version import Version as PypiVer
 from semantic_version import Version as SemVer
 
@@ -31,6 +30,7 @@
 from antsibull_core.ansible_core import get_ansible_core
 from antsibull_core.dependency_files import DepsFile, DependencyFileData
 from antsibull_core.galaxy import CollectionDownloader
+from antsibull_core.yaml import load_yaml_bytes, load_yaml_file
 
 
 class ChangelogData:
@@ -163,10 +163,10 @@ async def _get_changelog(self, version: SemVer,
                              collection_downloader: CollectionDownloader
                              ) -> t.Optional[ChangelogData]:
         path = await collection_downloader.download(self.collection, version)
-        changelog = read_changelog_file(path)
-        if changelog is None:
+        changelog_bytes = read_changelog_file(path)
+        if changelog_bytes is None:
             return None
-        changelog_data = yaml.load(changelog, Loader=yaml.SafeLoader)
+        changelog_data = load_yaml_bytes(changelog_bytes)
         return ChangelogData.collection(self.collection, str(version), changelog_data)
 
     async def _download_changelog_stream(self, start_version: SemVer,
@@ -245,15 +245,15 @@ async def _get_changelog_file(version: PypiVer,
             for root, dummy, files in os.walk(path):
                 if 'changelog.yaml' in files:
                     with open(os.path.join(root, 'changelog.yaml'), 'rb') as f:
-                        changelog = f.read()
-                    changelog_data = yaml.load(changelog, Loader=yaml.SafeLoader)
+                        changelog_bytes = f.read()
+                    changelog_data = load_yaml_bytes(changelog_bytes)
                     changelog = ChangelogData.ansible_core(changelog_data)
             return changelog
         if os.path.isfile(path) and path.endswith('.tar.gz'):
-            changelog = read_changelog_file(path, is_ansible_core=True)
-            if changelog is None:
+            maybe_changelog_bytes = read_changelog_file(path, is_ansible_core=True)
+            if maybe_changelog_bytes is None:
                 return None
-            changelog_data = yaml.load(changelog, Loader=yaml.SafeLoader)
+            changelog_data = load_yaml_bytes(maybe_changelog_bytes)
             return ChangelogData.ansible_core(changelog_data)
         return None
 
@@ -408,8 +408,7 @@ def __init__(self, deps_dir: t.Optional[str]):
         if deps_dir is not None:
             collection_meta_path = os.path.join(deps_dir, 'collection-meta.yaml')
             if os.path.exists(collection_meta_path):
-                with open(collection_meta_path, 'rb') as collection_meta_file:
-                    data = yaml.load(collection_meta_file, Loader=yaml.SafeLoader)
+                data = load_yaml_file(collection_meta_path)
                 if data and 'collections' in data:
                     for collection_name, collection_data in data['collections'].items():
                         self.data[collection_name] = CollectionMetadata(collection_data)

src/antsibull/cli/antsibull_build.py

@@ -10,7 +10,7 @@
 import sys
 from typing import List
 
-import twiggy
+import twiggy  # type: ignore[import]
 from packaging.version import Version as PypiVer
 
 from antsibull_core.logging import log, initialize_app_logging
@@ -351,19 +351,19 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
     subparsers.add_parser('build-multiple', add_help=False, parents=[build_multiple_parser])
     subparsers.add_parser('build-collection', add_help=False, parents=[collection_parser])
 
-    args: argparse.Namespace = parser.parse_args(args)
+    parsed_args: argparse.Namespace = parser.parse_args(args)
 
     # Validation and coercion
-    normalize_toplevel_options(args)
-    _normalize_commands(args)
-    _normalize_build_options(args)
-    _normalize_build_write_data_options(args)
-    _normalize_new_release_options(args)
-    _normalize_release_build_options(args)
-    _normalize_release_rebuild_options(args)
-    _normalize_collection_build_options(args)
+    normalize_toplevel_options(parsed_args)
+    _normalize_commands(parsed_args)
+    _normalize_build_options(parsed_args)
+    _normalize_build_write_data_options(parsed_args)
+    _normalize_new_release_options(parsed_args)
+    _normalize_release_build_options(parsed_args)
+    _normalize_release_rebuild_options(parsed_args)
+    _normalize_collection_build_options(parsed_args)
 
-    return args
+    return parsed_args
 
 
 def run(args: List[str]) -> int:
@@ -379,20 +379,20 @@ def run(args: List[str]) -> int:
 
     program_name = os.path.basename(args[0])
     try:
-        args: argparse.Namespace = parse_args(program_name, args[1:])
+        parsed_args: argparse.Namespace = parse_args(program_name, args[1:])
     except InvalidArgumentError as e:
         print(e)
         return 2
 
-    cfg = load_config(args.config_file)
+    cfg = load_config(parsed_args.config_file)
     flog.fields(config=cfg).info('Config loaded')
 
-    context_data = app_context.create_contexts(args=args, cfg=cfg)
+    context_data = app_context.create_contexts(args=parsed_args, cfg=cfg)
     with app_context.app_and_lib_context(context_data) as (app_ctx, dummy_):
         twiggy.dict_config(app_ctx.logging_cfg.dict())
         flog.debug('Set logging config')
 
-        return ARGS_MAP[args.command]()
+        return ARGS_MAP[parsed_args.command]()
 
 
 def main() -> int:

src/antsibull/new_ansible.py

@@ -9,7 +9,7 @@
 import os
 
 import aiohttp
-import asyncio_pool
+import asyncio_pool  # type: ignore[import]
 import semantic_version as semver
 
 from antsibull_core import app_context