Make mypy --strict happy

Makefile

@@ -6,4 +6,4 @@ lint:
 	flake8 .
 	black --check .
 	isort -c .
-	mypy .
+	mypy --strict .

mypy.ini

@@ -1,14 +1,8 @@
 [mypy]
-exclude = tests/test_.*
-
-[mypy-valohai_yaml.*]
-ignore_missing_imports = True
+exclude = tests/*
 
 [mypy-setuptools.*]
 ignore_missing_imports = True
 
 [mypy-tqdm.*]
 ignore_missing_imports = True
-
-[mypy-papi.*]
-ignore_missing_imports = True

requirements-dev.in

@@ -6,12 +6,15 @@ flake8-print
 flake8-return
 flake8-simplify
 isort
+jsonschema<4.0
 mypy
 pip-tools
 pydocstyle
 pytest
 pytest-cov
-requests
 requests-mock
+requests<2.26.0
 tqdm
+types-PyYAML
+types-requests
 valohai-yaml

requirements-dev.txt

@@ -1,105 +1,107 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements-dev.in
 #
-appdirs==1.4.4
-    # via black
 astor==0.8.1
     # via flake8-simplify
-attrs==20.3.0
+attrs==21.2.0
     # via
     #   flake8-bugbear
     #   jsonschema
     #   pytest
-black==20.8b1
+black==21.10b0
     # via -r requirements-dev.in
 bump2version==1.0.1
     # via -r requirements-dev.in
-certifi==2020.12.5
+certifi==2021.10.8
     # via requests
 chardet==4.0.0
     # via requests
-click==7.1.2
+click==8.0.3
     # via
     #   black
     #   pip-tools
-coverage==5.5
+coverage[toml]==6.1.1
     # via pytest-cov
-flake8-bugbear==21.4.3
+flake8==4.0.1
+    # via
+    #   -r requirements-dev.in
+    #   flake8-bugbear
+    #   flake8-print
+    #   flake8-simplify
+flake8-bugbear==21.9.2
     # via -r requirements-dev.in
-flake8-plugin-utils==1.3.1
+flake8-plugin-utils==1.3.2
     # via flake8-return
 flake8-print==4.0.0
     # via -r requirements-dev.in
-flake8-return==1.1.2
+flake8-return==1.1.3
     # via -r requirements-dev.in
-flake8-simplify==0.14.0
+flake8-simplify==0.14.2
     # via -r requirements-dev.in
-flake8==3.9.1
-    # via
-    #   -r requirements-dev.in
-    #   flake8-bugbear
-    #   flake8-print
-    #   flake8-simplify
 idna==2.10
     # via requests
 iniconfig==1.1.1
     # via pytest
-isort==5.8.0
+isort==5.10.0
     # via -r requirements-dev.in
 jsonschema==3.2.0
-    # via valohai-yaml
+    # via
+    #   -r requirements-dev.in
+    #   valohai-yaml
 mccabe==0.6.1
     # via flake8
+mypy==0.910
+    # via -r requirements-dev.in
 mypy-extensions==0.4.3
     # via
     #   black
     #   mypy
-mypy==0.812
-    # via -r requirements-dev.in
-packaging==20.9
+packaging==21.2
     # via pytest
-pathspec==0.8.1
+pathspec==0.9.0
     # via black
-pep517==0.10.0
+pep517==0.12.0
     # via pip-tools
-pip-tools==6.1.0
+pip-tools==6.4.0
     # via -r requirements-dev.in
-pluggy==0.13.1
+platformdirs==2.4.0
+    # via black
+pluggy==1.0.0
     # via pytest
 py==1.10.0
     # via pytest
-pycodestyle==2.7.0
+pycodestyle==2.8.0
     # via
     #   flake8
     #   flake8-print
-pydocstyle==6.0.0
+pydocstyle==6.1.1
     # via -r requirements-dev.in
-pyflakes==2.3.1
+pyflakes==2.4.0
     # via flake8
 pyparsing==2.4.7
     # via packaging
-pyrsistent==0.17.3
+pyrsistent==0.18.0
     # via jsonschema
-pytest-cov==2.11.1
-    # via -r requirements-dev.in
-pytest==6.2.3
+pytest==6.2.5
     # via
     #   -r requirements-dev.in
     #   pytest-cov
-pyyaml==5.4.1
+pytest-cov==3.0.0
+    # via -r requirements-dev.in
+pyyaml==6.0
     # via valohai-yaml
-regex==2021.4.4
+regex==2021.11.2
     # via black
-requests-mock==1.8.0
-    # via -r requirements-dev.in
 requests==2.25.1
     # via
     #   -r requirements-dev.in
     #   requests-mock
-six==1.15.0
+requests-mock==1.9.3
+    # via -r requirements-dev.in
+six==1.16.0
     # via
     #   flake8-print
     #   jsonschema
@@ -108,23 +110,29 @@ snowballstemmer==2.1.0
     # via pydocstyle
 toml==0.10.2
     # via
-    #   black
-    #   pep517
+    #   mypy
     #   pytest
-tqdm==4.60.0
-    # via -r requirements-dev.in
-typed-ast==1.4.3
+tomli==1.2.2
     # via
     #   black
-    #   mypy
-typing-extensions==3.7.4.3
+    #   coverage
+    #   pep517
+tqdm==4.62.3
+    # via -r requirements-dev.in
+types-pyyaml==6.0.0
+    # via -r requirements-dev.in
+types-requests==2.25.11
+    # via -r requirements-dev.in
+typing-extensions==3.10.0.2
     # via
     #   black
     #   mypy
-urllib3==1.26.5
+urllib3==1.26.7
     # via requests
-valohai-yaml==0.14.1
+valohai-yaml==0.20.0
     # via -r requirements-dev.in
+wheel==0.37.0
+    # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

setup.py

@@ -5,14 +5,14 @@
 import setuptools
 
 
-def get_version():
+def get_version() -> str:
     with open(
         os.path.join(os.path.dirname(__file__), "valohai", "__init__.py")
     ) as infp:
         match = re.search("__version__ = (.+?)$", infp.read(), re.M)
         if not match:
             raise ValueError("No version could be found")
-        return ast.literal_eval(match.group(1))
+        return str(ast.literal_eval(match.group(1)))
 
 
 setuptools.setup(

tests/test_pipeline.py

@@ -4,16 +4,17 @@
 import pytest
 from valohai_yaml import parse
 
-from tests.utils import read_yaml_test_data
+from tests.utils import compare_yaml, read_yaml_test_data
 from valohai.internals.pipeline import get_pipeline_from_source
-from valohai.yaml import config_to_yaml
 
 
 @pytest.mark.parametrize(
-    "original_yaml, source_python, expected_yaml",
+    "original_yaml, source_python, expected_yaml_filename",
     read_yaml_test_data("tests/test_pipeline_yaml"),
 )
-def test_yaml_update_from_source(tmpdir, original_yaml, source_python, expected_yaml):
+def test_pipeline_yaml_update_from_source(
+    tmpdir, original_yaml, source_python, expected_yaml_filename
+):
     yaml_path = os.path.join(tmpdir, "valohai.yaml")
     source_path = os.path.join(tmpdir, "test.py")
 
@@ -33,6 +34,5 @@ def test_yaml_update_from_source(tmpdir, original_yaml, source_python, expected_
     new_config = old_config.merge_with(new_config)
 
     # Check against expected result
-    with open(expected_yaml) as expected_yaml:
-        new_yaml = config_to_yaml(new_config)
-        assert new_yaml == expected_yaml.read()
+    with open(expected_yaml_filename) as fp:
+        compare_yaml(new_config, fp.read())

tests/test_yaml.py

@@ -4,17 +4,18 @@
 import pytest
 from valohai_yaml import parse
 
-from tests.utils import read_yaml_test_data
+from tests.utils import compare_yaml, read_yaml_test_data
 from valohai.internals.merge import python_to_yaml_merge_strategy
 from valohai.internals.yaml import generate_step, parse_config_from_source
-from valohai.yaml import config_to_yaml
 
 
 @pytest.mark.parametrize(
-    "original_yaml, source_python, expected_yaml",
+    "original_yaml, source_python, expected_yaml_filename",
     read_yaml_test_data("tests/test_yaml"),
 )
-def test_yaml_update_from_source(tmpdir, original_yaml, source_python, expected_yaml):
+def test_yaml_update_from_source(
+    tmpdir, original_yaml, source_python, expected_yaml_filename
+):
     yaml_path = os.path.join(tmpdir, "valohai.yaml")
     filename, file_extension = os.path.splitext(source_python)
     source_path = os.path.join(tmpdir, f"test{file_extension}")
@@ -38,9 +39,8 @@ def test_yaml_update_from_source(tmpdir, original_yaml, source_python, expected_
         new_config = old_config.merge_with(new_config, python_to_yaml_merge_strategy)
 
     # Check against expected result
-    with open(expected_yaml) as expected_yaml:
-        new_yaml = config_to_yaml(new_config)
-        assert new_yaml == expected_yaml.read()
+    with open(expected_yaml_filename) as fp:
+        compare_yaml(new_config, fp.read())
 
 
 def test_posix_path_separator(monkeypatch):

tests/utils.py

@@ -1,6 +1,11 @@
 import glob
 import json
 import os
+import re
+
+from valohai_yaml.objs import Config
+
+from valohai.yaml import config_to_yaml
 
 
 def read_source_files(path_without_ext):
@@ -56,3 +61,10 @@ def read_yaml_test_data(root_path):
             )
         )
     return test_data
+
+
+def compare_yaml(config: Config, fixture_yaml: str) -> None:
+    new_yaml = config_to_yaml(config)
+    # TODO: remove this when https://github.com/valohai/valohai-yaml/pull/59 lands
+    new_yaml = re.sub(r"\s+actions: \[]", "", new_yaml)
+    assert new_yaml == fixture_yaml

valohai/inputs.py

@@ -90,7 +90,7 @@ def streams(
         path_filter: Optional[str] = None,
         process_archives: bool = True,
         force_download: bool = False,
-    ) -> Iterator[IO]:
+    ) -> Iterator[IO[bytes]]:
         """Get file streams to all files for a given input name.
 
         Returns an Iterable for all the file IO streams for an input.
@@ -122,7 +122,7 @@ def stream(
         path_filter: Optional[str] = None,
         process_archives: bool = True,
         force_download: bool = False,
-    ) -> Optional[IO]:
+    ) -> Optional[IO[bytes]]:
         """Get a stream for to a file for a given input name.
 
         Returns an IO stream to a file for this input.

valohai/internals/compression.py

@@ -8,6 +8,8 @@
 from mimetypes import guess_type
 from typing import IO, Union
 
+FilenameOrStream = Union[str, IO[bytes]]
+
 # A short and likely incomplete list of file extensions that are likely not to
 # compress well or at all.
 INCOMPRESSIBLE_SUFFIXES = {
@@ -56,7 +58,7 @@ class BaseArchive:
     Base class for writable archives with an unified signature for adding files.
     """
 
-    def put(self, archive_name, source: Union[str, IO]):
+    def put(self, archive_name: str, source: FilenameOrStream) -> None:
         raise NotImplementedError("...")
 
 
@@ -91,7 +93,7 @@ def writestream(
                 shutil.copyfileobj(data, dest, 524288)
         assert zinfo.file_size
 
-    def put(self, archive_name: str, source: Union[str, IO]) -> None:
+    def put(self, archive_name: str, source: FilenameOrStream) -> None:
         compress_type = (
             zipfile.ZIP_DEFLATED
             if guess_compressible(archive_name)
@@ -112,7 +114,7 @@ def put(self, archive_name: str, source: Union[str, IO]) -> None:
 
 
 class TarArchive(BaseArchive, tarfile.TarFile):
-    def put(self, archive_name: str, source: Union[str, IO]) -> None:
+    def put(self, archive_name: str, source: FilenameOrStream) -> None:
         with contextlib.ExitStack() as es:
             if isinstance(source, str):
                 size = os.stat(source).st_size