Add mypy config, fix type errors in a few remaining places (iterative…

…#4951)

* Add mypy config, fix type errors in a few random places

* Add typing_extensions

* Use setup.cfg for mypy's config

dvc/parsing/context.py

@@ -42,7 +42,6 @@ def __init__(self, key, new, into):
             )
             return
 
-        assert isinstance(new, Node) and isinstance(into[key], Node)
         preexisting = into[key].meta.source
         new_src = new.meta.source
         path = new.meta.path()

dvc/tree/base.py

@@ -165,7 +165,7 @@ def open(self, path_info, mode: str = "r", encoding: str = None):
 
         raise RemoteActionNotImplemented("open", self.scheme)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info, use_dvcignore=True) -> bool:
         raise NotImplementedError
 
     # pylint: disable=unused-argument

dvc/tree/ssh/connection.py

@@ -70,16 +70,18 @@ def close(self):
         self._ssh.close()
 
     def st_mode(self, path):
+        lstat = None
         with suppress(FileNotFoundError):
-            return self.sftp.lstat(path).st_mode
+            lstat = self.sftp.lstat(path)
 
-        return 0
+        return lstat.st_mode if lstat else 0
 
     def getsize(self, path):
+        lstat = None
         with suppress(FileNotFoundError):
-            return self.sftp.lstat(path).st_size
+            lstat = self.sftp.lstat(path)
 
-        return 0
+        return lstat.st_size if lstat else 0
 
     def exists(self, path):
         return bool(self.st_mode(path))

dvc/utils/serialize/__init__.py

@@ -1,17 +1,22 @@
 from collections import defaultdict
+from typing import DefaultDict
 
 from ._common import *  # noqa, pylint: disable=wildcard-import
 from ._json import *  # noqa, pylint: disable=wildcard-import
 from ._py import *  # noqa, pylint: disable=wildcard-import
 from ._toml import *  # noqa, pylint: disable=wildcard-import
 from ._yaml import *  # noqa, pylint: disable=wildcard-import
 
-LOADERS = defaultdict(lambda: load_yaml)  # noqa: F405
+LOADERS: DefaultDict[str, LoaderFn] = defaultdict(  # noqa: F405
+    lambda: load_yaml  # noqa: F405
+)
 LOADERS.update(
     {".toml": load_toml, ".json": load_json, ".py": load_py}  # noqa: F405
 )
 
-MODIFIERS = defaultdict(lambda: modify_yaml)  # noqa: F405
+MODIFIERS: DefaultDict[str, ModifierFn] = defaultdict(  # noqa: F405
+    lambda: modify_yaml  # noqa: F405
+)
 MODIFIERS.update(
     {
         ".toml": modify_toml,  # noqa: F405

dvc/utils/serialize/_common.py

@@ -1,34 +1,70 @@
 """Common utilities for serialize."""
 import os
 from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Callable, ContextManager, Dict, Union
+
+from typing_extensions import Protocol
 
 from dvc.exceptions import DvcException
 from dvc.utils import relpath
 
+if TYPE_CHECKING:
+    from dvc.tree.base import BaseTree
+    from dvc.types import AnyPath
+
+
+class DumperFn(Protocol):
+    def __call__(
+        self, path: "AnyPath", data: Any, tree: "BaseTree" = None
+    ) -> Any:
+        ...
+
+
+class ModifierFn(Protocol):
+    def __call__(
+        self, path: "AnyPath", tree: "BaseTree" = None
+    ) -> ContextManager[Dict]:
+        ...
+
+
+class LoaderFn(Protocol):
+    def __call__(self, path: "AnyPath", tree: "BaseTree" = None) -> Any:
+        ...
+
+
+ReadType = Union[bytes, None, str]
+ParserFn = Callable[[ReadType, "AnyPath"], dict]
+
 
 class ParseError(DvcException):
     """Errors while parsing files"""
 
-    def __init__(self, path, message):
+    def __init__(self, path: "AnyPath", message: str):
         path = relpath(path)
         super().__init__(f"unable to read: '{path}', {message}")
 
 
-def _load_data(path, parser, tree=None):
+def _load_data(path: "AnyPath", parser: ParserFn, tree: "BaseTree" = None):
     open_fn = tree.open if tree else open
-    with open_fn(path, encoding="utf-8") as fd:
+    with open_fn(path, encoding="utf-8") as fd:  # type: ignore
         return parser(fd.read(), path)
 
 
-def _dump_data(path, data, dumper, tree=None):
+def _dump_data(path, data: Any, dumper: DumperFn, tree: "BaseTree" = None):
     open_fn = tree.open if tree else open
-    with open_fn(path, "w+", encoding="utf-8") as fd:
+    with open_fn(path, "w+", encoding="utf-8") as fd:  # type: ignore
         dumper(data, fd)
 
 
 @contextmanager
-def _modify_data(path, parser, dumper, tree=None):
-    exists = tree.exists if tree else os.path.exists
-    data = _load_data(path, parser=parser, tree=tree) if exists(path) else {}
+def _modify_data(
+    path: "AnyPath",
+    parser: ParserFn,
+    dumper: DumperFn,
+    tree: "BaseTree" = None,
+):
+    exists_fn = tree.exists if tree else os.path.exists
+    file_exists = exists_fn(path)  # type: ignore
+    data = _load_data(path, parser=parser, tree=tree) if file_exists else {}
     yield data
     dumper(path, data, tree=tree)

setup.cfg

@@ -24,3 +24,21 @@ timeout = 600
 timeout_method = thread
 log_level = debug
 addopts = -ra
+
+[mypy]
+# Error output
+show_column_numbers = True
+show_error_codes = True
+show_error_context = True
+show_traceback = True
+pretty = True
+
+# See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports.
+ignore_missing_imports = True
+check_untyped_defs = False
+
+# Warnings
+warn_no_return=True
+warn_redundant_casts=True
+warn_unused_ignores=True
+warn_unreachable = True

setup.py

@@ -84,6 +84,7 @@ def run(self):
     "dictdiffer>=0.8.1",
     "python-benedict>=0.21.1",
     "pyparsing==2.4.7",
+    "typing_extensions>=3.7.4",
 ]
 
 
@@ -152,6 +153,7 @@ def run(self):
     "wget",
     "filelock",
     "black==19.10b0",
+    "mypy",
     "wsgidav",
 ]