From d138bdd72a61abb23be0720907c97e02f1635d10 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Tue, 24 Nov 2020 14:19:55 +0545 Subject: [PATCH] Add mypy config, fix type errors in a few remaining places (#4951) * Add mypy config, fix type errors in a few random places * Add typing_extensions * Use setup.cfg for mypy's config --- dvc/parsing/context.py | 1 - dvc/tree/base.py | 2 +- dvc/tree/ssh/connection.py | 10 ++++--- dvc/utils/serialize/__init__.py | 9 ++++-- dvc/utils/serialize/_common.py | 52 ++++++++++++++++++++++++++++----- setup.cfg | 18 ++++++++++++ setup.py | 2 ++ 7 files changed, 78 insertions(+), 16 deletions(-) diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 4b01430853..fd42d3d94c 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -42,7 +42,6 @@ def __init__(self, key, new, into): ) return - assert isinstance(new, Node) and isinstance(into[key], Node) preexisting = into[key].meta.source new_src = new.meta.source path = new.meta.path() diff --git a/dvc/tree/base.py b/dvc/tree/base.py index 0848fd8e1b..1e76c6d36c 100644 --- a/dvc/tree/base.py +++ b/dvc/tree/base.py @@ -165,7 +165,7 @@ def open(self, path_info, mode: str = "r", encoding: str = None): raise RemoteActionNotImplemented("open", self.scheme) - def exists(self, path_info, use_dvcignore=True): + def exists(self, path_info, use_dvcignore=True) -> bool: raise NotImplementedError # pylint: disable=unused-argument diff --git a/dvc/tree/ssh/connection.py b/dvc/tree/ssh/connection.py index 37a4d6d749..c313eafe02 100644 --- a/dvc/tree/ssh/connection.py +++ b/dvc/tree/ssh/connection.py @@ -70,16 +70,18 @@ def close(self): self._ssh.close() def st_mode(self, path): + lstat = None with suppress(FileNotFoundError): - return self.sftp.lstat(path).st_mode + lstat = self.sftp.lstat(path) - return 0 + return lstat.st_mode if lstat else 0 def getsize(self, path): + lstat = None with suppress(FileNotFoundError): - return self.sftp.lstat(path).st_size + lstat = self.sftp.lstat(path) - return 0 + return lstat.st_size if lstat else 0 def exists(self, path): return bool(self.st_mode(path)) diff --git a/dvc/utils/serialize/__init__.py b/dvc/utils/serialize/__init__.py index a959f97c3f..b2b658454f 100644 --- a/dvc/utils/serialize/__init__.py +++ b/dvc/utils/serialize/__init__.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import DefaultDict from ._common import * # noqa, pylint: disable=wildcard-import from ._json import * # noqa, pylint: disable=wildcard-import @@ -6,12 +7,16 @@ from ._toml import * # noqa, pylint: disable=wildcard-import from ._yaml import * # noqa, pylint: disable=wildcard-import -LOADERS = defaultdict(lambda: load_yaml) # noqa: F405 +LOADERS: DefaultDict[str, LoaderFn] = defaultdict( # noqa: F405 + lambda: load_yaml # noqa: F405 +) LOADERS.update( {".toml": load_toml, ".json": load_json, ".py": load_py} # noqa: F405 ) -MODIFIERS = defaultdict(lambda: modify_yaml) # noqa: F405 +MODIFIERS: DefaultDict[str, ModifierFn] = defaultdict( # noqa: F405 + lambda: modify_yaml # noqa: F405 +) MODIFIERS.update( { ".toml": modify_toml, # noqa: F405 diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index e01b2c9572..828c32e158 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -1,34 +1,70 @@ """Common utilities for serialize.""" import os from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, Callable, ContextManager, Dict, Union + +from typing_extensions import Protocol from dvc.exceptions import DvcException from dvc.utils import relpath +if TYPE_CHECKING: + from dvc.tree.base import BaseTree + from dvc.types import AnyPath + + +class DumperFn(Protocol): + def __call__( + self, path: "AnyPath", data: Any, tree: "BaseTree" = None + ) -> Any: + ... + + +class ModifierFn(Protocol): + def __call__( + self, path: "AnyPath", tree: "BaseTree" = None + ) -> ContextManager[Dict]: + ... + + +class LoaderFn(Protocol): + def __call__(self, path: "AnyPath", tree: "BaseTree" = None) -> Any: + ... + + +ReadType = Union[bytes, None, str] +ParserFn = Callable[[ReadType, "AnyPath"], dict] + class ParseError(DvcException): """Errors while parsing files""" - def __init__(self, path, message): + def __init__(self, path: "AnyPath", message: str): path = relpath(path) super().__init__(f"unable to read: '{path}', {message}") -def _load_data(path, parser, tree=None): +def _load_data(path: "AnyPath", parser: ParserFn, tree: "BaseTree" = None): open_fn = tree.open if tree else open - with open_fn(path, encoding="utf-8") as fd: + with open_fn(path, encoding="utf-8") as fd: # type: ignore return parser(fd.read(), path) -def _dump_data(path, data, dumper, tree=None): +def _dump_data(path, data: Any, dumper: DumperFn, tree: "BaseTree" = None): open_fn = tree.open if tree else open - with open_fn(path, "w+", encoding="utf-8") as fd: + with open_fn(path, "w+", encoding="utf-8") as fd: # type: ignore dumper(data, fd) @contextmanager -def _modify_data(path, parser, dumper, tree=None): - exists = tree.exists if tree else os.path.exists - data = _load_data(path, parser=parser, tree=tree) if exists(path) else {} +def _modify_data( + path: "AnyPath", + parser: ParserFn, + dumper: DumperFn, + tree: "BaseTree" = None, +): + exists_fn = tree.exists if tree else os.path.exists + file_exists = exists_fn(path) # type: ignore + data = _load_data(path, parser=parser, tree=tree) if file_exists else {} yield data dumper(path, data, tree=tree) diff --git a/setup.cfg b/setup.cfg index 1c2a64993d..5642f6154e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,3 +24,21 @@ timeout = 600 timeout_method = thread log_level = debug addopts = -ra + +[mypy] +# Error output +show_column_numbers = True +show_error_codes = True +show_error_context = True +show_traceback = True +pretty = True + +# See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports. +ignore_missing_imports = True +check_untyped_defs = False + +# Warnings +warn_no_return=True +warn_redundant_casts=True +warn_unused_ignores=True +warn_unreachable = True diff --git a/setup.py b/setup.py index 8a5bf94b73..160e05640e 100644 --- a/setup.py +++ b/setup.py @@ -84,6 +84,7 @@ def run(self): "dictdiffer>=0.8.1", "python-benedict>=0.21.1", "pyparsing==2.4.7", + "typing_extensions>=3.7.4", ] @@ -152,6 +153,7 @@ def run(self): "wget", "filelock", "black==19.10b0", + "mypy", "wsgidav", ]