From d739dfd97b8b5acb4d71b4d544c5bc71c7b2c79d Mon Sep 17 00:00:00 2001 From: Nikita Kodenko Date: Wed, 29 Apr 2020 00:34:20 +0700 Subject: [PATCH] refactor: dvc/output class names unification --- dvc/dependency/__init__.py | 10 +++--- dvc/dependency/gs.py | 4 +-- dvc/dependency/hdfs.py | 4 +-- dvc/dependency/http.py | 4 +-- dvc/dependency/local.py | 4 +-- dvc/dependency/s3.py | 4 +-- dvc/dependency/ssh.py | 4 +-- dvc/output/__init__.py | 54 ++++++++++++++--------------- dvc/output/base.py | 2 +- dvc/output/gs.py | 4 +-- dvc/output/hdfs.py | 4 +-- dvc/output/local.py | 4 +-- dvc/output/s3.py | 4 +-- dvc/output/ssh.py | 4 +-- dvc/stage/__init__.py | 10 +++--- tests/func/test_repro.py | 6 ++-- tests/func/test_run_single_stage.py | 4 +-- tests/func/test_stage.py | 12 +++---- tests/unit/output/test_gs.py | 8 ++--- tests/unit/output/test_hdfs.py | 8 ++--- tests/unit/output/test_local.py | 24 ++++++------- tests/unit/output/test_output.py | 8 ++--- tests/unit/output/test_s3.py | 8 ++--- tests/unit/output/test_ssh.py | 8 ++--- 24 files changed, 103 insertions(+), 103 deletions(-) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 34efa984ab..911fb7e5cc 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -10,7 +10,7 @@ from dvc.dependency.s3 import S3Dependency from dvc.dependency.ssh import SSHDependency from dvc.dependency.param import ParamsDependency -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.remote import Remote from dvc.scheme import Schemes from .repo import RepoDependency @@ -42,8 +42,8 @@ # cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not # output is a metric file and how to parse it, see `-M` flag for `dvc run`). SCHEMA = output.SCHEMA.copy() -del SCHEMA[OutputBase.PARAM_CACHE] -del SCHEMA[OutputBase.PARAM_METRIC] +del SCHEMA[BaseOutput.PARAM_CACHE] +del SCHEMA[BaseOutput.PARAM_METRIC] SCHEMA.update(RepoDependency.REPO_SCHEMA) SCHEMA.update(ParamsDependency.PARAM_SCHEMA) @@ -71,7 +71,7 @@ def _get(stage, p, info): def loadd_from(stage, d_list): ret = [] for d in d_list: - p = d.pop(OutputBase.PARAM_PATH, None) + p = d.pop(BaseOutput.PARAM_PATH, None) ret.append(_get(stage, p, d)) return ret @@ -101,7 +101,7 @@ def loads_params(stage, s_list): for path, params in params_by_path.items(): d_list.append( { - OutputBase.PARAM_PATH: path, + BaseOutput.PARAM_PATH: path, ParamsDependency.PARAM_PARAMS: params, } ) diff --git a/dvc/dependency/gs.py b/dvc/dependency/gs.py index 38911fddd4..14dc5afd46 100644 --- a/dvc/dependency/gs.py +++ b/dvc/dependency/gs.py @@ -1,6 +1,6 @@ from dvc.dependency.base import BaseDependency -from dvc.output.gs import OutputGS +from dvc.output.gs import GSOutput -class GSDependency(BaseDependency, OutputGS): +class GSDependency(BaseDependency, GSOutput): pass diff --git a/dvc/dependency/hdfs.py b/dvc/dependency/hdfs.py index c7ddac5cb5..61a6475a12 100644 --- a/dvc/dependency/hdfs.py +++ b/dvc/dependency/hdfs.py @@ -1,6 +1,6 @@ from dvc.dependency.base import BaseDependency -from dvc.output.hdfs import OutputHDFS +from dvc.output.hdfs import HDFSOutput -class HDFSDependency(BaseDependency, OutputHDFS): +class HDFSDependency(BaseDependency, HDFSOutput): pass diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 126c538073..653a2f0138 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -1,7 +1,7 @@ from dvc.dependency.base import BaseDependency -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.remote.http import HTTPRemote -class HTTPDependency(BaseDependency, OutputBase): +class HTTPDependency(BaseDependency, BaseOutput): REMOTE = HTTPRemote diff --git a/dvc/dependency/local.py b/dvc/dependency/local.py index 6bbc435291..54df58f821 100644 --- a/dvc/dependency/local.py +++ b/dvc/dependency/local.py @@ -1,6 +1,6 @@ from dvc.dependency.base import BaseDependency -from dvc.output.local import OutputLOCAL +from dvc.output.local import LocalOutput -class LocalDependency(BaseDependency, OutputLOCAL): +class LocalDependency(BaseDependency, LocalOutput): pass diff --git a/dvc/dependency/s3.py b/dvc/dependency/s3.py index 432474c078..733a2316a8 100644 --- a/dvc/dependency/s3.py +++ b/dvc/dependency/s3.py @@ -1,6 +1,6 @@ from dvc.dependency.base import BaseDependency -from dvc.output.s3 import OutputS3 +from dvc.output.s3 import S3Output -class S3Dependency(BaseDependency, OutputS3): +class S3Dependency(BaseDependency, S3Output): pass diff --git a/dvc/dependency/ssh.py b/dvc/dependency/ssh.py index d5b69e1b98..00c0bebe86 100644 --- a/dvc/dependency/ssh.py +++ b/dvc/dependency/ssh.py @@ -1,6 +1,6 @@ from dvc.dependency.base import BaseDependency -from dvc.output.ssh import OutputSSH +from dvc.output.ssh import SSHOutput -class SSHDependency(BaseDependency, OutputSSH): +class SSHDependency(BaseDependency, SSHOutput): pass diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index fab6e5bdd8..00533ca791 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -1,12 +1,12 @@ from urllib.parse import urlparse from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo -from dvc.output.base import OutputBase -from dvc.output.gs import OutputGS -from dvc.output.hdfs import OutputHDFS -from dvc.output.local import OutputLOCAL -from dvc.output.s3 import OutputS3 -from dvc.output.ssh import OutputSSH +from dvc.output.base import BaseOutput +from dvc.output.gs import GSOutput +from dvc.output.hdfs import HDFSOutput +from dvc.output.local import LocalOutput +from dvc.output.s3 import S3Output +from dvc.output.ssh import SSHOutput from dvc.remote import Remote from dvc.remote.hdfs import HDFSRemote from dvc.remote.local import LocalRemote @@ -14,19 +14,19 @@ from dvc.scheme import Schemes OUTS = [ - OutputHDFS, - OutputS3, - OutputGS, - OutputSSH, - # NOTE: OutputLOCAL is the default choice + HDFSOutput, + S3Output, + GSOutput, + SSHOutput, + # NOTE: LocalOutput is the default choice ] OUTS_MAP = { - Schemes.HDFS: OutputHDFS, - Schemes.S3: OutputS3, - Schemes.GS: OutputGS, - Schemes.SSH: OutputSSH, - Schemes.LOCAL: OutputLOCAL, + Schemes.HDFS: HDFSOutput, + Schemes.S3: S3Output, + Schemes.GS: GSOutput, + Schemes.SSH: SSHOutput, + Schemes.LOCAL: LocalOutput, } CHECKSUM_SCHEMA = Any( @@ -52,11 +52,11 @@ TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA} SCHEMA = CHECKSUMS_SCHEMA.copy() -SCHEMA[Required(OutputBase.PARAM_PATH)] = str -SCHEMA[OutputBase.PARAM_CACHE] = bool -SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA -SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA -SCHEMA[OutputBase.PARAM_PERSIST] = bool +SCHEMA[Required(BaseOutput.PARAM_PATH)] = str +SCHEMA[BaseOutput.PARAM_CACHE] = bool +SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA +SCHEMA[BaseOutput.PARAM_TAGS] = TAGS_SCHEMA +SCHEMA[BaseOutput.PARAM_PERSIST] = bool def _get(stage, p, info, cache, metric, persist=False, tags=None): @@ -87,7 +87,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None): persist=persist, tags=tags, ) - return OutputLOCAL( + return LocalOutput( stage, p, info, @@ -102,11 +102,11 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None): def loadd_from(stage, d_list): ret = [] for d in d_list: - p = d.pop(OutputBase.PARAM_PATH) - cache = d.pop(OutputBase.PARAM_CACHE, True) - metric = d.pop(OutputBase.PARAM_METRIC, False) - persist = d.pop(OutputBase.PARAM_PERSIST, False) - tags = d.pop(OutputBase.PARAM_TAGS, None) + p = d.pop(BaseOutput.PARAM_PATH) + cache = d.pop(BaseOutput.PARAM_CACHE, True) + metric = d.pop(BaseOutput.PARAM_METRIC, False) + persist = d.pop(BaseOutput.PARAM_PERSIST, False) + tags = d.pop(BaseOutput.PARAM_TAGS, None) ret.append( _get( stage, diff --git a/dvc/output/base.py b/dvc/output/base.py index 191214136d..b70215449b 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -38,7 +38,7 @@ def __init__(self, path): super().__init__("Stage file '{}' cannot be an output.".format(path)) -class OutputBase(object): +class BaseOutput(object): IS_DEPENDENCY = False REMOTE = BaseRemote diff --git a/dvc/output/gs.py b/dvc/output/gs.py index 924fdf5e7b..00b3a14f9b 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -1,6 +1,6 @@ -from dvc.output.s3 import OutputS3 +from dvc.output.s3 import S3Output from dvc.remote.gs import GSRemote -class OutputGS(OutputS3): +class GSOutput(S3Output): REMOTE = GSRemote diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index a8db0d8adc..6c32787393 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -1,6 +1,6 @@ -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.remote.hdfs import HDFSRemote -class OutputHDFS(OutputBase): +class HDFSOutput(BaseOutput): REMOTE = HDFSRemote diff --git a/dvc/output/local.py b/dvc/output/local.py index 1ba98f1ff7..ea46ff01ff 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -4,7 +4,7 @@ from dvc.exceptions import DvcException from dvc.istextfile import istextfile -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.remote.local import LocalRemote from dvc.utils import relpath from dvc.compat import fspath_py35 @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -class OutputLOCAL(OutputBase): +class LocalOutput(BaseOutput): REMOTE = LocalRemote sep = os.sep diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 9e9bf221b6..dbd6ee8995 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -1,6 +1,6 @@ -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.remote.s3 import S3Remote -class OutputS3(OutputBase): +class S3Output(BaseOutput): REMOTE = S3Remote diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 317eb1ace0..29d0593f9c 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -1,6 +1,6 @@ -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.remote.ssh import SSHRemote -class OutputSSH(OutputBase): +class SSHOutput(BaseOutput): REMOTE = SSHRemote diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 91e8f46cf6..d79753dc11 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -455,7 +455,7 @@ def dumpd(self): } def _compute_md5(self): - from dvc.output.base import OutputBase + from dvc.output.base import BaseOutput d = self.dumpd() @@ -471,15 +471,15 @@ def _compute_md5(self): del d[self.PARAM_WDIR] # NOTE: excluding parameters that don't affect the state of the - # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if + # pipeline. Not excluding `LocalOutput.PARAM_CACHE`, because if # it has changed, we might not have that output in our cache. m = dict_md5( d, exclude=[ self.PARAM_LOCKED, - OutputBase.PARAM_METRIC, - OutputBase.PARAM_TAGS, - OutputBase.PARAM_PERSIST, + BaseOutput.PARAM_METRIC, + BaseOutput.PARAM_TAGS, + BaseOutput.PARAM_PERSIST, ], ) logger.debug("Computed {} md5: '{}'".format(self, m)) diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index ce13ddd9fc..5ea5dae81c 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -23,7 +23,7 @@ from dvc.exceptions import ReproductionError from dvc.exceptions import StagePathAsOutputError from dvc.main import main -from dvc.output.base import OutputBase +from dvc.output.base import BaseOutput from dvc.path_info import URLInfo from dvc.remote.local import LocalRemote from dvc.repo import Repo as DvcRepo @@ -1341,9 +1341,9 @@ def test_force_import(self): ) patch_checkout = patch.object( - OutputBase, + BaseOutput, "checkout", - side_effect=OutputBase.checkout, + side_effect=BaseOutput.checkout, autospec=True, ) diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 94c8130c09..a5e9a34d74 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -16,7 +16,7 @@ from dvc.exceptions import StagePathAsOutputError from dvc.dvcfile import DVC_FILE_SUFFIX from dvc.main import main -from dvc.output import OutputBase +from dvc.output import BaseOutput from dvc.output.base import OutputIsStageFileError from dvc.repo import Repo as DvcRepo from dvc.stage import Stage @@ -667,7 +667,7 @@ def run_command(self, file, file_content): def stage_should_contain_persist_flag(self, stage_file): stage_file_content = load_stage_file(stage_file) self.assertEqual( - True, stage_file_content["outs"][0][OutputBase.PARAM_PERSIST] + True, stage_file_content["outs"][0][BaseOutput.PARAM_PERSIST] ) def should_append_upon_repro(self, file, stage_file): diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index e1852fd8d1..c6a52b990a 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -3,7 +3,7 @@ import pytest from dvc.main import main -from dvc.output.local import OutputLOCAL +from dvc.output.local import LocalOutput from dvc.remote.local import LocalRemote from dvc.repo import Repo from dvc.stage import Stage @@ -54,15 +54,15 @@ def test_empty_list(): def test_list(): lst = [ - {OutputLOCAL.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"}, - {OutputLOCAL.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None}, - {OutputLOCAL.PARAM_PATH: "baz"}, + {LocalOutput.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"}, + {LocalOutput.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None}, + {LocalOutput.PARAM_PATH: "baz"}, ] d = {Stage.PARAM_DEPS: lst} SingleStageFile.validate(d) - lst[0][OutputLOCAL.PARAM_CACHE] = True - lst[1][OutputLOCAL.PARAM_CACHE] = False + lst[0][LocalOutput.PARAM_CACHE] = True + lst[1][LocalOutput.PARAM_CACHE] = False d = {Stage.PARAM_OUTS: lst} SingleStageFile.validate(d) diff --git a/tests/unit/output/test_gs.py b/tests/unit/output/test_gs.py index 69d20a4e8b..6868a8f813 100644 --- a/tests/unit/output/test_gs.py +++ b/tests/unit/output/test_gs.py @@ -1,7 +1,7 @@ -from dvc.output.gs import OutputGS -from tests.unit.output.test_local import TestOutputLOCAL +from dvc.output.gs import GSOutput +from tests.unit.output.test_local import TestLocalOutput -class TestOutputGS(TestOutputLOCAL): +class TestGSOutput(TestLocalOutput): def _get_cls(self): - return OutputGS + return GSOutput diff --git a/tests/unit/output/test_hdfs.py b/tests/unit/output/test_hdfs.py index a07e9fb942..8ac7027056 100644 --- a/tests/unit/output/test_hdfs.py +++ b/tests/unit/output/test_hdfs.py @@ -1,7 +1,7 @@ -from dvc.output.hdfs import OutputHDFS -from tests.unit.output.test_local import TestOutputLOCAL +from dvc.output.hdfs import HDFSOutput +from tests.unit.output.test_local import TestLocalOutput -class TestOutputHDFS(TestOutputLOCAL): +class TestHDFSOutput(TestLocalOutput): def _get_cls(self): - return OutputHDFS + return HDFSOutput diff --git a/tests/unit/output/test_local.py b/tests/unit/output/test_local.py index 6b1bdefd8b..76ebda0f39 100644 --- a/tests/unit/output/test_local.py +++ b/tests/unit/output/test_local.py @@ -1,16 +1,16 @@ import os from mock import patch -from dvc.output import OutputLOCAL +from dvc.output import LocalOutput from dvc.remote.local import LocalRemote from dvc.stage import Stage from dvc.utils import relpath from tests.basic_env import TestDvc -class TestOutputLOCAL(TestDvc): +class TestLocalOutput(TestDvc): def _get_cls(self): - return OutputLOCAL + return LocalOutput def _get_output(self): stage = Stage(self.dvc) @@ -25,19 +25,19 @@ def test_save_missing(self): def test_str_workdir_outside_repo(erepo_dir): stage = Stage(erepo_dir.dvc) - output = OutputLOCAL(stage, "path", cache=False) + output = LocalOutput(stage, "path", cache=False) assert relpath("path", erepo_dir.dvc.root_dir) == str(output) def test_str_workdir_inside_repo(dvc): stage = Stage(dvc) - output = OutputLOCAL(stage, "path", cache=False) + output = LocalOutput(stage, "path", cache=False) assert "path" == str(output) stage = Stage(dvc, wdir="some_folder") - output = OutputLOCAL(stage, "path", cache=False) + output = LocalOutput(stage, "path", cache=False) assert os.path.join("some_folder", "path") == str(output) @@ -47,7 +47,7 @@ def test_str_on_local_absolute_path(dvc): rel_path = os.path.join("path", "to", "file") abs_path = os.path.abspath(rel_path) - output = OutputLOCAL(stage, abs_path, cache=False) + output = LocalOutput(stage, abs_path, cache=False) assert output.def_path == rel_path assert output.path_info.fspath == abs_path @@ -59,7 +59,7 @@ def test_str_on_external_absolute_path(dvc): rel_path = os.path.join("..", "path", "to", "file") abs_path = os.path.abspath(rel_path) - output = OutputLOCAL(stage, abs_path, cache=False) + output = LocalOutput(stage, abs_path, cache=False) assert output.def_path == abs_path assert output.path_info.fspath == abs_path @@ -69,14 +69,14 @@ def test_str_on_external_absolute_path(dvc): class TestGetFilesNumber(TestDvc): def _get_output(self): stage = Stage(self.dvc) - return OutputLOCAL(stage, "path") + return LocalOutput(stage, "path") def test_return_0_on_no_cache(self): o = self._get_output() o.use_cache = False self.assertEqual(0, o.get_files_number()) - @patch.object(OutputLOCAL, "checksum", "12345678.dir") + @patch.object(LocalOutput, "checksum", "12345678.dir") @patch.object( LocalRemote, "get_dir_cache", @@ -87,8 +87,8 @@ def test_return_multiple_for_dir(self, mock_get_dir_cache): self.assertEqual(2, o.get_files_number()) - @patch.object(OutputLOCAL, "checksum", "12345678") - @patch.object(OutputLOCAL, "is_dir_checksum", False) + @patch.object(LocalOutput, "checksum", "12345678") + @patch.object(LocalOutput, "is_dir_checksum", False) def test_return_1_on_single_file_cache(self): o = self._get_output() diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py index ed4f637f95..c7cddb071a 100644 --- a/tests/unit/output/test_output.py +++ b/tests/unit/output/test_output.py @@ -6,7 +6,7 @@ from voluptuous import Schema, MultipleInvalid from dvc.cache import NamedCache -from dvc.output import CHECKSUM_SCHEMA, OutputBase +from dvc.output import CHECKSUM_SCHEMA, BaseOutput @pytest.mark.parametrize( @@ -74,15 +74,15 @@ def test_get_used_cache(exists, expected_message, mocker, caplog): mocker.patch.object(stage, "__str__", return_value="stage: 'stage.dvc'") mocker.patch.object(stage, "addressing", "stage.dvc") - output = OutputBase(stage, "path") + output = BaseOutput(stage, "path") mocker.patch.object(output, "use_cache", True) mocker.patch.object(stage, "is_repo_import", False) mocker.patch.object( - OutputBase, "checksum", new_callable=mocker.PropertyMock + BaseOutput, "checksum", new_callable=mocker.PropertyMock ).return_value = None mocker.patch.object( - OutputBase, "exists", new_callable=mocker.PropertyMock + BaseOutput, "exists", new_callable=mocker.PropertyMock ).return_value = exists with caplog.at_level(logging.WARNING, logger="dvc"): diff --git a/tests/unit/output/test_s3.py b/tests/unit/output/test_s3.py index 4b0b986f53..34b3080182 100644 --- a/tests/unit/output/test_s3.py +++ b/tests/unit/output/test_s3.py @@ -1,7 +1,7 @@ -from dvc.output.s3 import OutputS3 -from tests.unit.output.test_local import TestOutputLOCAL +from dvc.output.s3 import S3Output +from tests.unit.output.test_local import TestLocalOutput -class TestOutputS3(TestOutputLOCAL): +class TestS3Output(TestLocalOutput): def _get_cls(self): - return OutputS3 + return S3Output diff --git a/tests/unit/output/test_ssh.py b/tests/unit/output/test_ssh.py index 397e34ebc6..0f829c53c5 100644 --- a/tests/unit/output/test_ssh.py +++ b/tests/unit/output/test_ssh.py @@ -1,7 +1,7 @@ -from dvc.output.ssh import OutputSSH -from tests.unit.output.test_local import TestOutputLOCAL +from dvc.output.ssh import SSHOutput +from tests.unit.output.test_local import TestLocalOutput -class TestOutputSSH(TestOutputLOCAL): +class TestSSHOutput(TestLocalOutput): def _get_cls(self): - return OutputSSH + return SSHOutput