Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: dvc/output class names unification #3695

Merged
merged 1 commit into from
Apr 28, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions dvc/dependency/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from dvc.dependency.s3 import S3Dependency
from dvc.dependency.ssh import SSHDependency
from dvc.dependency.param import ParamsDependency
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.remote import Remote
from dvc.scheme import Schemes
from .repo import RepoDependency
Expand Down Expand Up @@ -42,8 +42,8 @@
# cached, see -o and -O flags for `dvc run`) and 'metric' (whether or not
# output is a metric file and how to parse it, see `-M` flag for `dvc run`).
SCHEMA = output.SCHEMA.copy()
del SCHEMA[OutputBase.PARAM_CACHE]
del SCHEMA[OutputBase.PARAM_METRIC]
del SCHEMA[BaseOutput.PARAM_CACHE]
del SCHEMA[BaseOutput.PARAM_METRIC]
SCHEMA.update(RepoDependency.REPO_SCHEMA)
SCHEMA.update(ParamsDependency.PARAM_SCHEMA)

Expand Down Expand Up @@ -71,7 +71,7 @@ def _get(stage, p, info):
def loadd_from(stage, d_list):
ret = []
for d in d_list:
p = d.pop(OutputBase.PARAM_PATH, None)
p = d.pop(BaseOutput.PARAM_PATH, None)
ret.append(_get(stage, p, d))
return ret

Expand Down Expand Up @@ -101,7 +101,7 @@ def loads_params(stage, s_list):
for path, params in params_by_path.items():
d_list.append(
{
OutputBase.PARAM_PATH: path,
BaseOutput.PARAM_PATH: path,
ParamsDependency.PARAM_PARAMS: params,
}
)
Expand Down
4 changes: 2 additions & 2 deletions dvc/dependency/gs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.dependency.base import BaseDependency
from dvc.output.gs import OutputGS
from dvc.output.gs import GSOutput


class GSDependency(BaseDependency, OutputGS):
class GSDependency(BaseDependency, GSOutput):
pass
4 changes: 2 additions & 2 deletions dvc/dependency/hdfs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.dependency.base import BaseDependency
from dvc.output.hdfs import OutputHDFS
from dvc.output.hdfs import HDFSOutput


class HDFSDependency(BaseDependency, OutputHDFS):
class HDFSDependency(BaseDependency, HDFSOutput):
pass
4 changes: 2 additions & 2 deletions dvc/dependency/http.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from dvc.dependency.base import BaseDependency
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.remote.http import HTTPRemote


class HTTPDependency(BaseDependency, OutputBase):
class HTTPDependency(BaseDependency, BaseOutput):
REMOTE = HTTPRemote
4 changes: 2 additions & 2 deletions dvc/dependency/local.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.dependency.base import BaseDependency
from dvc.output.local import OutputLOCAL
from dvc.output.local import LocalOutput


class LocalDependency(BaseDependency, OutputLOCAL):
class LocalDependency(BaseDependency, LocalOutput):
pass
4 changes: 2 additions & 2 deletions dvc/dependency/s3.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.dependency.base import BaseDependency
from dvc.output.s3 import OutputS3
from dvc.output.s3 import S3Output


class S3Dependency(BaseDependency, OutputS3):
class S3Dependency(BaseDependency, S3Output):
pass
4 changes: 2 additions & 2 deletions dvc/dependency/ssh.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.dependency.base import BaseDependency
from dvc.output.ssh import OutputSSH
from dvc.output.ssh import SSHOutput


class SSHDependency(BaseDependency, OutputSSH):
class SSHDependency(BaseDependency, SSHOutput):
pass
54 changes: 27 additions & 27 deletions dvc/output/__init__.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
from urllib.parse import urlparse
from voluptuous import Any, Required, Lower, Length, Coerce, And, SetTo

from dvc.output.base import OutputBase
from dvc.output.gs import OutputGS
from dvc.output.hdfs import OutputHDFS
from dvc.output.local import OutputLOCAL
from dvc.output.s3 import OutputS3
from dvc.output.ssh import OutputSSH
from dvc.output.base import BaseOutput
from dvc.output.gs import GSOutput
from dvc.output.hdfs import HDFSOutput
from dvc.output.local import LocalOutput
from dvc.output.s3 import S3Output
from dvc.output.ssh import SSHOutput
from dvc.remote import Remote
from dvc.remote.hdfs import HDFSRemote
from dvc.remote.local import LocalRemote
from dvc.remote.s3 import S3Remote
from dvc.scheme import Schemes

OUTS = [
OutputHDFS,
OutputS3,
OutputGS,
OutputSSH,
# NOTE: OutputLOCAL is the default choice
HDFSOutput,
S3Output,
GSOutput,
SSHOutput,
# NOTE: LocalOutput is the default choice
]

OUTS_MAP = {
Schemes.HDFS: OutputHDFS,
Schemes.S3: OutputS3,
Schemes.GS: OutputGS,
Schemes.SSH: OutputSSH,
Schemes.LOCAL: OutputLOCAL,
Schemes.HDFS: HDFSOutput,
Schemes.S3: S3Output,
Schemes.GS: GSOutput,
Schemes.SSH: SSHOutput,
Schemes.LOCAL: LocalOutput,
}

CHECKSUM_SCHEMA = Any(
Expand All @@ -52,11 +52,11 @@
TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}

SCHEMA = CHECKSUMS_SCHEMA.copy()
SCHEMA[Required(OutputBase.PARAM_PATH)] = str
SCHEMA[OutputBase.PARAM_CACHE] = bool
SCHEMA[OutputBase.PARAM_METRIC] = OutputBase.METRIC_SCHEMA
SCHEMA[OutputBase.PARAM_TAGS] = TAGS_SCHEMA
SCHEMA[OutputBase.PARAM_PERSIST] = bool
SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
SCHEMA[BaseOutput.PARAM_CACHE] = bool
SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
SCHEMA[BaseOutput.PARAM_TAGS] = TAGS_SCHEMA
SCHEMA[BaseOutput.PARAM_PERSIST] = bool


def _get(stage, p, info, cache, metric, persist=False, tags=None):
Expand Down Expand Up @@ -87,7 +87,7 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
persist=persist,
tags=tags,
)
return OutputLOCAL(
return LocalOutput(
stage,
p,
info,
Expand All @@ -102,11 +102,11 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
def loadd_from(stage, d_list):
ret = []
for d in d_list:
p = d.pop(OutputBase.PARAM_PATH)
cache = d.pop(OutputBase.PARAM_CACHE, True)
metric = d.pop(OutputBase.PARAM_METRIC, False)
persist = d.pop(OutputBase.PARAM_PERSIST, False)
tags = d.pop(OutputBase.PARAM_TAGS, None)
p = d.pop(BaseOutput.PARAM_PATH)
cache = d.pop(BaseOutput.PARAM_CACHE, True)
metric = d.pop(BaseOutput.PARAM_METRIC, False)
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
tags = d.pop(BaseOutput.PARAM_TAGS, None)
ret.append(
_get(
stage,
Expand Down
2 changes: 1 addition & 1 deletion dvc/output/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def __init__(self, path):
super().__init__("Stage file '{}' cannot be an output.".format(path))


class OutputBase(object):
class BaseOutput(object):
IS_DEPENDENCY = False

REMOTE = BaseRemote
Expand Down
4 changes: 2 additions & 2 deletions dvc/output/gs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.s3 import OutputS3
from dvc.output.s3 import S3Output
from dvc.remote.gs import GSRemote


class OutputGS(OutputS3):
class GSOutput(S3Output):
REMOTE = GSRemote
4 changes: 2 additions & 2 deletions dvc/output/hdfs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.remote.hdfs import HDFSRemote


class OutputHDFS(OutputBase):
class HDFSOutput(BaseOutput):
REMOTE = HDFSRemote
4 changes: 2 additions & 2 deletions dvc/output/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from dvc.exceptions import DvcException
from dvc.istextfile import istextfile
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.remote.local import LocalRemote
from dvc.utils import relpath
from dvc.compat import fspath_py35
Expand All @@ -14,7 +14,7 @@
logger = logging.getLogger(__name__)


class OutputLOCAL(OutputBase):
class LocalOutput(BaseOutput):
REMOTE = LocalRemote
sep = os.sep

Expand Down
4 changes: 2 additions & 2 deletions dvc/output/s3.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.remote.s3 import S3Remote


class OutputS3(OutputBase):
class S3Output(BaseOutput):
REMOTE = S3Remote
4 changes: 2 additions & 2 deletions dvc/output/ssh.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.remote.ssh import SSHRemote


class OutputSSH(OutputBase):
class SSHOutput(BaseOutput):
REMOTE = SSHRemote
10 changes: 5 additions & 5 deletions dvc/stage/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ def dumpd(self):
}

def _compute_md5(self):
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput

d = self.dumpd()

Expand All @@ -471,15 +471,15 @@ def _compute_md5(self):
del d[self.PARAM_WDIR]

# NOTE: excluding parameters that don't affect the state of the
# pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
# pipeline. Not excluding `LocalOutput.PARAM_CACHE`, because if
# it has changed, we might not have that output in our cache.
m = dict_md5(
d,
exclude=[
self.PARAM_LOCKED,
OutputBase.PARAM_METRIC,
OutputBase.PARAM_TAGS,
OutputBase.PARAM_PERSIST,
BaseOutput.PARAM_METRIC,
BaseOutput.PARAM_TAGS,
BaseOutput.PARAM_PERSIST,
],
)
logger.debug("Computed {} md5: '{}'".format(self, m))
Expand Down
6 changes: 3 additions & 3 deletions tests/func/test_repro.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from dvc.exceptions import ReproductionError
from dvc.exceptions import StagePathAsOutputError
from dvc.main import main
from dvc.output.base import OutputBase
from dvc.output.base import BaseOutput
from dvc.path_info import URLInfo
from dvc.remote.local import LocalRemote
from dvc.repo import Repo as DvcRepo
Expand Down Expand Up @@ -1341,9 +1341,9 @@ def test_force_import(self):
)

patch_checkout = patch.object(
OutputBase,
BaseOutput,
"checkout",
side_effect=OutputBase.checkout,
side_effect=BaseOutput.checkout,
autospec=True,
)

Expand Down
4 changes: 2 additions & 2 deletions tests/func/test_run_single_stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from dvc.exceptions import StagePathAsOutputError
from dvc.dvcfile import DVC_FILE_SUFFIX
from dvc.main import main
from dvc.output import OutputBase
from dvc.output import BaseOutput
from dvc.output.base import OutputIsStageFileError
from dvc.repo import Repo as DvcRepo
from dvc.stage import Stage
Expand Down Expand Up @@ -667,7 +667,7 @@ def run_command(self, file, file_content):
def stage_should_contain_persist_flag(self, stage_file):
stage_file_content = load_stage_file(stage_file)
self.assertEqual(
True, stage_file_content["outs"][0][OutputBase.PARAM_PERSIST]
True, stage_file_content["outs"][0][BaseOutput.PARAM_PERSIST]
)

def should_append_upon_repro(self, file, stage_file):
Expand Down
12 changes: 6 additions & 6 deletions tests/func/test_stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest

from dvc.main import main
from dvc.output.local import OutputLOCAL
from dvc.output.local import LocalOutput
from dvc.remote.local import LocalRemote
from dvc.repo import Repo
from dvc.stage import Stage
Expand Down Expand Up @@ -54,15 +54,15 @@ def test_empty_list():

def test_list():
lst = [
{OutputLOCAL.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"},
{OutputLOCAL.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None},
{OutputLOCAL.PARAM_PATH: "baz"},
{LocalOutput.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"},
{LocalOutput.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None},
{LocalOutput.PARAM_PATH: "baz"},
]
d = {Stage.PARAM_DEPS: lst}
SingleStageFile.validate(d)

lst[0][OutputLOCAL.PARAM_CACHE] = True
lst[1][OutputLOCAL.PARAM_CACHE] = False
lst[0][LocalOutput.PARAM_CACHE] = True
lst[1][LocalOutput.PARAM_CACHE] = False
d = {Stage.PARAM_OUTS: lst}
SingleStageFile.validate(d)

Expand Down
8 changes: 4 additions & 4 deletions tests/unit/output/test_gs.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from dvc.output.gs import OutputGS
from tests.unit.output.test_local import TestOutputLOCAL
from dvc.output.gs import GSOutput
from tests.unit.output.test_local import TestLocalOutput


class TestOutputGS(TestOutputLOCAL):
class TestGSOutput(TestLocalOutput):
def _get_cls(self):
return OutputGS
return GSOutput
8 changes: 4 additions & 4 deletions tests/unit/output/test_hdfs.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from dvc.output.hdfs import OutputHDFS
from tests.unit.output.test_local import TestOutputLOCAL
from dvc.output.hdfs import HDFSOutput
from tests.unit.output.test_local import TestLocalOutput


class TestOutputHDFS(TestOutputLOCAL):
class TestHDFSOutput(TestLocalOutput):
def _get_cls(self):
return OutputHDFS
return HDFSOutput
Loading