Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tests: remote: local: migrate to dir fixtures #3049

Merged
merged 1 commit into from
Jan 5, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 55 additions & 75 deletions tests/func/remote/test_local.py
Original file line number Diff line number Diff line change
@@ -1,84 +1,64 @@
import os

import mock
import pytest

import dvc
from dvc.exceptions import DvcException
from dvc.remote.local import RemoteLOCAL
from tests.utils import trees_equal


class TestUnpackedDirStatusOptimization(object):
@pytest.fixture(autouse=True)
def setUp(self, dvc_repo, repo_dir):
stages = dvc_repo.add(repo_dir.DATA_DIR)
assert len(stages) == 1
self.dir_out = stages[0].outs[0]
self.unpacked_dir = (
self.dir_out.cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
)

def test_should_not_fail_add_on_unpacked_dir_creation_exception(
self, dvc_repo
):
with mock.patch.object(
dvc.remote.local.RemoteLOCAL,
"_create_unpacked_dir",
side_effect=self._raise_dvc_exception,
) as unpacked_create_spy, dvc_repo.state:
assert not dvc_repo.cache.local.changed_cache(
self.dir_out.checksum
)
assert unpacked_create_spy.call_count == 1

def test_should_remove_unpacked_dir_on_create_exception(self, dvc_repo):
# artificial unpacked dir for test purpose
os.makedirs(self.unpacked_dir)
assert os.path.exists(self.unpacked_dir)

with mock.patch.object(
dvc.remote.local.RemoteLOCAL,
"_create_unpacked_dir",
side_effect=self._raise_dvc_exception,
), dvc_repo.state:
assert not dvc_repo.cache.local.changed_cache(
self.dir_out.checksum
)

assert not os.path.exists(self.unpacked_dir)

def test_should_create_unpacked_dir_on_status_check(
self, dvc_repo, repo_dir
):
assert not os.path.exists(self.unpacked_dir)

with dvc_repo.state:
assert not dvc_repo.cache.local.changed_cache(
self.dir_out.checksum
)

assert os.path.exists(self.unpacked_dir)

trees_equal(repo_dir.DATA_DIR, self.unpacked_dir)

def test_should_proceed_with_full_check_on_modified_cache(self, dvc_repo):

some_file_md5 = self.dir_out.dir_cache[0]["md5"]
assert not os.path.exists(self.unpacked_dir)

with dvc_repo.state:
assert not dvc_repo.cache.local.changed_cache(
self.dir_out.checksum
)
assert os.path.exists(self.unpacked_dir)

with open(dvc_repo.cache.local.get(some_file_md5), "a") as fobj:
fobj.write("string")

with dvc_repo.state:
assert dvc_repo.cache.local.changed_cache(self.dir_out.checksum)

@staticmethod
def _raise_dvc_exception(*args):
raise DvcException("msg")
def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc):
stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})

with mock.patch.object(
RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg")
) as unpacked_create_spy, dvc.state:
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
assert unpacked_create_spy.call_count == 1


def test_remove_unpacked_on_create_fail(tmp_dir, dvc):
stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX

# artificial unpacked dir for test purpose
os.makedirs(unpacked_dir)
assert os.path.exists(unpacked_dir)

with mock.patch.object(
RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg")
), dvc.state:
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)

assert not os.path.exists(unpacked_dir)


def test_create_unpacked_on_status(tmp_dir, dvc):
stage, = tmp_dir.dvc_gen({"dir": {"file": "file_content"}})
unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
assert not os.path.exists(unpacked_dir)

with dvc.state:
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
assert os.path.exists(unpacked_dir)
trees_equal("dir", unpacked_dir)


def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc):
stage, = tmp_dir.dvc_gen(
{"dir": {"file1": "file1 content", "file2": "file2 content"}}
)
unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX
assert not os.path.exists(unpacked_dir)
file_md5 = stage.outs[0].dir_cache[0]["md5"]

with dvc.state:
assert not dvc.cache.local.changed_cache(stage.outs[0].checksum)
assert os.path.exists(unpacked_dir)

cache_file_path = dvc.cache.local.get(file_md5)
with open(cache_file_path, "a") as fobj:
fobj.write("modification")

with dvc.state:
assert dvc.cache.local.changed_cache(stage.outs[0].checksum)