Skip to content

Commit

Permalink
run/repro: rename "build cache" -> "run cache"
Browse files Browse the repository at this point in the history
  • Loading branch information
efiop committed May 13, 2020
1 parent 3916d4f commit 88d29ed
Show file tree
Hide file tree
Showing 11 changed files with 45 additions and 35 deletions.
14 changes: 12 additions & 2 deletions dvc/command/repro.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ def run(self):
interactive=self.args.interactive,
pipeline=self.args.pipeline,
all_pipelines=self.args.all_pipelines,
ignore_build_cache=self.args.ignore_build_cache,
run_cache=not self.args.no_run_cache,
no_commit=self.args.no_commit,
downstream=self.args.downstream,
recursive=self.args.recursive,
force_downstream=self.args.force_downstream,
)

if len(stages) == 0:
Expand Down Expand Up @@ -136,7 +137,16 @@ def add_parser(subparsers, parent_parser):
help="Reproduce all stages in the specified directory.",
)
repro_parser.add_argument(
"--ignore-build-cache",
"--no-run-cache",
action="store_true",
default=False,
help=(
"Execute stage commands even if they have already been run with "
"the same command/dependencies/outputs/etc before."
),
)
repro_parser.add_argument(
"--force-downstream",
action="store_true",
default=False,
help="Reproduce all descendants of a changed stage even if their "
Expand Down
10 changes: 6 additions & 4 deletions dvc/command/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def run(self):
wdir=self.args.wdir,
no_exec=self.args.no_exec,
overwrite=self.args.overwrite_dvcfile,
ignore_build_cache=self.args.ignore_build_cache,
run_cache=not self.args.no_run_cache,
no_commit=self.args.no_commit,
outs_persist=self.args.outs_persist,
outs_persist_no_cache=self.args.outs_persist_no_cache,
Expand Down Expand Up @@ -165,11 +165,13 @@ def add_parser(subparsers, parent_parser):
help="Overwrite existing DVC-file without asking for confirmation.",
)
run_parser.add_argument(
"--ignore-build-cache",
"--no-run-cache",
action="store_true",
default=False,
help="Run this stage even if it has been already ran with the same "
"command/dependencies/outputs/etc before.",
help=(
"Execute the command even if this stage has already been run "
"with the same command/dependencies/outputs/etc before."
),
)
run_parser.add_argument(
"--no-commit",
Expand Down
3 changes: 2 additions & 1 deletion dvc/repo/reproduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,13 @@ def _reproduce_stages(
if stage not in pipeline:
pipeline.append(stage)

force_downstream = kwargs.pop("force_downstream", False)
result = []
for stage in pipeline:
try:
ret = _reproduce_stage(stage, **kwargs)

if len(ret) != 0 and kwargs.get("ignore_build_cache", False):
if len(ret) != 0 and force_downstream:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
Expand Down
2 changes: 1 addition & 1 deletion dvc/repo/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
else:
stage.run(
no_commit=kwargs.get("no_commit", False),
ignore_build_cache=kwargs.get("ignore_build_cache", False),
run_cache=kwargs.get("run_cache", True),
)

dvcfile.dump(stage, update_pipeline=True)
Expand Down
14 changes: 5 additions & 9 deletions dvc/stage/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,13 @@ def create_stage(cls, repo, path, **kwargs):

if stage and stage.dvcfile.exists():
has_persist_outs = any(out.persist for out in stage.outs)
ignore_build_cache = (
kwargs.get("ignore_build_cache", False) or has_persist_outs
ignore_run_cache = (
not kwargs.get("run_cache", True) or has_persist_outs
)
if has_persist_outs:
logger.warning("Build cache is ignored when persisting outputs.")

if not ignore_build_cache and stage.can_be_skipped:
if not ignore_run_cache and stage.can_be_skipped:
logger.info("Stage is cached, skipping.")
return None

Expand Down Expand Up @@ -623,9 +623,7 @@ def _run(self):
raise StageCmdFailedError(self, retcode)

@rwlocked(read=["deps"], write=["outs"])
def run(
self, dry=False, no_commit=False, force=False, ignore_build_cache=False
):
def run(self, dry=False, no_commit=False, force=False, run_cache=True):
if (self.cmd or self.is_import) and not self.locked and not dry:
self.remove_outs(ignore_remove=False, force=False)

Expand Down Expand Up @@ -670,9 +668,7 @@ def run(
if not stage_cached:
self._save_deps()
use_build_cache = (
not force
and not ignore_build_cache
and stage_cache.is_cached(self)
not force and run_cache and stage_cache.is_cached(self)
)

if use_build_cache:
Expand Down
4 changes: 2 additions & 2 deletions scripts/completion/dvc.bash
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ _dvc_remote_modify='--global --system --local -u --unset'
_dvc_remote_remove='--global --system --local'
_dvc_remove='-o --outs -p --purge -f --force'
_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -R --recursive --downstream'
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
_dvc_root=''
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --ignore-build-cache --no-commit -w --wdir'
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir'
_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
Expand Down
5 changes: 3 additions & 2 deletions scripts/completion/dvc.zsh
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,8 @@ _dvc_repro=(
{-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
{-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
{-R,--recursive}"[Reproduce all stages in the specified directory.]"
"--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
"--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
"--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--no-commit[Don't put files/directories into cache.]"
"--downstream[Start from the specified stages when reproducing pipelines.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
Expand All @@ -259,7 +260,7 @@ _dvc_run=(
"--no-exec[Only create stage file without actually running it.]"
{-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
"--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
"--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--remove-outs[Deprecated, this is now the default behavior]"
"--no-commit[Don't put files/directories into cache.]"
"--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
Expand Down
4 changes: 2 additions & 2 deletions tests/func/test_repro.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ def test(self):
self.assertEqual(len(stages), 3)


class TestReproIgnoreBuildCache(TestDvc):
class TestReproForceDownstream(TestDvc):
def test(self):
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
Expand Down Expand Up @@ -480,7 +480,7 @@ def test(self):
with open(code2, "a") as fobj:
fobj.write("\n\n")

stages = self.dvc.reproduce(file3_stage.path, ignore_build_cache=True)
stages = self.dvc.reproduce(file3_stage.path, force_downstream=True)
self.assertEqual(len(stages), 2)
self.assertEqual(stages[0].path, file2_stage.path)
self.assertEqual(stages[1].path, file3_stage.path)
Expand Down
13 changes: 6 additions & 7 deletions tests/func/test_run_single_stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ def test(self):
[
"run",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
Expand Down Expand Up @@ -401,7 +401,7 @@ def test(self):
[
"run",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
Expand Down Expand Up @@ -460,7 +460,7 @@ def test(self):
[
"run",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
Expand Down Expand Up @@ -545,7 +545,7 @@ def test(self):
"-d",
self.CODE,
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-o",
"out",
Expand Down Expand Up @@ -664,8 +664,7 @@ def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy):

assert run_copy("foo", "out", single_stage=True) is not None
assert (
run_copy("foo", "out", ignore_build_cache=True, single_stage=True)
is not None
run_copy("foo", "out", run_cache=False, single_stage=True) is not None
)


Expand Down Expand Up @@ -932,7 +931,7 @@ def test(self):


class TestPersistentOutput(TestDvc):
def test_ignore_build_cache(self):
def test_ignore_run_cache(self):
warning = "Build cache is ignored when persisting outputs."

with open("immutable", "w") as fobj:
Expand Down
3 changes: 2 additions & 1 deletion tests/unit/command/test_repro.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@
"downstream": False,
"dry": False,
"force": False,
"ignore_build_cache": False,
"run_cache": True,
"interactive": False,
"no_commit": False,
"pipeline": False,
"single_item": False,
"recursive": False,
"force_downstream": False,
}


Expand Down
8 changes: 4 additions & 4 deletions tests/unit/command/test_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def test_run(mocker, dvc):
"wdir",
"--no-exec",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--no-commit",
"--outs-persist",
"outs-persist",
Expand Down Expand Up @@ -58,7 +58,7 @@ def test_run(mocker, dvc):
wdir="wdir",
no_exec=True,
overwrite=True,
ignore_build_cache=True,
run_cache=False,
no_commit=True,
always_changed=True,
cmd="command",
Expand All @@ -85,7 +85,7 @@ def test_run_args_from_cli(mocker, dvc):
wdir=None,
no_exec=False,
overwrite=False,
ignore_build_cache=False,
run_cache=True,
no_commit=False,
always_changed=False,
cmd="echo foo",
Expand All @@ -112,7 +112,7 @@ def test_run_args_with_spaces(mocker, dvc):
wdir=None,
no_exec=False,
overwrite=False,
ignore_build_cache=False,
run_cache=True,
no_commit=False,
always_changed=False,
cmd='echo "foo bar"',
Expand Down

0 comments on commit 88d29ed

Please sign in to comment.