From 88d29edccc554bc872a428ce42fbaa69c69f4280 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 12 May 2020 19:12:57 +0300 Subject: [PATCH] run/repro: rename "build cache" -> "run cache" Part of #3777 --- dvc/command/repro.py | 14 ++++++++++++-- dvc/command/run.py | 10 ++++++---- dvc/repo/reproduce.py | 3 ++- dvc/repo/run.py | 2 +- dvc/stage/__init__.py | 14 +++++--------- scripts/completion/dvc.bash | 4 ++-- scripts/completion/dvc.zsh | 5 +++-- tests/func/test_repro.py | 4 ++-- tests/func/test_run_single_stage.py | 13 ++++++------- tests/unit/command/test_repro.py | 3 ++- tests/unit/command/test_run.py | 8 ++++---- 11 files changed, 45 insertions(+), 35 deletions(-) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 25789342f4..d5dc524821 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -33,10 +33,11 @@ def run(self): interactive=self.args.interactive, pipeline=self.args.pipeline, all_pipelines=self.args.all_pipelines, - ignore_build_cache=self.args.ignore_build_cache, + run_cache=not self.args.no_run_cache, no_commit=self.args.no_commit, downstream=self.args.downstream, recursive=self.args.recursive, + force_downstream=self.args.force_downstream, ) if len(stages) == 0: @@ -136,7 +137,16 @@ def add_parser(subparsers, parent_parser): help="Reproduce all stages in the specified directory.", ) repro_parser.add_argument( - "--ignore-build-cache", + "--no-run-cache", + action="store_true", + default=False, + help=( + "Execute stage commands even if they have already been run with " + "the same command/dependencies/outputs/etc before." + ), + ) + repro_parser.add_argument( + "--force-downstream", action="store_true", default=False, help="Reproduce all descendants of a changed stage even if their " diff --git a/dvc/command/run.py b/dvc/command/run.py index ef0f656f6d..9dbad5d508 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -42,7 +42,7 @@ def run(self): wdir=self.args.wdir, no_exec=self.args.no_exec, overwrite=self.args.overwrite_dvcfile, - ignore_build_cache=self.args.ignore_build_cache, + run_cache=not self.args.no_run_cache, no_commit=self.args.no_commit, outs_persist=self.args.outs_persist, outs_persist_no_cache=self.args.outs_persist_no_cache, @@ -165,11 +165,13 @@ def add_parser(subparsers, parent_parser): help="Overwrite existing DVC-file without asking for confirmation.", ) run_parser.add_argument( - "--ignore-build-cache", + "--no-run-cache", action="store_true", default=False, - help="Run this stage even if it has been already ran with the same " - "command/dependencies/outputs/etc before.", + help=( + "Execute the command even if this stage has already been run " + "with the same command/dependencies/outputs/etc before." + ), ) run_parser.add_argument( "--no-commit", diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 8fa2fc9384..fa20b6739e 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -162,12 +162,13 @@ def _reproduce_stages( if stage not in pipeline: pipeline.append(stage) + force_downstream = kwargs.pop("force_downstream", False) result = [] for stage in pipeline: try: ret = _reproduce_stage(stage, **kwargs) - if len(ret) != 0 and kwargs.get("ignore_build_cache", False): + if len(ret) != 0 and force_downstream: # NOTE: we are walking our pipeline from the top to the # bottom. If one stage is changed, it will be reproduced, # which tells us that we should force reproducing all of diff --git a/dvc/repo/run.py b/dvc/repo/run.py index 02cbf6fafd..b8be523fab 100644 --- a/dvc/repo/run.py +++ b/dvc/repo/run.py @@ -85,7 +85,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs): else: stage.run( no_commit=kwargs.get("no_commit", False), - ignore_build_cache=kwargs.get("ignore_build_cache", False), + run_cache=kwargs.get("run_cache", True), ) dvcfile.dump(stage, update_pipeline=True) diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 572faadc6b..a060343247 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -70,13 +70,13 @@ def create_stage(cls, repo, path, **kwargs): if stage and stage.dvcfile.exists(): has_persist_outs = any(out.persist for out in stage.outs) - ignore_build_cache = ( - kwargs.get("ignore_build_cache", False) or has_persist_outs + ignore_run_cache = ( + not kwargs.get("run_cache", True) or has_persist_outs ) if has_persist_outs: logger.warning("Build cache is ignored when persisting outputs.") - if not ignore_build_cache and stage.can_be_skipped: + if not ignore_run_cache and stage.can_be_skipped: logger.info("Stage is cached, skipping.") return None @@ -623,9 +623,7 @@ def _run(self): raise StageCmdFailedError(self, retcode) @rwlocked(read=["deps"], write=["outs"]) - def run( - self, dry=False, no_commit=False, force=False, ignore_build_cache=False - ): + def run(self, dry=False, no_commit=False, force=False, run_cache=True): if (self.cmd or self.is_import) and not self.locked and not dry: self.remove_outs(ignore_remove=False, force=False) @@ -670,9 +668,7 @@ def run( if not stage_cached: self._save_deps() use_build_cache = ( - not force - and not ignore_build_cache - and stage_cache.is_cached(self) + not force and run_cache and stage_cache.is_cached(self) ) if use_build_cache: diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index 771f233b09..cfa6478e04 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -68,10 +68,10 @@ _dvc_remote_modify='--global --system --local -u --unset' _dvc_remote_remove='--global --system --local' _dvc_remove='-o --outs -p --purge -f --force' _dvc_remove_COMPGEN=_dvc_compgen_DVCFiles -_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -R --recursive --downstream' +_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream' _dvc_repro_COMPGEN=_dvc_compgen_DVCFiles _dvc_root='' -_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --ignore-build-cache --no-commit -w --wdir' +_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir' _dvc_run_COMPGEN=_dvc_compgen_DVCFiles _dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud' _dvc_status_COMPGEN=_dvc_compgen_DVCFiles diff --git a/scripts/completion/dvc.zsh b/scripts/completion/dvc.zsh index 5fb44e7d14..67958cefd2 100644 --- a/scripts/completion/dvc.zsh +++ b/scripts/completion/dvc.zsh @@ -239,7 +239,8 @@ _dvc_repro=( {-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]" {-P,--all-pipelines}"[Reproduce all pipelines in the repo.]" {-R,--recursive}"[Reproduce all stages in the specified directory.]" - "--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" + "--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]" + "--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" "--no-commit[Don't put files/directories into cache.]" "--downstream[Start from the specified stages when reproducing pipelines.]" "*:Stages:_files -g '(*.dvc|Dvcfile)'" @@ -259,7 +260,7 @@ _dvc_run=( "--no-exec[Only create stage file without actually running it.]" {-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]" "--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]" - "--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" + "--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]" "--remove-outs[Deprecated, this is now the default behavior]" "--no-commit[Don't put files/directories into cache.]" "--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files" diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index ba9110355d..e0c1c45f95 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -437,7 +437,7 @@ def test(self): self.assertEqual(len(stages), 3) -class TestReproIgnoreBuildCache(TestDvc): +class TestReproForceDownstream(TestDvc): def test(self): stages = self.dvc.add(self.FOO) self.assertEqual(len(stages), 1) @@ -480,7 +480,7 @@ def test(self): with open(code2, "a") as fobj: fobj.write("\n\n") - stages = self.dvc.reproduce(file3_stage.path, ignore_build_cache=True) + stages = self.dvc.reproduce(file3_stage.path, force_downstream=True) self.assertEqual(len(stages), 2) self.assertEqual(stages[0].path, file2_stage.path) self.assertEqual(stages[1].path, file3_stage.path) diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 88a7479c9a..1d5fde96b7 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -343,7 +343,7 @@ def test(self): [ "run", "--overwrite-dvcfile", - "--ignore-build-cache", + "--no-run-cache", "--single-stage", "-d", self.CODE, @@ -401,7 +401,7 @@ def test(self): [ "run", "--overwrite-dvcfile", - "--ignore-build-cache", + "--no-run-cache", "--single-stage", "-d", self.CODE, @@ -460,7 +460,7 @@ def test(self): [ "run", "--overwrite-dvcfile", - "--ignore-build-cache", + "--no-run-cache", "--single-stage", "-d", self.CODE, @@ -545,7 +545,7 @@ def test(self): "-d", self.CODE, "--overwrite-dvcfile", - "--ignore-build-cache", + "--no-run-cache", "--single-stage", "-o", "out", @@ -664,8 +664,7 @@ def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy): assert run_copy("foo", "out", single_stage=True) is not None assert ( - run_copy("foo", "out", ignore_build_cache=True, single_stage=True) - is not None + run_copy("foo", "out", run_cache=False, single_stage=True) is not None ) @@ -932,7 +931,7 @@ def test(self): class TestPersistentOutput(TestDvc): - def test_ignore_build_cache(self): + def test_ignore_run_cache(self): warning = "Build cache is ignored when persisting outputs." with open("immutable", "w") as fobj: diff --git a/tests/unit/command/test_repro.py b/tests/unit/command/test_repro.py index 1e20a04cec..d29c027661 100644 --- a/tests/unit/command/test_repro.py +++ b/tests/unit/command/test_repro.py @@ -7,12 +7,13 @@ "downstream": False, "dry": False, "force": False, - "ignore_build_cache": False, + "run_cache": True, "interactive": False, "no_commit": False, "pipeline": False, "single_item": False, "recursive": False, + "force_downstream": False, } diff --git a/tests/unit/command/test_run.py b/tests/unit/command/test_run.py index e595b1db6e..00cd61fdfe 100644 --- a/tests/unit/command/test_run.py +++ b/tests/unit/command/test_run.py @@ -24,7 +24,7 @@ def test_run(mocker, dvc): "wdir", "--no-exec", "--overwrite-dvcfile", - "--ignore-build-cache", + "--no-run-cache", "--no-commit", "--outs-persist", "outs-persist", @@ -58,7 +58,7 @@ def test_run(mocker, dvc): wdir="wdir", no_exec=True, overwrite=True, - ignore_build_cache=True, + run_cache=False, no_commit=True, always_changed=True, cmd="command", @@ -85,7 +85,7 @@ def test_run_args_from_cli(mocker, dvc): wdir=None, no_exec=False, overwrite=False, - ignore_build_cache=False, + run_cache=True, no_commit=False, always_changed=False, cmd="echo foo", @@ -112,7 +112,7 @@ def test_run_args_with_spaces(mocker, dvc): wdir=None, no_exec=False, overwrite=False, - ignore_build_cache=False, + run_cache=True, no_commit=False, always_changed=False, cmd='echo "foo bar"',