From e1996e00b0583d59ca6c02fef62fe5aeaa6acb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Saugat=20Pachhai=20=28=E0=A4=B8=E0=A5=8C=E0=A4=97=E0=A4=BE?= =?UTF-8?q?=E0=A4=A4=29?= Date: Wed, 15 Sep 2021 14:01:37 +0545 Subject: [PATCH 1/2] exp init: add simple dvc exp init command --- dvc/command/experiments.py | 119 ++++++++++++++++++++++++++++ tests/func/experiments/test_init.py | 34 ++++++++ 2 files changed, 153 insertions(+) create mode 100644 tests/func/experiments/test_init.py diff --git a/dvc/command/experiments.py b/dvc/command/experiments.py index 1f74cd9f47..1adc4eee9e 100644 --- a/dvc/command/experiments.py +++ b/dvc/command/experiments.py @@ -1,5 +1,6 @@ import argparse import logging +import os from collections import Counter, OrderedDict, defaultdict from datetime import date, datetime from fnmatch import fnmatch @@ -789,6 +790,62 @@ def run(self): return 0 +class CmdExperimentsInit(CmdBase): + CODE = "src" + DATA = "data" + MODELS = "models" + DEFAULT_METRICS = "metrics.json" + DEFAULT_PARAMS = "params.yaml" + PLOTS = "plots" + DVCLIVE = "dvclive" + DEFAULT_NAME = "default" + + def run(self): + from dvc.command.stage import parse_cmd + + cmd = parse_cmd(self.args.cmd) + if not cmd: + raise InvalidArgumentError("command is not specified") + if self.args.interactive: + raise NotImplementedError( + "'-i/--interactive' is not implemented yet." + ) + if self.args.explicit: + raise NotImplementedError("'--explicit' is not implemented yet.") + if self.args.template: + raise NotImplementedError("template is not supported yet.") + + from dvc.utils.serialize import LOADERS + + code = self.args.code or self.CODE + data = self.args.data or self.DATA + models = self.args.models or self.MODELS + metrics = self.args.metrics or self.DEFAULT_METRICS + params_path = self.args.params or self.DEFAULT_PARAMS + plots = self.args.plots or self.PLOTS + dvclive = self.args.live or self.DVCLIVE + + _, ext = os.path.splitext(params_path) + params = list(LOADERS[ext](params_path)) + + name = self.args.name or self.DEFAULT_NAME + stage = self.repo.stage.add( + name=name, + cmd=cmd, + deps=[code, data], + outs=[models], + params=[{params_path: params}], + metrics_no_cache=[metrics], + plots_no_cache=[plots], + live=dvclive, + force=True, + ) + + if self.args.run: + return self.repo.experiments.run(targets=[stage.addressing]) + return 0 + + def add_parser(subparsers, parent_parser): EXPERIMENTS_HELP = "Commands to run and compare experiments." @@ -1303,6 +1360,68 @@ def add_parser(subparsers, parent_parser): ) experiments_remove_parser.set_defaults(func=CmdExperimentsRemove) + EXPERIMENTS_INIT_HELP = "Create experiments." + experiments_init_parser = experiments_subparsers.add_parser( + "init", + parents=[parent_parser], + description=append_doc_link(EXPERIMENTS_INIT_HELP, "exp/init"), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + experiments_init_parser.add_argument( + "cmd", + nargs=argparse.REMAINDER, + help="Command to execute.", + metavar="command", + ) + experiments_init_parser.add_argument( + "--run", + action="store_true", + help="Run the experiment after initializing it", + ) + experiments_init_parser.add_argument( + "--interactive", + "-i", + action="store_true", + help="Prompt for values that are not provided", + ) + experiments_init_parser.add_argument( + "--template", help="Stage template to use to fill with provided values" + ) + experiments_init_parser.add_argument( + "--explicit", help="Only use the path values explicitly provided" + ) + experiments_init_parser.add_argument( + "--name", "-n", help="Name of the stage to create" + ) + experiments_init_parser.add_argument( + "--code", + help="Path to the source file or directory " + "which your experiment depends", + ) + experiments_init_parser.add_argument( + "--data", + help="Path to the data file or directory " + "which your experiment depends", + ) + experiments_init_parser.add_argument( + "--models", + help="Path to the model file or directory for your experiments", + ) + experiments_init_parser.add_argument( + "--params", help="Path to the parameters file for your experiments" + ) + experiments_init_parser.add_argument( + "--metrics", help="Path to the metrics file for your experiments" + ) + experiments_init_parser.add_argument( + "--plots", + help="Path to the plots file or directory for your experiments", + ) + experiments_init_parser.add_argument( + "--live", help="Path to log dvclive outputs for your experiments" + ) + experiments_init_parser.set_defaults(func=CmdExperimentsInit) + def _add_run_common(parser): """Add common args for 'exp run' and 'exp resume'.""" diff --git a/tests/func/experiments/test_init.py b/tests/func/experiments/test_init.py new file mode 100644 index 0000000000..d55451d8f5 --- /dev/null +++ b/tests/func/experiments/test_init.py @@ -0,0 +1,34 @@ +import os + +from dvc.command.experiments import CmdExperimentsInit +from dvc.main import main +from dvc.utils.serialize import load_yaml + + +def test_init(tmp_dir, dvc): + tmp_dir.gen( + { + CmdExperimentsInit.CODE: {"copy.py": ""}, + "data": "data", + "params.yaml": '{"foo": 1}', + "dvclive": {}, + "plots": {}, + } + ) + code_path = os.path.join(CmdExperimentsInit.CODE, "copy.py") + script = f"python {code_path}" + + assert main(["exp", "init", script]) == 0 + assert load_yaml(tmp_dir / "dvc.yaml") == { + "stages": { + "default": { + "cmd": script, + "deps": ["data", "src"], + "live": {"dvclive": {"html": True, "summary": True}}, + "metrics": [{"metrics.json": {"cache": False}}], + "outs": ["models"], + "params": ["foo"], + "plots": [{"plots": {"cache": False}}], + } + } + } From 04c09f2fe36fb9e695cfcded8809044e5b60024e Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Thu, 16 Sep 2021 07:06:31 +0545 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Dave Berenbaum --- dvc/command/experiments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dvc/command/experiments.py b/dvc/command/experiments.py index 1adc4eee9e..e699832a4f 100644 --- a/dvc/command/experiments.py +++ b/dvc/command/experiments.py @@ -1360,7 +1360,7 @@ def add_parser(subparsers, parent_parser): ) experiments_remove_parser.set_defaults(func=CmdExperimentsRemove) - EXPERIMENTS_INIT_HELP = "Create experiments." + EXPERIMENTS_INIT_HELP = "Initialize experiments." experiments_init_parser = experiments_subparsers.add_parser( "init", parents=[parent_parser], @@ -1396,12 +1396,12 @@ def add_parser(subparsers, parent_parser): experiments_init_parser.add_argument( "--code", help="Path to the source file or directory " - "which your experiment depends", + "which your experiments depend", ) experiments_init_parser.add_argument( "--data", help="Path to the data file or directory " - "which your experiment depends", + "which your experiments depend", ) experiments_init_parser.add_argument( "--models",