dvcfile: add desc field for outs/stages (#4901)

Outs desc will be used by viewer right away. Stage desc is for people to
start using now, but we won't be using it it viewer until DAG support is
added there.

dvc/command/add.py

@@ -21,6 +21,7 @@ def run(self):
                 fname=self.args.file,
                 external=self.args.external,
                 glob=self.args.glob,
+                desc=self.args.desc,
             )
 
         except DvcException:
@@ -69,6 +70,15 @@ def add_parser(subparsers, parent_parser):
         help="Specify name of the DVC-file this command will generate.",
         metavar="<filename>",
     )
+    parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     parser.add_argument(
         "targets", nargs="+", help="Input files/directories to add.",
     ).complete = completion.FILE

dvc/command/imp.py

@@ -18,6 +18,7 @@ def run(self):
                 fname=self.args.file,
                 rev=self.args.rev,
                 no_exec=self.args.no_exec,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception(
@@ -72,4 +73,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Only create DVC-file without actually downloading it.",
     )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     import_parser.set_defaults(func=CmdImport)

dvc/command/imp_url.py

@@ -16,6 +16,7 @@ def run(self):
                 out=self.args.out,
                 fname=self.args.file,
                 no_exec=self.args.no_exec,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception(
@@ -67,4 +68,13 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Only create DVC-file without actually downloading it.",
     )
+    import_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the data (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/command/run.py

@@ -57,6 +57,7 @@ def run(self):
                 name=self.args.name,
                 single_stage=self.args.single_stage,
                 external=self.args.external,
+                desc=self.args.desc,
             )
         except DvcException:
             logger.exception("")
@@ -240,6 +241,15 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Allow outputs that are outside of the DVC repository.",
     )
+    run_parser.add_argument(
+        "--desc",
+        type=str,
+        metavar="<text>",
+        help=(
+            "User description of the stage (optional). "
+            "This doesn't affect any DVC operations."
+        ),
+    )
     run_parser.add_argument(
         "command", nargs=argparse.REMAINDER, help="Command to execute."
     )

dvc/dependency/__init__.py

@@ -56,6 +56,7 @@
 SCHEMA = output.SCHEMA.copy()
 del SCHEMA[BaseOutput.PARAM_CACHE]
 del SCHEMA[BaseOutput.PARAM_METRIC]
+del SCHEMA[BaseOutput.PARAM_DESC]
 SCHEMA.update(RepoDependency.REPO_SCHEMA)
 SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
 

dvc/output/__init__.py

@@ -68,6 +68,7 @@
 SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
 SCHEMA[HashInfo.PARAM_SIZE] = int
 SCHEMA[HashInfo.PARAM_NFILES] = int
+SCHEMA[BaseOutput.PARAM_DESC] = str
 
 
 def _get(
@@ -79,6 +80,7 @@ def _get(
     plot=False,
     persist=False,
     checkpoint=False,
+    desc=None,
 ):
     parsed = urlparse(p)
 
@@ -94,6 +96,7 @@ def _get(
             plot=plot,
             persist=persist,
             checkpoint=checkpoint,
+            desc=desc,
         )
 
     for o in OUTS:
@@ -108,6 +111,7 @@ def _get(
                 plot=plot,
                 persist=persist,
                 checkpoint=checkpoint,
+                desc=desc,
             )
     return LocalOutput(
         stage,
@@ -119,6 +123,7 @@ def _get(
         plot=plot,
         persist=persist,
         checkpoint=checkpoint,
+        desc=desc,
     )
 
 
@@ -131,6 +136,7 @@ def loadd_from(stage, d_list):
         plot = d.pop(BaseOutput.PARAM_PLOT, False)
         persist = d.pop(BaseOutput.PARAM_PERSIST, False)
         checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
+        desc = d.pop(BaseOutput.PARAM_DESC, False)
         ret.append(
             _get(
                 stage,
@@ -141,6 +147,7 @@ def loadd_from(stage, d_list):
                 plot=plot,
                 persist=persist,
                 checkpoint=checkpoint,
+                desc=desc,
             )
         )
     return ret

dvc/output/base.py

@@ -74,6 +74,7 @@ class BaseOutput:
     PARAM_PLOT_TITLE = "title"
     PARAM_PLOT_HEADER = "header"
     PARAM_PERSIST = "persist"
+    PARAM_DESC = "desc"
 
     METRIC_SCHEMA = Any(
         None,
@@ -102,6 +103,7 @@ def __init__(
         plot=False,
         persist=False,
         checkpoint=False,
+        desc=None,
     ):
         self._validate_output_path(path, stage)
         # This output (and dependency) objects have too many paths/urls
@@ -127,6 +129,7 @@ def __init__(
         self.plot = False if self.IS_DEPENDENCY else plot
         self.persist = persist
         self.checkpoint = checkpoint
+        self.desc = desc
 
         self.path_info = self._parse_path(tree, path)
         if self.use_cache and self.cache is None:
@@ -299,6 +302,9 @@ def dumpd(self):
         if self.IS_DEPENDENCY:
             return ret
 
+        if self.desc:
+            ret[self.PARAM_DESC] = self.desc
+
         if not self.use_cache:
             ret[self.PARAM_CACHE] = self.use_cache
 

dvc/repo/add.py

@@ -23,13 +23,7 @@
 @locked
 @scm_context
 def add(
-    repo,
-    targets,
-    recursive=False,
-    no_commit=False,
-    fname=None,
-    external=False,
-    glob=False,
+    repo, targets, recursive=False, no_commit=False, fname=None, **kwargs,
 ):
     if recursive and fname:
         raise RecursiveAddingWhileUsingFilename()
@@ -63,12 +57,7 @@ def add(
                 )
 
             stages = _create_stages(
-                repo,
-                sub_targets,
-                fname,
-                pbar=pbar,
-                external=external,
-                glob=glob,
+                repo, sub_targets, fname, pbar=pbar, **kwargs,
             )
 
             try:
@@ -161,7 +150,7 @@ def _find_all_targets(repo, target, recursive):
 
 
 def _create_stages(
-    repo, targets, fname, pbar=None, external=False, glob=False
+    repo, targets, fname, pbar=None, external=False, glob=False, desc=None,
 ):
     from glob import iglob
 
@@ -194,6 +183,8 @@ def _create_stages(
         )
         if stage:
             Dvcfile(repo, stage.path).remove()
+            if desc:
+                stage.outs[0].desc = desc
 
         repo._reset()  # pylint: disable=protected-access
 

dvc/repo/imp.py

@@ -1,8 +1,8 @@
-def imp(self, url, path, out=None, fname=None, rev=None, no_exec=False):
+def imp(self, url, path, out=None, fname=None, rev=None, **kwargs):
     erepo = {"url": url}
     if rev is not None:
         erepo["rev"] = rev
 
     return self.imp_url(
-        path, out=out, fname=fname, erepo=erepo, frozen=True, no_exec=no_exec
+        path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs
     )

dvc/repo/imp_url.py

@@ -11,7 +11,14 @@
 @locked
 @scm_context
 def imp_url(
-    self, url, out=None, fname=None, erepo=None, frozen=True, no_exec=False
+    self,
+    url,
+    out=None,
+    fname=None,
+    erepo=None,
+    frozen=True,
+    no_exec=False,
+    desc=None,
 ):
     from dvc.dvcfile import Dvcfile
     from dvc.stage import Stage, create_stage
@@ -40,6 +47,9 @@ def imp_url(
     if stage is None:
         return None
 
+    if desc:
+        stage.outs[0].desc = desc
+
     dvcfile = Dvcfile(self, stage.path)
     dvcfile.remove()
 

dvc/schema.py

@@ -17,6 +17,7 @@
     StageParams.PARAM_FROZEN: bool,
     StageParams.PARAM_META: object,
     StageParams.PARAM_ALWAYS_CHANGED: bool,
+    StageParams.PARAM_DESC: str,
 }
 
 DATA_SCHEMA = {
@@ -38,6 +39,7 @@
         BaseOutput.PARAM_CACHE: bool,
         BaseOutput.PARAM_PERSIST: bool,
         BaseOutput.PARAM_CHECKPOINT: bool,
+        BaseOutput.PARAM_DESC: str,
     }
 }
 
@@ -68,6 +70,7 @@
     ],
     Optional(StageParams.PARAM_FROZEN): bool,
     Optional(StageParams.PARAM_META): object,
+    Optional(StageParams.PARAM_DESC): str,
     Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
     Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)],
     Optional(StageParams.PARAM_METRICS): [

dvc/stage/__init__.py

@@ -52,6 +52,7 @@ def loads_from(cls, repo, path, wdir, data):
                 Stage.PARAM_FROZEN,
                 Stage.PARAM_ALWAYS_CHANGED,
                 Stage.PARAM_MD5,
+                Stage.PARAM_DESC,
                 "name",
             ],
         ),
@@ -111,6 +112,7 @@ def __init__(
         always_changed=False,
         stage_text=None,
         dvcfile=None,
+        desc=None,
     ):
         if deps is None:
             deps = []
@@ -128,6 +130,7 @@ def __init__(
         self.always_changed = always_changed
         self._stage_text = stage_text
         self._dvcfile = dvcfile
+        self.desc = desc
 
     @property
     def path(self):

dvc/stage/loader.py

@@ -58,6 +58,7 @@ def load_stage(cls, dvcfile, name, stage_data, lock_data=None):
         )
         stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data)
         stage.name = name
+        stage.desc = stage_data.get(Stage.PARAM_DESC)
 
         deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS])
         fill_stage_dependencies(stage, **deps)

dvc/stage/params.py

@@ -11,3 +11,4 @@ class StageParams:
     PARAM_PARAMS = "params"
     PARAM_METRICS = "metrics"
     PARAM_PLOTS = "plots"
+    PARAM_DESC = "desc"

dvc/stage/serialize.py

@@ -27,6 +27,7 @@
 PARAM_PLOT = BaseOutput.PARAM_PLOT
 PARAM_PERSIST = BaseOutput.PARAM_PERSIST
 PARAM_CHECKPOINT = BaseOutput.PARAM_CHECKPOINT
+PARAM_DESC = BaseOutput.PARAM_DESC
 
 DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
 
@@ -36,6 +37,8 @@
 
 @post_processing(OrderedDict)
 def _get_flags(out):
+    if out.desc:
+        yield PARAM_DESC, out.desc
     if not out.use_cache:
         yield PARAM_CACHE, False
     if out.checkpoint:
@@ -119,6 +122,7 @@ def to_pipeline_file(stage: "PipelineStage"):
 
     outs, metrics, plots = _serialize_outs(stage.outs)
     res = [
+        (stage.PARAM_DESC, stage.desc),
         (stage.PARAM_CMD, stage.cmd),
         (stage.PARAM_WDIR, wdir),
         (stage.PARAM_DEPS, deps),

dvc/stage/utils.py

@@ -199,6 +199,7 @@ def get_dump(stage):
     return {
         key: value
         for key, value in {
+            stage.PARAM_DESC: stage.desc,
             stage.PARAM_MD5: stage.md5,
             stage.PARAM_CMD: stage.cmd,
             stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path),

tests/func/test_dvcfile.py

@@ -317,6 +317,27 @@ def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy):
     assert dvcfile._load()[0]["stages"]["run_copy"]["meta"] == metadata
 
 
+def test_dvcfile_dump_preserves_desc(tmp_dir, dvc, run_copy):
+    tmp_dir.gen("foo", "foo")
+    stage_desc = "test stage description"
+    out_desc = "test out description"
+
+    stage = run_copy("foo", "bar", name="run_copy", desc=stage_desc)
+    dvcfile = stage.dvcfile
+
+    data = dvcfile._load()[0]
+    data["stages"]["run_copy"]["outs"][0] = {"bar": {"desc": out_desc}}
+    dump_yaml(dvcfile.path, data)
+
+    assert stage.desc == stage_desc
+    stage.outs[0].desc = out_desc
+    dvcfile.dump(stage)
+    loaded = dvcfile._load()[0]
+    assert loaded == data
+    assert loaded["stages"]["run_copy"]["desc"] == stage_desc
+    assert loaded["stages"]["run_copy"]["outs"][0]["bar"]["desc"] == out_desc
+
+
 def test_dvcfile_dump_preserves_comments(tmp_dir, dvc):
     text = textwrap.dedent(
         """\