live logs initial (#4902)

* plots: allow dir plots * first scenario * separate dvclive tests * rename index to step * another test * refactor * its ok to have step in data * sort plots * introduce --logs * add logs to run commands * variable name refactoring * fix logs option * fixup * cmd: run: fix unit tests * plots: modify: fix tests * improve dir handling * fixup * dvclive: logs summary * api: introduce logs summarization method * handle no repo case * fix granular dir plots * api: dvclive: simplify * dvclive: roll back logs flag * cmd: introduce logs * api: adjust to logs * test: dvclive fix * rename logs to dvclive * live: command: add uninitialized * outs: dvclive type * dvclive: support providing config via evn variables * dvclive: treat as metrics * dvclive integration: add support for summary on/off * dvc integration: add temporary test for configuration export * remove requirements * cmd: introduce logs * rename logs to dvclive * cmd: introduce logs * remove logs leftowvers * dvclive: env: adjust to chekpoints env variables * command: live: remove unnecessary help * fix linter suggestions * type hinting * review refactor * replace dvclive with live * stage: loading: move live loading to separate method * stage: serialize: allow only single live output * live: add show/diff * stage: serialize: roll back single live enforcing * fixup * rebase fixups * output: live: force single instance * live: command: add unit tests * Update dvc/output/base.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/stage/__init__.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/stage/serialize.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * review v2 * live: fix summary tracking by experiments * naming fixups * api: live: add tests * tests: api: separate live tests Co-authored-by: Ruslan Kuprieiev <[email protected]>
iterative · Jan 4, 2021 · 862ab8d · 862ab8d
1 parent 19606ba
commit 862ab8d
Show file tree

Hide file tree

Showing 36 changed files with 810 additions and 89 deletions.
diff --git a/dvc/api.py → dvc/api/__init__.py b/dvc/api.py → dvc/api/__init__.py
diff --git a/dvc/api/live.py b/dvc/api/live.py
@@ -0,0 +1,24 @@
+import logging
+import os
+from typing import List
+
+from dvc.exceptions import NotDvcRepoError
+from dvc.repo import Repo
+from dvc.utils.html import write
+
+logger = logging.getLogger(__name__)
+
+
+def summary(path: str, revs: List[str] = None):
+    try:
+        root = Repo.find_root()
+    except NotDvcRepoError:
+        root = os.getcwd()
+
+    metrics, plots = Repo(root_dir=root, uninitialized=True).live.show(
+        path, revs
+    )
+
+    html_path = path + ".html"
+    write(html_path, plots, metrics)
+    logger.info(f"\nfile://{os.path.abspath(html_path)}")
diff --git a/dvc/cli.py b/dvc/cli.py
@@ -27,6 +27,7 @@
     imp_url,
     init,
     install,
+    live,
     ls,
     metrics,
     move,
@@ -82,6 +83,7 @@
     plots,
     experiments,
     check_ignore,
+    live,
 ]
 
 

diff --git a/dvc/command/live.py b/dvc/command/live.py
@@ -0,0 +1,94 @@
+import argparse
+import logging
+import os
+
+from dvc.command import completion
+from dvc.command.base import CmdBase, fix_subparsers
+from dvc.utils.html import write
+
+logger = logging.getLogger(__name__)
+
+
+class CmdLive(CmdBase):
+    UNINITIALIZED = True
+
+    def _run(self, target, revs=None):
+        metrics, plots = self.repo.live.show(target=target, revs=revs)
+
+        html_path = self.args.target + ".html"
+        write(html_path, plots, metrics)
+
+        logger.info(f"\nfile://{os.path.abspath(html_path)}")
+
+        return 0
+
+
+class CmdLiveShow(CmdLive):
+    def run(self):
+        return self._run(self.args.target)
+
+
+class CmdLiveDiff(CmdLive):
+    def run(self):
+        return self._run(self.args.target, self.args.revs)
+
+
+def shared_parent_parser():
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument(
+        "target", help="Logs dir to produce summary from",
+    ).complete = completion.DIR
+    parent_parser.add_argument(
+        "-o",
+        "--out",
+        default=None,
+        help="Destination path to save plots to",
+        metavar="<path>",
+    ).complete = completion.DIR
+    return parent_parser
+
+
+def add_parser(subparsers, parent_parser):
+    LIVE_DESCRIPTION = (
+        "Commands to visualize and compare dvclive-produced logs."
+    )
+    live_parser = subparsers.add_parser(
+        "live",
+        parents=[parent_parser],
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=LIVE_DESCRIPTION,
+    )
+    live_subparsers = live_parser.add_subparsers(
+        dest="cmd",
+        help="Use `dvc live CMD --help` to display command-specific help.",
+    )
+
+    fix_subparsers(live_subparsers)
+
+    SHOW_HELP = "Visualize dvclive directory content."
+    live_show_parser = live_subparsers.add_parser(
+        "show",
+        parents=[parent_parser, shared_parent_parser()],
+        help=SHOW_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    live_show_parser.set_defaults(func=CmdLiveShow)
+
+    DIFF_HELP = (
+        "Show multiple versions of dvclive data, "
+        "by plotting it in single view."
+    )
+    live_diff_parser = live_subparsers.add_parser(
+        "diff",
+        parents=[parent_parser, shared_parent_parser()],
+        help=DIFF_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    live_diff_parser.add_argument(
+        "--revs",
+        nargs="*",
+        default=None,
+        help="Git revision (e.g. SHA, branch, tag)",
+        metavar="<commit>",
+    )
+    live_diff_parser.set_defaults(func=CmdLiveDiff)
diff --git a/dvc/command/plots.py b/dvc/command/plots.py
@@ -7,28 +7,10 @@
 from dvc.exceptions import DvcException
 from dvc.schema import PLOT_PROPS
 from dvc.utils import format_link
+from dvc.utils.html import write
 
 logger = logging.getLogger(__name__)
 
-PAGE_HTML = """<!DOCTYPE html>
-<html>
-<head>
-    <title>DVC Plot</title>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-    <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
-</head>
-<body>
-    {divs}
-</body>
-</html>"""
-
-DIV_HTML = """<div id = "{id}"></div>
-<script type = "text/javascript">
-    var spec = {vega_json};
-    vegaEmbed('#{id}', spec);
-</script>"""
-
 
 class CmdPlots(CmdBase):
     def _func(self, *args, **kwargs):
@@ -58,16 +40,10 @@ def run(self):
                 logger.info(plots[target])
                 return 0
 
-            divs = [
-                DIV_HTML.format(id=f"plot{i}", vega_json=plot)
-                for i, plot in enumerate(plots.values())
-            ]
-            html = PAGE_HTML.format(divs="\n".join(divs))
             path = self.args.out or "plots.html"
-
             path = os.path.join(os.getcwd(), path)
-            with open(path, "w") as fobj:
-                fobj.write(html)
+
+            write(path, plots)
 
             logger.info(f"file://{path}")
 

diff --git a/dvc/command/run.py b/dvc/command/run.py
@@ -42,6 +42,8 @@ def run(self):
                 metrics_no_cache=self.args.metrics_no_cache,
                 plots=self.args.plots,
                 plots_no_cache=self.args.plots_no_cache,
+                live=self.args.live,
+                live_summary=not self.args.live_no_summary,
                 deps=self.args.deps,
                 params=self.args.params,
                 fname=self.args.file,
@@ -162,6 +164,15 @@ def add_parser(subparsers, parent_parser):
         help="Declare output plot file (do not put into DVC cache).",
         metavar="<path>",
     )
+    run_parser.add_argument(
+        "--live", help=argparse.SUPPRESS, metavar="<path>",
+    )
+    run_parser.add_argument(
+        "--live-no-summary",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
     run_parser.add_argument(
         "--file", metavar="<filename>", help=argparse.SUPPRESS,
     )

diff --git a/dvc/env.py b/dvc/env.py
@@ -2,3 +2,5 @@
 DVC_DAEMON = "DVC_DAEMON"
 DVC_PAGER = "DVC_PAGER"
 DVC_ROOT = "DVC_ROOT"
+DVCLIVE_PATH = "DVCLIVE_PATH"
+DVCLIVE_SUMMARY = "DVCLIVE_SUMMARY"
diff --git a/dvc/exceptions.py b/dvc/exceptions.py
@@ -196,9 +196,9 @@ def __init__(self, command, run_options):
 class MetricDoesNotExistError(MetricsError):
     def __init__(self, targets: List[str]):
         if len(targets) == 1:
-            msg = "File: '{}' does not exist."
+            msg = "'{}' does not exist."
         else:
-            msg = "Files: '{}' do not exist."
+            msg = "'{}' do not exist."
         super().__init__(msg.format(", ".join(targets)))
 
 

diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py
@@ -81,6 +81,7 @@ def _get(
     plot=False,
     persist=False,
     checkpoint=False,
+    live=False,
     desc=None,
     isexec=False,
 ):
@@ -98,6 +99,7 @@ def _get(
             plot=plot,
             persist=persist,
             checkpoint=checkpoint,
+            live=live,
             desc=desc,
             isexec=isexec,
         )
@@ -114,6 +116,7 @@ def _get(
                 plot=plot,
                 persist=persist,
                 checkpoint=checkpoint,
+                live=live,
                 desc=desc,
                 isexec=isexec,
             )
@@ -127,6 +130,7 @@ def _get(
         plot=plot,
         persist=persist,
         checkpoint=checkpoint,
+        live=live,
         desc=desc,
         isexec=isexec,
     )
@@ -143,6 +147,7 @@ def loadd_from(stage, d_list):
         checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
         desc = d.pop(BaseOutput.PARAM_DESC, False)
         isexec = d.pop(BaseOutput.PARAM_ISEXEC, False)
+        live = d.pop(BaseOutput.PARAM_LIVE, False)
         ret.append(
             _get(
                 stage,
@@ -155,6 +160,7 @@ def loadd_from(stage, d_list):
                 checkpoint=checkpoint,
                 desc=desc,
                 isexec=isexec,
+                live=live,
             )
         )
     return ret
@@ -169,6 +175,7 @@ def loads_from(
     persist=False,
     checkpoint=False,
     isexec=False,
+    live=False,
 ):
     return [
         _get(
@@ -181,6 +188,7 @@ def loads_from(
             persist=persist,
             checkpoint=checkpoint,
             isexec=isexec,
+            live=live,
         )
         for s in s_list
     ]
@@ -209,21 +217,35 @@ def _merge_data(s_list):
 
 
 @collecting
-def load_from_pipeline(stage, s_list, typ="outs"):
-    if typ not in (stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS):
+def load_from_pipeline(stage, data, typ="outs"):
+    if typ not in (
+        stage.PARAM_OUTS,
+        stage.PARAM_METRICS,
+        stage.PARAM_PLOTS,
+        stage.PARAM_LIVE,
+    ):
         raise ValueError(f"'{typ}' key is not allowed for pipeline files.")
 
     metric = typ == stage.PARAM_METRICS
     plot = typ == stage.PARAM_PLOTS
+    live = typ == stage.PARAM_LIVE
 
-    d = _merge_data(s_list)
+    if live:
+        # `live` is single object
+        data = [data]
+
+    d = _merge_data(data)
 
     for path, flags in d.items():
-        plt_d = {}
+        plt_d, live_d = {}, {}
         if plot:
             from dvc.schema import PLOT_PROPS
 
             plt_d, flags = _split_dict(flags, keys=PLOT_PROPS.keys())
+        if live:
+            from dvc.schema import LIVE_PROPS
+
+            live_d, flags = _split_dict(flags, keys=LIVE_PROPS.keys())
         extra = project(
             flags,
             [
@@ -232,4 +254,13 @@ def load_from_pipeline(stage, s_list, typ="outs"):
                 BaseOutput.PARAM_CHECKPOINT,
             ],
         )
-        yield _get(stage, path, {}, plot=plt_d or plot, metric=metric, **extra)
+
+        yield _get(
+            stage,
+            path,
+            {},
+            plot=plt_d or plot,
+            metric=metric,
+            live=live_d or live,
+            **extra,
+        )