live: fix summary tracking by experiments

dvc/repo/reproduce.py

@@ -67,6 +67,12 @@ def _track_stage(stage):
     for out in stage.outs:
         if not out.use_scm_ignore and out.is_in_repo:
             stage.repo.scm.track_file(os.fspath(out.path_info))
+            if out.live:
+                from dvc.repo.live import summary_path_info
+
+                summary = summary_path_info(out)
+                if summary:
+                    stage.repo.scm.track_file(os.fspath(summary))
     stage.repo.scm.track_changed_files()
 
 

dvc/stage/utils.py

@@ -59,7 +59,7 @@ def fill_stage_outputs(stage, **kwargs):
     stage.outs = []
 
     stage.outs += _load_live_outputs(
-        stage, kwargs.get("live", []), kwargs.get("live_summary", False)
+        stage, kwargs.get("live", None), kwargs.get("live_summary", False)
     )
 
     for key in keys:
@@ -74,18 +74,15 @@ def fill_stage_outputs(stage, **kwargs):
         )
 
 
-def _load_live_outputs(stage, live_l, live_summary):
-    from dvc.exceptions import DvcException
+def _load_live_outputs(stage, live_l=None, live_summary=False):
     from dvc.output import BaseOutput
 
     outs = []
     if live_l:
-        if len(live_l) != 1:
-            raise DvcException("Only one live output allowed!")
 
         outs += output.loads_from(
             stage,
-            live_l,
+            [live_l],
             use_cache=False,
             live={BaseOutput.PARAM_LIVE_SUMMARY: live_summary},
         )

tests/func/test_live.py

@@ -1,35 +1,60 @@
-import subprocess
 from textwrap import dedent
 
 import pytest
 
+from dvc import stage as stage_module
 from dvc.exceptions import MetricsError
 
 LIVE_SCRITP = dedent(
     """
         from dvclive import dvclive
         import sys
-        r = 5
+        r = 2
         for i in range(r):
-           dvclive.log("loss", -i/5)
-           dvclive.log("accuracy", i/5)"""
+           dvclive.log("loss", 1-i/r)
+           dvclive.log("accuracy", i/r)
+           dvclive.next_step()"""
 )
 
 
+@pytest.fixture
+def live_stage(tmp_dir, scm, dvc):
+
+    pytest.skip("dvclive does not exist yet")
+
+    def make(summary=True):
+        tmp_dir.gen("train.py", LIVE_SCRITP)
+        tmp_dir.gen("params.yaml", "foo: 1")
+        stage = dvc.run(
+            cmd="python train.py",
+            params=["foo"],
+            deps=["train.py"],
+            name="live_stage",
+            live="logs",
+            live_summary=summary,
+        )
+
+        scm.add(["dvc.yaml", "dvc.lock", "train.py", "params.yaml"])
+        scm.commit("initial: live_stage")
+        return stage
+
+    yield make
+
+
 @pytest.mark.parametrize("summary", (True, False))
 def test_export_config_tmp(tmp_dir, dvc, mocker, summary):
-    proc_spy = mocker.spy(subprocess, "Popen")
+    run_spy = mocker.spy(stage_module.run, "_run")
     tmp_dir.gen("src", "dependency")
     dvc.run(
         cmd="mkdir logs && touch logs.json",
         deps=["src"],
         name="run_logger",
-        live=["logs"],
+        live="logs",
         live_summary=summary,
     )
 
-    assert proc_spy.call_count == 1
-    _, kwargs = proc_spy.call_args
+    assert run_spy.call_count == 1
+    _, kwargs = run_spy.call_args
 
     assert "DVCLIVE_PATH" in kwargs["env"]
     assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
@@ -38,20 +63,13 @@ def test_export_config_tmp(tmp_dir, dvc, mocker, summary):
     assert kwargs["env"]["DVCLIVE_SUMMARY"] == str(int(summary))
 
 
-@pytest.mark.skip(reason="dvclive does not exist yet")
 @pytest.mark.parametrize("summary", (True, False))
-def test_export_config(tmp_dir, dvc, mocker, summary):
-    proc_spy = mocker.spy(subprocess, "Popen")
-    tmp_dir.gen("log.py", LIVE_SCRITP.format(log_path="logs"))
-    dvc.run(
-        cmd="python log.py",
-        deps=["log.py"],
-        name="run_logger",
-        live=["logs"],
-        live_summary=summary,
-    )
-    assert proc_spy.call_count == 1
-    _, kwargs = proc_spy.call_args
+def test_export_config(tmp_dir, dvc, mocker, summary, live_stage):
+    run_spy = mocker.spy(stage_module.run, "_run")
+    live_stage(summary=summary)
+
+    assert run_spy.call_count == 1
+    _, kwargs = run_spy.call_args
 
     assert "DVCLIVE_PATH" in kwargs["env"]
     assert kwargs["env"]["DVCLIVE_PATH"] == "logs"
@@ -60,20 +78,12 @@ def test_export_config(tmp_dir, dvc, mocker, summary):
     assert kwargs["env"]["DVCLIVE_SUMMARY"] == str(int(summary))
 
 
-@pytest.mark.skip(reason="dvclive does not exist yet")
-def test_live_provides_metrics(tmp_dir, dvc):
-    tmp_dir.gen("log.py", LIVE_SCRITP.format(log_path="logs"))
-    dvc.run(
-        cmd="python log.py",
-        deps=["log.py"],
-        name="run_logger",
-        live=["logs"],
-        live_summary=True,
-    )
+def test_live_provides_metrics(tmp_dir, dvc, live_stage):
+    live_stage(summary=True)
 
     assert (tmp_dir / "logs.json").is_file()
     assert dvc.metrics.show() == {
-        "": {"logs.json": {"step": 3, "loss": -0.6, "accuracy": 0.6}}
+        "": {"logs.json": {"step": 1, "loss": 0.5, "accuracy": 0.5}}
     }
 
     assert (tmp_dir / "logs").is_dir()
@@ -82,16 +92,8 @@ def test_live_provides_metrics(tmp_dir, dvc):
     assert "logs/loss.tsv" in plots
 
 
-@pytest.mark.skip(reason="dvclive does not exist yet")
-def test_live_provides_no_metrics(tmp_dir, dvc):
-    tmp_dir.gen("log.py", LIVE_SCRITP.format(log_path="logs"))
-    dvc.run(
-        cmd="python log.py",
-        deps=["log.py"],
-        name="run_logger",
-        live=["logs"],
-        live_summary=False,
-    )
+def test_live_provides_no_metrics(tmp_dir, dvc, live_stage):
+    live_stage(summary=False)
 
     assert not (tmp_dir / "logs.json").is_file()
     with pytest.raises(MetricsError):
@@ -101,3 +103,15 @@ def test_live_provides_no_metrics(tmp_dir, dvc):
     plots = dvc.plots.show()
     assert "logs/accuracy.tsv" in plots
     assert "logs/loss.tsv" in plots
+
+
+def test_experiments_track_summary(tmp_dir, scm, dvc, live_stage):
+    live_stage(summary=True)
+    baseline_rev = scm.get_rev()
+
+    experiments = dvc.experiments.run(targets=["live_stage"], params=["foo=2"])
+    assert len(experiments) == 1
+    ((exp_rev, _),) = experiments.items()
+
+    res = dvc.experiments.show()
+    assert "logs.json" in res[baseline_rev][exp_rev]["metrics"].keys()