exp show --html: Handle fill_value.

Adds new argument `fill_value` to ParallelCoordinatesRenderer.

For scalar columns, prevent converting to categorical by explictly inserting None when value is missing (value == fill_value).

For categorical columns, replace `fill_value` with `"Missing"` and ensure it is the last item so it gets rendered on top of the column.

dvc/command/experiments.py

@@ -476,17 +476,17 @@ def show_experiments(
     if kwargs.get("only_changed", False) or html:
         td.drop_duplicates("cols")
 
-    extra_render_args = {}
+    html_args = {}
     if html:
-        td.dropna("rows")
+        td.dropna("rows", how="all")
         td.column("Experiment")[:] = [
             # remove tree characters
             str(x).encode("ascii", "ignore").strip().decode()
             for x in td.column("Experiment")
         ]
         rel = kwargs.get("out") or "dvc_plots"
-        extra_render_args["output_path"] = (Path.cwd() / rel).resolve()
-        extra_render_args["color_by"] = kwargs.get("sort_by", "Experiment")
+        html_args["output_path"] = (Path.cwd() / rel).resolve()
+        html_args["color_by"] = kwargs.get("sort_by") or "Experiment"
 
     td.render(
         pager=pager,
@@ -497,7 +497,7 @@ def show_experiments(
         csv=csv,
         markdown=markdown,
         html=html,
-        **extra_render_args,
+        **html_args,
     )
 
     if html and kwargs.get("open"):
@@ -507,7 +507,7 @@ def show_experiments(
         if "Microsoft" in uname().release:
             url = Path(rel) / "index.html"
         else:
-            url = Path(extra_render_args["output_path"]) / "index.html"
+            url = Path(html_args["output_path"]) / "index.html"
             url = url.as_uri()
 
         opened = webbrowser.open(url)

dvc/compare.py

@@ -159,7 +159,9 @@ def to_parallel_coordinates(self, output_path, color_by):
 
         index_path = write(
             output_path,
-            renderers=[ParallelCoordinatesRenderer(self, color_by)],
+            renderers=[
+                ParallelCoordinatesRenderer(self, color_by, self._fill_value)
+            ],
         )
         return index_path.as_uri()
 

dvc/render/plotly.py

@@ -24,11 +24,15 @@ class ParallelCoordinatesRenderer(Renderer):
 
     # pylint: disable=W0231
     def __init__(
-        self, tabular_data: "TabularData", color_by: Optional[str] = None
+        self,
+        tabular_data: "TabularData",
+        color_by: Optional[str] = None,
+        fill_value: str = "",
     ):
         self.tabular_data = tabular_data
         self.color_by = color_by
         self.filename = "experiments"
+        self.fill_value = fill_value
 
     def _convert(self, path):
         return self.as_json()
@@ -42,15 +46,23 @@ def as_json(self) -> str:
         trace: Dict[str, Any] = {"type": "parcoords", "dimensions": []}
         for label, values in tabular_dict.items():
             is_categorical = False
-
             try:
-                float_values = [float(x) for x in values]
+                float_values = [
+                    float(x) if x != self.fill_value else None for x in values
+                ]
             except ValueError:
                 is_categorical = True
-                unique_values = sorted(set(values))
 
             if is_categorical:
+                non_missing = [x for x in values if x != self.fill_value]
+                unique_values = sorted(set(non_missing))
+                unique_values.append(self.fill_value)
+
                 dummy_values = [unique_values.index(x) for x in values]
+
+                values = [
+                    x if x != self.fill_value else "Missing" for x in values
+                ]
                 trace["dimensions"].append(
                     {
                         "label": label,
@@ -68,9 +80,7 @@ def as_json(self) -> str:
                 trace["line"] = {
                     "color": dummy_values if is_categorical else float_values,
                     "showscale": True,
-                    "colorbar": {
-                        "title": self.color_by
-                    }
+                    "colorbar": {"title": self.color_by},
                 }
                 if is_categorical:
                     trace["line"]["colorbar"]["tickmode"] = "array"

tests/func/experiments/test_show.py

@@ -606,9 +606,7 @@ def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker):
     assert main(["exp", "show", "--html"]) == 0
     kwargs = show_experiments.call_args[1]
 
-    assert kwargs["color_by"] == "Experiment"
     html_text = (tmp_dir / "dvc_plots" / "index.html").read_text()
-
     assert all(rev in html_text for rev in ["workspace", "master", "[exp-"])
 
     assert (
@@ -621,11 +619,10 @@ def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker):
     assert '"label": "metrics.yaml:bar"' not in html_text
 
     assert (
-        main(["exp", "show", "--html", "--color-by", "metrics.yaml:foo"]) == 0
+        main(["exp", "show", "--html", "--sort-by", "metrics.yaml:foo"]) == 0
     )
     kwargs = show_experiments.call_args[1]
 
-    assert kwargs["color_by"] == "metrics.yaml:foo"
     html_text = (tmp_dir / "dvc_plots" / "index.html").read_text()
     assert '"line": {"color": [2.0, 1.0, 2.0]' in html_text
 
@@ -638,3 +635,9 @@ def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker):
     assert main(["exp", "show", "--html", "--open"]) == 0
 
     webbroser_open.assert_called()
+
+    params_data = {"foo": 1, "bar": 1, "foobar": 2}
+    (tmp_dir / params_file).dump(params_data)
+    assert main(["exp", "show", "--html"]) == 0
+    html_text = (tmp_dir / "dvc_plots" / "index.html").read_text()
+    assert '{"label": "foobar", "values": [2.0, null, null]}' in html_text

tests/unit/command/test_experiments.py

@@ -789,7 +789,7 @@ def test_show_experiments_html(tmp_dir, mocker):
 
     show_experiments(all_experiments, html=True)
 
-    td.dropna.assert_called_with("rows")
+    td.dropna.assert_called_with("rows", how="all")
 
     render_kwargs = td.render.call_args[1]
 

tests/unit/render/test_parallel_coordinates.py

@@ -17,8 +17,8 @@ def expected_format(result):
 
 
 def test_scalar_columns():
-    td = TabularData(["col-1", "col-2"])
-    td.extend([["0.1", "1"], ["2", "0.2"]])
+    td = TabularData(["col-1", "col-2", "col-3"])
+    td.extend([["0.1", "1", ""], ["2", "0.2", "0"]])
     renderer = ParallelCoordinatesRenderer(td)
 
     result = json.loads(renderer.as_json())
@@ -33,11 +33,15 @@ def test_scalar_columns():
         "label": "col-2",
         "values": [1.0, 0.2],
     }
+    assert result["data"][0]["dimensions"][2] == {
+        "label": "col-3",
+        "values": [None, 0],
+    }
 
 
 def test_categorical_columns():
-    td = TabularData(["col-1"])
-    td.extend([["foo"], ["bar"], ["foo"]])
+    td = TabularData(["col-1", "col-2"])
+    td.extend([["foo", ""], ["bar", "foobar"], ["foo", ""]])
     renderer = ParallelCoordinatesRenderer(td)
 
     result = json.loads(renderer.as_json())
@@ -50,6 +54,12 @@ def test_categorical_columns():
         "tickvals": [1, 0, 1],
         "ticktext": ["foo", "bar", "foo"],
     }
+    assert result["data"][0]["dimensions"][1] == {
+        "label": "col-2",
+        "values": [1, 0, 1],
+        "tickvals": [1, 0, 1],
+        "ticktext": ["Missing", "foobar", "Missing"],
+    }
 
 
 def test_mixed_columns():
@@ -84,9 +94,7 @@ def test_color_by_scalar():
     assert result["data"][0]["line"] == {
         "color": [0.1, 2.0],
         "showscale": True,
-        "colorbar": {
-            "title": "scalar"
-        }
+        "colorbar": {"title": "scalar"},
     }
 
 
@@ -125,3 +133,24 @@ def test_write_parallel_coordinates(tmp_dir):
         id="plot_experiments", partial=renderer.as_json()
     )
     assert div in html_text
+
+
+def test_fill_value():
+    td = TabularData(["categorical", "scalar"])
+    td.extend([["foo", "-"], ["-", "2"]])
+    renderer = ParallelCoordinatesRenderer(td, fill_value="-")
+
+    result = json.loads(renderer.as_json())
+
+    assert expected_format(result)
+
+    assert result["data"][0]["dimensions"][0] == {
+        "label": "categorical",
+        "values": [0, 1],
+        "tickvals": [0, 1],
+        "ticktext": ["foo", "Missing"],
+    }
+    assert result["data"][0]["dimensions"][1] == {
+        "label": "scalar",
+        "values": [None, 2.0],
+    }

tests/unit/test_tabular_data.py

@@ -279,6 +279,6 @@ def test_to_parallel_coordinates(tmp_dir, mocker):
 
     td.render(html=True, output_path="foo")
 
-    renderer_class.assert_called_with(td, None)
+    renderer_class.assert_called_with(td, None, td._fill_value)
 
     write.assert_called_with("foo", renderers=[renderer])