Skip to content

Commit

Permalink
Improve error clarity in TensorboardMetric
Browse files Browse the repository at this point in the history
Summary:
Add specific error messages to tb fetching for when data is missing.

Previously this would error out in a pandas operation which was confusing to many users. Now, we check to see whether the issue occurred because the tag was not present on the multiplexer at all (common) or because the tag was present but not data was found associated ie an empty tensor was attached (uncommon but possible).

Differential Revision: D57342319
  • Loading branch information
mpolson64 authored and facebook-github-bot committed May 14, 2024
1 parent 5216b09 commit 88d3e83
Showing 1 changed file with 21 additions and 5 deletions.
26 changes: 21 additions & 5 deletions ax/metrics/tensorboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ def bulk_fetch_trial_data(
for metric in tb_metrics
}

if len(mul.PluginRunToTagToContent("scalars")) == 0:
scalar_dict = mul.PluginRunToTagToContent("scalars")
if len(scalar_dict) == 0:
return {
metric.name: Err(
MetricFetchE(
Expand Down Expand Up @@ -135,14 +136,29 @@ def bulk_fetch_trial_data(
),
"sem": float("nan"),
}
for run_name, tb_metrics in mul.PluginRunToTagToContent(
"scalars"
).items()
for tag in tb_metrics
for run_name, tags in scalar_dict.items()
for tag in tags
if tag == metric.tag
for t in mul.Tensors(run_name, tag)
]

# If records is empty something has gone wrong: either the tag is
# not present on the multiplexer or the content referenced is empty
if len(records) == 0:
if metric.tag not in [
j for sub in scalar_dict.values() for j in sub
]:
raise KeyError(
f"Tag {metric.tag} not found on multiplexer {mul=}. "
"Did you specify this tag exactly as it appears in "
"the TensorBoard UI's Scalars tab?"
)
else:
raise ValueError(
f"Found tag {metric.tag}, but no data found for it. Is "
"the curve empty in the TensorBoard UI?"
)

df = (
pd.DataFrame(records)
# If a metric has multiple records for the same arm, metric, and
Expand Down

0 comments on commit 88d3e83

Please sign in to comment.