Skip to content

Commit

Permalink
Improve readibility of compare_to_baseline output message (#2025)
Browse files Browse the repository at this point in the history
Summary:

Integrating the following feedback into the construction of the compare_to_baseline output message

 " I'm noticing this could be a bit easier to digest. Luckily we can use markdown formatting as follows:
* add an asterisk before each item in comparison_list so that the user gets a bulleted list.
* put metric name + percent improvement first, followed by arm names + absolute metric values, since the former tends to be what the user most cares about.
* bolding the first part of each line item can help create visual anchors for the reader.
* adding backticks can help users identify variable names/values. "

Reviewed By: bernardbeckerman, mpolson64

Differential Revision: D51632256
  • Loading branch information
Matthew Grange authored and facebook-github-bot committed Nov 29, 2023
1 parent 14a5615 commit a345835
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 14 deletions.
9 changes: 5 additions & 4 deletions ax/service/tests/test_report_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1013,8 +1013,8 @@ def test_compare_to_baseline_moo(self) -> None:
comparison_arm_names = ["opt_0", "opt_1_min", "opt_3"]

preamble = (
"Each of the following arms optimizes a different "
"objective metric.<br>"
"Below is the greatest improvement, if any,"
" achieved for each objective metric \n"
)
output_text_0 = _format_comparison_string(
comparison_arm_name="opt_0",
Expand Down Expand Up @@ -1054,10 +1054,11 @@ def test_compare_to_baseline_moo(self) -> None:

expected_result = (
preamble
+ " * "
+ output_text_0
+ "<br>"
+ " * "
+ output_text_1
+ "<br>"
+ " * "
+ output_text_3
)

Expand Down
18 changes: 8 additions & 10 deletions ax/service/utils/report_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1148,12 +1148,10 @@ def _format_comparison_string(
digits: int,
) -> str:
return (
f"{comparison_arm_name=} "
+ "improves your objective metric "
+ f"{objective_name} by {percent_change:.2f}%. "
+ f" {baseline_arm_name=} was improved "
+ f"from {baseline_value=:.{digits}f}"
+ f" to {comparison_value=:.{digits}f}"
"**Metric "
f"`{objective_name}` improved {percent_change:.{digits}f}%** "
f"from `{baseline_value:.{digits}f}` in arm `'{baseline_arm_name}'` "
f"to `{comparison_value:.{digits}f}` in arm `'{comparison_arm_name}'`.\n "
)


Expand Down Expand Up @@ -1339,17 +1337,17 @@ def compare_to_baseline(
result_message = ""
if len(comparison_list) > 1:
result_message = (
"Each of the following arms optimizes a different "
+ "objective metric.<br>"
"Below is the greatest improvement, if any,"
" achieved for each objective metric \n"
)

for idx, result_tuple in enumerate(comparison_list):
for _, result_tuple in enumerate(comparison_list):
comparison_message = _construct_comparison_message(*result_tuple)
if comparison_message:
result_message = (
result_message
+ (" * " if len(comparison_list) > 1 else "")
+ not_none(comparison_message)
+ ("<br>" if idx != len(comparison_list) - 1 else "")
)

return result_message if result_message else None

0 comments on commit a345835

Please sign in to comment.