Standardize testing output: Response Selection (#5824)

* create plotting utils * write errors and successes for responses * fix file names * update plot filenames * add changelog entry * add missing docstrings * update tests * fix type * address deepsource issues * use io_utils * fix call to write_text_file * added complete intent and response key to the 'intent' key * Update rasa/nlu/test.py Co-authored-by: Daksh Varshneya <[email protected]> * Update rasa/nlu/test.py Co-authored-by: Daksh Varshneya <[email protected]> * use intent_target key for confustion matrix Co-authored-by: Daksh <[email protected]> Co-authored-by: Roberto <[email protected]>
RasaHQ · May 18, 2020 · dd954c3 · dd954c3
1 parent 6e23dd3
commit dd954c3
Show file tree

Hide file tree

Showing 8 changed files with 679 additions and 366 deletions.
diff --git a/changelog/5748.improvement.rst b/changelog/5748.improvement.rst
@@ -0,0 +1,6 @@
+Standardize testing output. The following test output can be produced for intents, responses and entities:
+- report: a detailed report with testing metrics per label (e.g. precision, recall, accuracy, etc.)
+- errors: a file that contains incorrect predictions
+- successes: a file that contains correct predictions
+- confusion matrix: plot of confusion matrix (only intents and responses)
+- histogram: plot of confidence distribution (only intents and responses)
diff --git a/rasa/cli/arguments/test.py b/rasa/cli/arguments/test.py
@@ -100,27 +100,13 @@ def add_test_nlu_argument_group(
         "--successes",
         action="store_true",
         default=False,
-        help="If set successful predictions (intent and entities) will be written "
-        "to a file.",
+        help="If set successful predictions will be written to a file.",
     )
     parser.add_argument(
         "--no-errors",
         action="store_true",
         default=False,
-        help="If set incorrect predictions (intent and entities) will NOT be written "
-        "to a file.",
-    )
-    parser.add_argument(
-        "--histogram",
-        required=False,
-        default="hist.png",
-        help="Output path for the confidence histogram.",
-    )
-    parser.add_argument(
-        "--confmat",
-        required=False,
-        default="confmat.png",
-        help="Output path for the confusion matrix plot.",
+        help="If set incorrect predictions will NOT be written to a file.",
     )
     parser.add_argument(
         "-c",
@@ -191,6 +177,6 @@ def add_no_plot_param(
         dest="disable_plotting",
         action="store_true",
         default=default,
-        help=f"Don't render evaluation plots",
+        help="Don't render evaluation plots.",
         required=required,
     )
diff --git a/rasa/core/test.py b/rasa/core/test.py
@@ -604,7 +604,7 @@ def plot_story_evaluation(
     from sklearn.metrics import confusion_matrix
     from sklearn.utils.multiclass import unique_labels
     import matplotlib.pyplot as plt
-    from rasa.nlu.test import plot_confusion_matrix
+    from rasa.utils.plotting import plot_confusion_matrix
 
     log_evaluation_table(
         test_y,