IntentTEDPolicy: rasa interactive updates

changelog/8421.improvement.md

@@ -0,0 +1 @@
+Print an information about `action_unlikely_intent` when in gets predicted in the interactive mode.

rasa/core/test.py

@@ -637,9 +637,7 @@ def _collect_action_executed_predictions(
             # that something else than the form was supposed to run.
             predicted_action = action.name()
 
-    predicted_warning = action_executed_eval_store.predicted_warning(
-        predicted_action
-    )
+    predicted_warning = action_executed_eval_store.predicted_warning(predicted_action)
     if not predicted_warning:
         action_executed_eval_store.add_to_store(
             action_predictions=[predicted_action], action_targets=[expected_action]

rasa/core/training/interactive.py

@@ -44,6 +44,7 @@
     LOOP_REJECTED,
     REQUESTED_SLOT,
     LOOP_INTERRUPTED,
+    ACTION_UNLIKELY_INTENT_NAME,
 )
 from rasa.core import run, utils
 import rasa.core.train
@@ -60,7 +61,12 @@
     UserUtteranceReverted,
 )
 import rasa.core.interpreter
-from rasa.shared.constants import INTENT_MESSAGE_PREFIX, DEFAULT_SENDER_ID, UTTER_PREFIX
+from rasa.shared.constants import (
+    INTENT_MESSAGE_PREFIX,
+    DEFAULT_SENDER_ID,
+    UTTER_PREFIX,
+    DOCS_URL_POLICIES,
+)
 from rasa.shared.core.trackers import EventVerbosity, DialogueStateTracker
 from rasa.shared.core.training_data import visualization
 from rasa.shared.core.training_data.visualization import (
@@ -1130,9 +1136,23 @@ async def _validate_action(
 
     Returns `True` if the prediction is correct, `False` otherwise."""
 
-    question = questionary.confirm(f"The bot wants to run '{action_name}', correct?")
+    if action_name == ACTION_UNLIKELY_INTENT_NAME:
+        question = questionary.confirm(
+            f"The bot wants to run '{action_name}' "
+            f"to indicate that the last user message was unexpected "
+            f"at this point in the conversation. "
+            f"Check out IntentTEDPolicy ({DOCS_URL_POLICIES}/#intent-ted-policy) "
+            f"to learn more."
+        )
+    else:
+        question = questionary.confirm(
+            f"The bot wants to run '{action_name}', correct?"
+        )
 
-    is_correct = await _ask_questions(question, conversation_id, endpoint)
+    is_correct = (
+        await _ask_questions(question, conversation_id, endpoint)
+        or action_name == ACTION_UNLIKELY_INTENT_NAME
+    )
 
     if not is_correct:
         action_name, is_new_action = await _request_action_from_user(

tests/core/training/test_interactive.py

@@ -5,6 +5,7 @@
 from collections import deque
 from pathlib import Path
 from typing import Any, Dict, List, Text, Tuple, Callable
+from tests.conftest import AsyncMock
 
 import mock
 import pytest
@@ -16,7 +17,11 @@
 import rasa.utils.io
 from rasa.core.actions import action
 from rasa.core.training import interactive
-from rasa.shared.constants import INTENT_MESSAGE_PREFIX, DEFAULT_SENDER_ID
+from rasa.shared.constants import (
+    INTENT_MESSAGE_PREFIX,
+    DEFAULT_SENDER_ID,
+    DOCS_URL_POLICIES,
+)
 from rasa.shared.core.constants import ACTION_LISTEN_NAME
 from rasa.shared.core.domain import Domain
 from rasa.shared.core.events import BotUttered, ActionExecuted, UserUttered
@@ -733,6 +738,84 @@ def test_retry_on_error_success(monkeypatch: MonkeyPatch):
     m.assert_called_once_with("export_path", 1, a=2)
 
 
+@pytest.mark.parametrize(
+    "action_name, question, is_marked_as_correct, sent_action_name",
+    [
+        (
+            "action_unlikely_intent",
+            f"The bot wants to run 'action_unlikely_intent' "
+            f"to indicate that the last user message was unexpected "
+            f"at this point in the conversation. "
+            f"Check out IntentTEDPolicy ({DOCS_URL_POLICIES}/#intent-ted-policy) "
+            f"to learn more.",
+            True,
+            "action_unlikely_intent",
+        ),
+        (
+            "action_unlikely_intent",
+            f"The bot wants to run 'action_unlikely_intent' "
+            f"to indicate that the last user message was unexpected "
+            f"at this point in the conversation. "
+            f"Check out IntentTEDPolicy ({DOCS_URL_POLICIES}/#intent-ted-policy) "
+            f"to learn more.",
+            False,
+            "action_unlikely_intent",
+        ),
+        (
+            "action_test",
+            "The bot wants to run 'action_test', correct?",
+            True,
+            "action_test",
+        ),
+        (
+            "action_test",
+            "The bot wants to run 'action_test', correct?",
+            False,
+            "action_another_one",
+        ),
+    ],
+)
+async def test_correct_question_for_action_name_was_asked(
+    monkeypatch: MonkeyPatch,
+    mock_endpoint: EndpointConfig,
+    action_name: Text,
+    question: Text,
+    is_marked_as_correct: bool,
+    sent_action_name: Text,
+):
+    conversation_id = "conversation_id"
+    policy = "policy"
+    tracker = DialogueStateTracker.from_events("some_sender", [])
+
+    monkeypatch.setattr(
+        interactive,
+        "retrieve_tracker",
+        AsyncMock(return_value=tracker.current_state()),
+    )
+    monkeypatch.setattr(
+        interactive, "_ask_questions", AsyncMock(return_value=is_marked_as_correct)
+    )
+    monkeypatch.setattr(
+        interactive,
+        "_request_action_from_user",
+        AsyncMock(return_value=("action_another_one", False,)),
+    )
+
+    mocked_send_action = AsyncMock()
+    monkeypatch.setattr(interactive, "send_action", mocked_send_action)
+
+    mocked_confirm = Mock(return_value=None)
+    monkeypatch.setattr(interactive.questionary, "confirm", mocked_confirm)
+
+    # validate the action and make sure that the correct question was asked
+    await interactive._validate_action(
+        action_name, policy, 1.0, [], mock_endpoint, conversation_id
+    )
+    mocked_confirm.assert_called_once_with(question)
+    args, kwargs = mocked_send_action.call_args_list[-1]
+    assert args[2] == sent_action_name
+
+
 def test_retry_on_error_three_retries(monkeypatch: MonkeyPatch):
     monkeypatch.setattr(interactive.questionary, "confirm", QuestionaryConfirmMock(3))