IntentTEDPolicy: Updated rasa test tests

rasa/cli/test.py

@@ -3,17 +3,17 @@
 import os
 from typing import List, Optional, Text, Dict, Union, Any
 
-from rasa.core.test import (
-    FAILED_STORIES_FILE,
-    SUCCESSFUL_STORIES_FILE,
-    STORIES_WITH_WARNINGS_FILE,
-)
 from rasa.cli import SubParsersAction
 import rasa.shared.data
 from rasa.shared.exceptions import YamlException
 import rasa.shared.utils.io
 import rasa.shared.utils.cli
 from rasa.cli.arguments import test as arguments
+from rasa.core.constants import (
+    FAILED_STORIES_FILE,
+    SUCCESSFUL_STORIES_FILE,
+    STORIES_WITH_WARNINGS_FILE,
+)
 from rasa.shared.constants import (
     CONFIG_SCHEMA_FILE,
     DEFAULT_E2E_TESTS_PATH,

rasa/core/constants.py

@@ -62,3 +62,10 @@
 # Names of the environment variables defining PostgreSQL pool size and max overflow
 POSTGRESQL_POOL_SIZE = "SQL_POOL_SIZE"
 POSTGRESQL_MAX_OVERFLOW = "SQL_MAX_OVERFLOW"
+
+# File names for testing
+CONFUSION_MATRIX_STORIES_FILE = "story_confusion_matrix.png"
+REPORT_STORIES_FILE = "story_report.json"
+FAILED_STORIES_FILE = "failed_test_stories.yml"
+SUCCESSFUL_STORIES_FILE = "successful_test_stories.yml"
+STORIES_WITH_WARNINGS_FILE = "stories_with_warnings.yml"

rasa/core/test.py

@@ -6,6 +6,13 @@
 from typing import Any, Dict, List, Optional, Text, Tuple
 
 from rasa import telemetry
+from rasa.core.constants import (
+    CONFUSION_MATRIX_STORIES_FILE,
+    REPORT_STORIES_FILE,
+    FAILED_STORIES_FILE,
+    SUCCESSFUL_STORIES_FILE,
+    STORIES_WITH_WARNINGS_FILE,
+)
 from rasa.core.policies.policy import PolicyPrediction
 from rasa.nlu.test import EntityEvaluationResult, evaluate_entities
 from rasa.shared.core.constants import (
@@ -78,12 +85,6 @@
         total=False,
     )
 
-CONFUSION_MATRIX_STORIES_FILE = "story_confusion_matrix.png"
-REPORT_STORIES_FILE = "story_report.json"
-FAILED_STORIES_FILE = "failed_test_stories.yml"
-SUCCESSFUL_STORIES_FILE = "successful_test_stories.yml"
-STORIES_WITH_WARNINGS_FILE = "stories_with_warnings.yml"
-
 logger = logging.getLogger(__name__)
 
 StoryEvaluation = namedtuple(

tests/cli/test_rasa_test.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from shutil import copyfile
 
-from rasa.core.test import CONFUSION_MATRIX_STORIES_FILE
+from rasa.core.constants import CONFUSION_MATRIX_STORIES_FILE
 from rasa.constants import RESULTS_FILE
 from rasa.shared.constants import DEFAULT_RESULTS_PATH
 from rasa.shared.utils.io import list_files, write_yaml, write_text_file

tests/core/test_evaluation.py

@@ -12,12 +12,14 @@
     _create_data_generator,
     _collect_story_predictions,
     test as evaluate_stories,
-    FAILED_STORIES_FILE,
+    _clean_entity_results,
+)
+from rasa.core.constants import (
     CONFUSION_MATRIX_STORIES_FILE,
     REPORT_STORIES_FILE,
+    FAILED_STORIES_FILE,
     SUCCESSFUL_STORIES_FILE,
     STORIES_WITH_WARNINGS_FILE,
-    _clean_entity_results,
 )
 from rasa.core.policies.memoization import MemoizationPolicy
 

tests/core/test_test.py

@@ -4,10 +4,39 @@
 import pytest
 
 import rasa.core.test
+from rasa.core.policies.ensemble import SimplePolicyEnsemble
+from rasa.core.policies.policy import PolicyPrediction
+from rasa.shared.core.events import UserUttered
+from _pytest.monkeypatch import MonkeyPatch
 from _pytest.capture import CaptureFixture
 from rasa.core.agent import Agent
 
 
+def _probabilities_with_action_unlikely_intent_for(intent_name: Text):
+    _original = SimplePolicyEnsemble.probabilities_using_best_policy
+
+    def probabilities_using_best_policy(
+        self, tracker, domain, interpreter, **kwargs,
+    ) -> PolicyPrediction:
+        latest_event = tracker.events[-1]
+        if (
+            isinstance(latest_event, UserUttered)
+            and latest_event.parse_data["intent"]["name"] == intent_name
+        ):
+            # Here we return `action_unlikely_intent` if the name of the latest intent
+            # matches `intent_name`.
+            # We need to do it because every time the tests are run,
+            # training will result in different model weights which might
+            # result in different predictions of `action_unlikely_intent`.
+            # Because we're not testing `IntentTEDPolicy` here we simply trigger it
+            # predicting `action_unlikely_intent` in a specified moment
+            # to make the tests deterministic.
+            return PolicyPrediction.for_action_name(domain, "action_unlikely_intent")
+        return _original(self, tracker, domain, interpreter, **kwargs)
+
+    return probabilities_using_best_policy
+
+
 async def test_testing_warns_if_action_unknown(
     capsys: CaptureFixture,
     e2e_bot_agent: Agent,
@@ -43,8 +72,14 @@ async def test_testing_valid_with_non_e2e_core_model(core_agent: Agent):
 
 
 async def test_action_unlikely_intent_1(
-    tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
+    monkeypatch: MonkeyPatch, tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
 ):
+    monkeypatch.setattr(
+        SimplePolicyEnsemble,
+        "probabilities_using_best_policy",
+        _probabilities_with_action_unlikely_intent_for("mood_unhappy"),
+    )
+
     file_name = tmp_path / "test_action_unlikely_intent_1.yml"
     file_name.write_text(
         """
@@ -71,8 +106,14 @@ async def test_action_unlikely_intent_1(
 
 
 async def test_action_unlikely_intent_2(
-    tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
+    monkeypatch: MonkeyPatch, tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
 ):
+    monkeypatch.setattr(
+        SimplePolicyEnsemble,
+        "probabilities_using_best_policy",
+        _probabilities_with_action_unlikely_intent_for("mood_unhappy"),
+    )
+
     file_name = tmp_path / "test_action_unlikely_intent_2.yml"
     file_name.write_text(
         """
@@ -100,8 +141,14 @@ async def test_action_unlikely_intent_2(
 
 
 async def test_action_unlikely_intent_complete(
-    tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
+    monkeypatch: MonkeyPatch, tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
 ):
+    monkeypatch.setattr(
+        SimplePolicyEnsemble,
+        "probabilities_using_best_policy",
+        _probabilities_with_action_unlikely_intent_for("mood_unhappy"),
+    )
+
     file_name = tmp_path / "test_action_unlikely_intent_complete.yml"
     file_name.write_text(
         """
@@ -159,8 +206,14 @@ async def test_action_unlikely_intent_complete(
 
 
 async def test_action_unlikely_intent_wrong_story(
-    tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
+    monkeypatch: MonkeyPatch, tmp_path: Path, intent_ted_policy_moodbot_agent: Agent
 ):
+    monkeypatch.setattr(
+        SimplePolicyEnsemble,
+        "probabilities_using_best_policy",
+        _probabilities_with_action_unlikely_intent_for("mood_unhappy"),
+    )
+
     file_name = tmp_path / "test_action_unlikely_intent_complete.yml"
     file_name.write_text(
         """