From 6babee5334ab5a5a881be9fa8091abf38b9b20d3 Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Wed, 7 Apr 2021 16:49:09 -0700 Subject: [PATCH 1/8] Fix for missing intent warnings when running rasa test --- rasa/core/test.py | 11 ++++++++++- rasa/shared/importers/rasa.py | 6 ++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index a888eadddf8c..9b8b85b56916 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -372,9 +372,18 @@ async def _create_data_generator( use_conversation_test_files: bool = False, ) -> "TrainingDataGenerator": from rasa.shared.core.generator import TrainingDataGenerator + from rasa.shared.constants import DEFAULT_DOMAIN_PATH + from rasa.model import get_model_subdirectories + + core_model, _ = get_model_subdirectories(agent.model_directory) + + if core_model: + domain_path = os.path.join(core_model, DEFAULT_DOMAIN_PATH) + else: + domain_path = None test_data_importer = TrainingDataImporter.load_from_dict( - training_data_paths=[resource_name] + training_data_paths=[resource_name], domain_path=domain_path ) if use_conversation_test_files: story_graph = await test_data_importer.get_conversation_tests() diff --git a/rasa/shared/importers/rasa.py b/rasa/shared/importers/rasa.py index 278007316329..e06269682aec 100644 --- a/rasa/shared/importers/rasa.py +++ b/rasa/shared/importers/rasa.py @@ -70,11 +70,13 @@ async def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData: async def get_domain(self) -> Domain: """Retrieves model domain (see parent class for full docstring).""" + from rasa.shared.constants import DEFAULT_DOMAIN_PATH + domain = Domain.empty() - # If domain path is None, return an empty domain + # If domain path is None, attempt to load the default domain if not self._domain_path: - return domain + self._domain_path = DEFAULT_DOMAIN_PATH try: domain = Domain.load(self._domain_path) except InvalidDomain as e: From dd139d9f4740d414da51ce9c2ef0fdb4a4cf7343 Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Thu, 22 Apr 2021 15:14:45 -0700 Subject: [PATCH 2/8] Added default domain file exist check --- rasa/core/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 9b8b85b56916..fca193959542 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -377,7 +377,7 @@ async def _create_data_generator( core_model, _ = get_model_subdirectories(agent.model_directory) - if core_model: + if core_model and os.path.exists(os.path.join(core_model, DEFAULT_DOMAIN_PATH)): domain_path = os.path.join(core_model, DEFAULT_DOMAIN_PATH) else: domain_path = None From 0f7ded45e39a65bfed7bd21ab47c6fcb201c5a3f Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Thu, 22 Apr 2021 17:10:49 -0700 Subject: [PATCH 3/8] Set default domain path for test data importers --- rasa/cli/test.py | 3 ++- rasa/nlu/test.py | 3 ++- rasa/shared/importers/rasa.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rasa/cli/test.py b/rasa/cli/test.py index 38f44f68b87c..01469f7ecc2f 100644 --- a/rasa/cli/test.py +++ b/rasa/cli/test.py @@ -16,6 +16,7 @@ DEFAULT_MODELS_PATH, DEFAULT_DATA_PATH, DEFAULT_RESULTS_PATH, + DEFAULT_DOMAIN_PATH, ) import rasa.shared.utils.validation as validation_utils import rasa.cli.utils @@ -158,7 +159,7 @@ async def run_nlu_test_async( data_path = rasa.cli.utils.get_validated_path(data_path, "nlu", DEFAULT_DATA_PATH) test_data_importer = TrainingDataImporter.load_from_dict( - training_data_paths=[data_path] + training_data_paths=[data_path], domain_path=DEFAULT_DOMAIN_PATH, ) nlu_data = await test_data_importer.get_nlu_data() diff --git a/rasa/nlu/test.py b/rasa/nlu/test.py index ff29e8cc6a07..efc30e558eda 100644 --- a/rasa/nlu/test.py +++ b/rasa/nlu/test.py @@ -1439,13 +1439,14 @@ async def run_evaluation( Returns: dictionary containing evaluation results """ import rasa.shared.nlu.training_data.loading + from rasa.shared.constants import DEFAULT_DOMAIN_PATH # get the metadata config from the package data interpreter = Interpreter.load(model_path, component_builder) interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline) test_data_importer = TrainingDataImporter.load_from_dict( - training_data_paths=[data_path] + training_data_paths=[data_path], domain_path=DEFAULT_DOMAIN_PATH, ) test_data = await test_data_importer.get_nlu_data() diff --git a/rasa/shared/importers/rasa.py b/rasa/shared/importers/rasa.py index e06269682aec..533604a62a0e 100644 --- a/rasa/shared/importers/rasa.py +++ b/rasa/shared/importers/rasa.py @@ -76,7 +76,7 @@ async def get_domain(self) -> Domain: # If domain path is None, attempt to load the default domain if not self._domain_path: - self._domain_path = DEFAULT_DOMAIN_PATH + return domain try: domain = Domain.load(self._domain_path) except InvalidDomain as e: From e20305866e344f5e39cee387c8e3a1654acf60dc Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Thu, 22 Apr 2021 17:12:37 -0700 Subject: [PATCH 4/8] Removed unused import --- rasa/shared/importers/rasa.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rasa/shared/importers/rasa.py b/rasa/shared/importers/rasa.py index 533604a62a0e..278007316329 100644 --- a/rasa/shared/importers/rasa.py +++ b/rasa/shared/importers/rasa.py @@ -70,11 +70,9 @@ async def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData: async def get_domain(self) -> Domain: """Retrieves model domain (see parent class for full docstring).""" - from rasa.shared.constants import DEFAULT_DOMAIN_PATH - domain = Domain.empty() - # If domain path is None, attempt to load the default domain + # If domain path is None, return an empty domain if not self._domain_path: return domain try: From 01cf7cafcdf1d2dd47f787196c5e2b04f78b0e45 Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Thu, 22 Apr 2021 17:15:45 -0700 Subject: [PATCH 5/8] Added changelog --- changelog/8388.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/8388.bugfix.md diff --git a/changelog/8388.bugfix.md b/changelog/8388.bugfix.md new file mode 100644 index 000000000000..828be565a4eb --- /dev/null +++ b/changelog/8388.bugfix.md @@ -0,0 +1 @@ +Fixed bug where missing intent warnings appear when running `rasa test` \ No newline at end of file From c20f8040bc29fbaa1d8b3af2049eefdc70d7601b Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Thu, 22 Apr 2021 17:36:39 -0700 Subject: [PATCH 6/8] Added Agent model directory check --- rasa/core/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index fca193959542..a4f25d0b444d 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -375,7 +375,9 @@ async def _create_data_generator( from rasa.shared.constants import DEFAULT_DOMAIN_PATH from rasa.model import get_model_subdirectories - core_model, _ = get_model_subdirectories(agent.model_directory) + core_model = None + if agent.model_directory: + core_model, _ = get_model_subdirectories(agent.model_directory) if core_model and os.path.exists(os.path.join(core_model, DEFAULT_DOMAIN_PATH)): domain_path = os.path.join(core_model, DEFAULT_DOMAIN_PATH) From 10f2980b0c10e3d53c12571f7b35a6efeeaa0589 Mon Sep 17 00:00:00 2001 From: b-quachtran Date: Tue, 11 May 2021 14:55:43 -0700 Subject: [PATCH 7/8] Added unit test --- tests/conftest.py | 1 + tests/core/test_test.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index d80dd0193a48..6111563a2afa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -192,6 +192,7 @@ async def _trained_default_agent( agent = Agent( "data/test_domains/default_with_slots.yml", policies=[AugmentedMemoizationPolicy(max_history=3), RulePolicy()], + model_directory=model_path, ) training_data = await agent.load_data(stories_path) diff --git a/tests/core/test_test.py b/tests/core/test_test.py index 302017bf4c40..77fa87197c8a 100644 --- a/tests/core/test_test.py +++ b/tests/core/test_test.py @@ -1,4 +1,7 @@ from pathlib import Path +from typing import Text + +import pytest import rasa.core.test from _pytest.capture import CaptureFixture @@ -17,3 +20,16 @@ async def test_testing_warns_if_action_unknown( assert "Test story" in output assert "contains the bot utterance" in output assert "which is not part of the training data / domain" in output + + +async def test_testing_does_not_warn_if_intent_in_domain( + default_agent: Agent, stories_path: Text, +): + with pytest.warns(UserWarning) as record: + await rasa.core.test.test(Path(stories_path), default_agent) + + assert all("Found intent" not in r.message.args[0] for r in record) + assert all( + "in stories which is not part of the domain" not in r.message.args[0] + for r in record + ) From 089fd9b21e346f60d3c850789523ba7b4a964408 Mon Sep 17 00:00:00 2001 From: Ben Quachtran <65514514+b-quachtran@users.noreply.github.com> Date: Mon, 17 May 2021 15:05:35 -0700 Subject: [PATCH 8/8] Update tests/core/test_test.py Co-authored-by: Joe Juzl --- tests/core/test_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_test.py b/tests/core/test_test.py index 77fa87197c8a..65f392b561ec 100644 --- a/tests/core/test_test.py +++ b/tests/core/test_test.py @@ -28,7 +28,7 @@ async def test_testing_does_not_warn_if_intent_in_domain( with pytest.warns(UserWarning) as record: await rasa.core.test.test(Path(stories_path), default_agent) - assert all("Found intent" not in r.message.args[0] for r in record) + assert not any("Found intent" in r.message.args[0] for r in record) assert all( "in stories which is not part of the domain" not in r.message.args[0] for r in record