diff --git a/changelog/7799.improvement.md b/changelog/7799.improvement.md new file mode 100644 index 000000000000..e18d1c90b34b --- /dev/null +++ b/changelog/7799.improvement.md @@ -0,0 +1 @@ +Updated validator used by `rasa data validate` to verify that actions used in stories and rules are present in the domain and that form slots match domain slots. diff --git a/rasa/cli/data.py b/rasa/cli/data.py index fc9189368445..64acb4bfcd3f 100644 --- a/rasa/cli/data.py +++ b/rasa/cli/data.py @@ -273,7 +273,11 @@ def validate_stories(args: argparse.Namespace) -> None: def _validate_domain(validator: "Validator") -> bool: - return validator.verify_domain_validity() + return ( + validator.verify_domain_validity() + and validator.verify_actions_in_stories_rules() + and validator.verify_form_slots() + ) def _validate_nlu(validator: "Validator", args: argparse.Namespace) -> bool: diff --git a/rasa/validator.py b/rasa/validator.py index d0d111fabb5e..c821f9347561 100644 --- a/rasa/validator.py +++ b/rasa/validator.py @@ -197,6 +197,35 @@ def verify_utterances_in_stories(self, ignore_warnings: bool = True) -> bool: return everything_is_alright + def verify_actions_in_stories_rules(self) -> bool: + """Verifies that actions used in stories and rules are present in the domain.""" + everything_is_alright = True + visited = set() + + for story in self.story_graph.story_steps: + for event in story.events: + if not isinstance(event, ActionExecuted): + continue + + if not event.action_name.startswith("action_"): + continue + + if event.action_name in visited: + # we already processed this one before, we only want to warn once + continue + + if event.action_name not in self.domain.action_names_or_texts: + rasa.shared.utils.io.raise_warning( + f"The action '{event.action_name}' is used in the '{story.block_name}' block, but it " + f"is not listed in the domain file. You should add it to your " + f"domain file!", + docs=DOCS_URL_DOMAINS, + ) + everything_is_alright = False + visited.add(event.action_name) + + return everything_is_alright + def verify_story_structure( self, ignore_warnings: bool = True, max_history: Optional[int] = None ) -> bool: @@ -249,6 +278,26 @@ def verify_nlu(self, ignore_warnings: bool = True) -> bool: stories_are_valid = self.verify_utterances_in_stories(ignore_warnings) return intents_are_valid and stories_are_valid and there_is_no_duplication + def verify_form_slots(self) -> bool: + """Verifies that form slots match the slot mappings in domain.""" + domain_slot_names = [slot.name for slot in self.domain.slots] + everything_is_alright = True + + for form in self.domain.form_names: + form_slots = self.domain.slot_mapping_for_form(form) + for slot in form_slots.keys(): + if slot in domain_slot_names: + continue + else: + rasa.shared.utils.io.raise_warning( + f"The form slot '{slot}' in form '{form}' is not present in the domain slots." + f"Please add the correct slot or check for typos.", + docs=DOCS_URL_DOMAINS, + ) + everything_is_alright = False + + return everything_is_alright + def verify_domain_validity(self) -> bool: """Checks whether the domain returned by the importer is empty. diff --git a/tests/cli/test_rasa_data.py b/tests/cli/test_rasa_data.py index 18d49e1df5d8..413ee8e8cbd6 100644 --- a/tests/cli/test_rasa_data.py +++ b/tests/cli/test_rasa_data.py @@ -146,6 +146,61 @@ async def mock_from_importer(importer: TrainingDataImporter) -> Validator: data.validate_files(args) +@pytest.mark.parametrize( + ("file_type", "data_type"), [("stories", "story"), ("rules", "rule")] +) +def test_validate_files_action_not_found_invalid_domain( + file_type: Text, data_type: Text, tmp_path: Path +): + file_name = tmp_path / f"{file_type}.yml" + file_name.write_text( + f""" + version: "2.0" + {file_type}: + - {data_type}: test path + steps: + - intent: goodbye + - action: action_test + """ + ) + args = { + "domain": "data/test_moodbot/domain.yml", + "data": [file_name], + "max_history": None, + "config": None, + } + with pytest.raises(SystemExit): + data.validate_files(namedtuple("Args", args.keys())(*args.values())) + + +def test_validate_files_form_slots_not_matching(tmp_path: Path): + domain_file_name = tmp_path / "domain.yml" + domain_file_name.write_text( + """ + version: "2.0" + forms: + name_form: + first_name: + - type: from_text + last_name: + - type: from_text + slots: + first_name: + type: text + last_nam: + type: text + """ + ) + args = { + "domain": domain_file_name, + "data": None, + "max_history": None, + "config": None, + } + with pytest.raises(SystemExit): + data.validate_files(namedtuple("Args", args.keys())(*args.values())) + + def test_validate_files_exit_early(): with pytest.raises(SystemExit) as pytest_e: args = { diff --git a/tests/test_validator.py b/tests/test_validator.py index 41cc130b8523..09e8628d1c1c 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -259,6 +259,92 @@ async def test_verify_there_is_not_example_repetition_in_intents(): assert validator.verify_example_repetition_in_intents(False) +async def test_verify_actions_in_stories_not_in_domain( + tmp_path: Path, domain_path: Text +): + story_file_name = tmp_path / "stories.yml" + story_file_name.write_text( + """ + version: "2.0" + stories: + - story: story path 1 + steps: + - intent: greet + - action: action_test_1 + """ + ) + + importer = RasaFileImporter( + domain_path=domain_path, training_data_paths=[story_file_name], + ) + validator = await Validator.from_importer(importer) + with pytest.warns(UserWarning) as warning: + validity = validator.verify_actions_in_stories_rules() + assert validity is False + + assert ( + "The action 'action_test_1' is used in the 'story path 1' block, " + "but it is not listed in the domain file." in warning[0].message.args[0] + ) + + +async def test_verify_actions_in_rules_not_in_domain(tmp_path: Path, domain_path: Text): + rules_file_name = tmp_path / "rules.yml" + rules_file_name.write_text( + """ + version: "2.0" + rules: + - rule: rule path 1 + steps: + - intent: goodbye + - action: action_test_2 + """ + ) + importer = RasaFileImporter( + domain_path=domain_path, training_data_paths=[rules_file_name], + ) + validator = await Validator.from_importer(importer) + with pytest.warns(UserWarning) as warning: + validity = validator.verify_actions_in_stories_rules() + assert validity is False + + assert ( + "The action 'action_test_2' is used in the 'rule path 1' block, " + "but it is not listed in the domain file." in warning[0].message.args[0] + ) + + +async def test_verify_form_slots_invalid_domain(tmp_path: Path): + domain = tmp_path / "domain.yml" + domain.write_text( + """ + version: "2.0" + forms: + name_form: + first_name: + - type: from_text + last_name: + - type: from_text + slots: + first_name: + type: text + last_nam: + type: text + """ + ) + importer = RasaFileImporter(domain_path=domain) + validator = await Validator.from_importer(importer) + with pytest.warns(UserWarning) as w: + validity = validator.verify_form_slots() + assert validity is False + + assert ( + w[0].message.args[0] + == "The form slot 'last_name' in form 'name_form' is not present in the domain slots." + "Please add the correct slot or check for typos." + ) + + async def test_response_selector_responses_in_domain_no_errors(): importer = RasaFileImporter( config_file="data/test_config/config_defaults.yml", @@ -278,3 +364,88 @@ async def test_invalid_domain_mapping_policy(): ) validator = await Validator.from_importer(importer) assert validator.verify_domain_validity() is False + + +@pytest.mark.parametrize( + ("file_name", "data_type"), [("stories", "story"), ("rules", "rule")] +) +async def test_valid_stories_rules_actions_in_domain( + file_name: Text, data_type: Text, tmp_path: Path +): + domain = tmp_path / "domain.yml" + domain.write_text( + """ + version: "2.0" + intents: + - greet + actions: + - action_greet + """ + ) + file_name = tmp_path / f"{file_name}.yml" + file_name.write_text( + f""" + version: "2.0" + {file_name}: + - {data_type}: test path + steps: + - intent: greet + - action: action_greet + """ + ) + importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name],) + validator = await Validator.from_importer(importer) + assert validator.verify_actions_in_stories_rules() + + +@pytest.mark.parametrize( + ("file_name", "data_type"), [("stories", "story"), ("rules", "rule")] +) +async def test_valid_stories_rules_default_actions( + file_name: Text, data_type: Text, tmp_path: Path +): + domain = tmp_path / "domain.yml" + domain.write_text( + """ + version: "2.0" + intents: + - greet + """ + ) + file_name = tmp_path / f"{file_name}.yml" + file_name.write_text( + f""" + version: "2.0" + {file_name}: + - {data_type}: test path + steps: + - intent: greet + - action: action_restart + """ + ) + importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name],) + validator = await Validator.from_importer(importer) + assert validator.verify_actions_in_stories_rules() + + +async def test_valid_form_slots_in_domain(tmp_path: Path): + domain = tmp_path / "domain.yml" + domain.write_text( + """ + version: "2.0" + forms: + name_form: + first_name: + - type: from_text + last_name: + - type: from_text + slots: + first_name: + type: text + last_name: + type: text + """ + ) + importer = RasaFileImporter(domain_path=domain) + validator = await Validator.from_importer(importer) + assert validator.verify_form_slots()