Merge pull request #8645 from RasaHQ/rasa-validate-7799

Add verify stories and rules actions in domain to rasa data validate

changelog/7799.improvement.md

@@ -0,0 +1 @@
+Updated validator used by `rasa data validate` to verify that actions used in stories and rules are present in the domain and that form slots match domain slots.

rasa/cli/data.py

@@ -273,7 +273,11 @@ def validate_stories(args: argparse.Namespace) -> None:
 
 
 def _validate_domain(validator: "Validator") -> bool:
-    return validator.verify_domain_validity()
+    return (
+        validator.verify_domain_validity()
+        and validator.verify_actions_in_stories_rules()
+        and validator.verify_form_slots()
+    )
 
 
 def _validate_nlu(validator: "Validator", args: argparse.Namespace) -> bool:

rasa/validator.py

@@ -197,6 +197,35 @@ def verify_utterances_in_stories(self, ignore_warnings: bool = True) -> bool:
 
         return everything_is_alright
 
+    def verify_actions_in_stories_rules(self) -> bool:
+        """Verifies that actions used in stories and rules are present in the domain."""
+        everything_is_alright = True
+        visited = set()
+
+        for story in self.story_graph.story_steps:
+            for event in story.events:
+                if not isinstance(event, ActionExecuted):
+                    continue
+
+                if not event.action_name.startswith("action_"):
+                    continue
+
+                if event.action_name in visited:
+                    # we already processed this one before, we only want to warn once
+                    continue
+
+                if event.action_name not in self.domain.action_names_or_texts:
+                    rasa.shared.utils.io.raise_warning(
+                        f"The action '{event.action_name}' is used in the '{story.block_name}' block, but it "
+                        f"is not listed in the domain file. You should add it to your "
+                        f"domain file!",
+                        docs=DOCS_URL_DOMAINS,
+                    )
+                    everything_is_alright = False
+                visited.add(event.action_name)
+
+        return everything_is_alright
+
     def verify_story_structure(
         self, ignore_warnings: bool = True, max_history: Optional[int] = None
     ) -> bool:
@@ -249,6 +278,26 @@ def verify_nlu(self, ignore_warnings: bool = True) -> bool:
         stories_are_valid = self.verify_utterances_in_stories(ignore_warnings)
         return intents_are_valid and stories_are_valid and there_is_no_duplication
 
+    def verify_form_slots(self) -> bool:
+        """Verifies that form slots match the slot mappings in domain."""
+        domain_slot_names = [slot.name for slot in self.domain.slots]
+        everything_is_alright = True
+
+        for form in self.domain.form_names:
+            form_slots = self.domain.slot_mapping_for_form(form)
+            for slot in form_slots.keys():
+                if slot in domain_slot_names:
+                    continue
+                else:
+                    rasa.shared.utils.io.raise_warning(
+                        f"The form slot '{slot}' in form '{form}' is not present in the domain slots."
+                        f"Please add the correct slot or check for typos.",
+                        docs=DOCS_URL_DOMAINS,
+                    )
+                    everything_is_alright = False
+
+        return everything_is_alright
+
     def verify_domain_validity(self) -> bool:
         """Checks whether the domain returned by the importer is empty.
 

tests/cli/test_rasa_data.py

@@ -146,6 +146,61 @@ async def mock_from_importer(importer: TrainingDataImporter) -> Validator:
         data.validate_files(args)
 
 
+@pytest.mark.parametrize(
+    ("file_type", "data_type"), [("stories", "story"), ("rules", "rule")]
+)
+def test_validate_files_action_not_found_invalid_domain(
+    file_type: Text, data_type: Text, tmp_path: Path
+):
+    file_name = tmp_path / f"{file_type}.yml"
+    file_name.write_text(
+        f"""
+        version: "2.0"
+        {file_type}:
+        - {data_type}: test path
+          steps:
+          - intent: goodbye
+          - action: action_test
+        """
+    )
+    args = {
+        "domain": "data/test_moodbot/domain.yml",
+        "data": [file_name],
+        "max_history": None,
+        "config": None,
+    }
+    with pytest.raises(SystemExit):
+        data.validate_files(namedtuple("Args", args.keys())(*args.values()))
+
+
+def test_validate_files_form_slots_not_matching(tmp_path: Path):
+    domain_file_name = tmp_path / "domain.yml"
+    domain_file_name.write_text(
+        """
+        version: "2.0"
+        forms:
+          name_form:
+             first_name:
+             - type: from_text
+             last_name:
+             - type: from_text
+        slots:
+             first_name:
+                type: text
+             last_nam:
+                type: text
+        """
+    )
+    args = {
+        "domain": domain_file_name,
+        "data": None,
+        "max_history": None,
+        "config": None,
+    }
+    with pytest.raises(SystemExit):
+        data.validate_files(namedtuple("Args", args.keys())(*args.values()))
+
+
 def test_validate_files_exit_early():
     with pytest.raises(SystemExit) as pytest_e:
         args = {

tests/test_validator.py

@@ -259,6 +259,92 @@ async def test_verify_there_is_not_example_repetition_in_intents():
     assert validator.verify_example_repetition_in_intents(False)
 
 
+async def test_verify_actions_in_stories_not_in_domain(
+    tmp_path: Path, domain_path: Text
+):
+    story_file_name = tmp_path / "stories.yml"
+    story_file_name.write_text(
+        """
+        version: "2.0"
+        stories:
+        - story: story path 1
+          steps:
+          - intent: greet
+          - action: action_test_1
+        """
+    )
+
+    importer = RasaFileImporter(
+        domain_path=domain_path, training_data_paths=[story_file_name],
+    )
+    validator = await Validator.from_importer(importer)
+    with pytest.warns(UserWarning) as warning:
+        validity = validator.verify_actions_in_stories_rules()
+        assert validity is False
+
+    assert (
+        "The action 'action_test_1' is used in the 'story path 1' block, "
+        "but it is not listed in the domain file." in warning[0].message.args[0]
+    )
+
+
+async def test_verify_actions_in_rules_not_in_domain(tmp_path: Path, domain_path: Text):
+    rules_file_name = tmp_path / "rules.yml"
+    rules_file_name.write_text(
+        """
+        version: "2.0"
+        rules:
+        - rule: rule path 1
+          steps:
+          - intent: goodbye
+          - action: action_test_2
+        """
+    )
+    importer = RasaFileImporter(
+        domain_path=domain_path, training_data_paths=[rules_file_name],
+    )
+    validator = await Validator.from_importer(importer)
+    with pytest.warns(UserWarning) as warning:
+        validity = validator.verify_actions_in_stories_rules()
+        assert validity is False
+
+    assert (
+        "The action 'action_test_2' is used in the 'rule path 1' block, "
+        "but it is not listed in the domain file." in warning[0].message.args[0]
+    )
+
+
+async def test_verify_form_slots_invalid_domain(tmp_path: Path):
+    domain = tmp_path / "domain.yml"
+    domain.write_text(
+        """
+        version: "2.0"
+        forms:
+          name_form:
+             first_name:
+             - type: from_text
+             last_name:
+             - type: from_text
+        slots:
+             first_name:
+                type: text
+             last_nam:
+                type: text
+        """
+    )
+    importer = RasaFileImporter(domain_path=domain)
+    validator = await Validator.from_importer(importer)
+    with pytest.warns(UserWarning) as w:
+        validity = validator.verify_form_slots()
+        assert validity is False
+
+    assert (
+        w[0].message.args[0]
+        == "The form slot 'last_name' in form 'name_form' is not present in the domain slots."
+        "Please add the correct slot or check for typos."
+    )
+
+
 async def test_response_selector_responses_in_domain_no_errors():
     importer = RasaFileImporter(
         config_file="data/test_config/config_defaults.yml",
@@ -278,3 +364,88 @@ async def test_invalid_domain_mapping_policy():
     )
     validator = await Validator.from_importer(importer)
     assert validator.verify_domain_validity() is False
+
+
+@pytest.mark.parametrize(
+    ("file_name", "data_type"), [("stories", "story"), ("rules", "rule")]
+)
+async def test_valid_stories_rules_actions_in_domain(
+    file_name: Text, data_type: Text, tmp_path: Path
+):
+    domain = tmp_path / "domain.yml"
+    domain.write_text(
+        """
+        version: "2.0"
+        intents:
+        - greet
+        actions:
+        - action_greet
+        """
+    )
+    file_name = tmp_path / f"{file_name}.yml"
+    file_name.write_text(
+        f"""
+        version: "2.0"
+        {file_name}:
+        - {data_type}: test path
+          steps:
+          - intent: greet
+          - action: action_greet
+        """
+    )
+    importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name],)
+    validator = await Validator.from_importer(importer)
+    assert validator.verify_actions_in_stories_rules()
+
+
+@pytest.mark.parametrize(
+    ("file_name", "data_type"), [("stories", "story"), ("rules", "rule")]
+)
+async def test_valid_stories_rules_default_actions(
+    file_name: Text, data_type: Text, tmp_path: Path
+):
+    domain = tmp_path / "domain.yml"
+    domain.write_text(
+        """
+        version: "2.0"
+        intents:
+        - greet
+        """
+    )
+    file_name = tmp_path / f"{file_name}.yml"
+    file_name.write_text(
+        f"""
+            version: "2.0"
+            {file_name}:
+            - {data_type}: test path
+              steps:
+              - intent: greet
+              - action: action_restart
+            """
+    )
+    importer = RasaFileImporter(domain_path=domain, training_data_paths=[file_name],)
+    validator = await Validator.from_importer(importer)
+    assert validator.verify_actions_in_stories_rules()
+
+
+async def test_valid_form_slots_in_domain(tmp_path: Path):
+    domain = tmp_path / "domain.yml"
+    domain.write_text(
+        """
+        version: "2.0"
+        forms:
+          name_form:
+             first_name:
+             - type: from_text
+             last_name:
+             - type: from_text
+        slots:
+             first_name:
+                type: text
+             last_name:
+                type: text
+        """
+    )
+    importer = RasaFileImporter(domain_path=domain)
+    validator = await Validator.from_importer(importer)
+    assert validator.verify_form_slots()