Merge pull request #6590 from RasaHQ/6566_mix_nlu_and_stories

Added ability to mix NLU and Stories in one file

data/test_mixed_yaml_training_data/training_data.yml

@@ -0,0 +1,24 @@
+version: "2.0"
+nlu:
+- intent: intent_name
+  examples: |
+    - how much CO2 will that use?
+    - how much carbon will a one way flight from [new york]{"entity": "city", "role": "from"} to california produce?
+
+stories:
+- story: simple_story_with_only_start
+  steps:
+  - checkpoint: check_greet
+  - intent: simple
+  - action: utter_default
+
+- rule: Say 'I am a bot' anytime the user challenges
+  steps:
+  - intent: bot_challenge
+  - action: utter_iamabot
+
+responses:
+  chitchat/ask_name:
+  - image: "https://i.imgur.com/zTvA58i.jpeg"
+    text: hello, my name is retrieval bot.
+  - text: Oh yeah, I am called the retrieval bot.

pyproject.toml

@@ -18,7 +18,7 @@ repository = "https://github.com/rasahq/rasa"
 documentation = "https://rasa.com/docs"
 classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Topic :: Software Development :: Libraries",]
 keywords = [ "nlp", "machine-learning", "machine-learning-library", "bot", "bots", "botkit", "rasa conversational-agents", "conversational-ai", "chatbot", "chatbot-framework", "bot-framework",]
-include = [ "LICENSE.txt", "README.md", "rasa/core/schemas/*", "rasa/core/training/visualization.html", "rasa/nlu/schemas/*", "rasa/cli/default_config.yml", "rasa/importers/*", "rasa/utils/schemas.yml",]
+include = [ "LICENSE.txt", "README.md", "rasa/core/training/visualization.html", "rasa/cli/default_config.yml", "rasa/importers/*", "rasa/utils/schemas/*",]
 readme = "README.md"
 license = "Apache-2.0"
 [[tool.poetry.source]]

rasa/constants.py

@@ -26,8 +26,8 @@
 PACKAGE_NAME = "rasa"
 
 CONFIG_SCHEMA_FILE = "nlu/schemas/config.yml"
-DOMAIN_SCHEMA_FILE = "core/schemas/domain.yml"
-SCHEMA_UTILS_FILE = "utils/schemas.yml"
+DOMAIN_SCHEMA_FILE = "utils/schemas/domain.yml"
+RESPONSES_SCHEMA_FILE = "utils/schemas/responses.yml"
 SCHEMA_EXTENSIONS_FILE = "utils/pykwalify_extensions.py"
 YAML_VERSION = (1, 2)
 

rasa/core/training/story_reader/yaml_story_reader.py

@@ -41,7 +41,7 @@
 KEY_RULE_FOR_CONVERSATION_START = "conversation_start"
 
 
-CORE_SCHEMA_FILE = "core/schemas/stories.yml"
+CORE_SCHEMA_FILE = "utils/schemas/stories.yml"
 
 
 class YAMLStoryReader(StoryReader):

rasa/nlu/training_data/formats/rasa_yaml.py

@@ -48,7 +48,7 @@
 
 MULTILINE_TRAINING_EXAMPLE_LEADING_SYMBOL = "-"
 
-NLU_SCHEMA_FILE = "nlu/schemas/nlu.yml"
+NLU_SCHEMA_FILE = "utils/schemas/nlu.yml"
 
 STRIP_SYMBOLS = "\n\r "
 

rasa/core/schemas/domain.yml → rasa/utils/schemas/domain.yml

@@ -25,7 +25,7 @@ mapping:
     - type: "str"
       required: True
   responses:
-    # see rasa/utils/schemas.yml
+    # see utils/schemas/responses.yml
     include: responses
 
   slots:

rasa/nlu/schemas/nlu.yml → rasa/utils/schemas/nlu.yml

@@ -49,6 +49,5 @@ mapping:
   responses:
     # see rasa/utils/schemas.yml
     include: responses
-  stories:
+  regex;(.*):
     type: "any"
-    required: False

rasa/core/schemas/stories.yml → rasa/utils/schemas/stories.yml

@@ -127,3 +127,5 @@ mapping:
         wait_for_user_input:
           type: "bool"
           allowempty: False
+  regex;(.*):
+    type: "any"

rasa/utils/validation.py

@@ -6,7 +6,7 @@
     PACKAGE_NAME,
     DOCS_URL_TRAINING_DATA_NLU,
     SCHEMA_EXTENSIONS_FILE,
-    SCHEMA_UTILS_FILE,
+    RESPONSES_SCHEMA_FILE,
 )
 
 
@@ -59,7 +59,7 @@ def validate_yaml_schema(
     try:
         schema_file = pkg_resources.resource_filename(PACKAGE_NAME, schema_path)
         schema_utils_file = pkg_resources.resource_filename(
-            PACKAGE_NAME, SCHEMA_UTILS_FILE
+            PACKAGE_NAME, RESPONSES_SCHEMA_FILE
         )
         schema_extensions = pkg_resources.resource_filename(
             PACKAGE_NAME, SCHEMA_EXTENSIONS_FILE

tests/core/training/story_reader/test_yaml_story_reader.py

@@ -345,3 +345,14 @@ def test_end_to_end_story_with_shortcut_intent():
         intent={"name": intent},
         entities=[{"entity": "name", "start": 6, "end": 22, "value": "test"}],
     )
+
+
+def test_read_mixed_training_data_file(default_domain: Domain):
+    training_data_file = "data/test_mixed_yaml_training_data/training_data.yml"
+
+    reader = YAMLStoryReader(default_domain)
+    yaml_content = rasa.utils.io.read_yaml_file(training_data_file)
+
+    with pytest.warns(None) as record:
+        reader.read_from_parsed_yaml(yaml_content)
+        assert not len(record)

tests/nlu/training_data/formats/test_rasa_yaml.py

@@ -356,3 +356,13 @@ def test_nlg_multimedia_load_dump_roundtrip():
 
     # dumping again should also not change the format
     assert dumped == RasaYAMLWriter().dumps(dumped_result)
+
+
+def test_read_mixed_training_data_file():
+    training_data_file = "data/test_mixed_yaml_training_data/training_data.yml"
+
+    reader = RasaYAMLReader()
+
+    with pytest.warns(None) as record:
+        reader.read(training_data_file)
+        assert not len(record)

tests/test_data.py

@@ -122,3 +122,9 @@ def test_is_md_file(path, is_md):
 )
 def test_is_json_file(path, is_json):
     assert data.is_likely_json_file(path) == is_json
+
+
+async def test_get_files_with_mixed_training_data():
+    default_data_path = "data/test_mixed_yaml_training_data/training_data.yml"
+    assert data.get_data_files(default_data_path, data.is_nlu_file)
+    assert data.get_data_files(default_data_path, data.is_story_file)