From df2ead65c6a80d43c58bd586cfe2d1aa7dce5a49 Mon Sep 17 00:00:00 2001 From: Lydia Kalkbrenner Date: Wed, 21 Oct 2020 11:03:52 +0200 Subject: [PATCH 1/3] make markdown_story_reader support response key for test stories --- .../story_reader/markdown_story_reader.py | 8 ++- .../test_story_markdown_to_yaml_converter.py | 72 +++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/rasa/shared/core/training_data/story_reader/markdown_story_reader.py b/rasa/shared/core/training_data/story_reader/markdown_story_reader.py index d7fbacbdbbe6..28e3b0f48cd4 100644 --- a/rasa/shared/core/training_data/story_reader/markdown_story_reader.py +++ b/rasa/shared/core/training_data/story_reader/markdown_story_reader.py @@ -250,12 +250,16 @@ def _parse_message(self, message: Text, line_num: int) -> UserUttered: if self.use_e2e: parsed = self.parse_e2e_message(message, self.is_used_for_conversion) text = parsed.get("text") - intent = {INTENT_NAME_KEY: parsed.get("intent")} + intent = { + INTENT_NAME_KEY: parsed.get( + "intent_response_key", default=parsed.get("intent") + ) + } entities = parsed.get("entities") parse_data = { "text": text, "intent": intent, - "intent_ranking": [{INTENT_NAME_KEY: parsed.get("intent")}], + "intent_ranking": [intent], "entities": entities, } else: diff --git a/tests/core/training/converters/test_story_markdown_to_yaml_converter.py b/tests/core/training/converters/test_story_markdown_to_yaml_converter.py index d0d8097f4d26..068238ef2a9a 100644 --- a/tests/core/training/converters/test_story_markdown_to_yaml_converter.py +++ b/tests/core/training/converters/test_story_markdown_to_yaml_converter.py @@ -116,3 +116,75 @@ async def test_test_stories(tmpdir: Path): " - action: respond_faq\n" " - action: action_set_faq_slot\n" ) + + +async def test_test_stories_response_key(tmpdir: Path): + converted_data_folder = tmpdir / "converted_data" + os.mkdir(converted_data_folder) + + test_data_folder = tmpdir / "tests" + os.makedirs(test_data_folder, exist_ok=True) + test_data_file = Path(test_data_folder / "test_stories.md") + + simple_story_md = """ + ## id + + * out_of_scope/other: hahaha + - utter_out_of_scope/other + """ + + with open(test_data_file, "w") as f: + f.write(simple_story_md) + + await StoryMarkdownToYamlConverter().convert_and_write( + test_data_file, converted_data_folder + ) + + assert len(os.listdir(converted_data_folder)) == 1 + with open(f"{converted_data_folder}/test_stories_converted.yml", "r") as f: + content = f.read() + assert content == ( + f'version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"\n' + "stories:\n" + "- story: id\n" + " steps:\n" + " - intent: out_of_scope/other\n" + " user: |-\n" + " hahaha\n" + " - action: utter_out_of_scope/other\n" + ) + + +async def test_stories_are_converted_response_key(tmpdir: Path): + converted_data_folder = tmpdir / "converted_data" + os.mkdir(converted_data_folder) + + training_data_folder = tmpdir / "data/core" + os.makedirs(training_data_folder, exist_ok=True) + training_data_file = Path(training_data_folder / "stories.md") + + simple_story_md = """ + ## id + * out_of_scope/other + - utter_out_of_scope/other + """ + + with open(training_data_file, "w") as f: + f.write(simple_story_md) + + await StoryMarkdownToYamlConverter().convert_and_write( + training_data_file, converted_data_folder + ) + + assert len(os.listdir(converted_data_folder)) == 1 + + with open(f"{converted_data_folder}/stories_converted.yml", "r") as f: + content = f.read() + assert content == ( + f'version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"\n' + "stories:\n" + "- story: id\n" + " steps:\n" + " - intent: out_of_scope/other\n" + " - action: utter_out_of_scope/other\n" + ) From f850a4b19e622cc67a0512c2d0cfd228fe1c11a5 Mon Sep 17 00:00:00 2001 From: Lydia Kalkbrenner Date: Wed, 21 Oct 2020 11:50:18 +0200 Subject: [PATCH 2/3] make lint happy --- .../converters/test_story_markdown_to_yaml_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/training/converters/test_story_markdown_to_yaml_converter.py b/tests/core/training/converters/test_story_markdown_to_yaml_converter.py index 068238ef2a9a..726144cda389 100644 --- a/tests/core/training/converters/test_story_markdown_to_yaml_converter.py +++ b/tests/core/training/converters/test_story_markdown_to_yaml_converter.py @@ -128,7 +128,7 @@ async def test_test_stories_response_key(tmpdir: Path): simple_story_md = """ ## id - + * out_of_scope/other: hahaha - utter_out_of_scope/other """ From 1476f5f79230d0493c0bc8d81c2c52208f74801e Mon Sep 17 00:00:00 2001 From: Lydia Kalkbrenner Date: Fri, 30 Oct 2020 10:26:26 +0100 Subject: [PATCH 3/3] changelog entry and test renaming --- changelog/7034.bugfix.md | 2 ++ .../converters/test_story_markdown_to_yaml_converter.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog/7034.bugfix.md diff --git a/changelog/7034.bugfix.md b/changelog/7034.bugfix.md new file mode 100644 index 000000000000..9227df2b436c --- /dev/null +++ b/changelog/7034.bugfix.md @@ -0,0 +1,2 @@ +The converter tool now converts test stories and stories that contain full retrieval intents correctly. +Previously the response keys were deleted during conversion to YAML. \ No newline at end of file diff --git a/tests/core/training/converters/test_story_markdown_to_yaml_converter.py b/tests/core/training/converters/test_story_markdown_to_yaml_converter.py index 726144cda389..04f9a30dd2a0 100644 --- a/tests/core/training/converters/test_story_markdown_to_yaml_converter.py +++ b/tests/core/training/converters/test_story_markdown_to_yaml_converter.py @@ -118,7 +118,7 @@ async def test_test_stories(tmpdir: Path): ) -async def test_test_stories_response_key(tmpdir: Path): +async def test_test_stories_conversion_response_key(tmpdir: Path): converted_data_folder = tmpdir / "converted_data" os.mkdir(converted_data_folder) @@ -155,7 +155,7 @@ async def test_test_stories_response_key(tmpdir: Path): ) -async def test_stories_are_converted_response_key(tmpdir: Path): +async def test_stories_conversion_response_key(tmpdir: Path): converted_data_folder = tmpdir / "converted_data" os.mkdir(converted_data_folder)