From f85134c104975d826323c51932044a29284aa167 Mon Sep 17 00:00:00 2001 From: martasls Date: Wed, 17 Mar 2021 13:18:23 +0000 Subject: [PATCH 01/11] Change --data type in 'rasa data validate' command --- rasa/cli/arguments/default_arguments.py | 4 ++-- tests/cli/test_rasa_data.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rasa/cli/arguments/default_arguments.py b/rasa/cli/arguments/default_arguments.py index 2157fcb10973..c7130881175a 100644 --- a/rasa/cli/arguments/default_arguments.py +++ b/rasa/cli/arguments/default_arguments.py @@ -103,9 +103,9 @@ def add_data_param( ) -> None: parser.add_argument( "--data", - type=str, default=default, - help=f"Path to the file or directory containing {data_type} data.", + nargs="+", + help=f"Paths to the files or directories containing {data_type} data.", required=required, ) diff --git a/tests/cli/test_rasa_data.py b/tests/cli/test_rasa_data.py index a1aa2f624f5e..cee8f444e19e 100644 --- a/tests/cli/test_rasa_data.py +++ b/tests/cli/test_rasa_data.py @@ -109,7 +109,7 @@ def test_data_validate_help(run: Callable[..., RunResult]): help_text = """usage: rasa data validate [-h] [-v] [-vv] [--quiet] [--max-history MAX_HISTORY] [-c CONFIG] - [--fail-on-warnings] [-d DOMAIN] [--data DATA]""" + [--fail-on-warnings] [-d DOMAIN] [--data DATA [DATA ...]]""" lines = help_text.split("\n") # expected help text lines should appear somewhere in the output From a3a6542c2be53c301e7319a53e083b8128915d19 Mon Sep 17 00:00:00 2001 From: martasls Date: Tue, 23 Mar 2021 20:24:14 +0000 Subject: [PATCH 02/11] Add changelog entry --- changelog/8225.improvement.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/8225.improvement.md diff --git a/changelog/8225.improvement.md b/changelog/8225.improvement.md new file mode 100644 index 000000000000..804e29d54140 --- /dev/null +++ b/changelog/8225.improvement.md @@ -0,0 +1 @@ +Changed --data option type in the ``rasa data validate``` command to allow more than one path to be passed. \ No newline at end of file From 2a9b90d4d6bffa30f7577e5120500c8424b8d37c Mon Sep 17 00:00:00 2001 From: martasls Date: Sat, 3 Apr 2021 19:38:50 +0100 Subject: [PATCH 03/11] Fix data path in convert data method --- data/test_config/config_defaults.yml | 2 -- other | Bin 0 -> 12288 bytes rasa/nlu/convert.py | 15 ++++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 other diff --git a/data/test_config/config_defaults.yml b/data/test_config/config_defaults.yml index 1d42a220213c..098e1f08c39c 100644 --- a/data/test_config/config_defaults.yml +++ b/data/test_config/config_defaults.yml @@ -14,12 +14,10 @@ pipeline: [] # - name: DIETClassifier # epochs: 100 # constrain_similarities: true -# model_confidence: cosine # - name: EntitySynonymMapper # - name: ResponseSelector # epochs: 100 # constrain_similarities: true -# model_confidence: cosine # - name: FallbackClassifier # threshold: 0.3 # ambiguity_threshold: 0.1 diff --git a/other b/other new file mode 100644 index 0000000000000000000000000000000000000000..b828e50def9ba0630d1273e3a166363988559be0 GIT binary patch literal 12288 zcmeI#O-sWt7zgmQlgSFc+{SLtaRu?<*xfqgEY?nIl?@Li+6V#LjV;XWD1I6JmZsJX z#kmhK{(-zanS(afzpzmhT}|VYO#H0S?rjIF=m@yF@4dBcxw@u+4`~mZ)vl` z?)#3}wRb<*Ueyd}5P$##AOHafKmY;|fB*y_0D*M^_ja}3YFKlXr*fudiJZz@Pm+l= z@im8sNRo^@YetnGTEVZHfUJUq)Xp&ae+im`VR@a0jb zrc(2fn0I+ZVK1ieHV6*LsV!^Lofmb^y{LKAZo9uy^n4_fTx4=hipu0fi)=(^LC=ee z!YbG1!hg>dd*V^6VZM4V60jIbd-v1l?hjBFsKmY;|fB*y_009U<00Izz0Aru_>Tjn2 literal 0 HcmV?d00001 diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index 4cc0c576ae1c..5774594d9349 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -8,14 +8,15 @@ def convert_training_data( - data_file: Text, out_file: Text, output_format: Text, language: Text + data_files: list, out_file: Text, output_format: Text, language: Text ): - if not os.path.exists(data_file): - print_error( - "Data file '{}' does not exist. Provide a valid NLU data file using " - "the '--data' argument.".format(data_file) - ) - return + for data_file in data_files: + if not os.path.exists(data_file): + print_error( + "Data file '{}' does not exist. Provide a valid NLU data file using " + "the '--data' argument.".format(data_file) + ) + return if output_format == "json": td = rasa.shared.nlu.training_data.loading.load_data(data_file, language) From 3364a3a7e14c8ca20f57b9001e7c9d5ad1ea6d28 Mon Sep 17 00:00:00 2001 From: martasls Date: Thu, 6 May 2021 22:33:41 +0100 Subject: [PATCH 04/11] Adapt code to unit tests --- rasa/cli/arguments/default_arguments.py | 1 + rasa/cli/data.py | 5 ++++- rasa/nlu/convert.py | 20 +++++++++++--------- tests/cli/test_rasa_data.py | 6 ++++-- tests/cli/test_rasa_x.py | 4 ++-- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/rasa/cli/arguments/default_arguments.py b/rasa/cli/arguments/default_arguments.py index c7130881175a..8e36ccbaaf01 100644 --- a/rasa/cli/arguments/default_arguments.py +++ b/rasa/cli/arguments/default_arguments.py @@ -105,6 +105,7 @@ def add_data_param( "--data", default=default, nargs="+", + type=str, help=f"Paths to the files or directories containing {data_type} data.", required=required, ) diff --git a/rasa/cli/data.py b/rasa/cli/data.py index 0fd3abb8f6c3..c6ef521d2399 100644 --- a/rasa/cli/data.py +++ b/rasa/cli/data.py @@ -377,9 +377,12 @@ def _migrate_responses(args: argparse.Namespace) -> None: async def _convert_to_yaml( - out_path: Text, data_path: Text, converter: "TrainingDataConverter" + out_path: Text, data_path: Union[list, Text], converter: "TrainingDataConverter" ) -> None: + if isinstance(data_path, list): + data_path = data_path[0] + output = Path(out_path) if not os.path.exists(output): rasa.shared.utils.cli.print_error_and_exit( diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index 5774594d9349..19f88cf45d50 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -8,15 +8,17 @@ def convert_training_data( - data_files: list, out_file: Text, output_format: Text, language: Text -): - for data_file in data_files: - if not os.path.exists(data_file): - print_error( - "Data file '{}' does not exist. Provide a valid NLU data file using " - "the '--data' argument.".format(data_file) - ) - return + data_file: list, out_file: Text, output_format: Text, language: Text +) -> None: + + data_file = data_file[0] + + if not os.path.exists(data_file): + print_error( + "Data file '{}' does not exist. Provide a valid NLU data file using " + "the '--data' argument.".format(data_file) + ) + return if output_format == "json": td = rasa.shared.nlu.training_data.loading.load_data(data_file, language) diff --git a/tests/cli/test_rasa_data.py b/tests/cli/test_rasa_data.py index 8d8e2d184dba..88782ee133d5 100644 --- a/tests/cli/test_rasa_data.py +++ b/tests/cli/test_rasa_data.py @@ -95,7 +95,7 @@ def test_data_convert_help(run: Callable[..., RunResult]): output = run("data", "convert", "nlu", "--help") help_text = """usage: rasa data convert nlu [-h] [-v] [-vv] [--quiet] [-f {json,md,yaml}] - --data DATA [--out OUT] [-l LANGUAGE]""" + --data DATA [DATA ...] [--out OUT] [-l LANGUAGE]""" lines = help_text.split("\n") # expected help text lines should appear somewhere in the output @@ -109,7 +109,9 @@ def test_data_validate_help(run: Callable[..., RunResult]): help_text = """usage: rasa data validate [-h] [-v] [-vv] [--quiet] [--max-history MAX_HISTORY] [-c CONFIG] - [--fail-on-warnings] [-d DOMAIN] [--data DATA [DATA ...]]""" + [--fail-on-warnings] [-d DOMAIN] + [--data DATA [DATA ...]] + {stories} ...""" lines = help_text.split("\n") # expected help text lines should appear somewhere in the output diff --git a/tests/cli/test_rasa_x.py b/tests/cli/test_rasa_x.py index 5a64dd67951f..d6322fdf4c38 100644 --- a/tests/cli/test_rasa_x.py +++ b/tests/cli/test_rasa_x.py @@ -16,8 +16,8 @@ def test_x_help(run: Callable[..., RunResult]): output = run("x", "--help") - help_text = """usage: rasa x [-h] [-v] [-vv] [--quiet] [-m MODEL] [--data DATA] [-c CONFIG] - [-d DOMAIN] [--no-prompt] [--production] + help_text = """usage: rasa x [-h] [-v] [-vv] [--quiet] [-m MODEL] [--data DATA [DATA ...]] + [-c CONFIG] [-d DOMAIN] [--no-prompt] [--production] [--rasa-x-port RASA_X_PORT] [--config-endpoint CONFIG_ENDPOINT] [--log-file LOG_FILE] [--endpoints ENDPOINTS] [-p PORT] [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]] [--enable-api] From 189fe7a136e54c7c51ae3a1254784398351f4c19 Mon Sep 17 00:00:00 2001 From: martasls Date: Fri, 7 May 2021 14:40:28 +0100 Subject: [PATCH 05/11] Add missing import --- rasa/cli/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/cli/data.py b/rasa/cli/data.py index 6343a9565f65..fc9189368445 100644 --- a/rasa/cli/data.py +++ b/rasa/cli/data.py @@ -3,7 +3,7 @@ import os import shutil from pathlib import Path -from typing import Dict, List, Text, TYPE_CHECKING +from typing import Dict, Union, List, Text, TYPE_CHECKING import rasa.shared.core.domain from rasa import telemetry From 4411a6c1d7051b0a6c3dda0ee161fd7dd2921bbf Mon Sep 17 00:00:00 2001 From: martasls Date: Sat, 8 May 2021 12:26:46 +0100 Subject: [PATCH 06/11] Check date_file type on convert_training_data --- rasa/nlu/convert.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index 80b6f8e04931..861ab9e0c7ab 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -1,6 +1,6 @@ import argparse import os -from typing import Text +from typing import Text, Union from rasa.shared.utils.cli import print_error import rasa.shared.nlu.training_data.loading @@ -8,10 +8,11 @@ def convert_training_data( - data_file: list, out_file: Text, output_format: Text, language: Text + data_file: Union[list, Text], out_file: Text, output_format: Text, language: Text ) -> None: - data_file = data_file[0] + if isinstance(data_file, list): + data_file = data_file[0] if not os.path.exists(data_file): print_error( From 69e34bc34d030bde5f5448605171034b39f152be Mon Sep 17 00:00:00 2001 From: martasls Date: Wed, 12 May 2021 11:26:23 +0100 Subject: [PATCH 07/11] Add docstrings to convert_training_data method --- rasa/nlu/convert.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index 861ab9e0c7ab..a2e6304436c7 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -10,6 +10,14 @@ def convert_training_data( data_file: Union[list, Text], out_file: Text, output_format: Text, language: Text ) -> None: + """Convert training data. + + Args: + data_file (Union[list, Text]): Path to the file or directory containing Rasa data. + out_file (Text): File or existing path where to save training data in Rasa format. + output_format (Text): Output fortmat the training data should be converted into. + language (Text): Language of the data. + """ if isinstance(data_file, list): data_file = data_file[0] From 5d35f6c4f152cef3111b9c8b841d2e2f99c05253 Mon Sep 17 00:00:00 2001 From: martasls Date: Wed, 12 May 2021 15:16:31 +0100 Subject: [PATCH 08/11] Update linting --- rasa/nlu/convert.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index a2e6304436c7..f27c1dc8b151 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -13,9 +13,12 @@ def convert_training_data( """Convert training data. Args: - data_file (Union[list, Text]): Path to the file or directory containing Rasa data. - out_file (Text): File or existing path where to save training data in Rasa format. - output_format (Text): Output fortmat the training data should be converted into. + data_file (Union[list, Text]): Path to the file or directory + containing Rasa data. + out_file (Text): File or existing path where to save + training data in Rasa format. + output_format (Text): Output format the training data + should be converted into. language (Text): Language of the data. """ From eaca48798afdacacae9681a98150d93751a60bd0 Mon Sep 17 00:00:00 2001 From: martasls Date: Wed, 12 May 2021 15:57:01 +0100 Subject: [PATCH 09/11] Remove other (binary) file --- other | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 other diff --git a/other b/other deleted file mode 100644 index b828e50def9ba0630d1273e3a166363988559be0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI#O-sWt7zgmQlgSFc+{SLtaRu?<*xfqgEY?nIl?@Li+6V#LjV;XWD1I6JmZsJX z#kmhK{(-zanS(afzpzmhT}|VYO#H0S?rjIF=m@yF@4dBcxw@u+4`~mZ)vl` z?)#3}wRb<*Ueyd}5P$##AOHafKmY;|fB*y_0D*M^_ja}3YFKlXr*fudiJZz@Pm+l= z@im8sNRo^@YetnGTEVZHfUJUq)Xp&ae+im`VR@a0jb zrc(2fn0I+ZVK1ieHV6*LsV!^Lofmb^y{LKAZo9uy^n4_fTx4=hipu0fi)=(^LC=ee z!YbG1!hg>dd*V^6VZM4V60jIbd-v1l?hjBFsKmY;|fB*y_009U<00Izz0Aru_>Tjn2 From a6020fc749c87906c0cbb93a7cd6de16fe0d0b1a Mon Sep 17 00:00:00 2001 From: martasls Date: Wed, 12 May 2021 16:24:13 +0100 Subject: [PATCH 10/11] Remove blank line after docstrings --- rasa/nlu/convert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index f27c1dc8b151..0639b09986a4 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -21,7 +21,6 @@ def convert_training_data( should be converted into. language (Text): Language of the data. """ - if isinstance(data_file, list): data_file = data_file[0] From abf0eefbbac5f3104b6ba920371a8e2f619fdbaf Mon Sep 17 00:00:00 2001 From: martasls Date: Wed, 12 May 2021 18:15:31 +0100 Subject: [PATCH 11/11] Make sure data_file is a string when passed to os.path.exists() --- rasa/nlu/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/nlu/convert.py b/rasa/nlu/convert.py index 0639b09986a4..30d1d4343e77 100644 --- a/rasa/nlu/convert.py +++ b/rasa/nlu/convert.py @@ -24,7 +24,7 @@ def convert_training_data( if isinstance(data_file, list): data_file = data_file[0] - if not os.path.exists(data_file): + if not os.path.exists(str(data_file)): print_error( "Data file '{}' does not exist. Provide a valid NLU data file using " "the '--data' argument.".format(data_file)