Add support for different recipes

data/test_config/config_response_selector_minimal.yml

@@ -5,3 +5,16 @@ pipeline:
    - name: CountVectorsFeaturizer
    - name: ResponseSelector
      epochs: 1
+policies:
+# # No configuration for policies was provided. The following default policies were used to train your model.
+# # If you'd like to customize them, uncomment and adjust the policies.
+# # See https://rasa.com/docs/rasa/policies for more information.
+#   - name: MemoizationPolicy
+#   - name: RulePolicy
+#   - name: UnexpecTEDIntentPolicy
+#     max_history: 5
+#     epochs: 100
+#   - name: TEDPolicy
+#     max_history: 5
+#     epochs: 100
+#     constrain_similarities: true

rasa/cli/interactive.py

@@ -10,7 +10,7 @@
 import rasa.cli.utils
 from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH, DEFAULT_MODELS_PATH
-from rasa.shared.importers.autoconfig import TrainingType
+from rasa.shared.data import TrainingType
 from rasa.shared.importers.importer import TrainingDataImporter
 import rasa.shared.utils.cli
 import rasa.utils.common

rasa/cli/shell.py

@@ -9,7 +9,7 @@
 from rasa.cli.arguments import shell as arguments
 from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.model import get_latest_model
-from rasa.shared.importers.autoconfig import TrainingType
+from rasa.shared.data import TrainingType
 from rasa.shared.utils.cli import print_error
 from rasa.exceptions import ModelNotFound
 

rasa/core/processor.py

@@ -14,7 +14,7 @@
 from rasa.engine.storage.local_model_storage import LocalModelStorage
 from rasa.engine.storage.storage import ModelMetadata
 from rasa.model import get_latest_model
-from rasa.shared.importers.autoconfig import TrainingType
+from rasa.shared.data import TrainingType
 import rasa.shared.utils.io
 import rasa.core.actions.action
 from rasa.core import jobs

rasa/engine/graph.py

@@ -16,7 +16,7 @@
 
 from rasa.engine.storage.storage import ModelStorage
 from rasa.shared.exceptions import InvalidConfigException, RasaException
-from rasa.shared.importers.autoconfig import TrainingType
+from rasa.shared.data import TrainingType
 
 logger = logging.getLogger(__name__)
 

rasa/engine/recipes/default_recipe.py

@@ -13,6 +13,7 @@
     CoreFeaturizationInputConverter,
     CoreFeaturizationCollector,
 )
+from rasa.shared.exceptions import FileNotFoundException
 from rasa.core.policies.ensemble import DefaultPolicyPredictionEnsemble
 
 from rasa.engine.graph import (
@@ -41,17 +42,36 @@
 from rasa.graph_components.providers.training_tracker_provider import (
     TrainingTrackerProvider,
 )
+import rasa.shared.constants
 from rasa.shared.exceptions import RasaException, InvalidConfigException
-from rasa.shared.importers.autoconfig import TrainingType
+from rasa.shared.data import TrainingType
 
 from rasa.utils.tensorflow.constants import EPOCHS
-import rasa.shared.utils.common
+from rasa.shared.utils.common import (
+    class_from_module_path,
+    transform_collection_to_sentence,
+)
 
 logger = logging.getLogger(__name__)
 
 
 DEFAULT_PREDICT_KWARGS = dict(constructor_name="load", eager=True, is_target=False,)
 
+COMMENTS_FOR_KEYS = {
+    "pipeline": (
+        f"# # No configuration for the NLU pipeline was provided. The following "
+        f"default pipeline was used to train your model.\n"
+        f"# # If you'd like to customize it, uncomment and adjust the pipeline.\n"
+        f"# # See {rasa.shared.constants.DOCS_URL_PIPELINE} for more information.\n"
+    ),
+    "policies": (
+        f"# # No configuration for policies was provided. The following default "
+        f"policies were used to train your model.\n"
+        f"# # If you'd like to customize them, uncomment and adjust the policies.\n"
+        f"# # See {rasa.shared.constants.DOCS_URL_POLICIES} for more information.\n"
+    ),
+}
+
 
 class DefaultV1RecipeRegisterException(RasaException):
     """If you register a class which is not of type `GraphComponent`."""
@@ -144,7 +164,7 @@ def _from_registry(cls, name: Text) -> RegisteredComponent:
             return cls._registered_components[name]
 
         if "." in name:
-            clazz = rasa.shared.utils.common.class_from_module_path(name)
+            clazz = class_from_module_path(name)
             if clazz.__name__ in cls._registered_components:
                 return cls._registered_components[clazz.__name__]
 
@@ -816,3 +836,250 @@ def _add_end_to_end_features_for_inference(
             config={},
         )
         return node_with_e2e_features
+
+    @staticmethod
+    def auto_configure(
+        config_file_path: Optional[Text],
+        config: Dict,
+        training_type: Optional[TrainingType] = TrainingType.BOTH,
+    ) -> Dict[Text, Any]:
+        """Determine configuration from a configuration file.
+
+        Keys that are provided and have a value in the file are kept. Keys that are not
+        provided are configured automatically.
+
+        Args:
+            config_file_path: The path to the configuration file.
+            config: Configuration in dictionary format.
+            training_type: Optional training type to auto-configure. By default
+            both core and NLU will be auto-configured.
+        """
+        missing_keys = DefaultV1Recipe._get_missing_config_keys(config, training_type)
+        keys_to_configure = DefaultV1Recipe._get_unspecified_autoconfigurable_keys(
+            config, training_type
+        )
+
+        if keys_to_configure:
+            config = DefaultV1Recipe.complete_config(config, keys_to_configure)
+            DefaultV1Recipe._dump_config(
+                config, config_file_path, missing_keys, keys_to_configure, training_type
+            )
+
+        return config
+
+    @staticmethod
+    def _get_unspecified_autoconfigurable_keys(
+        config: Dict[Text, Any],
+        training_type: Optional[TrainingType] = TrainingType.BOTH,
+    ) -> Set[Text]:
+        if training_type == TrainingType.NLU:
+            all_keys = rasa.shared.constants.CONFIG_AUTOCONFIGURABLE_KEYS_NLU
+        elif training_type == TrainingType.CORE:
+            all_keys = rasa.shared.constants.CONFIG_AUTOCONFIGURABLE_KEYS_CORE
+        else:
+            all_keys = rasa.shared.constants.CONFIG_AUTOCONFIGURABLE_KEYS
+
+        return {k for k in all_keys if config.get(k) is None}
+
+    @staticmethod
+    def _get_missing_config_keys(
+        config: Dict[Text, Any],
+        training_type: Optional[TrainingType] = TrainingType.BOTH,
+    ) -> Set[Text]:
+        if training_type == TrainingType.NLU:
+            all_keys = rasa.shared.constants.CONFIG_KEYS_NLU
+        elif training_type == TrainingType.CORE:
+            all_keys = rasa.shared.constants.CONFIG_KEYS_CORE
+        else:
+            all_keys = rasa.shared.constants.CONFIG_KEYS
+
+        return {k for k in all_keys if k not in config.keys()}
+
+    @staticmethod
+    def complete_config(
+        config: Dict[Text, Any], keys_to_configure: Set[Text]
+    ) -> Dict[Text, Any]:
+        """Complete a config by adding automatic configuration for the specified keys.
+
+        Args:
+            config: The provided configuration.
+            keys_to_configure: Keys to be configured automatically (e.g. `policies`).
+
+        Returns:
+            The resulting configuration including both the provided and
+            the automatically configured keys.
+        """
+        import pkg_resources
+
+        if keys_to_configure:
+            logger.debug(
+                f"The provided configuration does not contain the key(s) "
+                f"{transform_collection_to_sentence(keys_to_configure)}. "  # noqa: E501, W505
+                f"Values will be provided from the default configuration."
+            )
+
+        filename = "config_files/default_config.yml"
+
+        default_config_file = pkg_resources.resource_filename(__name__, filename)
+        default_config = rasa.shared.utils.io.read_config_file(default_config_file)
+
+        config = copy.deepcopy(config)
+        for key in keys_to_configure:
+            config[key] = default_config[key]
+
+        return config
+
+    @staticmethod
+    def _dump_config(
+        config: Dict[Text, Any],
+        config_file_path: Text,
+        missing_keys: Set[Text],
+        auto_configured_keys: Set[Text],
+        training_type: Optional[TrainingType] = TrainingType.BOTH,
+    ) -> None:
+        """Dump the automatically configured keys into the config file.
+
+        The configuration provided in the file is kept as it is (preserving the order of
+        keys and comments).
+        For keys that were automatically configured, an explanatory
+        comment is added and the automatically chosen configuration is
+        added commented-out.
+        If there are already blocks with comments from a previous auto
+        configuration run, they are replaced with the new auto
+        configuration.
+
+        Args:
+            config: The configuration including the automatically configured keys.
+            config_file_path: The file into which the configuration should be dumped.
+            missing_keys: Keys that need to be added to the config file.
+            auto_configured_keys: Keys for which a commented out auto
+            configuration section needs to be added to the config file.
+            training_type: NLU, CORE or BOTH depending on which is trained.
+        """
+        config_as_expected = DefaultV1Recipe._is_config_file_as_expected(
+            config_file_path, missing_keys, auto_configured_keys, training_type
+        )
+        if not config_as_expected:
+            rasa.shared.utils.cli.print_error(
+                f"The configuration file at '{config_file_path}' has been removed or "
+                f"modified while the automatic configuration was running. The current "
+                f"configuration will therefore not be dumped to the file. If you want "
+                f"your model to use the configuration provided in "
+                f"'{config_file_path}' you need to re-run training."
+            )
+            return
+
+        DefaultV1Recipe._add_missing_config_keys_to_file(config_file_path, missing_keys)
+
+        autoconfig_lines = DefaultV1Recipe._get_commented_out_autoconfig_lines(
+            config, auto_configured_keys
+        )
+
+        current_config_content = rasa.shared.utils.io.read_file(config_file_path)
+        current_config_lines = current_config_content.splitlines(keepends=True)
+
+        updated_lines = DefaultV1Recipe._get_lines_including_autoconfig(
+            current_config_lines, autoconfig_lines
+        )
+
+        rasa.shared.utils.io.write_text_file("".join(updated_lines), config_file_path)
+
+        auto_configured_keys = transform_collection_to_sentence(auto_configured_keys)
+        rasa.shared.utils.cli.print_info(
+            f"The configuration for {auto_configured_keys} was chosen automatically. "
+            "It was written into the config file at '{config_file_path}'."
+        )
+
+    @staticmethod
+    def _is_config_file_as_expected(
+        config_file_path: Text,
+        missing_keys: Set[Text],
+        auto_configured_keys: Set[Text],
+        training_type: Optional[TrainingType] = TrainingType.BOTH,
+    ) -> bool:
+        try:
+            content = rasa.shared.utils.io.read_config_file(config_file_path)
+        except FileNotFoundException:
+            content = ""
+
+        return (
+            bool(content)
+            and missing_keys
+            == DefaultV1Recipe._get_missing_config_keys(content, training_type)
+            and auto_configured_keys
+            == DefaultV1Recipe._get_unspecified_autoconfigurable_keys(
+                content, training_type
+            )
+        )
+
+    @staticmethod
+    def _add_missing_config_keys_to_file(
+        config_file_path: Text, missing_keys: Set[Text]
+    ) -> None:
+        if not missing_keys:
+            return
+        with open(
+            config_file_path, "a", encoding=rasa.shared.utils.io.DEFAULT_ENCODING
+        ) as f:
+            for key in missing_keys:
+                f.write(f"{key}:\n")
+
+    @staticmethod
+    def _get_lines_including_autoconfig(
+        lines: List[Text], autoconfig_lines: Dict[Text, List[Text]]
+    ) -> List[Text]:
+        auto_configured_keys = autoconfig_lines.keys()
+
+        lines_with_autoconfig = []
+        remove_comments_until_next_uncommented_line = False
+        for line in lines:
+            insert_section = None
+
+            # remove old auto configuration
+            if remove_comments_until_next_uncommented_line:
+                if line.startswith("#"):
+                    continue
+                remove_comments_until_next_uncommented_line = False
+
+            # add an explanatory comment to auto configured sections
+            for key in auto_configured_keys:
+                if line.startswith(f"{key}:"):  # start of next auto-section
+                    line = line + COMMENTS_FOR_KEYS[key]
+                    insert_section = key
+                    remove_comments_until_next_uncommented_line = True
+
+            lines_with_autoconfig.append(line)
+
+            if not insert_section:
+                continue
+
+            # add the auto configuration (commented out)
+            lines_with_autoconfig += autoconfig_lines[insert_section]
+
+        return lines_with_autoconfig
+
+    @staticmethod
+    def _get_commented_out_autoconfig_lines(
+        config: Dict[Text, Any], auto_configured_keys: Set[Text]
+    ) -> Dict[Text, List[Text]]:
+        import ruamel.yaml
+        import ruamel.yaml.compat
+
+        yaml_parser = ruamel.yaml.YAML()
+        yaml_parser.indent(mapping=2, sequence=4, offset=2)
+
+        autoconfig_lines = {}
+
+        for key in auto_configured_keys:
+            stream = ruamel.yaml.compat.StringIO()
+            yaml_parser.dump(config.get(key), stream)
+            dump = stream.getvalue()
+
+            lines = dump.split("\n")
+            if not lines[-1]:
+                lines = lines[:-1]  # yaml dump adds an empty line at the end
+            lines = [f"# {line}\n" for line in lines]
+
+            autoconfig_lines[key] = lines
+
+        return autoconfig_lines

rasa/engine/recipes/graph_recipe.py

@@ -0,0 +1,20 @@
+from rasa.engine.recipes.recipe import Recipe
+from rasa.engine.graph import GraphModelConfiguration
+from rasa.shared.data import TrainingType
+
+from typing import Dict, Text, Any
+
+
+class GraphV1Recipe(Recipe):
+    """Recipe which converts the graph model config to train and predict graph."""
+
+    name = "graph.v1"
+
+    def graph_config_for_recipe(
+        self,
+        config: Dict,
+        cli_parameters: Dict[Text, Any],
+        training_type: TrainingType = TrainingType.BOTH,
+        is_finetuning: bool = False,
+    ) -> GraphModelConfiguration:
+        """Converts the default config to graphs (see interface for full docstring)."""