Merge pull request #7499 from RasaHQ/merge_2.1.3_to_master

Merge 2.1.3 to master

CHANGELOG.mdx

@@ -16,6 +16,41 @@ https://github.com/RasaHQ/rasa/tree/master/changelog/ . -->
 
 <!-- TOWNCRIER -->
 
+## [2.1.3] - 2020-12-04
+
+
+### Improvements
+- [#7426](https://github.com/rasahq/rasa/issues/7426): Removed `multidict` from the project dependencies. `multidict` continues to be a second
+  order dependency of Rasa Open Source but will be determined by the dependencies which
+  use it instead of by Rasa Open Source directly.
+
+  This resolves issues like the following:
+
+  ```bash
+  sanic 20.9.1 has requirement multidict==5.0.0, but you'll have multidict 4.6.0 which is incompatible.
+  ```
+
+### Bugfixes
+- [#7316](https://github.com/rasahq/rasa/issues/7316): `SingleStateFeaturizer` checks whether it was trained with `RegexInterpreter` as
+  nlu interpreter. If that is the case, `RegexInterpreter` is used during prediction.
+- [#7390](https://github.com/rasahq/rasa/issues/7390): Make sure the `responses` are synced between NLU training data and the Domain even if there're no retrieval intents in the NLU training data.
+- [#7417](https://github.com/rasahq/rasa/issues/7417): Categorical slots will have a default value set when just updating nlg data in the domain.
+
+  Previously this resulted in `InvalidDomain` being thrown.
+- [#7418](https://github.com/rasahq/rasa/issues/7418): - Preserve `domain` slot ordering while dumping it back to the file.
+  - Preserve multiline `text` examples of `responses` defined in `domain` and `NLU` training data.
+
+
+## [2.1.2] - 2020-11-27
+
+
+### Bugfixes
+- [#7235](https://github.com/rasahq/rasa/issues/7235): Slots that use `initial_value` won't cause rule contradiction errors when `conversation_start: true` is used. Previously, two rules that differed only in their use of `conversation_start` would be flagged as contradicting when a slot used `initial_value`.
+
+  In checking for incomplete rules, an action will be required to have set _only_ those slots that the same action has set in another rule. Previously, an action was expected to have set also slots which, despite being present after this action in another rule, were not actually set by this action.
+- [#7345](https://github.com/rasahq/rasa/issues/7345): Fixed Rasa Open Source not being able to fetch models from certain URLs.
+
+
 ## [2.1.1] - 2020-11-23
 
 

data/test_domains/domain_with_categorical_slot.yml

@@ -0,0 +1,10 @@
+slots:
+  category_slot:
+    type: categorical
+    values:
+      - value_one
+      - value_two
+
+responses:
+  utter_greet:
+    - text: "hey there!"

data/test_nlg/test_responses.yml

@@ -0,0 +1,3 @@
+responses:
+  utter_rasa:
+    - text: this is utter_rasa!

rasa/core/agent.py

@@ -355,9 +355,7 @@ def __init__(
         self.policy_ensemble = self._create_ensemble(policies)
 
         if self.domain is not None:
-            self.domain.add_requested_slot()
-            self.domain.add_knowledge_base_slots()
-            self.domain.add_categorical_slot_default_value()
+            self.domain.setup_slots()
 
         PolicyEnsemble.check_domain_ensemble_compatibility(
             self.policy_ensemble, self.domain

rasa/model.py

@@ -315,7 +315,7 @@ async def model_fingerprint(file_importer: "TrainingDataImporter") -> Fingerprin
     domain = copy.copy(domain)
     # don't include the response texts in the fingerprint.
     # Their fingerprint is separate.
-    domain.templates = []
+    domain.templates = {}
 
     return {
         FINGERPRINT_CONFIG_KEY: _get_fingerprint_of_config(
@@ -518,5 +518,5 @@ async def update_model_with_new_domain(
 
     model_path = Path(unpacked_model_path) / DEFAULT_CORE_SUBDIRECTORY_NAME
     domain = await importer.get_domain()
-
+    domain.setup_slots()
     domain.persist(model_path / DEFAULT_DOMAIN_PATH)

rasa/shared/core/domain.py

@@ -52,6 +52,7 @@
 KEY_ACTIONS = "actions"
 KEY_FORMS = "forms"
 KEY_E2E_ACTIONS = "e2e_actions"
+KEY_RESPONSES_TEXT = "text"
 
 ALL_DOMAIN_KEYS = [
     KEY_SLOTS,
@@ -286,8 +287,8 @@ def collect_slots(slot_dict: Dict[Text, Any]) -> List[Slot]:
         slots = []
         # make a copy to not alter the input dictionary
         slot_dict = copy.deepcopy(slot_dict)
-        # Sort the slots so that the order of the slot states is consistent
-        for slot_name in sorted(slot_dict):
+        # Don't sort the slots, see https://github.com/RasaHQ/rasa-x/issues/3900
+        for slot_name in slot_dict:
             slot_type = slot_dict[slot_name].pop("type", None)
             slot_class = Slot.resolve_by_type(slot_type)
 
@@ -683,6 +684,12 @@ def is_retrieval_intent_template(
         """
         return rasa.shared.nlu.constants.RESPONSE_IDENTIFIER_DELIMITER in template[0]
 
+    def setup_slots(self) -> None:
+        """Sets up the default slots and slot values for the domain."""
+        self.add_requested_slot()
+        self.add_knowledge_base_slots()
+        self.add_categorical_slot_default_value()
+
     def add_categorical_slot_default_value(self) -> None:
         """Add a default value to all categorical slots.
 
@@ -1051,6 +1058,33 @@ def as_dict(self) -> Dict[Text, Any]:
             KEY_E2E_ACTIONS: self.action_texts,
         }
 
+    @staticmethod
+    def get_responses_with_multilines(
+        responses: Dict[Text, List[Dict[Text, Any]]]
+    ) -> Dict[Text, List[Dict[Text, Any]]]:
+        """Returns `responses` with preserved multilines in the `text` key.
+
+        Args:
+            responses: Original `responses`.
+
+        Returns:
+            `responses` with preserved multilines in the `text` key.
+        """
+        from ruamel.yaml.scalarstring import LiteralScalarString
+
+        final_responses = responses.copy()
+        for response_name, examples in final_responses.items():
+            for i, example in enumerate(examples):
+                response_text = example.get(KEY_RESPONSES_TEXT, "")
+                if not response_text or "\n" not in response_text:
+                    continue
+                # Has new lines, use `LiteralScalarString`
+                final_responses[response_name][i][
+                    KEY_RESPONSES_TEXT
+                ] = LiteralScalarString(response_text)
+
+        return final_responses
+
     def _transform_intents_for_file(self) -> List[Union[Text, Dict[Text, Any]]]:
         """Transform intent properties for displaying or writing into a domain file.
 
@@ -1194,6 +1228,10 @@ def as_yaml(self, clean_before_dump: bool = False) -> Text:
             domain_data.update(self.cleaned_domain())
         else:
             domain_data.update(self.as_dict())
+        if domain_data.get(KEY_RESPONSES, {}):
+            domain_data[KEY_RESPONSES] = self.get_responses_with_multilines(
+                domain_data[KEY_RESPONSES]
+            )
 
         return rasa.shared.utils.io.dump_obj_as_yaml_to_string(
             domain_data, should_preserve_key_order=True

rasa/shared/importers/importer.py

@@ -153,7 +153,7 @@ def load_from_dict(
                 )
             ]
 
-        return E2EImporter(RetrievalModelsDataImporter(CombinedDataImporter(importers)))
+        return E2EImporter(ResponsesSyncImporter(CombinedDataImporter(importers)))
 
     @staticmethod
     def _importer_from_dict(
@@ -293,8 +293,8 @@ async def get_nlu_data(self, language: Optional[Text] = "en") -> TrainingData:
         )
 
 
-class RetrievalModelsDataImporter(TrainingDataImporter):
-    """A `TrainingDataImporter` that sets up the data for training retrieval models.
+class ResponsesSyncImporter(TrainingDataImporter):
+    """Importer that syncs `responses` between Domain and NLU training data.
 
     Synchronizes response templates between Domain and NLU
     and adds retrieval intent properties from the NLU training data
@@ -314,19 +314,18 @@ async def get_domain(self) -> Domain:
         existing_domain = await self._importer.get_domain()
         existing_nlu_data = await self._importer.get_nlu_data()
 
-        # Check if NLU data has any retrieval intents, if yes
-        # add corresponding retrieval actions with `utter_` prefix automatically
-        # to an empty domain, update the properties of existing retrieval intents
-        # and merge response templates
-        if existing_nlu_data.retrieval_intents:
-
-            domain_with_retrieval_intents = self._get_domain_with_retrieval_intents(
-                existing_nlu_data.retrieval_intents,
-                existing_nlu_data.responses,
-                existing_domain,
-            )
+        # Merge responses from NLU data with responses in the domain.
+        # If NLU data has any retrieval intents, then add corresponding
+        # retrieval actions with `utter_` prefix automatically to the
+        # final domain, update the properties of existing retrieval intents.
+        domain_with_retrieval_intents = self._get_domain_with_retrieval_intents(
+            existing_nlu_data.retrieval_intents,
+            existing_nlu_data.responses,
+            existing_domain,
+        )
 
-            existing_domain = existing_domain.merge(domain_with_retrieval_intents)
+        existing_domain = existing_domain.merge(domain_with_retrieval_intents)
+        existing_domain.check_missing_templates()
 
         return existing_domain
 
@@ -351,16 +350,19 @@ def _get_domain_with_retrieval_intents(
         response_templates: Dict[Text, List[Dict[Text, Any]]],
         existing_domain: Domain,
     ) -> Domain:
-        """Construct a domain consisting of retrieval intents listed in the NLU training data.
+        """Construct a domain consisting of retrieval intents.
+
+         The result domain will have retrieval intents that are listed
+         in the NLU training data.
 
         Args:
             retrieval_intents: Set of retrieval intents defined in NLU training data.
+            response_templates: Response templates defined in NLU training data.
             existing_domain: Domain which is already loaded from the domain file.
 
         Returns: Domain with retrieval actions added to action names and properties
-        for retrieval intents updated.
+          for retrieval intents updated.
         """
-
         # Get all the properties already defined
         # for each retrieval intent in other domains
         # and add the retrieval intent property to them
@@ -379,9 +381,7 @@ def _get_domain_with_retrieval_intents(
             [],
             [],
             response_templates,
-            RetrievalModelsDataImporter._construct_retrieval_action_names(
-                retrieval_intents
-            ),
+            ResponsesSyncImporter._construct_retrieval_action_names(retrieval_intents),
             {},
         )
 

rasa/shared/importers/rasa.py

@@ -65,7 +65,6 @@ async def get_domain(self) -> Domain:
             return domain
         try:
             domain = Domain.load(self._domain_path)
-            domain.check_missing_templates()
         except InvalidDomain as e:
             rasa.shared.utils.io.raise_warning(
                 f"Loading domain from '{self._domain_path}' failed. Using "

rasa/shared/nlu/training_data/formats/rasa_yaml.py

@@ -4,6 +4,7 @@
 from typing import Text, Any, List, Dict, Tuple, Union, Iterator, Optional
 
 import rasa.shared.data
+from rasa.shared.core.domain import Domain
 from rasa.shared.exceptions import YamlException
 from rasa.shared.utils import validation
 from ruamel.yaml import StringIO
@@ -409,7 +410,9 @@ def training_data_to_dict(
             result[KEY_NLU] = nlu_items
 
         if training_data.responses:
-            result[KEY_RESPONSES] = training_data.responses
+            result[KEY_RESPONSES] = Domain.get_responses_with_multilines(
+                training_data.responses
+            )
 
         return result
 

tests/conftest.py

@@ -35,6 +35,7 @@
     DEFAULT_DOMAIN_PATH_WITH_SLOTS,
     DEFAULT_STACK_CONFIG,
     DEFAULT_STORIES_FILE,
+    DOMAIN_WITH_CATEGORICAL_SLOT,
     END_TO_END_STORY_FILE,
     INCORRECT_NLU_DATA,
 )
@@ -125,6 +126,11 @@ def default_domain_path() -> Text:
     return DEFAULT_DOMAIN_PATH_WITH_SLOTS
 
 
+@pytest.fixture(scope="session")
+def domain_with_categorical_slot_path() -> Text:
+    return DOMAIN_WITH_CATEGORICAL_SLOT
+
+
 @pytest.fixture(scope="session")
 def default_domain() -> Domain:
     return Domain.load(DEFAULT_DOMAIN_PATH_WITH_SLOTS)

tests/core/conftest.py

@@ -24,6 +24,8 @@
 
 DEFAULT_DOMAIN_PATH_WITH_SLOTS = "data/test_domains/default_with_slots.yml"
 
+DOMAIN_WITH_CATEGORICAL_SLOT = "data/test_domains/domain_with_categorical_slot.yml"
+
 DEFAULT_DOMAIN_PATH_WITH_SLOTS_AND_NO_ACTIONS = (
     "data/test_domains/default_with_slots_and_no_actions.yml"
 )

tests/core/test_model.py

@@ -418,3 +418,28 @@ async def get_domain() -> Domain:
     actual = Domain.load(tmpdir / DEFAULT_CORE_SUBDIRECTORY_NAME / DEFAULT_DOMAIN_PATH)
 
     assert actual.is_empty()
+
+
+async def test_update_with_new_domain_preserves_domain(
+    tmpdir: Path, domain_with_categorical_slot_path
+):
+    domain = Domain.load(domain_with_categorical_slot_path)
+    domain.setup_slots()
+
+    core_directory = tmpdir / DEFAULT_CORE_SUBDIRECTORY_NAME
+    core_directory.mkdir()
+
+    domain.persist(str(core_directory / DEFAULT_DOMAIN_PATH))
+    domain.persist_specification(core_directory)
+
+    mocked_importer = Mock()
+
+    async def get_domain() -> Domain:
+        return Domain.load(domain_with_categorical_slot_path)
+
+    mocked_importer.get_domain = get_domain
+
+    await model.update_model_with_new_domain(mocked_importer, tmpdir)
+
+    new_persisted = Domain.load(core_directory / DEFAULT_DOMAIN_PATH)
+    new_persisted.compare_with_specification(str(core_directory))