diff --git a/rasa/core/actions/action.py b/rasa/core/actions/action.py index 28cb70fe2558..ac298f18a931 100644 --- a/rasa/core/actions/action.py +++ b/rasa/core/actions/action.py @@ -21,6 +21,7 @@ OPEN_UTTERANCE_PREDICTION_KEY, RESPONSE_SELECTOR_PROPERTY_NAME, INTENT_RANKING_KEY, + INTENT_NAME_KEY, ) from rasa.core.events import ( @@ -722,14 +723,14 @@ async def run( tracker: "DialogueStateTracker", domain: "Domain", ) -> List[Event]: - intent_to_affirm = tracker.latest_message.intent.get("name") + intent_to_affirm = tracker.latest_message.intent.get(INTENT_NAME_KEY) intent_ranking = tracker.latest_message.intent.get(INTENT_RANKING_KEY, []) if ( intent_to_affirm == DEFAULT_NLU_FALLBACK_INTENT_NAME and len(intent_ranking) > 1 ): - intent_to_affirm = intent_ranking[1]["name"] + intent_to_affirm = intent_ranking[1][INTENT_NAME_KEY] affirmation_message = f"Did you mean '{intent_to_affirm}'?" diff --git a/rasa/core/domain.py b/rasa/core/domain.py index a1e898f3d68d..4bf03c18f4c2 100644 --- a/rasa/core/domain.py +++ b/rasa/core/domain.py @@ -10,6 +10,7 @@ from ruamel.yaml import YAMLError import rasa.core.constants +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.common import ( raise_warning, lazy_property, @@ -675,7 +676,7 @@ def get_parsing_states(self, tracker: "DialogueStateTracker") -> Dict[Text, floa if not latest_message: return state_dict - intent_name = latest_message.intent.get("name") + intent_name = latest_message.intent.get(INTENT_NAME_KEY) if intent_name: for entity_name in self._get_featurized_entities(latest_message): @@ -699,18 +700,18 @@ def get_parsing_states(self, tracker: "DialogueStateTracker") -> Dict[Text, floa if "intent_ranking" in latest_message.parse_data: for intent in latest_message.parse_data["intent_ranking"]: - if intent.get("name"): - intent_id = "intent_{}".format(intent["name"]) + if intent.get(INTENT_NAME_KEY): + intent_id = "intent_{}".format(intent[INTENT_NAME_KEY]) state_dict[intent_id] = intent["confidence"] elif intent_name: - intent_id = "intent_{}".format(latest_message.intent["name"]) + intent_id = "intent_{}".format(latest_message.intent[INTENT_NAME_KEY]) state_dict[intent_id] = latest_message.intent.get("confidence", 1.0) return state_dict def _get_featurized_entities(self, latest_message: UserUttered) -> Set[Text]: - intent_name = latest_message.intent.get("name") + intent_name = latest_message.intent.get(INTENT_NAME_KEY) intent_config = self.intent_config(intent_name) entities = latest_message.entities entity_names = { diff --git a/rasa/core/events/__init__.py b/rasa/core/events/__init__.py index 1083562bdfe1..4437cbe2945e 100644 --- a/rasa/core/events/__init__.py +++ b/rasa/core/events/__init__.py @@ -18,6 +18,7 @@ EXTERNAL_MESSAGE_PREFIX, ACTION_NAME_SENDER_ID_CONNECTOR_STR, ) +from rasa.nlu.constants import INTENT_NAME_KEY if typing.TYPE_CHECKING: from rasa.core.trackers import DialogueStateTracker @@ -258,7 +259,11 @@ def _from_parse_data( def __hash__(self) -> int: return hash( - (self.text, self.intent.get("name"), jsonpickle.encode(self.entities)) + ( + self.text, + self.intent.get(INTENT_NAME_KEY), + jsonpickle.encode(self.entities), + ) ) def __eq__(self, other) -> bool: @@ -267,11 +272,11 @@ def __eq__(self, other) -> bool: else: return ( self.text, - self.intent.get("name"), + self.intent.get(INTENT_NAME_KEY), [jsonpickle.encode(ent) for ent in self.entities], ) == ( other.text, - other.intent.get("name"), + other.intent.get(INTENT_NAME_KEY), [jsonpickle.encode(ent) for ent in other.entities], ) @@ -324,11 +329,11 @@ def as_story_string(self, e2e: bool = False) -> Text: ent_string = "" parse_string = "{intent}{entities}".format( - intent=self.intent.get("name", ""), entities=ent_string + intent=self.intent.get(INTENT_NAME_KEY, ""), entities=ent_string ) if e2e: message = md_format_message(self.text, self.intent, self.entities) - return "{}: {}".format(self.intent.get("name"), message) + return "{}: {}".format(self.intent.get(INTENT_NAME_KEY), message) else: return parse_string else: @@ -344,7 +349,7 @@ def create_external( ) -> "UserUttered": return UserUttered( text=f"{EXTERNAL_MESSAGE_PREFIX}{intent_name}", - intent={"name": intent_name}, + intent={INTENT_NAME_KEY: intent_name}, metadata={IS_EXTERNAL: True}, entities=entity_list or [], ) diff --git a/rasa/core/interpreter.py b/rasa/core/interpreter.py index d283aef27d24..b0ac7395091b 100644 --- a/rasa/core/interpreter.py +++ b/rasa/core/interpreter.py @@ -11,6 +11,7 @@ from rasa.core import constants from rasa.core.trackers import DialogueStateTracker from rasa.core.constants import INTENT_MESSAGE_PREFIX +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.common import raise_warning, class_from_module_path from rasa.utils.endpoints import EndpointConfig @@ -171,8 +172,8 @@ def synchronous_parse( return { "text": message_text, - "intent": {"name": intent, "confidence": confidence}, - "intent_ranking": [{"name": intent, "confidence": confidence}], + "intent": {INTENT_NAME_KEY: intent, "confidence": confidence}, + "intent_ranking": [{INTENT_NAME_KEY: intent, "confidence": confidence}], "entities": entities, } @@ -195,7 +196,7 @@ async def parse( Return a default value if the parsing of the text failed.""" default_return = { - "intent": {"name": "", "confidence": 0.0}, + "intent": {INTENT_NAME_KEY: "", "confidence": 0.0}, "entities": [], "text": "", } diff --git a/rasa/core/policies/mapping_policy.py b/rasa/core/policies/mapping_policy.py index 8f04f6675c11..6749eeb64796 100644 --- a/rasa/core/policies/mapping_policy.py +++ b/rasa/core/policies/mapping_policy.py @@ -6,6 +6,7 @@ from rasa.constants import DOCS_URL_POLICIES, DOCS_URL_MIGRATION_GUIDE import rasa.utils.io +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils import common as common_utils from rasa.core.actions.action import ( @@ -108,7 +109,7 @@ def predict_action_probabilities( result = self._default_predictions(domain) - intent = tracker.latest_message.intent.get("name") + intent = tracker.latest_message.intent.get(INTENT_NAME_KEY) if intent == USER_INTENT_RESTART: action = ACTION_RESTART_NAME elif intent == USER_INTENT_BACK: diff --git a/rasa/core/policies/two_stage_fallback.py b/rasa/core/policies/two_stage_fallback.py index 9bbddd42fa13..3ba31c13386f 100644 --- a/rasa/core/policies/two_stage_fallback.py +++ b/rasa/core/policies/two_stage_fallback.py @@ -22,6 +22,7 @@ from rasa.core.policies.policy import confidence_scores_for from rasa.core.trackers import DialogueStateTracker from rasa.core.constants import FALLBACK_POLICY_PRIORITY +from rasa.nlu.constants import INTENT_NAME_KEY if typing.TYPE_CHECKING: from rasa.core.policies.ensemble import PolicyEnsemble @@ -121,7 +122,7 @@ def predict_action_probabilities( """Predicts the next action if NLU confidence is low.""" nlu_data = tracker.latest_message.parse_data - last_intent_name = nlu_data["intent"].get("name", None) + last_intent_name = nlu_data["intent"].get(INTENT_NAME_KEY, None) should_nlu_fallback = self.should_nlu_fallback( nlu_data, tracker.latest_action_name ) diff --git a/rasa/core/processor.py b/rasa/core/processor.py index 7dc842b44b33..d8561b78906c 100644 --- a/rasa/core/processor.py +++ b/rasa/core/processor.py @@ -51,6 +51,7 @@ from rasa.core.policies.ensemble import PolicyEnsemble from rasa.core.tracker_store import TrackerStore from rasa.core.trackers import DialogueStateTracker, EventVerbosity +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.common import raise_warning from rasa.utils.endpoints import EndpointConfig @@ -436,7 +437,7 @@ def _check_for_unseen_features(self, parse_data: Dict[Text, Any]) -> None: if not self.domain or self.domain.is_empty(): return - intent = parse_data["intent"]["name"] + intent = parse_data["intent"][INTENT_NAME_KEY] if intent: known_intents = self.domain.intents + DEFAULT_INTENTS if intent not in known_intents: @@ -520,7 +521,7 @@ async def _handle_message_with_tracker( def _should_handle_message(tracker: DialogueStateTracker): return ( not tracker.is_paused() - or tracker.latest_message.intent.get("name") == USER_INTENT_RESTART + or tracker.latest_message.intent.get(INTENT_NAME_KEY) == USER_INTENT_RESTART ) def is_action_limit_reached( diff --git a/rasa/core/tracker_store.py b/rasa/core/tracker_store.py index 33ab6b024d5f..f1668a423a8e 100644 --- a/rasa/core/tracker_store.py +++ b/rasa/core/tracker_store.py @@ -33,6 +33,7 @@ from rasa.core.events import SessionStarted from rasa.core.trackers import ActionExecuted, DialogueStateTracker, EventVerbosity import rasa.cli.utils as rasa_cli_utils +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.common import class_from_module_path, raise_warning, arguments_of from rasa.utils.endpoints import EndpointConfig import sqlalchemy as sa @@ -910,7 +911,9 @@ def save(self, tracker: DialogueStateTracker) -> None: for event in events: data = event.as_dict() - intent = data.get("parse_data", {}).get("intent", {}).get("name") + intent = ( + data.get("parse_data", {}).get("intent", {}).get(INTENT_NAME_KEY) + ) action = data.get("name") timestamp = data.get("timestamp") diff --git a/rasa/core/training/interactive.py b/rasa/core/training/interactive.py index 276792aab15f..3fc44db62bf2 100644 --- a/rasa/core/training/interactive.py +++ b/rasa/core/training/interactive.py @@ -12,6 +12,7 @@ from aiohttp import ClientError from colorclass import Color +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.nlu.training_data.loading import MARKDOWN, RASA from sanic import Sanic, response from sanic.exceptions import NotFound @@ -322,12 +323,19 @@ def _selection_choices_from_intent_prediction( ) -> List[Dict[Text, Any]]: """"Given a list of ML predictions create a UI choice list.""" - sorted_intents = sorted(predictions, key=lambda k: (-k["confidence"], k["name"])) + sorted_intents = sorted( + predictions, key=lambda k: (-k["confidence"], k[INTENT_NAME_KEY]) + ) choices = [] for p in sorted_intents: - name_with_confidence = f'{p.get("confidence"):03.2f} {p.get("name"):40}' - choice = {"name": name_with_confidence, "value": p.get("name")} + name_with_confidence = ( + f'{p.get("confidence"):03.2f} {p.get(INTENT_NAME_KEY):40}' + ) + choice = { + INTENT_NAME_KEY: name_with_confidence, + "value": p.get(INTENT_NAME_KEY), + } choices.append(choice) return choices @@ -416,15 +424,15 @@ async def _request_intent_from_user( predictions = latest_message.get("parse_data", {}).get("intent_ranking", []) - predicted_intents = {p["name"] for p in predictions} + predicted_intents = {p[INTENT_NAME_KEY] for p in predictions} for i in intents: if i not in predicted_intents: - predictions.append({"name": i, "confidence": 0.0}) + predictions.append({INTENT_NAME_KEY: i, "confidence": 0.0}) # convert intents to ui list and add as a free text alternative choices = [ - {"name": "", "value": OTHER_INTENT} + {INTENT_NAME_KEY: "", "value": OTHER_INTENT} ] + _selection_choices_from_intent_prediction(predictions) intent_name = await _request_selection_from_intents( @@ -433,11 +441,12 @@ async def _request_intent_from_user( if intent_name == OTHER_INTENT: intent_name = await _request_free_text_intent(conversation_id, endpoint) - selected_intent = {"name": intent_name, "confidence": 1.0} + selected_intent = {INTENT_NAME_KEY: intent_name, "confidence": 1.0} else: # returns the selected intent with the original probability value selected_intent = next( - (x for x in predictions if x["name"] == intent_name), {"name": None} + (x for x in predictions if x[INTENT_NAME_KEY] == intent_name), + {INTENT_NAME_KEY: None}, ) return selected_intent @@ -479,7 +488,7 @@ def colored(txt: Text, color: Text) -> Text: def format_user_msg(user_event: UserUttered, max_width: int) -> Text: intent = user_event.intent or {} - intent_name = intent.get("name", "") + intent_name = intent.get(INTENT_NAME_KEY, "") _confidence = intent.get("confidence", 1.0) _md = _as_md_message(user_event.parse_data) @@ -745,7 +754,9 @@ def _collect_messages(events: List[Dict[Text, Any]]) -> List[Message]: if event.get("event") == UserUttered.type_name: data = event.get("parse_data", {}) rasa_nlu_training_data_utils.remove_untrainable_entities_from(data) - msg = Message.build(data["text"], data["intent"]["name"], data["entities"]) + msg = Message.build( + data["text"], data["intent"][INTENT_NAME_KEY], data["entities"] + ) messages.append(msg) elif event.get("event") == UserUtteranceReverted.type_name and messages: messages.pop() # user corrected the nlu, remove incorrect example @@ -1117,7 +1128,7 @@ def _validate_user_regex(latest_message: Dict[Text, Any], intents: List[Text]) - `/greet`. Return `True` if the intent is a known one.""" parse_data = latest_message.get("parse_data", {}) - intent = parse_data.get("intent", {}).get("name") + intent = parse_data.get("intent", {}).get(INTENT_NAME_KEY) if intent in intents: return True @@ -1134,7 +1145,7 @@ async def _validate_user_text( parse_data = latest_message.get("parse_data", {}) text = _as_md_message(parse_data) - intent = parse_data.get("intent", {}).get("name") + intent = parse_data.get("intent", {}).get(INTENT_NAME_KEY) entities = parse_data.get("entities", []) if entities: message = ( diff --git a/rasa/core/training/story_reader/markdown_story_reader.py b/rasa/core/training/story_reader/markdown_story_reader.py index bc240a354a34..98c28065214f 100644 --- a/rasa/core/training/story_reader/markdown_story_reader.py +++ b/rasa/core/training/story_reader/markdown_story_reader.py @@ -16,6 +16,7 @@ from rasa.core.training.story_reader.story_reader import StoryReader from rasa.core.training.structures import StoryStep, FORM_PREFIX from rasa.data import MARKDOWN_FILE_EXTENSION +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.common import raise_warning logger = logging.getLogger(__name__) @@ -209,7 +210,7 @@ async def _parse_message(self, message: Text, line_num: int) -> UserUttered: utterance = UserUttered( message, parse_data.get("intent"), parse_data.get("entities"), parse_data ) - intent_name = utterance.intent.get("name") + intent_name = utterance.intent.get(INTENT_NAME_KEY) if self.domain and intent_name not in self.domain.intents: raise_warning( f"Found unknown intent '{intent_name}' on line {line_num}. " diff --git a/rasa/core/training/story_reader/yaml_story_reader.py b/rasa/core/training/story_reader/yaml_story_reader.py index 2e4dd6c93af9..ffa9274695c2 100644 --- a/rasa/core/training/story_reader/yaml_story_reader.py +++ b/rasa/core/training/story_reader/yaml_story_reader.py @@ -13,6 +13,7 @@ from rasa.core.training.story_reader.story_reader import StoryReader from rasa.core.training.structures import StoryStep from rasa.data import YAML_FILE_EXTENSIONS +from rasa.nlu.constants import INTENT_NAME_KEY logger = logging.getLogger(__name__) @@ -249,7 +250,7 @@ def _parse_user_utterance(self, step: Dict[Text, Any]) -> None: self.current_step_builder.add_user_messages([utterance]) def _validate_that_utterance_is_in_domain(self, utterance: UserUttered) -> None: - intent_name = utterance.intent.get("name") + intent_name = utterance.intent.get(INTENT_NAME_KEY) if not self.domain: logger.debug( diff --git a/rasa/nlu/classifiers/fallback_classifier.py b/rasa/nlu/classifiers/fallback_classifier.py index 7e607f9ba8be..e6ec20dd98a2 100644 --- a/rasa/nlu/classifiers/fallback_classifier.py +++ b/rasa/nlu/classifiers/fallback_classifier.py @@ -1,13 +1,22 @@ -from typing import Any, List, Type, Text, Dict, Union +import logging +from typing import Any, List, Type, Text, Dict, Union, Tuple, Optional from rasa.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME from rasa.core.constants import DEFAULT_NLU_FALLBACK_THRESHOLD from rasa.nlu.classifiers.classifier import IntentClassifier from rasa.nlu.components import Component from rasa.nlu.training_data import Message -from rasa.nlu.constants import INTENT_RANKING_KEY, INTENT, INTENT_CONFIDENCE_KEY +from rasa.nlu.constants import ( + INTENT_RANKING_KEY, + INTENT, + INTENT_CONFIDENCE_KEY, + INTENT_NAME_KEY, +) THRESHOLD_KEY = "threshold" +AMBIGUITY_THRESHOLD_KEY = "ambiguity_threshold" + +logger = logging.getLogger(__name__) class FallbackClassifier(Component): @@ -16,7 +25,10 @@ class FallbackClassifier(Component): defaults = { # If all intent confidence scores are beyond this threshold, set the current # intent to `FALLBACK_INTENT_NAME` - THRESHOLD_KEY: DEFAULT_NLU_FALLBACK_THRESHOLD + THRESHOLD_KEY: DEFAULT_NLU_FALLBACK_THRESHOLD, + # If the confidence scores for the top two intent predictions are closer than + # `AMBIGUITY_THRESHOLD_KEY`, then `FALLBACK_INTENT_NAME ` is predicted. + AMBIGUITY_THRESHOLD_KEY: 0.1, } @classmethod @@ -47,15 +59,60 @@ def process(self, message: Message, **kwargs: Any) -> None: message.data[INTENT_RANKING_KEY].insert(0, _fallback_intent()) def _should_fallback(self, message: Message) -> bool: - return ( - message.data[INTENT].get(INTENT_CONFIDENCE_KEY) - < self.component_config[THRESHOLD_KEY] - ) + """Check if the fallback intent should be predicted. + + Args: + message: The current message and its intent predictions. + + Returns: + `True` if the fallback intent should be predicted. + """ + intent_name = message.data[INTENT].get(INTENT_NAME_KEY) + below_threshold, nlu_confidence = self._nlu_confidence_below_threshold(message) + + if below_threshold: + logger.debug( + f"NLU confidence {nlu_confidence} for intent '{intent_name}' is lower " + f"than NLU threshold {self.component_config[THRESHOLD_KEY]:.2f}." + ) + return True + + ambiguous_prediction, confidence_delta = self._nlu_prediction_ambiguous(message) + if ambiguous_prediction: + logger.debug( + f"The difference in NLU confidences " + f"for the top two intents ({confidence_delta}) is lower than " + f"the ambiguity threshold " + f"{self.component_config[AMBIGUITY_THRESHOLD_KEY]:.2f}. Predicting " + f"intent '{DEFAULT_NLU_FALLBACK_INTENT_NAME}' instead of " + f"'{intent_name}'." + ) + return True + + return False + + def _nlu_confidence_below_threshold(self, message: Message) -> Tuple[bool, float]: + nlu_confidence = message.data[INTENT].get(INTENT_CONFIDENCE_KEY) + return nlu_confidence < self.component_config[THRESHOLD_KEY], nlu_confidence + + def _nlu_prediction_ambiguous( + self, message: Message + ) -> Tuple[bool, Optional[float]]: + intents = message.data.get(INTENT_RANKING_KEY, []) + if len(intents) >= 2: + first_confidence = intents[0].get(INTENT_CONFIDENCE_KEY, 1.0) + second_confidence = intents[1].get(INTENT_CONFIDENCE_KEY, 1.0) + difference = first_confidence - second_confidence + return ( + difference < self.component_config[AMBIGUITY_THRESHOLD_KEY], + difference, + ) + return False, None def _fallback_intent() -> Dict[Text, Union[Text, float]]: return { - "name": DEFAULT_NLU_FALLBACK_INTENT_NAME, + INTENT_NAME_KEY: DEFAULT_NLU_FALLBACK_INTENT_NAME, # TODO: Re-consider how we represent the confidence here INTENT_CONFIDENCE_KEY: 1.0, } diff --git a/rasa/nlu/constants.py b/rasa/nlu/constants.py index 6d2506e65730..94741ee2207d 100644 --- a/rasa/nlu/constants.py +++ b/rasa/nlu/constants.py @@ -64,6 +64,7 @@ INTENT_RANKING_KEY = "intent_ranking" INTENT_CONFIDENCE_KEY = "confidence" +INTENT_NAME_KEY = "name" FEATURE_TYPE_SENTENCE = "sentence" FEATURE_TYPE_SEQUENCE = "sequence" diff --git a/rasa/nlu/emulators/dialogflow.py b/rasa/nlu/emulators/dialogflow.py index d1a3ad31dabf..bfe9a8a44685 100644 --- a/rasa/nlu/emulators/dialogflow.py +++ b/rasa/nlu/emulators/dialogflow.py @@ -2,6 +2,7 @@ from datetime import datetime from typing import Any, Dict, Text +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.nlu.emulators.no_emulator import NoEmulator @@ -28,7 +29,7 @@ def normalise_response_json(self, data: Dict[Text, Any]) -> Dict[Text, Any]: "result": { "source": "agent", "resolvedQuery": data["text"], - "action": data["intent"]["name"], + "action": data["intent"][INTENT_NAME_KEY], "actionIncomplete": False, "parameters": entities, "contexts": [], diff --git a/rasa/nlu/model.py b/rasa/nlu/model.py index 0c86fe9137cc..a5957ec514f4 100644 --- a/rasa/nlu/model.py +++ b/rasa/nlu/model.py @@ -10,6 +10,7 @@ from rasa.nlu import components, utils # pytype: disable=pyi-error from rasa.nlu.components import Component, ComponentBuilder # pytype: disable=pyi-error from rasa.nlu.config import RasaNLUModelConfig, component_config_from_pipeline +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.nlu.persistor import Persistor from rasa.nlu.training_data import Message, TrainingData from rasa.nlu.utils import write_json_to_file @@ -253,7 +254,7 @@ class Interpreter: # that will be returned by `parse` @staticmethod def default_output_attributes() -> Dict[Text, Any]: - return {"intent": {"name": None, "confidence": 0.0}, "entities": []} + return {"intent": {INTENT_NAME_KEY: None, "confidence": 0.0}, "entities": []} @staticmethod def ensure_model_compatibility( diff --git a/rasa/nlu/test.py b/rasa/nlu/test.py index 5d12678edb0b..ca269a4b49c6 100644 --- a/rasa/nlu/test.py +++ b/rasa/nlu/test.py @@ -38,6 +38,7 @@ ENTITY_ATTRIBUTE_CONFIDENCE_TYPE, ENTITY_ATTRIBUTE_CONFIDENCE_ROLE, ENTITY_ATTRIBUTE_CONFIDENCE_GROUP, + INTENT_NAME_KEY, ) from rasa.model import get_model from rasa.nlu.components import ComponentBuilder @@ -173,7 +174,7 @@ def write_intent_successes( "text": r.message, "intent": r.intent_target, "intent_prediction": { - "name": r.intent_prediction, + INTENT_NAME_KEY: r.intent_prediction, "confidence": r.confidence, }, } @@ -203,7 +204,7 @@ def write_intent_errors( "text": r.message, "intent": r.intent_target, "intent_prediction": { - "name": r.intent_prediction, + INTENT_NAME_KEY: r.intent_prediction, "confidence": r.confidence, }, } @@ -1310,7 +1311,7 @@ def get_eval_data( intent_results.append( IntentEvaluationResult( example.get(INTENT, ""), - intent_prediction.get("name"), + intent_prediction.get(INTENT_NAME_KEY), result.get(TEXT, {}), intent_prediction.get("confidence"), ) diff --git a/tests/core/test_interpreter.py b/tests/core/test_interpreter.py index e8a40e44ac22..6892f61c9686 100644 --- a/tests/core/test_interpreter.py +++ b/tests/core/test_interpreter.py @@ -6,6 +6,7 @@ RasaNLUHttpInterpreter, RegexInterpreter, ) +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.endpoints import EndpointConfig from tests.utilities import latest_request, json_of_latest_request @@ -21,7 +22,9 @@ async def test_regex_interpreter_intent(regex_interpreter): assert result["text"] == text assert len(result["intent_ranking"]) == 1 assert ( - result["intent"]["name"] == result["intent_ranking"][0]["name"] == "my_intent" + result["intent"][INTENT_NAME_KEY] + == result["intent_ranking"][0][INTENT_NAME_KEY] + == "my_intent" ) assert ( result["intent"]["confidence"] @@ -37,7 +40,9 @@ async def test_regex_interpreter_entities(regex_interpreter): assert result["text"] == text assert len(result["intent_ranking"]) == 1 assert ( - result["intent"]["name"] == result["intent_ranking"][0]["name"] == "my_intent" + result["intent"][INTENT_NAME_KEY] + == result["intent_ranking"][0][INTENT_NAME_KEY] + == "my_intent" ) assert ( result["intent"]["confidence"] @@ -55,7 +60,9 @@ async def test_regex_interpreter_confidence(regex_interpreter): assert result["text"] == text assert len(result["intent_ranking"]) == 1 assert ( - result["intent"]["name"] == result["intent_ranking"][0]["name"] == "my_intent" + result["intent"][INTENT_NAME_KEY] + == result["intent_ranking"][0][INTENT_NAME_KEY] + == "my_intent" ) assert ( result["intent"]["confidence"] @@ -71,7 +78,9 @@ async def test_regex_interpreter_confidence_and_entities(regex_interpreter): assert result["text"] == text assert len(result["intent_ranking"]) == 1 assert ( - result["intent"]["name"] == result["intent_ranking"][0]["name"] == "my_intent" + result["intent"][INTENT_NAME_KEY] + == result["intent_ranking"][0][INTENT_NAME_KEY] + == "my_intent" ) assert ( result["intent"]["confidence"] diff --git a/tests/core/test_policies.py b/tests/core/test_policies.py index 997a4437430a..fb40b3d0f4f5 100644 --- a/tests/core/test_policies.py +++ b/tests/core/test_policies.py @@ -35,6 +35,7 @@ from rasa.core.policies.memoization import AugmentedMemoizationPolicy, MemoizationPolicy from rasa.core.policies.sklearn_policy import SklearnPolicy from rasa.core.trackers import DialogueStateTracker +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.tensorflow.constants import ( SIMILARITY_TYPE, RANKING_LENGTH, @@ -789,7 +790,7 @@ async def test_affirmation(self, default_channel, default_nlg, default_domain): events, default_channel, default_nlg, default_domain ) - assert "greet" == tracker.latest_message.parse_data["intent"]["name"] + assert "greet" == tracker.latest_message.parse_data["intent"][INTENT_NAME_KEY] assert tracker.export_stories() == ( "## sender\n* greet\n - utter_hello\n* greet\n" ) @@ -825,7 +826,7 @@ async def test_successful_rephrasing( events, default_channel, default_nlg, default_domain ) - assert "bye" == tracker.latest_message.parse_data["intent"]["name"] + assert "bye" == tracker.latest_message.parse_data["intent"][INTENT_NAME_KEY] assert tracker.export_stories() == "## sender\n* bye\n" def test_affirm_rephrased_intent(self, trained_policy, default_domain): @@ -865,7 +866,7 @@ async def test_affirmed_rephrasing( events, default_channel, default_nlg, default_domain ) - assert "bye" == tracker.latest_message.parse_data["intent"]["name"] + assert "bye" == tracker.latest_message.parse_data["intent"][INTENT_NAME_KEY] assert tracker.export_stories() == "## sender\n* bye\n" def test_denied_rephrasing_affirmation(self, trained_policy, default_domain): @@ -905,7 +906,7 @@ async def test_rephrasing_instead_affirmation( events, default_channel, default_nlg, default_domain ) - assert "bye" == tracker.latest_message.parse_data["intent"]["name"] + assert "bye" == tracker.latest_message.parse_data["intent"][INTENT_NAME_KEY] assert tracker.export_stories() == ( "## sender\n* greet\n - utter_hello\n* bye\n" ) diff --git a/tests/core/test_processor.py b/tests/core/test_processor.py index ed0877b0c0e9..fa0b7f6b0557 100644 --- a/tests/core/test_processor.py +++ b/tests/core/test_processor.py @@ -34,6 +34,7 @@ from rasa.core.slots import Slot from rasa.core.tracker_store import InMemoryTrackerStore from rasa.core.trackers import DialogueStateTracker +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.endpoints import EndpointConfig from tests.utilities import latest_request @@ -69,7 +70,7 @@ async def test_message_id_logging(default_processor: MessageProcessor): async def test_parsing(default_processor: MessageProcessor): message = UserMessage('/greet{"name": "boy"}') parsed = await default_processor._parse_message(message) - assert parsed["intent"]["name"] == "greet" + assert parsed["intent"][INTENT_NAME_KEY] == "greet" assert parsed["entities"][0]["entity"] == "name" @@ -126,7 +127,7 @@ async def mocked_parse(self, text, message_id=None, tracker=None): value from the tracker's state.""" return { - "intent": {"name": "", "confidence": 0.0}, + "intent": {INTENT_NAME_KEY: "", "confidence": 0.0}, "entities": [], "text": text, "requested_language": tracker.get_slot("requested_language"), @@ -177,7 +178,8 @@ async def test_reminder_scheduled( assert t.events[-4] == ActionExecuted("action_schedule_reminder") assert isinstance(t.events[-3], ReminderScheduled) assert t.events[-2] == UserUttered( - f"{EXTERNAL_MESSAGE_PREFIX}remind", intent={"name": "remind", IS_EXTERNAL: True} + f"{EXTERNAL_MESSAGE_PREFIX}remind", + intent={INTENT_NAME_KEY: "remind", IS_EXTERNAL: True}, ) assert t.events[-1] == ActionExecuted("action_listen") @@ -261,7 +263,7 @@ async def test_reminder_cancelled_multi_user( assert ( UserUttered( f"{EXTERNAL_MESSAGE_PREFIX}greet", - intent={"name": "greet", IS_EXTERNAL: True}, + intent={INTENT_NAME_KEY: "greet", IS_EXTERNAL: True}, ) not in tracker_0.events ) @@ -271,7 +273,7 @@ async def test_reminder_cancelled_multi_user( assert ( UserUttered( f"{EXTERNAL_MESSAGE_PREFIX}greet", - intent={"name": "greet", IS_EXTERNAL: True}, + intent={INTENT_NAME_KEY: "greet", IS_EXTERNAL: True}, ) in tracker_1.events ) @@ -617,7 +619,7 @@ async def test_handle_message_with_session_start( ActionExecuted(ACTION_LISTEN_NAME), UserUttered( f"/greet{json.dumps(slot_1)}", - {"name": "greet", "confidence": 1.0}, + {INTENT_NAME_KEY: "greet", "confidence": 1.0}, [{"entity": entity, "start": 6, "end": 22, "value": "Core"}], ), SlotSet(entity, slot_1[entity]), @@ -631,7 +633,7 @@ async def test_handle_message_with_session_start( ActionExecuted(ACTION_LISTEN_NAME), UserUttered( f"/greet{json.dumps(slot_2)}", - {"name": "greet", "confidence": 1.0}, + {INTENT_NAME_KEY: "greet", "confidence": 1.0}, [ { "entity": entity, diff --git a/tests/core/training/story_reader/test_common_story_reader.py b/tests/core/training/story_reader/test_common_story_reader.py index e782211c340b..f1d86cb5fde6 100644 --- a/tests/core/training/story_reader/test_common_story_reader.py +++ b/tests/core/training/story_reader/test_common_story_reader.py @@ -14,6 +14,7 @@ MaxHistoryTrackerFeaturizer, BinarySingleStateFeaturizer, ) +from rasa.nlu.constants import INTENT_NAME_KEY @pytest.mark.parametrize( @@ -36,11 +37,11 @@ async def test_can_read_test_story(stories_file: Text, default_domain: Domain): assert tracker.events[0] == ActionExecuted("action_listen") assert tracker.events[1] == UserUttered( "simple", - intent={"name": "simple", "confidence": 1.0}, + intent={INTENT_NAME_KEY: "simple", "confidence": 1.0}, parse_data={ "text": "/simple", - "intent_ranking": [{"confidence": 1.0, "name": "simple"}], - "intent": {"confidence": 1.0, "name": "simple"}, + "intent_ranking": [{"confidence": 1.0, INTENT_NAME_KEY: "simple"}], + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "simple"}, "entities": [], }, ) diff --git a/tests/core/utilities.py b/tests/core/utilities.py index 2906ea18afad..96790631e009 100644 --- a/tests/core/utilities.py +++ b/tests/core/utilities.py @@ -11,6 +11,7 @@ from rasa.core.domain import Domain from rasa.core.events import UserUttered, Event from rasa.core.trackers import DialogueStateTracker +from rasa.nlu.constants import INTENT_NAME_KEY from tests.core.conftest import DEFAULT_DOMAIN_PATH_WITH_SLOTS if typing.TYPE_CHECKING: @@ -71,7 +72,7 @@ def user_uttered( metadata: Dict[Text, Any] = None, timestamp: Optional[float] = None, ) -> UserUttered: - parse_data = {"intent": {"name": text, "confidence": confidence}} + parse_data = {"intent": {INTENT_NAME_KEY: text, "confidence": confidence}} return UserUttered( text="Random", intent=parse_data["intent"], diff --git a/tests/nlu/classifiers/test_fallback_classifier.py b/tests/nlu/classifiers/test_fallback_classifier.py index 7e684d672bf2..5e2a339c6f12 100644 --- a/tests/nlu/classifiers/test_fallback_classifier.py +++ b/tests/nlu/classifiers/test_fallback_classifier.py @@ -1,34 +1,87 @@ import copy +from typing import Dict + +import pytest from rasa.constants import DEFAULT_NLU_FALLBACK_INTENT_NAME from rasa.core.constants import DEFAULT_NLU_FALLBACK_THRESHOLD -from rasa.nlu.classifiers.fallback_classifier import FallbackClassifier, THRESHOLD_KEY +from rasa.nlu.classifiers.fallback_classifier import ( + FallbackClassifier, + THRESHOLD_KEY, + AMBIGUITY_THRESHOLD_KEY, +) from rasa.nlu.training_data import Message -from rasa.nlu.constants import INTENT_RANKING_KEY, INTENT, INTENT_CONFIDENCE_KEY +from rasa.nlu.constants import ( + INTENT_RANKING_KEY, + INTENT, + INTENT_CONFIDENCE_KEY, + INTENT_NAME_KEY, +) -def test_predict_fallback_intent(): - threshold = 0.5 - message = Message( - "some message", - data={ - INTENT: {"name": "greet", INTENT_CONFIDENCE_KEY: 0.234891876578331}, - INTENT_RANKING_KEY: [ - {"name": "greet", INTENT_CONFIDENCE_KEY: 0.234891876578331}, - {"name": "stop", INTENT_CONFIDENCE_KEY: threshold - 0.0001}, - {"name": "affirm", INTENT_CONFIDENCE_KEY: 0}, - {"name": "inform", INTENT_CONFIDENCE_KEY: -100}, - {"name": "deny", INTENT_CONFIDENCE_KEY: 0.0879683718085289}, - ], - }, - ) +@pytest.mark.parametrize( + "message, component_config", + [ + ( + Message( + "some message", + data={ + INTENT: { + INTENT_NAME_KEY: "greet", + INTENT_CONFIDENCE_KEY: 0.234891876578331, + }, + INTENT_RANKING_KEY: [ + { + INTENT_NAME_KEY: "greet", + INTENT_CONFIDENCE_KEY: 0.234891876578331, + }, + {INTENT_NAME_KEY: "stop", INTENT_CONFIDENCE_KEY: 0.5 - 0.0001}, + {INTENT_NAME_KEY: "affirm", INTENT_CONFIDENCE_KEY: 0}, + {INTENT_NAME_KEY: "inform", INTENT_CONFIDENCE_KEY: -100}, + { + INTENT_NAME_KEY: "deny", + INTENT_CONFIDENCE_KEY: 0.0879683718085289, + }, + ], + }, + ), + {THRESHOLD_KEY: 0.5}, + ), + ( + Message( + "some message", + data={ + INTENT: {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 1}, + INTENT_RANKING_KEY: [ + {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 1}, + {INTENT_NAME_KEY: "stop", INTENT_CONFIDENCE_KEY: 0.9}, + ], + }, + ), + {THRESHOLD_KEY: 0.5, AMBIGUITY_THRESHOLD_KEY: 0.1}, + ), + ( + Message( + "some message", + data={ + INTENT: {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 1}, + INTENT_RANKING_KEY: [ + {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 1}, + {INTENT_NAME_KEY: "stop", INTENT_CONFIDENCE_KEY: 0.5}, + ], + }, + ), + {THRESHOLD_KEY: 0.5, AMBIGUITY_THRESHOLD_KEY: 0.51}, + ), + ], +) +def test_predict_fallback_intent(message: Message, component_config: Dict): old_message_state = copy.deepcopy(message) - - classifier = FallbackClassifier(component_config={THRESHOLD_KEY: threshold}) + classifier = FallbackClassifier(component_config=component_config) classifier.process(message) expected_intent = { - "name": DEFAULT_NLU_FALLBACK_INTENT_NAME, + INTENT_NAME_KEY: DEFAULT_NLU_FALLBACK_INTENT_NAME, INTENT_CONFIDENCE_KEY: 1.0, } assert message.data[INTENT] == expected_intent @@ -41,30 +94,57 @@ def test_predict_fallback_intent(): assert current_intent_ranking[0] == expected_intent -def test_not_predict_fallback_intent(): - threshold = 0.5 - message = Message( - "some message", - data={ - INTENT: {"name": "greet", INTENT_CONFIDENCE_KEY: threshold}, - INTENT_RANKING_KEY: [ - {"name": "greet", INTENT_CONFIDENCE_KEY: 0.234891876578331}, - {"name": "stop", INTENT_CONFIDENCE_KEY: 0.1}, - {"name": "affirm", INTENT_CONFIDENCE_KEY: 0}, - {"name": "inform", INTENT_CONFIDENCE_KEY: -100}, - {"name": "deny", INTENT_CONFIDENCE_KEY: 0.0879683718085289}, - ], - }, - ) +@pytest.mark.parametrize( + "message, component_config", + [ + ( + Message( + "some message", + data={ + INTENT: {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 0.5}, + INTENT_RANKING_KEY: [ + { + INTENT_NAME_KEY: "greet", + INTENT_CONFIDENCE_KEY: 0.234891876578331, + }, + {INTENT_NAME_KEY: "stop", INTENT_CONFIDENCE_KEY: 0.1}, + {INTENT_NAME_KEY: "affirm", INTENT_CONFIDENCE_KEY: 0}, + {INTENT_NAME_KEY: "inform", INTENT_CONFIDENCE_KEY: -100}, + { + INTENT_NAME_KEY: "deny", + INTENT_CONFIDENCE_KEY: 0.0879683718085289, + }, + ], + }, + ), + {THRESHOLD_KEY: 0.5}, + ), + ( + Message( + "some message", + data={ + INTENT: {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 1}, + INTENT_RANKING_KEY: [ + {INTENT_NAME_KEY: "greet", INTENT_CONFIDENCE_KEY: 1}, + {INTENT_NAME_KEY: "stop", INTENT_CONFIDENCE_KEY: 0.89}, + ], + }, + ), + {THRESHOLD_KEY: 0.5, AMBIGUITY_THRESHOLD_KEY: 0.1}, + ), + ], +) +def test_not_predict_fallback_intent(message: Message, component_config: Dict): old_message_state = copy.deepcopy(message) - classifier = FallbackClassifier(component_config={THRESHOLD_KEY: threshold}) + classifier = FallbackClassifier(component_config=component_config) classifier.process(message) assert message == old_message_state -def test_default_threshold(): +def test_defaults(): classifier = FallbackClassifier({}) assert classifier.component_config[THRESHOLD_KEY] == DEFAULT_NLU_FALLBACK_THRESHOLD + assert classifier.component_config[AMBIGUITY_THRESHOLD_KEY] == 0.1 diff --git a/tests/test_server.py b/tests/test_server.py index 60c8d94ec6d1..d1551369ce96 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -27,6 +27,7 @@ from rasa.core.events import Event, UserUttered, SlotSet, BotUttered from rasa.core.trackers import DialogueStateTracker from rasa.model import unpack_model +from rasa.nlu.constants import INTENT_NAME_KEY from rasa.utils.endpoints import EndpointConfig from sanic import Sanic from sanic.testing import SanicTestClient @@ -41,7 +42,7 @@ "event": UserUttered.type_name, "text": "/goodbye", "parse_data": { - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "entities": [], }, } @@ -262,7 +263,7 @@ def is_server_ready() -> bool: "/model/parse", { "entities": [], - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "text": "hello", }, payload={"text": "hello"}, @@ -271,7 +272,7 @@ def is_server_ready() -> bool: "/model/parse", { "entities": [], - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "text": "hello", }, payload={"text": "hello"}, @@ -280,7 +281,7 @@ def is_server_ready() -> bool: "/model/parse", { "entities": [], - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "text": "hello ńöñàśçií", }, payload={"text": "hello ńöñàśçií"}, @@ -304,7 +305,7 @@ def test_parse(rasa_app, response_test): "/model/parse?emulation_mode=wit", { "entities": [], - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "text": "hello", }, payload={"text": "hello"}, @@ -313,7 +314,7 @@ def test_parse(rasa_app, response_test): "/model/parse?emulation_mode=dialogflow", { "entities": [], - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "text": "hello", }, payload={"text": "hello"}, @@ -322,7 +323,7 @@ def test_parse(rasa_app, response_test): "/model/parse?emulation_mode=luis", { "entities": [], - "intent": {"confidence": 1.0, "name": "greet"}, + "intent": {"confidence": 1.0, INTENT_NAME_KEY: "greet"}, "text": "hello ńöñàśçií", }, payload={"text": "hello ńöñàśçií"}, @@ -618,7 +619,7 @@ def test_predict(rasa_app: SanicTestClient): "text": "hello", "parse_data": { "entities": [], - "intent": {"confidence": 0.57, "name": "greet"}, + "intent": {"confidence": 0.57, INTENT_NAME_KEY: "greet"}, "text": "hello", }, }, @@ -654,7 +655,7 @@ def test_requesting_non_existent_tracker(rasa_app: SanicTestClient): {"event": "session_started", "timestamp": 1514764800}, { "event": "action", - "name": "action_listen", + INTENT_NAME_KEY: "action_listen", "policy": None, "confidence": None, "timestamp": 1514764800, @@ -929,7 +930,7 @@ def test_load_model_invalid_configuration(rasa_app: SanicTestClient): def test_execute(rasa_app: SanicTestClient): _create_tracker_for_sender(rasa_app, "test_execute") - data = {"name": "utter_greet"} + data = {INTENT_NAME_KEY: "utter_greet"} _, response = rasa_app.post("/conversations/test_execute/execute", json=data) assert response.status == 200 @@ -960,7 +961,7 @@ def test_execute_with_not_existing_action(rasa_app: SanicTestClient): def test_trigger_intent(rasa_app: SanicTestClient): - data = {"name": "greet"} + data = {INTENT_NAME_KEY: "greet"} _, response = rasa_app.post("/conversations/test_trigger/trigger_intent", json=data) assert response.status == 200 @@ -985,7 +986,7 @@ def test_trigger_intent_with_not_existing_intent(rasa_app: SanicTestClient): test_sender = "test_trigger_intent_with_not_existing_intent" _create_tracker_for_sender(rasa_app, test_sender) - data = {"name": "ka[pa[opi[opj[oj[oija"} + data = {INTENT_NAME_KEY: "ka[pa[opi[opj[oj[oija"} _, response = rasa_app.post( f"/conversations/{test_sender}/trigger_intent", json=data )