diff --git a/changelog/6453.removal.md b/changelog/6453.removal.md new file mode 100644 index 000000000000..f34ab7b8923b --- /dev/null +++ b/changelog/6453.removal.md @@ -0,0 +1,34 @@ +Removed support for `queue` argument in `PikaEventBroker` (use `queues` instead). + +Domain file: +- Removed support for `templates` key (use `responses` instead). +- Removed support for string `responses` (use dictionaries instead). + +NLU `Component`: +- Removed support for `provides` attribute, it's not needed anymore. +- Removed support for `requires` attribute (use `required_components()` instead). + +Removed `_guess_format()` utils method from `rasa.nlu.training_data.loading` (use `guess_format` instead). + +Removed several config options for [TED Policy](./policies#ted-policy), [DIETClassifier](./components/intent-classifiers#dietclassifier) and [ResponseSelector](./components/selectors#responseselector): +- `hidden_layers_sizes_pre_dial` +- `hidden_layers_sizes_bot` +- `droprate` +- `droprate_a` +- `droprate_b` +- `hidden_layers_sizes_a` +- `hidden_layers_sizes_b` +- `num_transformer_layers` +- `num_heads` +- `dense_dim` +- `embed_dim` +- `num_neg` +- `mu_pos` +- `mu_neg` +- `use_max_sim_neg` +- `C2` +- `C_emb` +- `evaluate_every_num_epochs` +- `evaluate_on_num_examples` + +Please check the documentation for more information. diff --git a/data/test_domains/default_deprecated_templates.yml b/data/test_domains/default_deprecated_templates.yml deleted file mode 100644 index 40f18dc2c977..000000000000 --- a/data/test_domains/default_deprecated_templates.yml +++ /dev/null @@ -1,26 +0,0 @@ -intents: - - greet: {use_entities: [name]} - - default: {ignore_entities : [unrelated_recognized_entity]} - - goodbye: {use_entities: null} - - thank: {use_entities: False} - - ask: {use_entities: True} - - why: {use_entities: []} - - pure_intent - -entities: - - name - - unrelated_recognized_entity - - other - -templates: - utter_greet: - - hey there! - utter_goodbye: - - goodbye :( - utter_default: - - default message - -actions: - - utter_default - - utter_greet - - utter_goodbye diff --git a/data/test_domains/default_unfeaturized_entities.yml b/data/test_domains/default_unfeaturized_entities.yml index 2d0a4672efbd..cf897ef7a48a 100644 --- a/data/test_domains/default_unfeaturized_entities.yml +++ b/data/test_domains/default_unfeaturized_entities.yml @@ -14,8 +14,8 @@ entities: responses: utter_greet: - - hey there! + - text: hey there! utter_goodbye: - - goodbye :( + - text: goodbye :( utter_default: - - default message + - text: default message diff --git a/data/test_domains/empty_response_format.yml b/data/test_domains/empty_response_format.yml new file mode 100644 index 000000000000..f843a2318d66 --- /dev/null +++ b/data/test_domains/empty_response_format.yml @@ -0,0 +1,25 @@ +intents: + - greet + - default + - goodbye + +slots: + cuisine: + type: text + location: + type: text + +entities: + - name + +responses: + utter_greet: + utter_goodbye: + - text: goodbye :( + utter_default: + - text: default message + +actions: + - utter_default + - utter_greet + - utter_goodbye diff --git a/data/test_domains/wrong_custom_response_format.yml b/data/test_domains/wrong_custom_response_format.yml new file mode 100644 index 000000000000..37a152112388 --- /dev/null +++ b/data/test_domains/wrong_custom_response_format.yml @@ -0,0 +1,26 @@ +intents: + - greet + - default + - goodbye + +slots: + cuisine: + type: text + location: + type: text + +entities: + - name + +responses: + utter_greet: + - super: cool + utter_goodbye: + - text: goodbye :( + utter_default: + - text: default message + +actions: + - utter_default + - utter_greet + - utter_goodbye diff --git a/data/test_domains/wrong_response_format.yml b/data/test_domains/wrong_response_format.yml new file mode 100644 index 000000000000..d9419df2b095 --- /dev/null +++ b/data/test_domains/wrong_response_format.yml @@ -0,0 +1,26 @@ +intents: + - greet + - default + - goodbye + +slots: + cuisine: + type: text + location: + type: text + +entities: + - name + +responses: + utter_greet: + - hey there! + utter_goodbye: + - goodbye :( + utter_default: + - stuff: default message + +actions: + - utter_default + - utter_greet + - utter_goodbye diff --git a/docs/docs/policies.mdx b/docs/docs/policies.mdx index 463245d65dfe..53b9bb53f1c3 100644 --- a/docs/docs/policies.mdx +++ b/docs/docs/policies.mdx @@ -172,8 +172,6 @@ in your policy configuration. The functionality of the policy stayed the same. ::: - - ## TED Policy The Transformer Embedding Dialogue (TED) Policy is described in diff --git a/pyproject.toml b/pyproject.toml index 35dec835a295..226c4a017042 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ repository = "https://github.com/rasahq/rasa" documentation = "https://rasa.com/docs" classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Topic :: Software Development :: Libraries",] keywords = [ "nlp", "machine-learning", "machine-learning-library", "bot", "bots", "botkit", "rasa conversational-agents", "conversational-ai", "chatbot", "chatbot-framework", "bot-framework",] -include = [ "LICENSE.txt", "README.md", "rasa/core/schemas/*", "rasa/core/training/visualization.html", "rasa/nlu/schemas/*", "rasa/cli/default_config.yml", "rasa/importers/*",] +include = [ "LICENSE.txt", "README.md", "rasa/core/schemas/*", "rasa/core/training/visualization.html", "rasa/nlu/schemas/*", "rasa/cli/default_config.yml", "rasa/importers/*", "rasa/utils/schemas.yml",] readme = "README.md" license = "Apache-2.0" [[tool.poetry.source]] diff --git a/rasa/constants.py b/rasa/constants.py index e3fc0dda8c48..154c80c2f32d 100644 --- a/rasa/constants.py +++ b/rasa/constants.py @@ -26,6 +26,8 @@ CONFIG_SCHEMA_FILE = "nlu/schemas/config.yml" DOMAIN_SCHEMA_FILE = "core/schemas/domain.yml" +SCHEMA_UTILS_FILE = "utils/schemas.yml" +SCHEMA_EXTENSIONS_FILE = "utils/pykwalify_extensions.py" YAML_VERSION = (1, 2) DEFAULT_RASA_X_PORT = 5002 @@ -62,6 +64,8 @@ MINIMUM_COMPATIBLE_VERSION = "1.11.0a3" +NEXT_MAJOR_VERSION_FOR_DEPRECATIONS = "3.0.0" + LATEST_TRAINING_DATA_FORMAT_VERSION = "2.0" GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml") diff --git a/rasa/core/brokers/pika.py b/rasa/core/brokers/pika.py index 0899ef33a372..f5b8ecc412ac 100644 --- a/rasa/core/brokers/pika.py +++ b/rasa/core/brokers/pika.py @@ -291,7 +291,7 @@ def __init__( self.password = password self.port = port self.channel: Optional["Channel"] = None - self.queues = self._get_queues_from_args(queues, kwargs) + self.queues = self._get_queues_from_args(queues) self.should_keep_unpublished_messages = should_keep_unpublished_messages self.raise_on_failure = raise_on_failure @@ -315,37 +315,24 @@ def rasa_environment(self) -> Optional[Text]: @staticmethod def _get_queues_from_args( - queues_arg: Union[List[Text], Tuple[Text], Text, None], kwargs: Any + queues_arg: Union[List[Text], Tuple[Text], Text, None] ) -> Union[List[Text], Tuple[Text]]: """Get queues for this event broker. The preferred argument defining the RabbitMQ queues the `PikaEventBroker` should - publish to is `queues` (as of Rasa Open Source version 1.8.2). This function - ensures backwards compatibility with the old `queue` argument. This method + publish to is `queues` (as of Rasa Open Source version 1.8.2). This method can be removed in the future, and `self.queues` should just receive the value of the `queues` kwarg in the constructor. Args: queues_arg: Value of the supplied `queues` argument. - kwargs: Additional kwargs supplied to the `PikaEventBroker` constructor. - If `queues_arg` is not supplied, the `queue` kwarg will be used instead. Returns: Queues this event broker publishes to. Raises: - `ValueError` if no valid `queue` or `queues` argument was found. + `ValueError` if no valid `queues` argument was found. """ - queue_arg = kwargs.pop("queue", None) - - if queue_arg: - raise_warning( - "Your Pika event broker config contains the deprecated `queue` key. " - "Please use the `queues` key instead.", - FutureWarning, - docs=DOCS_URL_PIKA_EVENT_BROKER, - ) - if queues_arg and isinstance(queues_arg, (list, tuple)): return queues_arg @@ -357,14 +344,8 @@ def _get_queues_from_args( ) return [queues_arg] - if queue_arg and isinstance(queue_arg, str): - return [queue_arg] - - if queue_arg: - return queue_arg # pytype: disable=bad-return-type - raise_warning( - f"No `queues` or `queue` argument provided. It is suggested to " + f"No `queues` argument provided. It is suggested to " f"explicitly specify a queue as described in " f"{DOCS_URL_PIKA_EVENT_BROKER}. " f"Using the default queue '{DEFAULT_QUEUE_NAME}' for now." diff --git a/rasa/core/channels/mattermost.py b/rasa/core/channels/mattermost.py index 6eb51515934c..7a33f58d1d7e 100644 --- a/rasa/core/channels/mattermost.py +++ b/rasa/core/channels/mattermost.py @@ -11,7 +11,7 @@ from rasa.core.channels.channel import UserMessage, OutputChannel, InputChannel from sanic.response import HTTPResponse -from rasa.utils.common import raise_warning +from rasa.utils import common as common_utils logger = logging.getLogger(__name__) @@ -132,12 +132,11 @@ def from_credentials(cls, credentials: Optional[Dict[Text, Any]]) -> InputChanne # pytype: disable=attribute-error if credentials.get("pw") is not None or credentials.get("user") is not None: - raise_warning( + common_utils.raise_deprecation_warning( "Mattermost recently switched to bot accounts. 'user' and 'pw' " "should not be used anymore, you should rather convert your " "account to a bot account and use a token. Password based " "authentication will be removed in a future Rasa Open Source version.", - FutureWarning, docs=DOCS_URL_CONNECTORS + "mattermost/", ) token = MattermostBot.token_from_login( diff --git a/rasa/core/domain.py b/rasa/core/domain.py index b6ddaa718dd7..1d8dc25c0cbd 100644 --- a/rasa/core/domain.py +++ b/rasa/core/domain.py @@ -164,18 +164,7 @@ def from_yaml(cls, yaml: Text, original_filename: Text = "") -> "Domain": @classmethod def from_dict(cls, data: Dict) -> "Domain": - utter_templates = cls.collect_templates(data.get(KEY_RESPONSES, {})) - if "templates" in data: - raise_warning( - "Your domain file contains the key: 'templates'. This has been " - "deprecated and renamed to 'responses'. The 'templates' key will " - "no longer work in future versions of Rasa. Please replace " - "'templates' with 'responses'", - FutureWarning, - docs=DOCS_URL_DOMAINS, - ) - utter_templates = cls.collect_templates(data.get("templates", {})) - + utter_templates = data.get(KEY_RESPONSES, {}) slots = cls.collect_slots(data.get(KEY_SLOTS, {})) additional_arguments = data.get("config", {}) session_config = cls._get_session_config(data.get(SESSION_CONFIG_KEY, {})) @@ -418,46 +407,6 @@ def _add_default_intents( _, properties = cls._intent_properties(intent_name, entities) intent_properties.update(properties) - @staticmethod - def collect_templates( - yml_templates: Dict[Text, List[Any]] - ) -> Dict[Text, List[Dict[Text, Any]]]: - """Go through the templates and make sure they are all in dict format.""" - templates = {} - for template_key, template_variations in yml_templates.items(): - validated_variations = [] - if template_variations is None: - raise InvalidDomain( - "Response '{}' does not have any defined variations.".format( - template_key - ) - ) - - for t in template_variations: - - # responses should be a dict with options - if isinstance(t, str): - raise_warning( - f"Responses should not be strings anymore. " - f"Response '{template_key}' should contain " - f"either a '- text: ' or a '- custom: ' " - f"attribute to be a proper response.", - FutureWarning, - docs=DOCS_URL_DOMAINS + "#responses", - ) - validated_variations.append({"text": t}) - elif "text" not in t and "custom" not in t: - raise InvalidDomain( - f"Response '{template_key}' needs to contain either " - f"'- text: ' or '- custom: ' attribute to be a proper " - f"response." - ) - else: - validated_variations.append(t) - - templates[template_key] = validated_variations - return templates - def __init__( self, intents: Union[Set[Text], List[Union[Text, Dict[Text, Any]]]], diff --git a/rasa/core/policies/fallback.py b/rasa/core/policies/fallback.py index cb4f1cf855c2..c55f43525435 100644 --- a/rasa/core/policies/fallback.py +++ b/rasa/core/policies/fallback.py @@ -58,10 +58,9 @@ def __init__( self.core_threshold = core_threshold self.fallback_action_name = fallback_action_name - common_utils.raise_warning( + common_utils.raise_deprecation_warning( f"'{self.__class__.__name__}' is deprecated and will be removed " "in the future. It is recommended to use the 'RulePolicy' instead.", - category=FutureWarning, docs=DOCS_URL_MIGRATION_GUIDE, ) diff --git a/rasa/core/policies/form_policy.py b/rasa/core/policies/form_policy.py index a221f5411cc8..78f6a8f1dc88 100644 --- a/rasa/core/policies/form_policy.py +++ b/rasa/core/policies/form_policy.py @@ -35,10 +35,9 @@ def __init__( featurizer=featurizer, priority=priority, max_history=2, lookup=lookup ) - common_utils.raise_warning( + common_utils.raise_deprecation_warning( f"'{FormPolicy.__name__}' is deprecated and will be removed in " "in the future. It is recommended to use the 'RulePolicy' instead.", - category=FutureWarning, docs=DOCS_URL_MIGRATION_GUIDE, ) diff --git a/rasa/core/policies/mapping_policy.py b/rasa/core/policies/mapping_policy.py index 6749eeb64796..f189c5d2a72e 100644 --- a/rasa/core/policies/mapping_policy.py +++ b/rasa/core/policies/mapping_policy.py @@ -52,10 +52,9 @@ def __init__(self, priority: int = MAPPING_POLICY_PRIORITY) -> None: super().__init__(priority=priority) - common_utils.raise_warning( + common_utils.raise_deprecation_warning( f"'{MappingPolicy.__name__}' is deprecated and will be removed in " "the future. It is recommended to use the 'RulePolicy' instead.", - category=FutureWarning, docs=DOCS_URL_MIGRATION_GUIDE, ) diff --git a/rasa/core/schemas/domain.yml b/rasa/core/schemas/domain.yml index 4428c93e0fba..be5bdcde4a97 100644 --- a/rasa/core/schemas/domain.yml +++ b/rasa/core/schemas/domain.yml @@ -25,8 +25,9 @@ mapping: - type: "str" required: True responses: - type: "map" - allowempty: True + # see rasa/utils/schemas.yml + include: responses + slots: type: "map" allowempty: True diff --git a/rasa/core/tracker_store.py b/rasa/core/tracker_store.py index f1668a423a8e..ac1125370b12 100644 --- a/rasa/core/tracker_store.py +++ b/rasa/core/tracker_store.py @@ -34,7 +34,7 @@ from rasa.core.trackers import ActionExecuted, DialogueStateTracker, EventVerbosity import rasa.cli.utils as rasa_cli_utils from rasa.nlu.constants import INTENT_NAME_KEY -from rasa.utils.common import class_from_module_path, raise_warning, arguments_of +from rasa.utils import common as common_utils from rasa.utils.endpoints import EndpointConfig import sqlalchemy as sa @@ -186,11 +186,11 @@ def _deserialise_dialogue_from_pickle( sender_id: Text, serialised_tracker: bytes ) -> Dialogue: - logger.warning( + common_utils.raise_deprecation_warning( f"Found pickled tracker for " f"conversation ID '{sender_id}'. Deserialisation of pickled " - f"trackers will be deprecated in version 2.0. Rasa will perform any " - f"future save operations of this tracker using json serialisation." + f"trackers is deprecated. Rasa will perform any " + f"future save operations of this tracker using json serialisation.", ) return pickle.loads(serialised_tracker) @@ -1086,8 +1086,8 @@ def _load_from_module_name_in_endpoint_config( """ try: - tracker_store_class = class_from_module_path(store.type) - init_args = arguments_of(tracker_store_class.__init__) + tracker_store_class = common_utils.class_from_module_path(store.type) + init_args = common_utils.arguments_of(tracker_store_class.__init__) if "url" in init_args and "host" not in init_args: # DEPRECATION EXCEPTION - remove in 2.1 raise Exception( @@ -1102,7 +1102,7 @@ def _load_from_module_name_in_endpoint_config( domain=domain, event_broker=event_broker, **store.kwargs ) except (AttributeError, ImportError): - raise_warning( + common_utils.raise_warning( f"Tracker store with type '{store.type}' not found. " f"Using `InMemoryTrackerStore` instead." ) diff --git a/rasa/nlu/components.py b/rasa/nlu/components.py index 6df9778eb822..da438ec0770f 100644 --- a/rasa/nlu/components.py +++ b/rasa/nlu/components.py @@ -3,7 +3,6 @@ import typing from typing import Any, Dict, Hashable, List, Optional, Set, Text, Tuple, Type, Iterable -from rasa.constants import DOCS_URL_MIGRATION_GUIDE from rasa.nlu.constants import TRAINABLE_EXTRACTORS from rasa.nlu.config import RasaNLUModelConfig, override_defaults, InvalidConfigError from rasa.nlu.training_data import Message, TrainingData @@ -121,32 +120,6 @@ def _required_component_in_pipeline( return False -def _check_deprecated_attributes(component: "Component") -> None: - """Checks that the component doesn't have deprecated attributes. - - Args: - component: The :class:`rasa.nlu.components.Component`. - """ - - if hasattr(component, "provides"): - raise_warning( - f"'{component.name}' contains property 'provides', " - f"which is deprecated. There is no need to specify " - f"the list of attributes that a component provides.", - category=FutureWarning, - docs=DOCS_URL_MIGRATION_GUIDE, - ) - if hasattr(component, "requires"): - raise_warning( - f"'{component.name}' contains property 'requires', " - f"which is deprecated. Use 'required_components()' method " - f"to specify which components are required to be present " - f"in the pipeline by this component.", - category=FutureWarning, - docs=DOCS_URL_MIGRATION_GUIDE, - ) - - def validate_required_components(pipeline: List["Component"]) -> None: """Validates that all required components are present in the pipeline. @@ -155,7 +128,6 @@ def validate_required_components(pipeline: List["Component"]) -> None: """ for i, component in enumerate(pipeline): - _check_deprecated_attributes(component) missing_components = [] for required_component in component.required_components(): diff --git a/rasa/nlu/schemas/nlu.yml b/rasa/nlu/schemas/nlu.yml index 370fc666deab..49f0150db1b7 100644 --- a/rasa/nlu/schemas/nlu.yml +++ b/rasa/nlu/schemas/nlu.yml @@ -47,9 +47,8 @@ mapping: type: "str" examples: *examples_anchor responses: - type: "map" - allowempty: True - required: False + # see rasa/utils/schemas.yml + include: responses stories: type: "any" required: False diff --git a/rasa/nlu/training_data/formats/markdown.py b/rasa/nlu/training_data/formats/markdown.py index f40416202437..706d22555732 100644 --- a/rasa/nlu/training_data/formats/markdown.py +++ b/rasa/nlu/training_data/formats/markdown.py @@ -49,8 +49,6 @@ def __init__(self) -> None: self.regex_features = [] self.lookup_tables = [] - self._deprecated_synonym_format_was_used = False - def reads(self, s: Text, **kwargs: Any) -> "TrainingData": """Read markdown string and create TrainingData object""" from rasa.nlu.training_data import TrainingData @@ -65,19 +63,6 @@ def reads(self, s: Text, **kwargs: Any) -> "TrainingData": self._parse_item(line) self._load_files(line) - if self._deprecated_synonym_format_was_used: - raise_warning( - "You are using the deprecated training data format to declare synonyms." - " Please use the following format: \n" - '[]{"entity": "", "value": ' - '""}.' - "\nYou can use the following command to update your training data file:" - "\nsed -i -E 's/\\[([^)]+)\\]\\(([^)]+):([^)]+)\\)/[\\1]{" - '"entity": "\\2", "value": "\\3"}/g\' nlu.md', - category=FutureWarning, - docs=DOCS_URL_TRAINING_DATA_NLU, - ) - return TrainingData( self.training_examples, self.entity_synonyms, diff --git a/rasa/nlu/training_data/formats/rasa_yaml.py b/rasa/nlu/training_data/formats/rasa_yaml.py index f27f1908825a..1a4544d64130 100644 --- a/rasa/nlu/training_data/formats/rasa_yaml.py +++ b/rasa/nlu/training_data/formats/rasa_yaml.py @@ -100,7 +100,7 @@ def reads(self, string: Text, **kwargs: Any) -> "TrainingData": if key == KEY_NLU: self._parse_nlu(value) elif key == KEY_RESPONSES: - self._parse_responses(value) + self.responses = value return TrainingData( self.training_examples, @@ -145,11 +145,6 @@ def _parse_nlu(self, nlu_data: Optional[List[Dict[Text, Any]]]) -> None: docs=DOCS_URL_TRAINING_DATA_NLU, ) - def _parse_responses(self, responses_data: Dict[Text, List[Any]]) -> None: - from rasa.core.domain import Domain - - self.responses = Domain.collect_templates(responses_data) - def _parse_intent(self, data: Dict[Text, Any]) -> None: from rasa.nlu.training_data import Message import rasa.nlu.training_data.entities_parser as entities_parser diff --git a/rasa/nlu/training_data/loading.py b/rasa/nlu/training_data/loading.py index 041bc55f1396..282c88575b98 100644 --- a/rasa/nlu/training_data/loading.py +++ b/rasa/nlu/training_data/loading.py @@ -179,11 +179,3 @@ def guess_format(filename: Text) -> Text: logger.debug(f"Training data format of '{filename}' is '{guess}'.") return guess - - -def _guess_format(filename: Text) -> Text: - logger.warning( - "Using '_guess_format()' is deprecated since Rasa 1.1.5. " - "Please use 'guess_format()' instead." - ) - return guess_format(filename) diff --git a/rasa/server.py b/rasa/server.py index c4d32fdfa299..164c1a11dbcc 100644 --- a/rasa/server.py +++ b/rasa/server.py @@ -1162,11 +1162,10 @@ def _validate_json_training_payload(rjs: Dict): ) if "force" in rjs or "save_to_default_model_directory" in rjs: - common_utils.raise_warning( + common_utils.raise_deprecation_warning( "Specifying 'force' and 'save_to_default_model_directory' as part of the " "JSON payload is deprecated. Please use the header arguments " "'force_training' and 'save_to_default_model_directory'.", - category=FutureWarning, docs=_docs("/api/http-api"), ) diff --git a/rasa/utils/common.py b/rasa/utils/common.py index cbea0e731453..7cacce253279 100644 --- a/rasa/utils/common.py +++ b/rasa/utils/common.py @@ -15,6 +15,7 @@ ENV_LOG_LEVEL, ENV_LOG_LEVEL_LIBRARIES, GLOBAL_USER_CONFIG_PATH, + NEXT_MAJOR_VERSION_FOR_DEPRECATIONS, ) logger = logging.getLogger(__name__) @@ -385,6 +386,28 @@ def formatwarning( warnings.formatwarning = original_formatter +def raise_deprecation_warning( + message: Text, + warn_until_version: Text = NEXT_MAJOR_VERSION_FOR_DEPRECATIONS, + docs: Optional[Text] = None, + **kwargs: Any, +) -> None: + """ + Thin wrapper around `raise_warning()` to raise a deprecation warning. It requires + a version until which we'll warn, and after which the support for the feature will + be removed. + """ + if warn_until_version not in message: + message = f"{message} (will be removed in {warn_until_version})" + + # need the correct stacklevel now + kwargs.setdefault("stacklevel", 3) + # we're raising a `FutureWarning` instead of a `DeprecationWarning` because + # we want these warnings to be visible in the terminal of our users + # https://docs.python.org/3/library/warnings.html#warning-categories + raise_warning(message, FutureWarning, docs, **kwargs) + + class RepeatedLogFilter(logging.Filter): """Filter repeated log records.""" diff --git a/rasa/utils/pykwalify_extensions.py b/rasa/utils/pykwalify_extensions.py new file mode 100644 index 000000000000..c152c47795ca --- /dev/null +++ b/rasa/utils/pykwalify_extensions.py @@ -0,0 +1,26 @@ +""" +This module regroups custom validation functions, and it is +loaded as an extension of the pykwalify library: + +https://pykwalify.readthedocs.io/en/latest/extensions.html#extensions +""" +from typing import Any, List, Dict, Text + +from pykwalify.errors import SchemaError + + +def require_response_keys( + responses: List[Dict[Text, Any]], rule_obj: Dict, path: Text +) -> bool: + """ + Validate that response dicts have either the "text" key or the "custom" key. + """ + for response in responses: + if not isinstance(response, dict): + # this is handled by other validation rules + continue + + if not response.get("text") and not response.get("custom"): + raise SchemaError("Missing 'text' or 'custom' key in response.") + + return True diff --git a/rasa/utils/schemas.yml b/rasa/utils/schemas.yml new file mode 100644 index 000000000000..3b105e6f8850 --- /dev/null +++ b/rasa/utils/schemas.yml @@ -0,0 +1,32 @@ +schema;responses: + type: "map" + allowempty: True + mapping: + regex;(.+): + type: "seq" + required: False + nullable: False + func: require_response_keys + sequence: + - type: "map" + required: True + allowempty: False + mapping: + text: + type: "str" + image: + type: "str" + custom: + type: "map" + allowempty: True + buttons: + type: "seq" + sequence: + - type: "map" + mapping: + title: + type: "str" + payload: + type: "str" + channel: + type: "str" diff --git a/rasa/utils/train_utils.py b/rasa/utils/train_utils.py index bc19adb39867..343c1c2fb8c4 100644 --- a/rasa/utils/train_utils.py +++ b/rasa/utils/train_utils.py @@ -1,31 +1,17 @@ -import numpy as np -import logging from typing import Optional, Text, Dict, Any, Union, List, Tuple -from rasa.core.constants import DIALOGUE -from rasa.nlu.constants import TEXT, NUMBER_OF_SUB_TOKENS +import numpy as np + +from rasa.constants import NEXT_MAJOR_VERSION_FOR_DEPRECATIONS +from rasa.nlu.constants import NUMBER_OF_SUB_TOKENS from rasa.nlu.tokenizers.tokenizer import Token import rasa.utils.io as io_utils +from rasa.utils import common as common_utils from rasa.utils.tensorflow.constants import ( - LABEL, - HIDDEN_LAYERS_SIZES, - NUM_TRANSFORMER_LAYERS, - NUM_HEADS, - DENSE_DIMENSION, LOSS_TYPE, SIMILARITY_TYPE, - NUM_NEG, EVAL_NUM_EXAMPLES, EVAL_NUM_EPOCHS, - REGULARIZATION_CONSTANT, - USE_MAX_NEG_SIM, - MAX_NEG_SIM, - MAX_POS_SIM, - EMBEDDING_DIMENSION, - DROP_RATE_DIALOGUE, - DROP_RATE_LABEL, - NEGATIVE_MARGIN_SCALE, - DROP_RATE, EPOCHS, SOFTMAX, MARGIN, @@ -35,9 +21,6 @@ ) -logger = logging.getLogger(__name__) - - def normalize(values: np.ndarray, ranking_length: Optional[int] = 0) -> np.ndarray: """Normalizes an array of positive numbers over the top `ranking_length` values. Other values will be set to 0. @@ -162,20 +145,25 @@ def load_tf_hub_model(model_url: Text) -> Any: def _replace_deprecated_option( - old_option: Text, new_option: Union[Text, List[Text]], config: Dict[Text, Any] + old_option: Text, + new_option: Union[Text, List[Text]], + config: Dict[Text, Any], + warn_until_version: Text = NEXT_MAJOR_VERSION_FOR_DEPRECATIONS, ) -> Dict[Text, Any]: if old_option in config: if isinstance(new_option, str): - logger.warning( + common_utils.raise_deprecation_warning( f"Option '{old_option}' got renamed to '{new_option}'. " - f"Please update your configuration file." + f"Please update your configuration file.", + warn_until_version=warn_until_version, ) config[new_option] = config[old_option] else: - logger.warning( + common_utils.raise_deprecation_warning( f"Option '{old_option}' got renamed to " f"a dictionary '{new_option[0]}' with a key '{new_option[1]}'. " - f"Please update your configuration file." + f"Please update your configuration file.", + warn_until_version=warn_until_version, ) option_dict = config.get(new_option[0], {}) option_dict[new_option[1]] = config[old_option] @@ -194,38 +182,6 @@ def check_deprecated_options(config: Dict[Text, Any]) -> Dict[Text, Any]: Returns: updated model configuration """ - config = _replace_deprecated_option( - "hidden_layers_sizes_pre_dial", [HIDDEN_LAYERS_SIZES, DIALOGUE], config - ) - config = _replace_deprecated_option( - "hidden_layers_sizes_bot", [HIDDEN_LAYERS_SIZES, LABEL], config - ) - config = _replace_deprecated_option("droprate", DROP_RATE, config) - config = _replace_deprecated_option("droprate_a", DROP_RATE_DIALOGUE, config) - config = _replace_deprecated_option("droprate_b", DROP_RATE_LABEL, config) - config = _replace_deprecated_option( - "hidden_layers_sizes_a", [HIDDEN_LAYERS_SIZES, TEXT], config - ) - config = _replace_deprecated_option( - "hidden_layers_sizes_b", [HIDDEN_LAYERS_SIZES, LABEL], config - ) - config = _replace_deprecated_option( - "num_transformer_layers", NUM_TRANSFORMER_LAYERS, config - ) - config = _replace_deprecated_option("num_heads", NUM_HEADS, config) - config = _replace_deprecated_option("dense_dim", DENSE_DIMENSION, config) - config = _replace_deprecated_option("embed_dim", EMBEDDING_DIMENSION, config) - config = _replace_deprecated_option("num_neg", NUM_NEG, config) - config = _replace_deprecated_option("mu_pos", MAX_POS_SIM, config) - config = _replace_deprecated_option("mu_neg", MAX_NEG_SIM, config) - config = _replace_deprecated_option("use_max_sim_neg", USE_MAX_NEG_SIM, config) - config = _replace_deprecated_option("C2", REGULARIZATION_CONSTANT, config) - config = _replace_deprecated_option("C_emb", NEGATIVE_MARGIN_SCALE, config) - config = _replace_deprecated_option( - "evaluate_every_num_epochs", EVAL_NUM_EPOCHS, config - ) - config = _replace_deprecated_option( - "evaluate_on_num_examples", EVAL_NUM_EXAMPLES, config - ) + # note: call _replace_deprecated_option() here when there are options to deprecate return config diff --git a/rasa/utils/validation.py b/rasa/utils/validation.py index d4b70fa11ea4..df3373b8725d 100644 --- a/rasa/utils/validation.py +++ b/rasa/utils/validation.py @@ -2,7 +2,12 @@ from ruamel.yaml.constructor import DuplicateKeyError -from rasa.constants import PACKAGE_NAME, DOCS_URL_TRAINING_DATA_NLU +from rasa.constants import ( + PACKAGE_NAME, + DOCS_URL_TRAINING_DATA_NLU, + SCHEMA_EXTENSIONS_FILE, + SCHEMA_UTILS_FILE, +) class InvalidYamlFileError(ValueError): @@ -53,8 +58,18 @@ def validate_yaml_schema( try: schema_file = pkg_resources.resource_filename(PACKAGE_NAME, schema_path) + schema_utils_file = pkg_resources.resource_filename( + PACKAGE_NAME, SCHEMA_UTILS_FILE + ) + schema_extensions = pkg_resources.resource_filename( + PACKAGE_NAME, SCHEMA_EXTENSIONS_FILE + ) - c = Core(source_data=source_data, schema_files=[schema_file]) + c = Core( + source_data=source_data, + schema_files=[schema_file, schema_utils_file], + extensions=[schema_extensions], + ) c.validate(raise_exception=True) except SchemaError: raise InvalidYamlFileError( diff --git a/tests/core/actions/test_two_stage_fallback.py b/tests/core/actions/test_two_stage_fallback.py index e7899ddfdc0a..3b860d1c624d 100644 --- a/tests/core/actions/test_two_stage_fallback.py +++ b/tests/core/actions/test_two_stage_fallback.py @@ -144,7 +144,7 @@ async def test_ask_rephrase_after_failed_affirmation(): f""" responses: utter_ask_rephrase: - - {rephrase_text} + - text: {rephrase_text} """ ) action = TwoStageFallbackAction() diff --git a/tests/core/test_broker.py b/tests/core/test_broker.py index 05484146c49c..a3008c3803b1 100644 --- a/tests/core/test_broker.py +++ b/tests/core/test_broker.py @@ -54,26 +54,17 @@ def test_pika_message_property_app_id(monkeypatch: MonkeyPatch): @pytest.mark.parametrize( - "queue_arg,queues_arg,expected,warning", + "queues_arg,expected,warning", [ # default case - (None, ["q1"], ["q1"], None), - # only provide `queue` - ("q1", None, ["q1"], FutureWarning), - # supplying a list for `queue` works too - (["q1", "q2"], None, ["q1", "q2"], FutureWarning), - # `queues` arg supplied, takes precedence - ("q1", "q2", ["q2"], FutureWarning), - # same, but with a list - ("q1", ["q2", "q3"], ["q2", "q3"], FutureWarning), - # only supplying `queues` works, and queues is a string - (None, "q1", ["q1"], None), + (["q1", "q2"], ["q1", "q2"], None), + # `queues` arg supplied, as string + ("q1", ["q1"], None), # no queues provided. Use default queue and print warning. - (None, None, [DEFAULT_QUEUE_NAME], UserWarning), + (None, [DEFAULT_QUEUE_NAME], UserWarning), ], ) def test_pika_queues_from_args( - queue_arg: Union[Text, List[Text], None], queues_arg: Union[Text, List[Text], None], expected: List[Text], warning: Optional[Type[Warning]], @@ -83,7 +74,7 @@ def test_pika_queues_from_args( monkeypatch.setattr(PikaEventBroker, "_run_pika", lambda _: None) with pytest.warns(warning): - pika_producer = PikaEventBroker("", "", "", queues=queues_arg, queue=queue_arg) + pika_producer = PikaEventBroker("", "", "", queues=queues_arg) assert pika_producer.queues == expected diff --git a/tests/core/test_domain.py b/tests/core/test_domain.py index 43998c857dc1..987497f1ee70 100644 --- a/tests/core/test_domain.py +++ b/tests/core/test_domain.py @@ -295,44 +295,6 @@ def test_domain_to_yaml(): assert actual_yaml.strip() == test_yaml.strip() -def test_domain_to_yaml_deprecated_templates(): - test_yaml = f"""actions: -- utter_greet -config: - store_entities_as_slots: true -entities: [] -forms: [] -intents: [] -templates: - utter_greet: - - text: hey there! -session_config: - carry_over_slots_to_new_session: true - session_expiration_time: {DEFAULT_SESSION_EXPIRATION_TIME_IN_MINUTES} -slots: {{}}""" - - target_yaml = f"""actions: -- utter_greet -config: - store_entities_as_slots: true -entities: [] -forms: [] -intents: [] -responses: - utter_greet: - - text: hey there! -session_config: - carry_over_slots_to_new_session: true - session_expiration_time: {DEFAULT_SESSION_EXPIRATION_TIME_IN_MINUTES} -slots: {{}}""" - - domain = Domain.from_yaml(test_yaml) - # python 3 and 2 are different here, python 3 will have a leading set - # of --- at the beginning of the yml - assert domain.as_yaml().strip().endswith(target_yaml.strip()) - assert Domain.from_yaml(domain.as_yaml()) is not None - - def test_merge_yaml_domains(): test_yaml_1 = """config: store_entities_as_slots: true @@ -761,35 +723,6 @@ def test_clean_domain_for_file(): assert cleaned == expected -def test_clean_domain_deprecated_templates(): - domain_path = "data/test_domains/default_deprecated_templates.yml" - cleaned = Domain.load(domain_path).cleaned_domain() - - expected = { - "intents": [ - {"greet": {USE_ENTITIES_KEY: ["name"]}}, - {"default": {IGNORE_ENTITIES_KEY: ["unrelated_recognized_entity"]}}, - {"goodbye": {USE_ENTITIES_KEY: []}}, - {"thank": {USE_ENTITIES_KEY: []}}, - "ask", - {"why": {USE_ENTITIES_KEY: []}}, - "pure_intent", - ], - "entities": ["name", "unrelated_recognized_entity", "other"], - "responses": { - "utter_greet": [{"text": "hey there!"}], - "utter_goodbye": [{"text": "goodbye :("}], - "utter_default": [{"text": "default message"}], - }, - "actions": ["utter_default", "utter_greet", "utter_goodbye"], - } - - expected = Domain.from_dict(expected) - actual = Domain.from_dict(cleaned) - - assert actual.as_dict() == expected.as_dict() - - def test_add_knowledge_base_slots(default_domain): # don't modify default domain as it is used in other tests test_domain = copy.deepcopy(default_domain) @@ -874,42 +807,6 @@ def test_are_sessions_enabled(session_config: SessionConfig, enabled: bool): assert session_config.are_sessions_enabled() == enabled -def test_domain_utterance_actions_deprecated_templates(): - new_yaml = f"""config: - store_entities_as_slots: true -entities: [] -forms: [] -intents: [] -templates: - utter_greet: - - text: hey there! - utter_goodbye: - - text: bye! -session_config: - carry_over_slots_to_new_session: true - session_expiration_time: {DEFAULT_SESSION_EXPIRATION_TIME_IN_MINUTES} -slots: {{}}""" - - old_yaml = f"""config: - store_entities_as_slots: true -entities: [] -forms: [] -intents: [] -responses: - utter_greet: - - text: hey there! - utter_goodbye: - - text: bye! -session_config: - carry_over_slots_to_new_session: true - session_expiration_time: {DEFAULT_SESSION_EXPIRATION_TIME_IN_MINUTES} -slots: {{}}""" - - old_domain = Domain.from_yaml(old_yaml) - new_domain = Domain.from_yaml(new_yaml) - assert hash(old_domain) == hash(new_domain) - - def test_domain_from_dict_does_not_change_input(): input_before = { "intents": [ diff --git a/tests/core/test_tracker_stores.py b/tests/core/test_tracker_stores.py index 9c38c3dbfae4..fe27a38ea924 100644 --- a/tests/core/test_tracker_stores.py +++ b/tests/core/test_tracker_stores.py @@ -253,7 +253,7 @@ def test_tracker_serialisation(): ) -def test_deprecated_pickle_deserialisation(caplog: LogCaptureFixture): +def test_deprecated_pickle_deserialisation(): def pickle_serialise_tracker(_tracker): # mocked version of TrackerStore.serialise_tracker() that uses # the deprecated pickle serialisation @@ -269,13 +269,14 @@ def pickle_serialise_tracker(_tracker): # deprecation warning should be emitted - caplog.clear() # avoid counting debug messages - with caplog.at_level(logging.WARNING): + with pytest.warns(FutureWarning) as record: assert tracker == store.deserialise_tracker( UserMessage.DEFAULT_SENDER_ID, serialised ) - assert len(caplog.records) == 1 - assert "Deserialisation of pickled trackers will be deprecated" in caplog.text + assert len(record) == 1 + assert ( + "Deserialisation of pickled trackers is deprecated" in record[0].message.args[0] + ) @pytest.mark.parametrize( diff --git a/tests/nlu/training_data/formats/test_rasa_yaml.py b/tests/nlu/training_data/formats/test_rasa_yaml.py index 89175728e448..7c415518c8ca 100644 --- a/tests/nlu/training_data/formats/test_rasa_yaml.py +++ b/tests/nlu/training_data/formats/test_rasa_yaml.py @@ -288,7 +288,6 @@ def test_nlg_reads_any_multimedia(): chitchat/ask_weather: - text: Where do you want to check the weather? image: https://example.com/weather.jpg - temperature: 25°C """ ) @@ -300,7 +299,6 @@ def test_nlg_reads_any_multimedia(): { "text": "Where do you want to check the weather?", "image": "https://example.com/weather.jpg", - "temperature": "25°C", } ] } @@ -329,7 +327,7 @@ def test_nlg_fails_on_empty_response(): reader = RasaYAMLReader() - with pytest.raises(InvalidDomain): + with pytest.raises(ValueError): reader.reads(responses_yml) @@ -340,7 +338,6 @@ def test_nlg_multimedia_load_dump_roundtrip(): chitchat/ask_weather: - text: Where do you want to check the weather? image: https://example.com/weather.jpg - temperature: 25°C chitchat/ask_name: - text: My name is Sara. diff --git a/tests/test_validator.py b/tests/test_validator.py index b8f6cb40c53e..b7133068a867 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -70,7 +70,7 @@ async def test_fail_on_invalid_utterances(tmpdir): invalid_domain = str(tmpdir / "invalid_domain.yml") io_utils.write_yaml( { - "responses": {"utter_greet": {"text": "hello"}}, + "responses": {"utter_greet": [{"text": "hello"}]}, "actions": [ "utter_greet", "utter_non_existent", # error: utter template odes not exist diff --git a/tests/utils/test_common.py b/tests/utils/test_common.py index de550e179600..9706146b4ebd 100644 --- a/tests/utils/test_common.py +++ b/tests/utils/test_common.py @@ -3,8 +3,10 @@ import pytest +from rasa.constants import NEXT_MAJOR_VERSION_FOR_DEPRECATIONS from rasa.utils.common import ( raise_warning, + raise_deprecation_warning, sort_list_of_dicts_by_first_key, transform_collection_to_sentence, RepeatedLogFilter, @@ -74,6 +76,44 @@ def test_raise_deprecation(): assert isinstance(record[0].message, DeprecationWarning) +def test_raise_deprecation_warning(): + with pytest.warns(FutureWarning) as record: + raise_deprecation_warning( + "This feature is deprecated.", warn_until_version="3.0.0" + ) + + assert len(record) == 1 + assert ( + record[0].message.args[0] + == "This feature is deprecated. (will be removed in 3.0.0)" + ) + + +def test_raise_deprecation_warning_version_already_in_message(): + with pytest.warns(FutureWarning) as record: + raise_deprecation_warning( + "This feature is deprecated and will be removed in 3.0.0!", + warn_until_version="3.0.0", + ) + + assert len(record) == 1 + assert ( + record[0].message.args[0] + == "This feature is deprecated and will be removed in 3.0.0!" + ) + + +def test_raise_deprecation_warning_default(): + with pytest.warns(FutureWarning) as record: + raise_deprecation_warning("This feature is deprecated.") + + assert len(record) == 1 + assert record[0].message.args[0] == ( + f"This feature is deprecated. " + f"(will be removed in {NEXT_MAJOR_VERSION_FOR_DEPRECATIONS})" + ) + + def test_repeated_log_filter(): log_filter = RepeatedLogFilter() record1 = logging.LogRecord( diff --git a/tests/utils/test_validation.py b/tests/utils/test_validation.py index 22af4827feb6..00f589e4f4cf 100644 --- a/tests/utils/test_validation.py +++ b/tests/utils/test_validation.py @@ -28,6 +28,9 @@ def test_validate_yaml_schema(file, schema): "file, schema", [ ("data/test_domains/invalid_format.yml", DOMAIN_SCHEMA_FILE), + ("data/test_domains/wrong_response_format.yml", DOMAIN_SCHEMA_FILE), + ("data/test_domains/wrong_custom_response_format.yml", DOMAIN_SCHEMA_FILE), + ("data/test_domains/empty_response_format.yml", DOMAIN_SCHEMA_FILE), ("data/test_config/example_config.yaml", CONFIG_SCHEMA_FILE), ], )