diff --git a/changelog/7684.bugfix.md b/changelog/7684.bugfix.md new file mode 100644 index 000000000000..27998136c35a --- /dev/null +++ b/changelog/7684.bugfix.md @@ -0,0 +1,9 @@ +Fixed a problem when using the `POST /model/test/intents` endpoint together with a +[model server](model-storage.mdx#load-model-from-server). The error looked as follows: + +``` +ERROR rasa.core.agent:agent.py:327 Could not load model due to Detected inconsistent loop usage. Trying to schedule a task on a new event loop, but scheduler was created with a different event loop. Make sure there is only one event loop in use and that the scheduler is running on that one. +``` + +This also fixes a problem where testing a model from a model server would change the +production model. diff --git a/rasa/server.py b/rasa/server.py index b87ad5ec434f..67d720ee8aa1 100644 --- a/rasa/server.py +++ b/rasa/server.py @@ -1134,7 +1134,11 @@ async def _evaluate_model_using_test_set( if model_path: model_server = app.agent.model_server if model_server is not None: + model_server = model_server.copy() model_server.url = model_path + # Set wait time between pulls to `0` so that the agent does not schedule + # a job to pull the model from the server + model_server.kwargs["wait_time_between_pulls"] = 0 eval_agent = await _load_agent( model_path, model_server, app.agent.remote_storage ) diff --git a/tests/test_server.py b/tests/test_server.py index 42e562305859..7a005543ac13 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,39 +1,38 @@ import asyncio import json import os +import time +import urllib.parse +import uuid +from contextlib import ExitStack from http import HTTPStatus +from multiprocessing import Process, Manager from multiprocessing.managers import DictProxy from pathlib import Path -from unittest.mock import Mock, ANY -import requests -import time -import uuid -import urllib.parse - from typing import List, Text, Type, Generator, NoReturn, Dict, Optional -from contextlib import ExitStack +from unittest.mock import Mock, ANY +import pytest +import requests from _pytest import pathlib from _pytest.monkeypatch import MonkeyPatch from aioresponses import aioresponses - -import pytest from freezegun import freeze_time from mock import MagicMock -from multiprocessing import Process, Manager +from ruamel.yaml import StringIO +from sanic import Sanic +from sanic.testing import SanicASGITestClient import rasa import rasa.constants +import rasa.core.jobs +import rasa.nlu +import rasa.server import rasa.shared.constants import rasa.shared.utils.io import rasa.utils.io -import rasa.server -import rasa.nlu from rasa.core import utils -from rasa.core.tracker_store import InMemoryTrackerStore -from rasa.nlu.test import CVEvaluationResult -from rasa.shared.core import events -from rasa.core.agent import Agent +from rasa.core.agent import Agent, load_agent from rasa.core.channels import ( channel, CollectingOutputChannel, @@ -41,8 +40,11 @@ SlackInput, CallbackInput, ) -from rasa.train import TrainingResult from rasa.core.channels.slack import SlackBot +from rasa.core.tracker_store import InMemoryTrackerStore +from rasa.model import unpack_model +from rasa.nlu.test import CVEvaluationResult +from rasa.shared.core import events from rasa.shared.core.constants import ( ACTION_SESSION_START_NAME, ACTION_LISTEN_NAME, @@ -58,17 +60,12 @@ SessionStarted, ) from rasa.shared.core.trackers import DialogueStateTracker -from rasa.model import unpack_model from rasa.shared.nlu.constants import INTENT_NAME_KEY +from rasa.train import TrainingResult from rasa.utils.endpoints import EndpointConfig -from sanic import Sanic -from sanic.testing import SanicASGITestClient - from tests.core.conftest import DEFAULT_STACK_CONFIG from tests.nlu.utilities import ResponseTest from tests.utilities import json_of_latest_request, latest_request -from ruamel.yaml import StringIO - # a couple of event instances that we can use for testing test_events = [ @@ -122,6 +119,12 @@ def rasa_secured_app(rasa_server_secured: Sanic) -> SanicASGITestClient: return rasa_server_secured.asgi_client +@pytest.fixture() +async def tear_down_scheduler() -> Generator[None, None, None]: + yield None + rasa.core.jobs.__scheduler = None + + async def test_root(rasa_app: SanicASGITestClient): _, response = await rasa_app.get("/") assert response.status == HTTPStatus.OK @@ -810,7 +813,7 @@ async def test_evaluate_intent_on_just_nlu_model( async def test_evaluate_intent_with_model_param( - rasa_app: SanicASGITestClient, trained_nlu_model, default_nlu_data: Text + rasa_app: SanicASGITestClient, trained_nlu_model: Text, default_nlu_data: Text ): _, response = await rasa_app.get("/status") previous_model_file = response.json()["model_file"] @@ -834,6 +837,59 @@ async def test_evaluate_intent_with_model_param( assert previous_model_file == response.json()["model_file"] +async def test_evaluate_intent_with_model_server( + rasa_app: SanicASGITestClient, + trained_rasa_model: Text, + default_nlu_data: Text, + tear_down_scheduler: None, +): + production_model_server_url = ( + "https://example.com/webhooks/actions?model=production" + ) + test_model_server_url = "https://example.com/webhooks/actions?model=test" + + nlu_data = rasa.shared.utils.io.read_file(default_nlu_data) + + with aioresponses() as mocked: + # Mock retrieving the production model from the model server + mocked.get( + production_model_server_url, + body=Path(trained_rasa_model).read_bytes(), + headers={"ETag": "production"}, + ) + # Mock retrieving the test model from the model server + mocked.get( + test_model_server_url, + body=Path(trained_rasa_model).read_bytes(), + headers={"ETag": "test"}, + ) + + agent_with_model_server = await load_agent( + model_server=EndpointConfig(production_model_server_url) + ) + rasa_app.app.agent = agent_with_model_server + + _, response = await rasa_app.post( + f"/model/test/intents?model={test_model_server_url}", + data=nlu_data, + headers={"Content-type": rasa.server.YAML_CONTENT_TYPE}, + ) + + assert response.status == HTTPStatus.OK + assert set(response.json().keys()) == { + "intent_evaluation", + "entity_evaluation", + "response_selection_evaluation", + } + + production_model_server = rasa_app.app.agent.model_server + # Assert that the model server URL for the test didn't override the production + # model server URL + assert production_model_server.url == production_model_server_url + # Assert the tests didn't break pulling the models + assert production_model_server.kwargs.get("wait_time_between_pulls") != 0 + + async def test_cross_validation( rasa_app_nlu: SanicASGITestClient, default_nlu_data: Text ): @@ -1348,7 +1404,7 @@ async def test_load_model(rasa_app: SanicASGITestClient, trained_core_model: Tex async def test_load_model_from_model_server( - rasa_app: SanicASGITestClient, trained_core_model: Text + rasa_app: SanicASGITestClient, trained_core_model: Text, tear_down_scheduler: None ): _, response = await rasa_app.get("/status") @@ -1380,10 +1436,6 @@ async def test_load_model_from_model_server( assert old_fingerprint != response.json()["fingerprint"] - import rasa.core.jobs - - rasa.core.jobs.__scheduler = None - async def test_load_model_invalid_request_body(rasa_app: SanicASGITestClient): _, response = await rasa_app.put("/model")