Added configurable prefixes for Redis Tracker and Lock stores

changelog/6498.improvement.md

@@ -0,0 +1,2 @@
+Adding configurable prefixes to Redis [Tracker](./tracker-stores.mdx) and [Lock Stores](./lock-stores.mdx) so that a single Redis instance (and logical DB) can support multiple conversation trackers and locks.
+By default, conversations will be prefixed with `tracker:...` and all locks prefixed with `lock:...`. Additionally, you can add an alphanumeric-only `prefix: value` in `endpoints.yml` such that keys in redis will take the form `value:tracker:...` and `value:lock:...` respectively.

data/test_endpoints/example_endpoints.yml

@@ -14,6 +14,7 @@ tracker_store:
   port: 6379
   db: 0
   password: password
+  key_prefix: conversation
   record_exp: 30000
 # example of mongoDB external tracker store config
 #tracker_store:

docs/docs/lock-stores.mdx

@@ -57,6 +57,7 @@ address the same node when sending messages for a given conversation ID.
          port: <port of your redis instance, usually 6379>
          password: <password used for authentication>
          db: <number of your database within redis, e.g. 0>
+         key_prefix: <alphanumeric value to prepend to lock store keys>
      ```
 
   3. To start the Rasa Core server using your Redis backend, add the `--endpoints`
@@ -76,10 +77,13 @@ address the same node when sending messages for a given conversation ID.
 
   * `db` (default: `1`): The number of your redis database
 
+  * `key_prefix` (default: `None`): The prefix to prepend to lock store keys. Must
+     be alphanumeric
+
   * `password` (default: `None`): Password used for authentication
     (`None` equals no authentication)
 
   * `use_ssl` (default: `False`): Whether or not the communication is encrypted
 
   * `socket_timeout` (default: `10`): Time in seconds after which an
-     error is raised if Redis doesn't answer
+     error is raised if Redis doesn't answer

docs/docs/tracker-stores.mdx

@@ -179,14 +179,12 @@ To set up Rasa Open Source with Redis the following steps are required:
      type: redis
      url: <url of the redis instance, e.g. localhost>
      port: <port of your redis instance, usually 6379>
+     key_prefix: <alphanumeric value to prepend to tracker store keys>
      db: <number of your database within redis, e.g. 0>
      password: <password used for authentication>
      use_ssl: <whether or not the communication is encrypted, default `false`>
  ```
 
-3. To start the Rasa server using your configured Redis instance,
- add the `--endpoints` flag, e.g.:
-
  ```bash
  rasa run -m models --endpoints endpoints.yml
  ```
@@ -207,18 +205,20 @@ To set up Rasa Open Source with Redis the following steps are required:
       url: <url of the redis instance, e.g. localhost>
       port: <port of your redis instance, usually 6379>
       db: <number of your database within redis, e.g. 0>
+      key_prefix: <alphanumeric value to prepend to tracker store keys>
       password: <password used for authentication>
       use_ssl: <whether or not the communication is encrypted, default `false`>
  ```
 
-#### Configuration Parameters
-
 * `url` (default: `localhost`): The url of your redis instance
 
 * `port` (default: `6379`): The port which redis is running on
 
 * `db` (default: `0`): The number of your redis database
 
+* `key_prefix` (default: `None`): The prefix to prepend to tracker store keys. Must
+    be alphanumeric
+
 * `password` (default: `None`): Password used for authentication
 (`None` equals no authentication)
 

rasa/core/lock_store.py

@@ -22,6 +22,8 @@ def _get_lock_lifetime() -> int:
 LOCK_LIFETIME = _get_lock_lifetime()
 DEFAULT_SOCKET_TIMEOUT_IN_SECONDS = 10
 
+DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX = "lock:"
+
 
 # noinspection PyUnresolvedReferences
 class LockError(RasaException):
@@ -199,6 +201,7 @@ def __init__(
         db: int = 1,
         password: Optional[Text] = None,
         use_ssl: bool = False,
+        key_prefix: Optional[Text] = None,
         socket_timeout: float = DEFAULT_SOCKET_TIMEOUT_IN_SECONDS,
     ) -> None:
         """Create a lock store which uses Redis for persistence.
@@ -211,6 +214,8 @@ def __init__(
             password: The password which should be used for authentication with the
                 Redis database.
             use_ssl: `True` if SSL should be used for the connection to Redis.
+            key_prefix: prefix to prepend to all keys used by the lock store. Must be
+                alphanumeric.
             socket_timeout: Timeout in seconds after which an exception will be raised
                 in case Redis doesn't respond within `socket_timeout` seconds.
         """
@@ -224,19 +229,36 @@ def __init__(
             ssl=use_ssl,
             socket_timeout=socket_timeout,
         )
+
+        self.key_prefix = DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX
+        if key_prefix:
+            logger.debug(f"Setting non-default redis key prefix: '{key_prefix}'.")
+            self._set_key_prefix(key_prefix)
+
         super().__init__()
 
+    def _set_key_prefix(self, key_prefix: Text) -> None:
+        if isinstance(key_prefix, str) and key_prefix.isalnum():
+            self.key_prefix = key_prefix + ":" + DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX
+        else:
+            logger.warning(
+                f"Omitting provided non-alphanumeric redis key prefix: '{key_prefix}'. Using default '{self.key_prefix}' instead."
+            )
+
+    def _get_key_prefix(self) -> Text:
+        return self.key_prefix
+
     def get_lock(self, conversation_id: Text) -> Optional[TicketLock]:
-        serialised_lock = self.red.get(conversation_id)
+        serialised_lock = self.red.get(self.key_prefix + conversation_id)
         if serialised_lock:
             return TicketLock.from_dict(json.loads(serialised_lock))
 
     def delete_lock(self, conversation_id: Text) -> None:
-        deletion_successful = self.red.delete(conversation_id)
+        deletion_successful = self.red.delete(self.key_prefix + conversation_id)
         self._log_deletion(conversation_id, deletion_successful)
 
     def save_lock(self, lock: TicketLock) -> None:
-        self.red.set(lock.conversation_id, lock.dumps())
+        self.red.set(self.key_prefix + lock.conversation_id, lock.dumps())
 
 
 class InMemoryLockStore(LockStore):

rasa/core/tracker_store.py

@@ -58,6 +58,9 @@
 POSTGRESQL_DEFAULT_MAX_OVERFLOW = 100
 POSTGRESQL_DEFAULT_POOL_SIZE = 50
 
+# default value for key prefix in RedisTrackerStore
+DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX = "tracker:"
+
 
 class TrackerStore:
     """Class to hold all of the TrackerStore classes"""
@@ -305,6 +308,7 @@ def __init__(
         password: Optional[Text] = None,
         event_broker: Optional[EventBroker] = None,
         record_exp: Optional[float] = None,
+        key_prefix: Optional[Text] = None,
         use_ssl: bool = False,
         **kwargs: Dict[Text, Any],
     ) -> None:
@@ -314,8 +318,25 @@ def __init__(
             host=host, port=port, db=db, password=password, ssl=use_ssl
         )
         self.record_exp = record_exp
+
+        self.key_prefix = DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX
+        if key_prefix:
+            logger.debug(f"Setting non-default redis key prefix: '{key_prefix}'.")
+            self._set_key_prefix(key_prefix)
+
         super().__init__(domain, event_broker, **kwargs)
 
+    def _set_key_prefix(self, key_prefix: Text) -> None:
+        if isinstance(key_prefix, str) and key_prefix.isalnum():
+            self.key_prefix = key_prefix + ":" + DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX
+        else:
+            logger.warning(
+                f"Omitting provided non-alphanumeric redis key prefix: '{key_prefix}'. Using default '{self.key_prefix}' instead."
+            )
+
+    def _get_key_prefix(self) -> Text:
+        return self.key_prefix
+
     def save(self, tracker, timeout=None):
         """Saves the current conversation state"""
         if self.event_broker:
@@ -325,18 +346,18 @@ def save(self, tracker, timeout=None):
             timeout = self.record_exp
 
         serialised_tracker = self.serialise_tracker(tracker)
-        self.red.set(tracker.sender_id, serialised_tracker, ex=timeout)
+        self.red.set(self.prefix + tracker.sender_id, serialised_tracker, ex=timeout)
 
     def retrieve(self, sender_id: Text) -> Optional[DialogueStateTracker]:
-        stored = self.red.get(sender_id)
+        stored = self.red.get(self.prefix + sender_id)
         if stored is not None:
             return self.deserialise_tracker(sender_id, stored)
         else:
             return None
 
     def keys(self) -> Iterable[Text]:
         """Returns keys of the Redis Tracker Store"""
-        return self.red.keys()
+        return self.red.keys(self.prefix + "*")
 
 
 class DynamoTrackerStore(TrackerStore):

tests/core/test_lock_store.py

@@ -16,7 +16,13 @@
 from rasa.core.constants import DEFAULT_LOCK_LIFETIME
 from rasa.shared.constants import INTENT_MESSAGE_PREFIX
 from rasa.core.lock import TicketLock
-from rasa.core.lock_store import InMemoryLockStore, LockError, LockStore, RedisLockStore
+from rasa.core.lock_store import (
+    InMemoryLockStore,
+    LockError,
+    LockStore,
+    RedisLockStore,
+    DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX,
+)
 
 
 class FakeRedisLockStore(RedisLockStore):
@@ -32,6 +38,8 @@ def __init__(self):
         # added in redis==3.3.0, but not yet in fakeredis
         self.red.connection_pool.connection_class.health_check_interval = 0
 
+        self.key_prefix = DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX
+
 
 def test_issue_ticket():
     lock = TicketLock("random id 0")
@@ -287,3 +295,46 @@ async def test_redis_lock_store_timeout(monkeypatch: MonkeyPatch):
     with pytest.raises(LockError):
         async with lock_store.lock("some sender"):
             pass
+
+
+async def test_redis_lock_store_with_invalid_prefix(monkeypatch: MonkeyPatch):
+    import redis.exceptions
+
+    lock_store = FakeRedisLockStore()
+
+    prefix = "!asdf234 34#"
+    lock_store._set_key_prefix(prefix)
+    assert lock_store._get_key_prefix() == DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX
+
+    monkeypatch.setattr(
+        lock_store,
+        lock_store.get_or_create_lock.__name__,
+        Mock(side_effect=redis.exceptions.TimeoutError),
+    )
+
+    with pytest.raises(LockError):
+        async with lock_store.lock("some sender"):
+            pass
+
+
+async def test_redis_lock_store_with_valid_prefix(monkeypatch: MonkeyPatch):
+    import redis.exceptions
+
+    lock_store = FakeRedisLockStore()
+
+    prefix = "chatbot42"
+    lock_store._set_key_prefix(prefix)
+    assert (
+        lock_store._get_key_prefix()
+        == prefix + ":" + DEFAULT_REDIS_LOCK_STORE_KEY_PREFIX
+    )
+
+    monkeypatch.setattr(
+        lock_store,
+        lock_store.get_or_create_lock.__name__,
+        Mock(side_effect=redis.exceptions.TimeoutError),
+    )
+
+    with pytest.raises(LockError):
+        async with lock_store.lock("some sender"):
+            pass

tests/core/test_tracker_stores.py

@@ -35,6 +35,7 @@
     TrackerStore,
     InMemoryTrackerStore,
     RedisTrackerStore,
+    DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX,
     SQLTrackerStore,
     DynamoTrackerStore,
     FailSafeTrackerStore,
@@ -148,6 +149,42 @@ def test_create_tracker_store_from_endpoint_config(default_domain: Domain):
     assert isinstance(tracker_store, type(TrackerStore.create(store, default_domain)))
 
 
+def test_redis_tracker_store_invalid_key_prefix(default_domain: Domain):
+
+    test_invalid_key_prefix = "$$ &!"
+
+    tracker_store = RedisTrackerStore(
+        domain=default_domain,
+        host="localhost",
+        port=6379,
+        db=0,
+        password="password",
+        key_prefix=test_invalid_key_prefix,
+        record_exp=3000,
+    )
+
+    assert tracker_store._get_key_prefix() == DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX
+
+
+def test_redis_tracker_store_valid_key_prefix(default_domain: Domain):
+    test_valid_key_prefix = "spanish"
+
+    tracker_store = RedisTrackerStore(
+        domain=default_domain,
+        host="localhost",
+        port=6379,
+        db=0,
+        password="password",
+        key_prefix=test_valid_key_prefix,
+        record_exp=3000,
+    )
+
+    assert (
+        tracker_store._get_key_prefix()
+        == f"{test_valid_key_prefix}:{DEFAULT_REDIS_TRACKER_STORE_KEY_PREFIX}"
+    )
+
+
 def test_exception_tracker_store_from_endpoint_config(
     default_domain: Domain, monkeypatch: MonkeyPatch
 ):

tests/shared/core/test_trackers.py

@@ -83,6 +83,9 @@ def __init__(self, _domain: Domain) -> None:
         # added in redis==3.3.0, but not yet in fakeredis
         self.red.connection_pool.connection_class.health_check_interval = 0
 
+        # Defined in RedisTrackerStore but needs to be added for the MockRedisTrackerStore
+        self.prefix = "tracker:"
+
         TrackerStore.__init__(self, _domain)