Fix usage of empty spaCy model

changelog/5638.bugfix.rst

@@ -0,0 +1 @@
+Fix: DIET breaks with empty spaCy model

rasa/nlu/featurizers/dense_featurizer/spacy_featurizer.py

@@ -1,5 +1,6 @@
 import numpy as np
 import typing
+import logging
 from typing import Any, Optional, Text, Dict, List, Type
 
 from rasa.nlu.config import RasaNLUModelConfig
@@ -20,6 +21,9 @@
     from spacy.tokens import Doc
 
 
+logger = logging.getLogger(__name__)
+
+
 class SpacyFeaturizer(DenseFeaturizer):
     @classmethod
     def required_components(cls) -> List[Type[Component]]:
@@ -52,25 +56,29 @@ def train(
                 self._set_spacy_features(example, attribute)
 
     def get_doc(self, message: Message, attribute: Text) -> Any:
-
         return message.get(SPACY_DOCS[attribute])
 
     def process(self, message: Message, **kwargs: Any) -> None:
-
         self._set_spacy_features(message)
 
-    def _set_spacy_features(self, message: Message, attribute: Text = TEXT):
+    def _set_spacy_features(self, message: Message, attribute: Text = TEXT) -> None:
         """Adds the spacy word vectors to the messages features."""
+        doc = self.get_doc(message, attribute)
+
+        if doc is None:
+            return
 
-        message_attribute_doc = self.get_doc(message, attribute)
+        # in case an empty spaCy model was used, no vectors are present
+        if doc.vocab.vectors_length == 0:
+            logger.debug("No features present. You are using an empty spaCy model.")
+            return
 
-        if message_attribute_doc is not None:
-            features = self._features_for_doc(message_attribute_doc)
+        features = self._features_for_doc(doc)
 
-            cls_token_vec = self._calculate_cls_vector(features, self.pooling_operation)
-            features = np.concatenate([features, cls_token_vec])
+        cls_token_vec = self._calculate_cls_vector(features, self.pooling_operation)
+        features = np.concatenate([features, cls_token_vec])
 
-            features = self._combine_with_existing_dense_features(
-                message, features, DENSE_FEATURE_NAMES[attribute]
-            )
-            message.set(DENSE_FEATURE_NAMES[attribute], features)
+        features = self._combine_with_existing_dense_features(
+            message, features, DENSE_FEATURE_NAMES[attribute]
+        )
+        message.set(DENSE_FEATURE_NAMES[attribute], features)

tests/nlu/featurizers/test_spacy_featurizer.py

@@ -99,11 +99,11 @@ def test_spacy_featurizer_sequence(sentence, expected, spacy_nlp):
     greet = {"intent": "greet", "text_features": [0.5]}
 
     message = Message(sentence, greet)
-    message.set("text_spacy_doc", doc)
+    message.set(SPACY_DOCS[TEXT], doc)
 
     ftr._set_spacy_features(message)
 
-    vecs = message.get("text_dense_features")[0][:5]
+    vecs = message.get(DENSE_FEATURE_NAMES[TEXT])[0][:5]
 
     assert np.allclose(token_vectors[0][:5], vecs, atol=1e-4)
     assert np.allclose(vecs, expected, atol=1e-4)
@@ -165,3 +165,24 @@ def test_spacy_featurizer_train(spacy_nlp):
     vecs = message.get(DENSE_FEATURE_NAMES[INTENT])
 
     assert vecs is None
+
+
+def test_spacy_featurizer_using_empty_model():
+    from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
+    import spacy
+
+    sentence = "This test is using an empty spaCy model"
+
+    model = spacy.blank("en")
+    doc = model(sentence)
+
+    ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
+
+    message = Message(sentence)
+    message.set(SPACY_DOCS[TEXT], doc)
+
+    ftr._set_spacy_features(message)
+
+    vecs = message.get(DENSE_FEATURE_NAMES[TEXT])
+
+    assert vecs is None