Merge pull request #8714 from RasaHQ/architecture-prototype/featuriza…

…tion-fixes Architecture prototype/featurization fixes
RasaHQ · May 20, 2021 · d812376 · d812376
2 parents 6b3bdf6 + c95f49b
commit d812376
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 5 deletions.
diff --git a/rasa/architecture_prototype/config_to_graph.py b/rasa/architecture_prototype/config_to_graph.py
@@ -62,6 +62,8 @@ def _train_and_process_component(
             process_task_name: {
                 "uses": component_class,
                 "fn": process_function,
+                "constructor_name": "load",
+                "eager": False,
                 "config": config,
                 "needs": {
                     "resource_name": train_task_name,

diff --git a/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py
@@ -737,6 +737,13 @@ def train(
         else:
             self._train_with_independent_vocab(attribute_texts)
 
+        return self.persist()
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        # process sentences and collect data for all attributes
+        processed_attribute_tokens = self._get_all_attributes_processed_tokens(
+            training_data
+        )
         # transform for all attributes
         for attribute in self._attributes:
             sequence_features, sentence_features = self._get_featurized_attribute(
@@ -750,11 +757,6 @@ def train(
                     sentence_features,
                     training_data.training_examples,
                 )
-
-        return self.persist()
-
-    def process_training_data(self, training_data: TrainingData) -> TrainingData:
-        self.process(training_data.training_examples)
         return training_data
 
     def process(self, messages: List[Message], **kwargs: Any) -> List[Message]:

diff --git a/tests/architecture_prototype/graph_schema.py b/tests/architecture_prototype/graph_schema.py
@@ -101,6 +101,8 @@
     "process_core_RegexFeaturizer_1": {
         "uses": RegexFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {},
         "needs": {
             "resource_name": "train_core_RegexFeaturizer_1",
@@ -116,6 +118,8 @@
     "process_core_LexicalSyntacticFeaturizer_2": {
         "uses": LexicalSyntacticFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {},
         "needs": {
             "resource_name": "train_core_LexicalSyntacticFeaturizer_2",
@@ -131,6 +135,8 @@
     "process_core_CountVectorsFeaturizer_3": {
         "uses": CountVectorsFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {},
         "needs": {
             "resource_name": "train_core_CountVectorsFeaturizer_3",
@@ -146,6 +152,8 @@
     "process_core_CountVectorsFeaturizer_4": {
         "uses": CountVectorsFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {"analyzer": "char_wb", "min_ngram": 1, "max_ngram": 4},
         "needs": {
             "resource_name": "train_core_CountVectorsFeaturizer_4",
@@ -182,6 +190,8 @@
     "process_RegexFeaturizer_1": {
         "uses": RegexFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {},
         "needs": {
             "resource_name": "train_RegexFeaturizer_1",
@@ -197,6 +207,8 @@
     "process_LexicalSyntacticFeaturizer_2": {
         "uses": LexicalSyntacticFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {},
         "needs": {
             "resource_name": "train_LexicalSyntacticFeaturizer_2",
@@ -212,6 +224,8 @@
     "process_CountVectorsFeaturizer_3": {
         "uses": CountVectorsFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {},
         "needs": {
             "resource_name": "train_CountVectorsFeaturizer_3",
@@ -227,6 +241,8 @@
     "process_CountVectorsFeaturizer_4": {
         "uses": CountVectorsFeaturizer,
         "fn": "process_training_data",
+        "constructor_name": "load",
+        "eager": False,
         "config": {"analyzer": "char_wb", "min_ngram": 1, "max_ngram": 4},
         "needs": {
             "resource_name": "train_CountVectorsFeaturizer_4",