Merge pull request #5825 from RasaHQ/fix-ensure-consistent-bilou-tagging

Fix error in 'ensure_consistent_bilou_tagging'

changelog/5825.bugfix.rst

@@ -0,0 +1 @@
+Fix list index out of range error in ``ensure_consistent_bilou_tagging``.

rasa/nlu/utils/bilou_utils.py

@@ -295,6 +295,15 @@ def _find_bilou_end(start_idx: int, predicted_tags: List[Text]) -> int:
     start_tag = tag_without_prefix(predicted_tags[start_idx])
 
     while not finished:
+        if current_idx >= len(predicted_tags):
+            logger.debug(
+                "Inconsistent BILOU tagging found, B- tag not closed by L- tag, "
+                "i.e [B-a, I-a, O] instead of [B-a, L-a, O].\n"
+                "Assuming last tag is L- instead of I-."
+            )
+            current_idx -= 1
+            break
+
         current_label = predicted_tags[current_idx]
         prefix = bilou_prefix_from_tag(current_label)
         tag = tag_without_prefix(current_label)

tests/nlu/utils/test_bilou_utils.py

@@ -163,6 +163,22 @@ def test_apply_bilou_schema():
             "B- tag, L- tag pair encloses multiple entity classes",
         ),
         (["O", "B-person", "O"], ["O", "U-person", "O"], "B- tag not closed"),
+        (["O", "B-person"], ["O", "U-person"], "B- tag not closed"),
+        (
+            ["O", "B-person", "I-person"],
+            ["O", "B-person", "L-person"],
+            "B- tag not closed",
+        ),
+        (
+            ["O", "B-person", "I-location"],
+            ["O", "B-person", "L-person"],
+            "B- tag not closed",
+        ),
+        (
+            ["O", "B-person", "B-location"],
+            ["O", "U-person", "U-location"],
+            "B- tag not closed",
+        ),
     ],
 )
 def test_check_consistent_bilou_tagging(