Feat: Bag of words for testing metric

test_unstructured/metrics/test_text_extraction.py

@@ -123,6 +123,21 @@ def test_calculate_edit_distance_with_filename(filename, expected_score, expecte
             "I have a dog and a cat, I love my dog.",
             {"i": 2, "have": 1, "a": 2, "dog": 2, "and": 1, "cat": 1, "love": 1, "my": 1},
         ),
+        (
+            "My dog's hair is red, but the dogs' houses are blue.",
+            {
+                "my": 1,
+                "dog's": 1,
+                "hair": 1,
+                "is": 2,
+                "red": 1,
+                "but": 1,
+                "the": 1,
+                "dogs": 1,
+                "house": 1,
+                "blue": 1,
+            },
+        ),
     ],
 )
 def test_bag_of_words(text, expected):

unstructured/metrics/text_extraction.py

@@ -4,6 +4,8 @@
 
 from rapidfuzz.distance import Levenshtein
 
+from unstructured.nlp.patterns import ENDS_IN_PUNCT_RE
+
 
 def calculate_edit_distance(
     output: str,
@@ -75,6 +77,12 @@ def bag_of_words(text: str) -> Dict[str, int]:
     incorrect_word: str = ""
     words = remove_punctuation(text.lower(), ["-", "'"]).split()
 
+    # Remove remaining punctuation
+    for idx in range(len(words)):
+        punct = ENDS_IN_PUNCT_RE.findall(words[idx])
+        if punct:
+            words[idx] = words[idx].replace(punct[0], "")
+
     i = 0
     while i < len(words):
         if len(words[i]) > 1: