alpha2digits keeps newlines. Fixes #94.

allo-media · Aug 14, 2023 · bfcb54d · bfcb54d
1 parent 497d461
commit bfcb54d
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/tests/test_text_to_num_fr.py b/tests/test_text_to_num_fr.py
@@ -201,3 +201,6 @@ def test_article(self):
     def test_un_pronoun(self):
         source = "Je n'en veux qu'un. J'annonce: le un"
         self.assertEqual(alpha2digit(source, "fr"), source)
+
+    def test_alpha2digit_newline(self):
+        self.assertEqual(alpha2digit("dix + deux\n= douze", "fr"), "10 + 2\n= 12")
diff --git a/text_to_num/transforms.py b/text_to_num/transforms.py
@@ -36,6 +36,7 @@
 
 omg = OrdinalsMerger()
 USE_PT_ORDINALS_MERGER = True
+WORD_SEP = re.compile(r"\s*[\.,;\(\)…\[\]:!\?]+\s*|\n")
 
 
 def look_ahead(sequence: Sequence[Any]) -> Iterator[Tuple[Any, Any]]:
@@ -108,10 +109,8 @@ def alpha2digit(
         raise Exception("Language not supported")
 
     language = LANG[lang]
-    segments = re.split(
-        r"\s*[\.,;\(\)…\[\]:!\?]+\s*", text
-    )
-    punct = re.findall(r"\s*[\.,;\(\)…\[\]:!\?]+\s*", text)
+    segments = WORD_SEP.split(text)
+    punct = WORD_SEP.findall(text)
     if len(punct) < len(segments):
         punct.append("")