Support Already Converted Llava Tokenizers

openvinotoolkit · Dec 3, 2024 · 3fcb72a · 3fcb72a
1 parent 7ce02af
commit 3fcb72a
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/src/regex_normalization.cpp b/src/regex_normalization.cpp
@@ -31,7 +31,8 @@ std::string reformat_replace_pattern(std::string replace_pattern) {
 
 const std::map<std::string, std::string> search_pattern_rewrites = {
     {R"( ([\\.\\?\\!,])| ('[ms])| (') | ('[rv]e)| (n't))", R"((?| ([\\.\\?\\!,])| ('[ms])| (') | ('[rv]e)| (n't)))"},
-    {R"((^)(.))", R"((^)([\s\S]))"}
+    {R"((^)(.))", R"((^)([\s\S]))"},
+    {R"((^)(.+))", R"((^)([\s\S]))"}
 };
 
 /**

diff --git a/tests/layer_tests.py b/tests/layer_tests.py
@@ -154,6 +154,14 @@ def test_charsmap_normalizartion(test_string, hf_charsmap_tokenizer, precompiled
                 replace_term=r"▁\2",
             )
         ),
+        (  # test backward compatibility with old regex
+            "\n",
+            "▁\n",
+            RegexNormalizationStep(
+                regex_search_pattern=r"(^)(.+)",
+                replace_term=r"▁$2",
+            )
+        ),
     ]
 )
 def test_regex_normalization(test_string, expected, layer):