From 14fd5616de4ce0722eb9d532cb35c8f0d56ba761 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 3 Dec 2024 22:32:15 +0400 Subject: [PATCH] Support Already Converted Llava Tokenizers (#332) --- src/regex_normalization.cpp | 3 ++- tests/layer_tests.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/regex_normalization.cpp b/src/regex_normalization.cpp index dd5b5b79..37b4ed87 100644 --- a/src/regex_normalization.cpp +++ b/src/regex_normalization.cpp @@ -31,7 +31,8 @@ std::string reformat_replace_pattern(std::string replace_pattern) { const std::map search_pattern_rewrites = { {R"( ([\\.\\?\\!,])| ('[ms])| (') | ('[rv]e)| (n't))", R"((?| ([\\.\\?\\!,])| ('[ms])| (') | ('[rv]e)| (n't)))"}, - {R"((^)(.))", R"((^)([\s\S]))"} + {R"((^)(.))", R"((^)([\s\S]))"}, + {R"((^)(.+))", R"((^)([\s\S]))"} }; /** diff --git a/tests/layer_tests.py b/tests/layer_tests.py index 17ed84f4..f46b1be9 100644 --- a/tests/layer_tests.py +++ b/tests/layer_tests.py @@ -154,6 +154,14 @@ def test_charsmap_normalizartion(test_string, hf_charsmap_tokenizer, precompiled replace_term=r"▁\2", ) ), + ( # test backward compatibility with old regex + "\n", + "▁\n", + RegexNormalizationStep( + regex_search_pattern=r"(^)(.+)", + replace_term=r"▁$2", + ) + ), ] ) def test_regex_normalization(test_string, expected, layer):