Readable tokenizer file

benbrandt · Jan 20, 2024 · f2fe1ac · f2fe1ac
1 parent 40aa6f6
commit f2fe1ac
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 1 deletion.
diff --git a/tests/text_splitter.rs b/tests/text_splitter.rs
@@ -133,7 +133,7 @@ fn random_chunk_range() {
 #[test]
 fn huggingface_small_chunk_behavior() {
     let tokenizer =
-        tokenizers::Tokenizer::from_file("./tests/tokenizers/alpha-03-128k.json").unwrap();
+        tokenizers::Tokenizer::from_file("./tests/tokenizers/huggingface.json").unwrap();
     let splitter = TextSplitter::new(tokenizer);
 
     let text = "notokenexistsforthisword";

diff --git a/tests/tokenizers/alpha-03-128k.json → tests/tokenizers/huggingface.json b/tests/tokenizers/alpha-03-128k.json → tests/tokenizers/huggingface.json