remove compiled-in tokenizers

microsoft · Feb 3, 2024 · ec4c172 · ec4c172
1 parent 8c2ac7a
commit ec4c172
Show file tree

Hide file tree

Showing 15 changed files with 0 additions and 1,095,990 deletions.
diff --git a/aicirt/src/hf-tokenizers/falcon.json b/aicirt/src/hf-tokenizers/falcon.json
diff --git a/aicirt/src/hf-tokenizers/fix.js b/aicirt/src/hf-tokenizers/fix.js
diff --git a/aicirt/src/hf-tokenizers/gpt2.json b/aicirt/src/hf-tokenizers/gpt2.json