diff --git a/Cargo.toml b/Cargo.toml index 0fffcd3f..11977f8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,9 +71,7 @@ rust_tokenizers = { version = "8", optional = true } strum = { version = "0.26", features = ["derive"] } thiserror = "1.0.63" tiktoken-rs = { version = "0.5", optional = true } -tokenizers = { version = "0.20", default-features = false, features = [ - "onig", -], optional = true } +tokenizers = { version = "0.20", default-features = false, optional = true } tree-sitter = { version = "0.23", optional = true } unicode-segmentation = "1.11" @@ -102,7 +100,7 @@ code = ["dep:tree-sitter"] markdown = ["dep:pulldown-cmark"] rust-tokenizers = ["dep:rust_tokenizers"] tiktoken-rs = ["dep:tiktoken-rs"] -tokenizers = ["dep:tokenizers"] +tokenizers = ["dep:tokenizers", "tokenizers/onig"] [lints] workspace = true