diff --git a/Cargo.lock b/Cargo.lock index adc4878..65f93df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1379,9 +1379,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3429c3955ff2d769edd699768796dafd1bbff0005ddca66882a1a9fce1b8c84" +checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" dependencies = [ "aho-corasick", "derive_builder", diff --git a/Cargo.toml b/Cargo.toml index c691866..0de76a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ once_cell = "1.19.0" pulldown-cmark = { version = "0.10.2", default-features = false, optional = true } regex = "1.10.4" tiktoken-rs = { version = "0.5.8", optional = true } -tokenizers = { version = "0.19.0", default_features = false, features = [ +tokenizers = { version = "0.19.1", default_features = false, features = [ "onig", ], optional = true } unicode-segmentation = "1.11.0" @@ -61,7 +61,7 @@ divan = "0.1.14" fake = "2.9.2" insta = { version = "1.38.0", features = ["glob", "yaml"] } more-asserts = "0.3.1" -tokenizers = { version = "0.19.0", default-features = false, features = [ +tokenizers = { version = "0.19.1", default-features = false, features = [ "onig", "http", ] } diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 8e0defb..4a240f1 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -23,6 +23,6 @@ text-splitter = { path = "../..", features = [ "tokenizers", ] } tiktoken-rs = "0.5.8" -tokenizers = { version = "0.19.0", default_features = false, features = [ +tokenizers = { version = "0.19.1", default_features = false, features = [ "onig", ] }