diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 5716a68..a910076 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -768,9 +768,9 @@ dependencies = [ [[package]] name = "tiktoken-rs" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9ae5a3c24361e5f038af22517ba7f8e3af4099e30e78a3d56f86b48238fce9d" +checksum = "2d9bfda9bba35383feec631f997c8fd49a2d744b189103abf8354270ad668bfb" dependencies = [ "anyhow", "base64 0.21.4", diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 340758c..25b0138 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.20.0", features = ["abi3-py37"] } text-splitter = { version = "0.4.4", features = ["tiktoken-rs", "tokenizers"] } -tiktoken-rs = "0.5.4" +tiktoken-rs = "0.5.5" tokenizers = { version = "0.14.1", default_features = false, features = [ "onig", ] }