Bump tokenizers from 0.15.1 to 0.15.2 in /bindings/python (#98)

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.15.1 to 0.15.2. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](huggingface/tokenizers@v0.15.1...v0.15.2) --- updated-dependencies: - dependency-name: tokenizers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
benbrandt · Feb 12, 2024 · 756f8e2 · 756f8e2
1 parent 53dc6fa
commit 756f8e2
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 13 deletions.
diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -17,6 +17,6 @@ crate-type = ["cdylib"]
 pyo3 = { version = "0.20.2", features = ["abi3-py38"] }
 text-splitter = { path = "../..", features = ["tiktoken-rs", "tokenizers"] }
 tiktoken-rs = "0.5.8"
-tokenizers = { version = "0.15.1", default_features = false, features = [
+tokenizers = { version = "0.15.2", default_features = false, features = [
     "onig",
 ] }