Bump tokenizers from 0.15.0 to 0.15.1 in /bindings/python (#88)

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.15.0 to 0.15.1. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](huggingface/tokenizers@v0.15.0...v0.15.1) --- updated-dependencies: - dependency-name: tokenizers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
benbrandt · Jan 23, 2024 · b8469f1 · b8469f1
1 parent 2c474b4
commit b8469f1
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -17,6 +17,6 @@ crate-type = ["cdylib"]
 pyo3 = { version = "0.20.2", features = ["abi3-py38"] }
 text-splitter = { path = "../..", features = ["tiktoken-rs", "tokenizers"] }
 tiktoken-rs = "0.5.8"
-tokenizers = { version = "0.15.0", default_features = false, features = [
+tokenizers = { version = "0.15.1", default_features = false, features = [
     "onig",
 ] }