From ae744ad4dc24630a1d3bc9e585dbc9d84a0f9311 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 18 Apr 2024 10:30:03 +0000
Subject: [PATCH] Bump tokenizers from 0.19.0 to 0.19.1

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.19.0 to 0.19.1.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.19.0...v0.19.1)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock                 | 4 ++--
 Cargo.toml                 | 4 ++--
 bindings/python/Cargo.toml | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index adc4878..65f93df 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1379,9 +1379,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.19.0"
+version = "0.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3429c3955ff2d769edd699768796dafd1bbff0005ddca66882a1a9fce1b8c84"
+checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
 dependencies = [
  "aho-corasick",
  "derive_builder",
diff --git a/Cargo.toml b/Cargo.toml
index c691866..0de76a0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -51,7 +51,7 @@ once_cell = "1.19.0"
 pulldown-cmark = { version = "0.10.2", default-features = false, optional = true }
 regex = "1.10.4"
 tiktoken-rs = { version = "0.5.8", optional = true }
-tokenizers = { version = "0.19.0", default_features = false, features = [
+tokenizers = { version = "0.19.1", default_features = false, features = [
     "onig",
 ], optional = true }
 unicode-segmentation = "1.11.0"
@@ -61,7 +61,7 @@ divan = "0.1.14"
 fake = "2.9.2"
 insta = { version = "1.38.0", features = ["glob", "yaml"] }
 more-asserts = "0.3.1"
-tokenizers = { version = "0.19.0", default-features = false, features = [
+tokenizers = { version = "0.19.1", default-features = false, features = [
     "onig",
     "http",
 ] }
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index 8e0defb..4a240f1 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -23,6 +23,6 @@ text-splitter = { path = "../..", features = [
     "tokenizers",
 ] }
 tiktoken-rs = "0.5.8"
-tokenizers = { version = "0.19.0", default_features = false, features = [
+tokenizers = { version = "0.19.1", default_features = false, features = [
     "onig",
 ] }