From 756f8e237b04e4bba20869504eddfb9458299d78 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Feb 2024 16:16:31 +0100
Subject: [PATCH] Bump tokenizers from 0.15.1 to 0.15.2 in /bindings/python
 (#98)

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.15.1 to 0.15.2.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.15.1...v0.15.2)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 bindings/python/Cargo.lock | 18 ++++++------------
 bindings/python/Cargo.toml |  2 +-
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock
index f6fd584b..282a371e 100644
--- a/bindings/python/Cargo.lock
+++ b/bindings/python/Cargo.lock
@@ -600,7 +600,7 @@ dependencies = [
  "aho-corasick",
  "memchr",
  "regex-automata",
- "regex-syntax 0.8.2",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -611,15 +611,9 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.8.2",
+ "regex-syntax",
 ]
 
-[[package]]
-name = "regex-syntax"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
-
 [[package]]
 name = "regex-syntax"
 version = "0.8.2"
@@ -788,15 +782,15 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.15.1"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db445cceba5dfeb0f9702be7d6bfd91801ddcbe8fe8722defe7f2e96da75812"
+checksum = "3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d"
 dependencies = [
  "aho-corasick",
  "derive_builder",
  "esaxx-rs",
  "getrandom",
- "itertools 0.11.0",
+ "itertools 0.12.0",
  "lazy_static",
  "log",
  "macro_rules_attribute",
@@ -807,7 +801,7 @@ dependencies = [
  "rayon",
  "rayon-cond",
  "regex",
- "regex-syntax 0.7.5",
+ "regex-syntax",
  "serde",
  "serde_json",
  "spm_precompiled",
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index f1a5fd17..acfafc9e 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -17,6 +17,6 @@ crate-type = ["cdylib"]
 pyo3 = { version = "0.20.2", features = ["abi3-py38"] }
 text-splitter = { path = "../..", features = ["tiktoken-rs", "tokenizers"] }
 tiktoken-rs = "0.5.8"
-tokenizers = { version = "0.15.1", default_features = false, features = [
+tokenizers = { version = "0.15.2", default_features = false, features = [
     "onig",
 ] }