From 7c136a6d38a405f1b2ebdbe97c4ddd6dabf9647e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:03:53 +0000 Subject: [PATCH] Bump tokenizers from 0.15.2 to 0.19.0 Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.15.2 to 0.19.0. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](https://github.com/huggingface/tokenizers/compare/v0.15.2...v0.19.0) --- updated-dependencies: - dependency-name: tokenizers dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 83 +++++++++++++++++--------------------- Cargo.toml | 4 +- bindings/python/Cargo.toml | 2 +- 3 files changed, 39 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12bccd2..adc4878 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,7 +51,7 @@ dependencies = [ "derive_utils", "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -219,9 +219,9 @@ checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "darling" -version = "0.14.4" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ "darling_core", "darling_macro", @@ -229,58 +229,58 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.4" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 1.0.109", + "syn", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "derive_builder" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "derive_builder_macro" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 1.0.109", + "syn", ] [[package]] @@ -291,7 +291,7 @@ checksum = "61bb5a1014ce6dfc2a378578509abe775a5aa06bff584a547555d9efdb81b926" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -343,7 +343,7 @@ checksum = "27540baf49be0d484d8f0130d7d8da3011c32a44d4fc873368154f1510e574a2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -668,9 +668,9 @@ dependencies = [ [[package]] name = "monostate" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c2a1f1c70e5724fa28f101ca787b6a7e8ad5c5e4ae4ca3b0fa4a419fa9075" +checksum = "a20fffcd8ca4c69d31e036a71abc400147b41f90895df4edcb36497a1f8af8bf" dependencies = [ "monostate-impl", "serde", @@ -678,13 +678,13 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f686d68a09079e63b1d2c64aa305095887ce50565f00a922ebfaeeee0d9ba6ce" +checksum = "bf307cbbbd777a9c10cec88ddafee572b3484caad5cce0c9236523c3803105a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -778,7 +778,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -860,9 +860,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] @@ -925,7 +925,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -938,7 +938,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -1212,7 +1212,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -1270,20 +1270,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.58" +version = "2.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" dependencies = [ "proc-macro2", "quote", @@ -1355,7 +1344,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] @@ -1390,9 +1379,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.15.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d" +checksum = "a3429c3955ff2d769edd699768796dafd1bbff0005ddca66882a1a9fce1b8c84" dependencies = [ "aho-corasick", "derive_builder", @@ -1737,7 +1726,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 770b626..c691866 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ once_cell = "1.19.0" pulldown-cmark = { version = "0.10.2", default-features = false, optional = true } regex = "1.10.4" tiktoken-rs = { version = "0.5.8", optional = true } -tokenizers = { version = "0.15.2", default_features = false, features = [ +tokenizers = { version = "0.19.0", default_features = false, features = [ "onig", ], optional = true } unicode-segmentation = "1.11.0" @@ -61,7 +61,7 @@ divan = "0.1.14" fake = "2.9.2" insta = { version = "1.38.0", features = ["glob", "yaml"] } more-asserts = "0.3.1" -tokenizers = { version = "0.15.2", default-features = false, features = [ +tokenizers = { version = "0.19.0", default-features = false, features = [ "onig", "http", ] } diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index a994e4a..8e0defb 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -23,6 +23,6 @@ text-splitter = { path = "../..", features = [ "tokenizers", ] } tiktoken-rs = "0.5.8" -tokenizers = { version = "0.15.2", default_features = false, features = [ +tokenizers = { version = "0.19.0", default_features = false, features = [ "onig", ] }