Prep 0.12.1 release

benbrandt · Apr 26, 2024 · 0d6b722 · 0d6b722
1 parent 3edd69b
commit 0d6b722
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## v0.12.1
+
+### What's New
+
+- [`rust_tokenizers`](https://crates.io/crates/rust_tokenizers) support has been added to the Rust crate.
+
 ## v0.12.0
 
 ### What's New

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 members = ["bindings/*"]
 
 [workspace.package]
-version = "0.12.0"
+version = "0.12.1"
 authors = ["Ben Brandt <[email protected]>"]
 edition = "2021"
 description = "Split text into semantic chunks, up to a desired chunk size. Supports calculating length by characters and tokens, and is callable from Rust and Python."
@@ -81,8 +81,8 @@ harness = false
 [features]
 markdown = ["dep:pulldown-cmark"]
 rust-tokenizers = ["dep:rust_tokenizers"]
-tokenizers = ["dep:tokenizers"]
 tiktoken-rs = ["dep:tiktoken-rs"]
+tokenizers = ["dep:tokenizers"]
 
 [lints.rust]
 future_incompatible = "warn"

diff --git a/README.md b/README.md
@@ -170,6 +170,7 @@ There are lots of methods of determining sentence breaks, all to varying degrees
 
 | Dependency Feature | Version Supported | Description                                                                                                                                                                    |
 | ------------------ | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `rust_tokenizers`  | `8.1.1`           | Enables `(Text/Markdown)Splitter::new` to take any of the provided tokenizers as an argument.                                                                                  |
 | `tiktoken-rs`      | `0.5.8`           | Enables `(Text/Markdown)Splitter::new` to take `tiktoken_rs::CoreBPE` as an argument. This is useful for splitting text for OpenAI models.                                     |
 | `tokenizers`       | `0.19.1`          | Enables `(Text/Markdown)Splitter::new` to take `tokenizers::Tokenizer` as an argument. This is useful for splitting text models that have a Hugging Face-compatible tokenizer. |