.Net: Bump Microsoft.ML.Tokenizers from 0.22.0-preview.24271.1 to 0.2…

…2.0-preview.24378.1 in /dotnet (microsoft#7894)

Bumps
[Microsoft.ML.Tokenizers](https://github.com/dotnet/machinelearning)
from 0.22.0-preview.24271.1 to 0.22.0-preview.24378.1.
<details>
<summary>Commits</summary>
<ul>
<li>See full diff in <a
href="https://github.com/dotnet/machinelearning/commits">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Microsoft.ML.Tokenizers&package-manager=nuget&previous-version=0.22.0-preview.24271.1&new-version=0.22.0-preview.24378.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Roger Barreto <[email protected]>

dotnet/Directory.Packages.props

@@ -39,7 +39,7 @@
     <PackageVersion Include="System.Threading.Tasks.Extensions" Version="4.5.4" />
     <PackageVersion Include="System.ValueTuple" Version="4.5.0" />
     <!-- Tokenizers -->
-    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24271.1" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
     <PackageVersion Include="Microsoft.DeepDev.TokenizerLib" Version="1.3.3" />
     <PackageVersion Include="SharpToken" Version="2.0.3" />
     <!-- Microsoft.Extensions.* -->

dotnet/samples/Concepts/Memory/TextChunkerUsage.cs

@@ -8,7 +8,7 @@ namespace Memory;
 
 public class TextChunkerUsage(ITestOutputHelper output) : BaseTest(output)
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4");
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
 
     [Fact]
     public void RunExample()

dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs

@@ -9,7 +9,7 @@ namespace Memory;
 public class TextChunkingAndEmbedding(ITestOutputHelper output) : BaseTest(output)
 {
     private const string EmbeddingModelName = "text-embedding-ada-002";
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel(EmbeddingModelName);
+    private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel(EmbeddingModelName);
 
     [Fact]
     public async Task RunAsync()

dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerInternationalTests.cs

@@ -15,7 +15,7 @@ public sealed class TextChunkerInternationalTests
     public sealed class StatefulTokenCounter
     {
         private readonly Dictionary<string, int> _callStats = [];
-        private readonly Tokenizer _tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4");
+        private readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
 
         public int Count(string input)
         {
@@ -29,7 +29,7 @@ public int Count(string input)
 
     private static TokenCounter StatelessTokenCounter => (string input) =>
     {
-        var tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4");
+        var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
         return tokenizer.CountTokens(input);
     };