From 52251473fb8d2bc0beca35bfc941fac9458e5bdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:34:24 +0100 Subject: [PATCH] .Net: Bump Microsoft.ML.Tokenizers from 0.22.0-preview.24271.1 to 0.22.0-preview.24378.1 in /dotnet (#7894) Bumps [Microsoft.ML.Tokenizers](https://github.com/dotnet/machinelearning) from 0.22.0-preview.24271.1 to 0.22.0-preview.24378.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Microsoft.ML.Tokenizers&package-manager=nuget&previous-version=0.22.0-preview.24271.1&new-version=0.22.0-preview.24378.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> --- dotnet/Directory.Packages.props | 2 +- dotnet/samples/Concepts/Memory/TextChunkerUsage.cs | 2 +- dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs | 2 +- .../Text/TextChunkerInternationalTests.cs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index a6783e305b33..b1c7dc58eddc 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -39,7 +39,7 @@ - + diff --git a/dotnet/samples/Concepts/Memory/TextChunkerUsage.cs b/dotnet/samples/Concepts/Memory/TextChunkerUsage.cs index a42e769ae916..51d50c619903 100644 --- a/dotnet/samples/Concepts/Memory/TextChunkerUsage.cs +++ b/dotnet/samples/Concepts/Memory/TextChunkerUsage.cs @@ -8,7 +8,7 @@ namespace Memory; public class TextChunkerUsage(ITestOutputHelper output) : BaseTest(output) { - private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4"); + private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); [Fact] public void RunExample() diff --git a/dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs b/dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs index 013bb4961621..04a74656e948 100644 --- a/dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs +++ b/dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs @@ -9,7 +9,7 @@ namespace Memory; public class TextChunkingAndEmbedding(ITestOutputHelper output) : BaseTest(output) { private const string EmbeddingModelName = "text-embedding-ada-002"; - private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel(EmbeddingModelName); + private static readonly Tokenizer s_tokenizer = TiktokenTokenizer.CreateForModel(EmbeddingModelName); [Fact] public async Task RunAsync() diff --git a/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerInternationalTests.cs b/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerInternationalTests.cs index ce3be9193191..47d61e74b164 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerInternationalTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerInternationalTests.cs @@ -15,7 +15,7 @@ public sealed class TextChunkerInternationalTests public sealed class StatefulTokenCounter { private readonly Dictionary _callStats = []; - private readonly Tokenizer _tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4"); + private readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); public int Count(string input) { @@ -29,7 +29,7 @@ public int Count(string input) private static TokenCounter StatelessTokenCounter => (string input) => { - var tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4"); + var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4"); return tokenizer.CountTokens(input); };