Merge branch 'main' into feature_agent_serialization

.github/workflows/dotnet-build-and-test.yml

@@ -134,6 +134,8 @@ jobs:
           AzureOpenAITextToImage__Endpoint: ${{ secrets.AZUREOPENAITEXTTOIMAGE__ENDPOINT }}
           AzureOpenAITextToImage__DeploymentName: ${{ vars.AZUREOPENAITEXTTOIMAGE__DEPLOYMENTNAME }}
           Bing__ApiKey: ${{ secrets.BING__APIKEY }}
+          Google__SearchEngineId: ${{ secrets.GOOGLE__SEARCHENGINEID }}
+          Google__ApiKey: ${{ secrets.GOOGLE__APIKEY }}
           OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }}
           OpenAI__ChatModelId: ${{ vars.OPENAI__CHATMODELID }}
           AzureAIInference__ApiKey: ${{ secrets.AZUREAIINFERENCE__APIKEY }}

dotnet/Directory.Packages.props

@@ -13,7 +13,7 @@
     <PackageVersion Include="Azure.Identity" Version="1.12.0" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.3.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
-    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.5" />
+    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.6" />
     <PackageVersion Include="Markdig" Version="0.37.0" />
     <PackageVersion Include="Handlebars.Net" Version="2.1.6" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.11.67" />
@@ -40,7 +40,7 @@
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
     <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0" />
     <PackageVersion Include="System.Text.Json" Version="8.0.5" />
-    <PackageVersion Include="OllamaSharp" Version="3.0.10" />
+    <PackageVersion Include="OllamaSharp" Version="3.0.12" />
     <!-- Tokenizers -->
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
     <PackageVersion Include="Microsoft.DeepDev.TokenizerLib" Version="1.3.3" />
@@ -69,15 +69,15 @@
     <PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
     <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="8.0.0" />
     <PackageVersion Include="Verify.Xunit" Version="23.5.2" />
-    <PackageVersion Include="xunit" Version="2.9.0" />
+    <PackageVersion Include="xunit" Version="2.9.2" />
     <PackageVersion Include="xunit.abstractions" Version="2.0.3" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
     <PackageVersion Include="xretry" Version="1.9.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.2" />
     <PackageVersion Include="Docker.DotNet" Version="3.125.15" />
     <!-- Plugins -->
     <PackageVersion Include="DocumentFormat.OpenXml" Version="3.1.0" />
-    <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
+    <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.10" />
     <PackageVersion Include="DuckDB.NET.Data.Full" Version="1.1.1" />
     <PackageVersion Include="DuckDB.NET.Data" Version="1.1.1" />
     <PackageVersion Include="MongoDB.Driver" Version="2.28.0" />

dotnet/samples/Demos/StepwisePlannerMigration/README.md

@@ -55,7 +55,7 @@ IChatCompletionService chatCompletionService = kernel.GetRequiredService<IChatCo
 ChatHistory chatHistory = [];
 chatHistory.AddUserMessage("Check current UTC time and return current weather in Boston city.");
 
-OpenAIPromptExecutionSettings executionSettings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };
+OpenAIPromptExecutionSettings executionSettings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() };
 
 await chatCompletionService.GetChatMessageContentAsync(chatHistory, executionSettings, kernel);
 
@@ -87,7 +87,7 @@ Kernel kernel = Kernel
     .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey"))
     .Build();
 
-OpenAIPromptExecutionSettings executionSettings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };
+OpenAIPromptExecutionSettings executionSettings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() };
 
 FunctionResult result = await kernel.InvokePromptAsync("Check current UTC time and return current weather in Boston city.", new(executionSettings));
 
@@ -124,7 +124,7 @@ IChatCompletionService chatCompletionService = kernel.GetRequiredService<IChatCo
 
 ChatHistory existingPlan = GetExistingPlan(); // plan can be stored in database for reusability.
 
-OpenAIPromptExecutionSettings executionSettings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };
+OpenAIPromptExecutionSettings executionSettings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() };
 
 ChatMessageContent result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, executionSettings, kernel);
 

dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Net;
 using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Embeddings;
 using UglyToad.PdfPig;
 using UglyToad.PdfPig.Content;
@@ -21,14 +23,14 @@ internal sealed class DataLoader<TKey>(
     ITextEmbeddingGenerationService textEmbeddingGenerationService) : IDataLoader where TKey : notnull
 {
     /// <inheritdoc/>
-    public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
+    public async Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken)
     {
         // Create the collection if it doesn't exist.
         await vectorStoreRecordCollection.CreateCollectionIfNotExistsAsync(cancellationToken).ConfigureAwait(false);
 
         // Load the paragraphs from the PDF file and split them into batches.
         var sections = LoadParagraphs(pdfPath, cancellationToken);
-        var batches = sections.Chunk(10);
+        var batches = sections.Chunk(batchSize);
 
         // Process each batch of paragraphs.
         foreach (var batch in batches)
@@ -40,7 +42,7 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
                 Text = section.ParagraphText,
                 ReferenceDescription = $"{new FileInfo(pdfPath).Name}#page={section.PageNumber}",
                 ReferenceLink = $"{new Uri(new FileInfo(pdfPath).FullName).AbsoluteUri}#page={section.PageNumber}",
-                TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(section.ParagraphText, cancellationToken: cancellationToken).ConfigureAwait(false)
+                TextEmbedding = await GenerateEmbeddingsWithRetryAsync(textEmbeddingGenerationService, section.ParagraphText, cancellationToken: cancellationToken).ConfigureAwait(false)
             });
 
             // Upsert the records into the vector store.
@@ -50,6 +52,8 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
             {
                 Console.WriteLine($"Upserted record '{key}' into VectorDB");
             }
+
+            await Task.Delay(betweenBatchDelayInMs, cancellationToken).ConfigureAwait(false);
         }
     }
 
@@ -83,4 +87,39 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
             }
         }
     }
+
+    /// <summary>
+    /// Add a simple retry mechanism to embedding generation.
+    /// </summary>
+    /// <param name="textEmbeddingGenerationService">The embedding generation service.</param>
+    /// <param name="text">The text to generate the embedding for.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>The generated embedding.</returns>
+    private static async Task<ReadOnlyMemory<float>> GenerateEmbeddingsWithRetryAsync(ITextEmbeddingGenerationService textEmbeddingGenerationService, string text, CancellationToken cancellationToken)
+    {
+        var tries = 0;
+
+        while (true)
+        {
+            try
+            {
+                return await textEmbeddingGenerationService.GenerateEmbeddingAsync(text, cancellationToken: cancellationToken).ConfigureAwait(false);
+            }
+            catch (HttpOperationException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests)
+            {
+                tries++;
+
+                if (tries < 3)
+                {
+                    Console.WriteLine($"Failed to generate embedding. Error: {ex}");
+                    Console.WriteLine("Retrying embedding generation...");
+                    await Task.Delay(10_000, cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+        }
+    }
 }

dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs

@@ -11,7 +11,9 @@ internal interface IDataLoader
     /// Load the text from a PDF file into the data store.
     /// </summary>
     /// <param name="pdfPath">The pdf file to load.</param>
+    /// <param name="batchSize">Maximum number of parallel threads to generate embeddings and upload records.</param>
+    /// <param name="betweenBatchDelayInMs">The number of milliseconds to delay between batches to avoid throttling.</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
     /// <returns>An async task that completes when the loading is complete.</returns>
-    Task LoadPdf(string pdfPath, CancellationToken cancellationToken);
+    Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken);
 }

dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs

@@ -22,4 +22,10 @@ internal sealed class RagConfig
 
     [Required]
     public string CollectionName { get; set; } = string.Empty;
+
+    [Required]
+    public int DataLoadingBatchSize { get; set; } = 2;
+
+    [Required]
+    public int DataLoadingBetweenBatchDelayInMilliseconds { get; set; } = 0;
 }

dotnet/samples/Demos/VectorStoreRAG/Program.cs

@@ -13,13 +13,17 @@
 
 HostApplicationBuilder builder = Host.CreateApplicationBuilder(args);
 
-builder.Configuration
-    .AddUserSecrets<Program>();
-
+// Configure configuration and load the application configuration.
+builder.Configuration.AddUserSecrets<Program>();
 builder.Services.Configure<RagConfig>(builder.Configuration.GetSection(RagConfig.ConfigSectionName));
-
 var appConfig = new ApplicationConfig(builder.Configuration);
 
+// Create a cancellation token and source to pass to the application service to allow them
+// to request a graceful application shutdown.
+CancellationTokenSource appShutdownCancellationTokenSource = new();
+CancellationToken appShutdownCancellationToken = appShutdownCancellationTokenSource.Token;
+builder.Services.AddKeyedSingleton("AppShutdown", appShutdownCancellationTokenSource);
+
 // Register the kernel with the dependency injection container
 // and add Chat Completion and Text Embedding Generation services.
 var kernelBuilder = builder.Services.AddKernel()
@@ -54,6 +58,10 @@
             appConfig.AzureCosmosDBNoSQLConfig.ConnectionString,
             appConfig.AzureCosmosDBNoSQLConfig.DatabaseName);
         break;
+    case "InMemory":
+        kernelBuilder.AddInMemoryVectorStoreRecordCollection<string, TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName);
+        break;
     case "Qdrant":
         kernelBuilder.AddQdrantVectorStoreRecordCollection<Guid, TextSnippet<Guid>>(
             appConfig.RagConfig.CollectionName,
@@ -84,6 +92,7 @@
     case "AzureAISearch":
     case "AzureCosmosDBMongoDB":
     case "AzureCosmosDBNoSQL":
+    case "InMemory":
     case "Redis":
         RegisterServices<string>(builder, kernelBuilder, appConfig);
         break;
@@ -97,7 +106,7 @@
 
 // Build and run the host.
 using IHost host = builder.Build();
-await host.RunAsync().ConfigureAwait(false);
+await host.RunAsync(appShutdownCancellationToken).ConfigureAwait(false);
 
 static void RegisterServices<TKey>(HostApplicationBuilder builder, IKernelBuilder kernelBuilder, ApplicationConfig vectorStoreRagConfig)
     where TKey : notnull

dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 using Microsoft.SemanticKernel;
@@ -17,11 +18,13 @@ namespace VectorStoreRAG;
 /// <param name="vectorStoreTextSearch">Used to search the vector store.</param>
 /// <param name="kernel">Used to make requests to the LLM.</param>
 /// <param name="ragConfigOptions">The configuration options for the application.</param>
+/// <param name="appShutdownCancellationTokenSource">Used to gracefully shut down the entire application when cancelled.</param>
 internal sealed class RAGChatService<TKey>(
     IDataLoader dataLoader,
     VectorStoreTextSearch<TextSnippet<TKey>> vectorStoreTextSearch,
     Kernel kernel,
-    IOptions<RagConfig> ragConfigOptions) : IHostedService
+    IOptions<RagConfig> ragConfigOptions,
+    [FromKeyedServices("AppShutdown")] CancellationTokenSource appShutdownCancellationTokenSource) : IHostedService
 {
     private Task? _dataLoaded;
     private Task? _chatLoop;
@@ -83,6 +86,9 @@ private async Task ChatLoopAsync(CancellationToken cancellationToken)
 
         Console.WriteLine("PDF loading complete\n");
 
+        Console.ForegroundColor = ConsoleColor.Green;
+        Console.WriteLine("Assistant > Press enter with no prompt to exit.");
+
         // Add a search plugin to the kernel which we will use in the template below
         // to do a vector search for related information to the user query.
         kernel.Plugins.Add(vectorStoreTextSearch.CreateWithGetTextSearchResults("SearchPlugin"));
@@ -99,6 +105,13 @@ private async Task ChatLoopAsync(CancellationToken cancellationToken)
             Console.Write("User > ");
             var question = Console.ReadLine();
 
+            // Exit the application if the user didn't type anything.
+            if (string.IsNullOrWhiteSpace(question))
+            {
+                appShutdownCancellationTokenSource.Cancel();
+                break;
+            }
+
             // Invoke the LLM with a template that uses the search plugin to
             // 1. get related information to the user query from the vector store
             // 2. add the information to the LLM prompt.
@@ -158,7 +171,11 @@ private async Task LoadDataAsync(CancellationToken cancellationToken)
             foreach (var pdfFilePath in ragConfigOptions.Value.PdfFilePaths ?? [])
             {
                 Console.WriteLine($"Loading PDF into vector store: {pdfFilePath}");
-                await dataLoader.LoadPdf(pdfFilePath, cancellationToken).ConfigureAwait(false);
+                await dataLoader.LoadPdf(
+                    pdfFilePath,
+                    ragConfigOptions.Value.DataLoadingBatchSize,
+                    ragConfigOptions.Value.DataLoadingBetweenBatchDelayInMilliseconds,
+                    cancellationToken).ConfigureAwait(false);
             }
         }
         catch (Exception ex)

dotnet/samples/Demos/VectorStoreRAG/README.md

@@ -11,12 +11,15 @@ The sample can be configured in various ways:
    1. AzureAISearch
    1. AzureCosmosDBMongoDB
    1. AzureCosmosDBNoSQL
+   1. InMemory
    1. Qdrant
    1. Redis
    1. Weaviate
 1. You can choose whether to load data into the vector store by setting the `Rag:BuildCollection` configuration setting in the `appsettings.json` file to `true`. If you set this to `false`, the sample will assume that data was already loaded previously and it will go straight into the chat experience.
 1. You can choose the name of the collection to use by setting the `Rag:CollectionName` configuration setting in the `appsettings.json` file.
 1. You can choose the pdf file to load into the vector store by setting the `Rag:PdfFilePaths` array in the `appsettings.json` file.
+1. You can choose the number of records to process per batch when loading data into the vector store by setting the `Rag:DataLoadingBatchSize` configuration setting in the `appsettings.json` file.
+1. You can choose the number of milliseconds to wait between batches when loading data into the vector store by setting the `Rag:DataLoadingBetweenBatchDelayInMilliseconds` configuration setting in the `appsettings.json` file.
 
 ## Dependency Setup
 
@@ -45,6 +48,7 @@ For Azure OpenAI, you need to add the following secrets:
 
 ```cli
 dotnet user-secrets set "AIServices:AzureOpenAI:Endpoint" "https://<yourservice>.openai.azure.com"
+dotnet user-secrets set "AIServices:AzureOpenAI:ChatDeploymentName" "<your deployment name>"
 ```
 
 Note that the code doesn't use an API Key to communicate with Azure Open AI, but rather an `AzureCliCredential` so no api key secret is required.
@@ -55,6 +59,7 @@ For Azure OpenAI Embeddings, you need to add the following secrets:
 
 ```cli
 dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:Endpoint" "https://<yourservice>.openai.azure.com"
+dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:DeploymentName" "<your deployment name>"
 ```
 
 Note that the code doesn't use an API Key to communicate with Azure Open AI, but rather an `AzureCliCredential` so no api key secret is required.

dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj

@@ -21,6 +21,7 @@
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.AzureAISearch\Connectors.Memory.AzureAISearch.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.AzureCosmosDBMongoDB\Connectors.Memory.AzureCosmosDBMongoDB.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.AzureCosmosDBNoSQL\Connectors.Memory.AzureCosmosDBNoSQL.csproj" />
+    <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.InMemory\Connectors.Memory.InMemory.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj" />

dotnet/samples/Demos/VectorStoreRAG/appsettings.json

@@ -43,7 +43,9 @@
   "Rag": {
     "BuildCollection": true,
     "PdfFilePaths": [ "sourcedocument.pdf" ],
-    "VectorStoreType": "Qdrant",
-    "CollectionName": "pdfcontent"
+    "VectorStoreType": "InMemory",
+    "CollectionName": "pdfcontent",
+    "DataLoadingBatchSize": 2,
+    "DataLoadingBetweenBatchDelayInMilliseconds": 1000
   }
 }

dotnet/samples/GettingStarted/README.md

@@ -8,7 +8,7 @@ The examples can be run as integration tests but their code can also be copied t
 
 Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI,
 Bing and other resources. We suggest using .NET
-[Secret Manager](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets)
+[Secret Manager](https://learn.microsoft.com/aspnet/core/security/app-secrets)
 to avoid the risk of leaking secrets into the repository, branches and pull requests.
 You can also use environment variables if you prefer.
 

dotnet/samples/GettingStartedWithTextSearch/README.md

@@ -8,10 +8,13 @@ The examples can be run as integration tests but their code can also be copied t
 
 Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI,
 Bing and other resources. We suggest using .NET
-[Secret Manager](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets)
+[Secret Manager](https://learn.microsoft.com/aspnet/core/security/app-secrets)
 to avoid the risk of leaking secrets into the repository, branches and pull requests.
 You can also use environment variables if you prefer.
 
+**NOTE**
+The `Step2_Search_For_RAG.RagWithBingTextSearchUsingFullPagesAsync` sample requires a large context window so we recommend using `gpt-4o` or `gpt-4o-mini` models.
+
 To set your secrets with Secret Manager:
 
 ```
@@ -23,6 +26,10 @@ dotnet user-secrets set "OpenAI:EmbeddingModelId" "..."
 dotnet user-secrets set "OpenAI:ChatModelId" "..."
 dotnet user-secrets set "OpenAI:ApiKey" "..."
 
+dotnet user-secrets set "Bing:ApiKey" "..."
+
+dotnet user-secrets set "Google:SearchEngineId" "..."
+dotnet user-secrets set "Google:ApiKey" "..."
 ```
 
 To set your secrets with environment variables, use these names:
@@ -31,4 +38,9 @@ To set your secrets with environment variables, use these names:
 OpenAI__EmbeddingModelId
 OpenAI__ChatModelId
 OpenAI__ApiKey
+
+Bing__ApiKey
+
+Google__SearchEngineId
+Google__ApiKey
 ```