.Net: Tool filters

dotnet/docs/EXPERIMENTS.md

@@ -25,6 +25,7 @@ You can use the following diagnostic IDs to ignore warnings or errors for a part
 - SKEXP0013: OpenAI parameters
 - SKEXP0014: OpenAI chat history extension
 - SKEXP0015: OpenAI file service
+- SKEXP0016: OpenAI tool call filters
 
 ## Memory connectors
 

dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs

@@ -175,25 +175,27 @@ internal async IAsyncEnumerable<StreamingTextContent> GetStreamingTextContentsAs
         };
     }
 
-    private static Dictionary<string, object?> GetResponseMetadata(ChatCompletions completions)
+    private static Dictionary<string, object?> GetResponseMetadata(ChatCompletions completions, int iterations)
     {
-        return new Dictionary<string, object?>(5)
+        return new Dictionary<string, object?>(6)
         {
             { nameof(completions.Id), completions.Id },
             { nameof(completions.Created), completions.Created },
             { nameof(completions.PromptFilterResults), completions.PromptFilterResults },
             { nameof(completions.SystemFingerprint), completions.SystemFingerprint },
             { nameof(completions.Usage), completions.Usage },
+            { "Iterations", iterations },
         };
     }
 
-    private static Dictionary<string, object?> GetResponseMetadata(StreamingChatCompletionsUpdate completions)
+    private static Dictionary<string, object?> GetResponseMetadata(StreamingChatCompletionsUpdate completions, int iterations)
     {
-        return new Dictionary<string, object?>(3)
+        return new Dictionary<string, object?>(4)
         {
             { nameof(completions.Id), completions.Id },
             { nameof(completions.Created), completions.Created },
             { nameof(completions.SystemFingerprint), completions.SystemFingerprint },
+            { "Iterations", iterations },
         };
     }
 
@@ -265,7 +267,7 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                 throw new KernelException("Chat completions not found");
             }
 
-            IReadOnlyDictionary<string, object?> metadata = GetResponseMetadata(responseData);
+            IReadOnlyDictionary<string, object?> metadata = GetResponseMetadata(responseData, iteration);
 
             // If we don't want to attempt to invoke any functions, just return the result.
             // Or if we are auto-invoking but we somehow end up with other than 1 choice even though only 1 was requested, similarly bail.
@@ -329,6 +331,27 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                     continue;
                 }
 
+                try
+                {
+                    // Invoke the pre-invocation filter.
+                    var invokingContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokingFilter(openAIFunctionToolCall, chat, iteration);
+                    if (invokingContext is not null)
+                    {
+                        // Need to update the chat options in case chat history has changed
+                        this.UpdateChatHistory(chat, chatOptions, chatExecutionSettings);
+
+                        // Check if filter has requested a stop
+                        this.HandleStopBehavior(invokingContext, chatOptions, ref autoInvoke);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // Add cancellation message to chat history, turn off tools, and bail out of any remaining tool calls
+                    AddResponseMessage(chatOptions, chat, null, "A tool filter requested cancellation before tool invocation.", toolCall.Id, this.Logger);
+                    chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                    break;
+                }
+
                 // Make sure the requested function is one we requested. If we're permitting any kernel function to be invoked,
                 // then we don't need to check this, as it'll be handled when we look up the function in the kernel to be able
                 // to invoke it. If we're permitting only a specific list of functions, though, then we need to explicitly check.
@@ -357,7 +380,7 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                     functionResult = (await function.InvokeAsync(kernel, functionArgs, cancellationToken: cancellationToken).ConfigureAwait(false)).GetValue<object>() ?? string.Empty;
                 }
 #pragma warning disable CA1031 // Do not catch general exception types
-                catch (Exception e)
+                catch (Exception e) when (!e.IsCriticalException())
 #pragma warning restore CA1031
                 {
                     AddResponseMessage(chatOptions, chat, null, $"Error: Exception while invoking function. {e.Message}", toolCall.Id, this.Logger);
@@ -369,6 +392,26 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                 }
                 AddResponseMessage(chatOptions, chat, functionResult as string ?? JsonSerializer.Serialize(functionResult), errorMessage: null, toolCall.Id, this.Logger);
 
+                try
+                {
+                    // Invoke the post-invocation filter.
+                    var invokedContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokedFilter(openAIFunctionToolCall, functionResult, chat, iteration);
+                    if (invokedContext is not null)
+                    {
+                        // Need to update the chat options in case chat history has changed
+                        this.UpdateChatHistory(chat, chatOptions, chatExecutionSettings);
+
+                        // Check if filter has requested a stop
+                        this.HandleStopBehavior(invokedContext, chatOptions, ref autoInvoke);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // The tool call already happened so we can't cancel it, but turn off tools and bail out of any remaining tool calls
+                    chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                    break;
+                }
+
                 static void AddResponseMessage(ChatCompletionsOptions chatOptions, ChatHistory chat, string? result, string? errorMessage, string toolId, ILogger logger)
                 {
                     // Log any error
@@ -409,6 +452,37 @@ static void AddResponseMessage(ChatCompletionsOptions chatOptions, ChatHistory c
         }
     }
 
+    private void HandleStopBehavior(ToolFilterContext context, ChatCompletionsOptions chatOptions, ref bool autoInvoke)
+    {
+        switch (context.StopBehavior)
+        {
+            case ToolFilterStopBehavior.StopAutoInvoke:
+                autoInvoke = false;
+                break;
+            case ToolFilterStopBehavior.StopTools:
+                chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                break;
+            case ToolFilterStopBehavior.Cancel:
+                throw new OperationCanceledException();
+        }
+    }
+
+    private void UpdateChatHistory(ChatHistory chatHistory, ChatCompletionsOptions options, OpenAIPromptExecutionSettings executionSettings)
+    {
+        // Clear out messages, then copy over from chat history
+        options.Messages.Clear();
+
+        if (!string.IsNullOrWhiteSpace(executionSettings?.ChatSystemPrompt) && !chatHistory.Any(m => m.Role == AuthorRole.System))
+        {
+            options.Messages.Add(GetRequestMessage(new ChatMessageContent(AuthorRole.System, executionSettings!.ChatSystemPrompt)));
+        }
+
+        foreach (var message in chatHistory)
+        {
+            options.Messages.Add(GetRequestMessage(message));
+        }
+    }
+
     internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingChatMessageContentsAsync(
         ChatHistory chat,
         PromptExecutionSettings? executionSettings,
@@ -447,7 +521,7 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
             CompletionsFinishReason finishReason = default;
             await foreach (StreamingChatCompletionsUpdate update in response.ConfigureAwait(false))
             {
-                metadata ??= GetResponseMetadata(update);
+                metadata ??= GetResponseMetadata(update, iteration);
                 streamedRole ??= update.Role;
                 finishReason = update.FinishReason ?? default;
 
@@ -519,6 +593,27 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
                     continue;
                 }
 
+                try
+                {
+                    // Invoke the pre-invocation filter.
+                    var invokingContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokingFilter(openAIFunctionToolCall, chat, iteration);
+                    if (invokingContext is not null)
+                    {
+                        // Need to update the chat options in case chat history has changed
+                        this.UpdateChatHistory(chat, chatOptions, chatExecutionSettings);
+
+                        // Check if filter has requested a stop
+                        this.HandleStopBehavior(invokingContext, chatOptions, ref autoInvoke);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // Add cancellation message to chat history, turn off tools, and bail out of any remaining tool calls
+                    AddResponseMessage(chatOptions, chat, streamedRole, toolCall, metadata, null, "A tool filter requested cancellation before tool invocation.", this.Logger);
+                    chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                    break;
+                }
+
                 // Make sure the requested function is one we requested. If we're permitting any kernel function to be invoked,
                 // then we don't need to check this, as it'll be handled when we look up the function in the kernel to be able
                 // to invoke it. If we're permitting only a specific list of functions, though, then we need to explicitly check.
@@ -547,7 +642,7 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
                     functionResult = (await function.InvokeAsync(kernel, functionArgs, cancellationToken: cancellationToken).ConfigureAwait(false)).GetValue<object>() ?? string.Empty;
                 }
 #pragma warning disable CA1031 // Do not catch general exception types
-                catch (Exception e)
+                catch (Exception e) when (!e.IsCriticalException())
 #pragma warning restore CA1031
                 {
                     AddResponseMessage(chatOptions, chat, streamedRole, toolCall, metadata, result: null, $"Error: Exception while invoking function. {e.Message}", this.Logger);
@@ -559,6 +654,26 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
                 }
                 AddResponseMessage(chatOptions, chat, streamedRole, toolCall, metadata, functionResult as string ?? JsonSerializer.Serialize(functionResult), errorMessage: null, this.Logger);
 
+                try
+                {
+                    // Invoke the post-invocation filter.
+                    var invokedContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokedFilter(openAIFunctionToolCall, functionResult, chat, iteration);
+                    if (invokedContext is not null)
+                    {
+                        // Need to update the chat options in case chat history has changed
+                        this.UpdateChatHistory(chat, chatOptions, chatExecutionSettings);
+
+                        // Check if filter has requested a stop
+                        this.HandleStopBehavior(invokedContext, chatOptions, ref autoInvoke);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    // This tool call already happened so we can't cancel it, but turn off tools and bail out of any remaining tool calls
+                    chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                    break;
+                }
+
                 static void AddResponseMessage(
                     ChatCompletionsOptions chatOptions, ChatHistory chat, ChatRole? streamedRole, ChatCompletionsToolCall tool, IReadOnlyDictionary<string, object?>? metadata,
                     string? result, string? errorMessage, ILogger logger)

dotnet/src/Connectors/Connectors.OpenAI/Connectors.OpenAI.csproj

@@ -6,7 +6,7 @@
     <RootNamespace>$(AssemblyName)</RootNamespace>
     <TargetFramework>netstandard2.0</TargetFramework>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <NoWarn>$(NoWarn);NU5104;SKEXP0013,SKEXP0014</NoWarn>
+    <NoWarn>$(NoWarn);NU5104;SKEXP0013,SKEXP0014,SKEXP0016</NoWarn>
     <EnablePackageValidation>true</EnablePackageValidation>
   </PropertyGroup>
 

dotnet/src/Connectors/Connectors.OpenAI/IToolFilter.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Connectors.OpenAI;
+
+/// <summary>
+/// Interface for tool filters.
+/// </summary>
+[Experimental("SKEXP0016")]
+public interface IToolFilter
+{
+    /// <summary>
+    /// Method which is executed before tool invocation.
+    /// </summary>
+    /// <param name="context">Data related to tool before invocation.</param>
+    void OnToolInvoking(ToolInvokingContext context);
+
+    /// <summary>
+    /// Method which is executed after tool invocation.
+    /// </summary>
+    /// <param name="context">Data related to tool after invocation.</param>
+    void OnToolInvoked(ToolInvokedContext context);
+}

dotnet/src/Connectors/Connectors.OpenAI/ToolCallBehavior.cs

@@ -2,8 +2,10 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Azure.AI.OpenAI;
+using Microsoft.SemanticKernel.ChatCompletion;
 
 namespace Microsoft.SemanticKernel.Connectors.OpenAI;
 
@@ -35,6 +37,12 @@ public abstract class ToolCallBehavior
     /// </remarks>
     private const int DefaultMaximumAutoInvokeAttempts = 5;
 
+    /// <summary>
+    /// Gets the collection of filters that will be applied to tool calls.
+    /// </summary>
+    [Experimental("SKEXP0016")]
+    public IList<IToolFilter> Filters { get; } = new List<IToolFilter>();
+
     /// <summary>
     /// Gets an instance that will provide all of the <see cref="Kernel"/>'s plugins' function information.
     /// Function call requests from the model will be propagated back to the caller.
@@ -236,4 +244,40 @@ internal override void ConfigureOptions(Kernel? kernel, ChatCompletionsOptions o
         /// </remarks>
         internal override int MaximumUseAttempts => 1;
     }
+
+    #region Filters
+    internal ToolInvokingContext? OnToolInvokingFilter(OpenAIFunctionToolCall toolCall, ChatHistory chatHistory, int iteration)
+    {
+        ToolInvokingContext? context = null;
+
+        if (this.Filters is { Count: > 0 })
+        {
+            context = new(toolCall, chatHistory, iteration);
+
+            for (int i = 0; i < this.Filters.Count; i++)
+            {
+                this.Filters[i].OnToolInvoking(context);
+            }
+        }
+
+        return context;
+    }
+
+    internal ToolInvokedContext? OnToolInvokedFilter(OpenAIFunctionToolCall toolCall, object? result, ChatHistory chatHistory, int iteration)
+    {
+        ToolInvokedContext? context = null;
+
+        if (this.Filters is { Count: > 0 })
+        {
+            context = new(toolCall, result, chatHistory, iteration);
+
+            for (int i = 0; i < this.Filters.Count; i++)
+            {
+                this.Filters[i].OnToolInvoked(context);
+            }
+        }
+
+        return context;
+    }
+    #endregion
 }