.Net: Tool filters

dotnet/docs/EXPERIMENTS.md

@@ -26,6 +26,7 @@ You can use the following diagnostic IDs to ignore warnings or errors for a part
 - SKEXP0013: OpenAI parameters
 - SKEXP0014: OpenAI chat history extension
 - SKEXP0015: OpenAI file service
+- SKEXP0016: OpenAI tool call filters
 
 ## Memory connectors
 

dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs

@@ -29,6 +29,8 @@ namespace Microsoft.SemanticKernel.Connectors.OpenAI;
 /// </summary>
 internal abstract class ClientCore
 {
+    private const string ModelIterationsCompletedKey = "ModelIterationsCompleted";
+
     private const int MaxResultsPerPrompt = 128;
 
     /// <summary>
@@ -176,25 +178,27 @@ internal async IAsyncEnumerable<StreamingTextContent> GetStreamingTextContentsAs
         };
     }
 
-    private static Dictionary<string, object?> GetResponseMetadata(ChatCompletions completions)
+    private static Dictionary<string, object?> GetResponseMetadata(ChatCompletions completions, int modelIterations)
     {
-        return new Dictionary<string, object?>(5)
+        return new Dictionary<string, object?>(6)
         {
             { nameof(completions.Id), completions.Id },
             { nameof(completions.Created), completions.Created },
             { nameof(completions.PromptFilterResults), completions.PromptFilterResults },
             { nameof(completions.SystemFingerprint), completions.SystemFingerprint },
             { nameof(completions.Usage), completions.Usage },
+            { ModelIterationsCompletedKey, modelIterations },
         };
     }
 
-    private static Dictionary<string, object?> GetResponseMetadata(StreamingChatCompletionsUpdate completions)
+    private static Dictionary<string, object?> GetResponseMetadata(StreamingChatCompletionsUpdate completions, int modelIterations)
     {
-        return new Dictionary<string, object?>(3)
+        return new Dictionary<string, object?>(4)
         {
             { nameof(completions.Id), completions.Id },
             { nameof(completions.Created), completions.Created },
             { nameof(completions.SystemFingerprint), completions.SystemFingerprint },
+            { ModelIterationsCompletedKey, modelIterations },
         };
     }
 
@@ -303,7 +307,7 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                 throw new KernelException("Chat completions not found");
             }
 
-            IReadOnlyDictionary<string, object?> metadata = GetResponseMetadata(responseData);
+            IReadOnlyDictionary<string, object?> metadata = GetResponseMetadata(responseData, iteration);
 
             // If we don't want to attempt to invoke any functions, just return the result.
             // Or if we are auto-invoking but we somehow end up with other than 1 choice even though only 1 was requested, similarly bail.
@@ -367,6 +371,19 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                     continue;
                 }
 
+                try
+                {
+                    // Invoke the pre-invocation filter.
+                    var invokingContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokingFilter(openAIFunctionToolCall, chat, iteration);
+                    this.ApplyToolFilterContextChanges(invokingContext, chatOptions, chat, chatExecutionSettings, ref autoInvoke);
+                }
+                catch (OperationCanceledException)
+                {
+                    // Add cancellation message to chat history and bail out of any remaining tool calls
+                    AddResponseMessage(chatOptions, chat, null, $"A tool filter requested cancellation before tool invocation. Model iterations completed: {iteration}", toolCall.Id, this.Logger);
+                    break;
+                }
+
                 // Make sure the requested function is one we requested. If we're permitting any kernel function to be invoked,
                 // then we don't need to check this, as it'll be handled when we look up the function in the kernel to be able
                 // to invoke it. If we're permitting only a specific list of functions, though, then we need to explicitly check.
@@ -395,7 +412,7 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                     functionResult = (await function.InvokeAsync(kernel, functionArgs, cancellationToken: cancellationToken).ConfigureAwait(false)).GetValue<object>() ?? string.Empty;
                 }
 #pragma warning disable CA1031 // Do not catch general exception types
-                catch (Exception e)
+                catch (Exception e) when (!e.IsCriticalException())
 #pragma warning restore CA1031
                 {
                     AddResponseMessage(chatOptions, chat, null, $"Error: Exception while invoking function. {e.Message}", toolCall.Id, this.Logger);
@@ -407,6 +424,18 @@ internal async Task<IReadOnlyList<ChatMessageContent>> GetChatMessageContentsAsy
                 }
                 AddResponseMessage(chatOptions, chat, functionResult as string ?? JsonSerializer.Serialize(functionResult), errorMessage: null, toolCall.Id, this.Logger);
 
+                try
+                {
+                    // Invoke the post-invocation filter.
+                    var invokedContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokedFilter(openAIFunctionToolCall, functionResult, chat, iteration);
+                    this.ApplyToolFilterContextChanges(invokedContext, chatOptions, chat, chatExecutionSettings, ref autoInvoke);
+                }
+                catch (OperationCanceledException)
+                {
+                    // The tool call already happened so we can't cancel it, but bail out of any remaining tool calls
+                    break;
+                }
+
                 static void AddResponseMessage(ChatCompletionsOptions chatOptions, ChatHistory chat, string? result, string? errorMessage, string toolId, ILogger logger)
                 {
                     // Log any error
@@ -447,6 +476,58 @@ static void AddResponseMessage(ChatCompletionsOptions chatOptions, ChatHistory c
         }
     }
 
+    private void ApplyToolFilterContextChanges(
+        ToolFilterContext? context,
+        ChatCompletionsOptions chatOptions,
+        ChatHistory chatHistory,
+        OpenAIPromptExecutionSettings executionSettings,
+        ref bool autoInvoke)
+    {
+        if (context is not null)
+        {
+            // Since the tool filter has access to the chat history, the chat history may have been modified.
+            // We want to make sure any subsequent requests to the model reflect these changes. The chatOptions object
+            // contains all the configuration information for a chat request, including a copy of the chat history.
+            // So we need to update the chat history stored in the chatOptions object to match what is in the chatHistory object.
+            this.UpdateChatOptions(chatOptions, chatHistory, executionSettings);
+
+            // Check if filter has requested a stop
+            this.HandleStopBehavior(context, chatOptions, ref autoInvoke);
+        }
+    }
+
+    private void HandleStopBehavior(ToolFilterContext context, ChatCompletionsOptions chatOptions, ref bool autoInvoke)
+    {
+        switch (context.StopBehavior)
+        {
+            case ToolFilterStopBehavior.StopAutoInvoke:
+                autoInvoke = false;
+                break;
+            case ToolFilterStopBehavior.StopTools:
+                chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                break;
+            case ToolFilterStopBehavior.Cancel:
+                chatOptions.ToolChoice = ChatCompletionsToolChoice.None;
+                throw new OperationCanceledException();
+        }
+    }
+
+    private void UpdateChatOptions(ChatCompletionsOptions options, ChatHistory chatHistory, OpenAIPromptExecutionSettings executionSettings)
+    {
+        // Clear out messages, then copy over from chat history
+        options.Messages.Clear();
+
+        if (!string.IsNullOrWhiteSpace(executionSettings?.ChatSystemPrompt) && !chatHistory.Any(m => m.Role == AuthorRole.System))
+        {
+            options.Messages.Add(GetRequestMessage(new ChatMessageContent(AuthorRole.System, executionSettings!.ChatSystemPrompt)));
+        }
+
+        foreach (var message in chatHistory)
+        {
+            options.Messages.Add(GetRequestMessage(message));
+        }
+    }
+
     internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingChatMessageContentsAsync(
         ChatHistory chat,
         PromptExecutionSettings? executionSettings,
@@ -485,7 +566,7 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
             CompletionsFinishReason finishReason = default;
             await foreach (StreamingChatCompletionsUpdate update in response.ConfigureAwait(false))
             {
-                metadata ??= GetResponseMetadata(update);
+                metadata ??= GetResponseMetadata(update, iteration);
                 streamedRole ??= update.Role;
                 finishReason = update.FinishReason ?? default;
 
@@ -557,6 +638,19 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
                     continue;
                 }
 
+                try
+                {
+                    // Invoke the pre-invocation filter.
+                    var invokingContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokingFilter(openAIFunctionToolCall, chat, iteration);
+                    this.ApplyToolFilterContextChanges(invokingContext, chatOptions, chat, chatExecutionSettings, ref autoInvoke);
+                }
+                catch (OperationCanceledException)
+                {
+                    // Add cancellation message to chat history and bail out of any remaining tool calls
+                    AddResponseMessage(chatOptions, chat, streamedRole, toolCall, metadata, null, $"A tool filter requested cancellation before tool invocation. Model iterations completed: {iteration}", this.Logger);
+                    break;
+                }
+
                 // Make sure the requested function is one we requested. If we're permitting any kernel function to be invoked,
                 // then we don't need to check this, as it'll be handled when we look up the function in the kernel to be able
                 // to invoke it. If we're permitting only a specific list of functions, though, then we need to explicitly check.
@@ -585,7 +679,7 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
                     functionResult = (await function.InvokeAsync(kernel, functionArgs, cancellationToken: cancellationToken).ConfigureAwait(false)).GetValue<object>() ?? string.Empty;
                 }
 #pragma warning disable CA1031 // Do not catch general exception types
-                catch (Exception e)
+                catch (Exception e) when (!e.IsCriticalException())
 #pragma warning restore CA1031
                 {
                     AddResponseMessage(chatOptions, chat, streamedRole, toolCall, metadata, result: null, $"Error: Exception while invoking function. {e.Message}", this.Logger);
@@ -597,6 +691,18 @@ internal async IAsyncEnumerable<OpenAIStreamingChatMessageContent> GetStreamingC
                 }
                 AddResponseMessage(chatOptions, chat, streamedRole, toolCall, metadata, functionResult as string ?? JsonSerializer.Serialize(functionResult), errorMessage: null, this.Logger);
 
+                try
+                {
+                    // Invoke the post-invocation filter.
+                    var invokedContext = chatExecutionSettings.ToolCallBehavior?.OnToolInvokedFilter(openAIFunctionToolCall, functionResult, chat, iteration);
+                    this.ApplyToolFilterContextChanges(invokedContext, chatOptions, chat, chatExecutionSettings, ref autoInvoke);
+                }
+                catch (OperationCanceledException)
+                {
+                    // This tool call already happened so we can't cancel it, but bail out of any remaining tool calls
+                    break;
+                }
+
                 static void AddResponseMessage(
                     ChatCompletionsOptions chatOptions, ChatHistory chat, ChatRole? streamedRole, ChatCompletionsToolCall tool, IReadOnlyDictionary<string, object?>? metadata,
                     string? result, string? errorMessage, ILogger logger)

dotnet/src/Connectors/Connectors.OpenAI/Connectors.OpenAI.csproj

@@ -6,7 +6,7 @@
     <RootNamespace>$(AssemblyName)</RootNamespace>
     <TargetFramework>netstandard2.0</TargetFramework>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
-    <NoWarn>$(NoWarn);NU5104;SKEXP0005,SKEXP0013,SKEXP0014</NoWarn>
+    <NoWarn>$(NoWarn);NU5104;SKEXP0005,SKEXP0013,SKEXP0014,SKEXP0016</NoWarn>
     <EnablePackageValidation>true</EnablePackageValidation>
   </PropertyGroup>
 

dotnet/src/Connectors/Connectors.OpenAI/IToolFilter.cs

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.SemanticKernel.Connectors.OpenAI;
+
+/// <summary>
+/// Interface for tool filters.
+/// </summary>
+[Experimental("SKEXP0016")]
+public interface IToolFilter
+{
+    /// <summary>
+    /// Method which is executed before tool invocation.
+    /// </summary>
+    /// <param name="context">Data related to tool before invocation.</param>
+    void OnToolInvoking(ToolInvokingContext context);
+
+    /// <summary>
+    /// Method which is executed after tool invocation.
+    /// </summary>
+    /// <param name="context">Data related to tool after invocation.</param>
+    void OnToolInvoked(ToolInvokedContext context);
+}

dotnet/src/Connectors/Connectors.OpenAI/ToolCallBehavior.cs

@@ -2,8 +2,10 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Azure.AI.OpenAI;
+using Microsoft.SemanticKernel.ChatCompletion;
 
 namespace Microsoft.SemanticKernel.Connectors.OpenAI;
 
@@ -35,6 +37,12 @@ public abstract class ToolCallBehavior
     /// </remarks>
     private const int DefaultMaximumAutoInvokeAttempts = 5;
 
+    /// <summary>
+    /// Gets the collection of filters that will be applied to tool calls.
+    /// </summary>
+    [Experimental("SKEXP0016")]
+    public IList<IToolFilter> Filters { get; } = new List<IToolFilter>();
+
     /// <summary>
     /// Gets an instance that will provide all of the <see cref="Kernel"/>'s plugins' function information.
     /// Function call requests from the model will be propagated back to the caller.
@@ -236,4 +244,40 @@ internal override void ConfigureOptions(Kernel? kernel, ChatCompletionsOptions o
         /// </remarks>
         internal override int MaximumUseAttempts => 1;
     }
+
+    #region Filters
+    internal ToolInvokingContext? OnToolInvokingFilter(OpenAIFunctionToolCall toolCall, ChatHistory chatHistory, int iteration)
+    {
+        ToolInvokingContext? context = null;
+
+        if (this.Filters is { Count: > 0 })
+        {
+            context = new(toolCall, chatHistory, iteration);
+
+            for (int i = 0; i < this.Filters.Count; i++)
+            {
+                this.Filters[i].OnToolInvoking(context);
+            }
+        }
+
+        return context;
+    }
+
+    internal ToolInvokedContext? OnToolInvokedFilter(OpenAIFunctionToolCall toolCall, object? result, ChatHistory chatHistory, int iteration)
+    {
+        ToolInvokedContext? context = null;
+
+        if (this.Filters is { Count: > 0 })
+        {
+            context = new(toolCall, result, chatHistory, iteration);
+
+            for (int i = 0; i < this.Filters.Count; i++)
+            {
+                this.Filters[i].OnToolInvoked(context);
+            }
+        }
+
+        return context;
+    }
+    #endregion
 }