.Net: Tokens information in telemetry

dotnet/docs/TELEMETRY.md

@@ -64,6 +64,10 @@ Available meters:
   - `SK.<SkillName><FunctionName>.ExecutionTotal` - total number of function executions
   - `SK.<SkillName><FunctionName>.ExecutionSuccess` - number of successful function executions
   - `SK.<SkillName><FunctionName>.ExecutionFailure` - number of failed function executions
+- _Microsoft.SemanticKernel.Connectors.AI.OpenAI_ - captures metrics for OpenAI functionality. List of metrics:
+  - `SK.Connectors.OpenAI.PromptTokens` - number of prompt tokens used.
+  - `SK.Connectors.OpenAI.CompletionTokens` - number of completion tokens used.
+  - `SK.Connectors.OpenAI.TotalTokens` - total number of tokens used.
 
 ### Examples
 

dotnet/samples/ApplicationInsightsExample/Program.cs

@@ -40,16 +40,16 @@ public static async Task Main()
         var serviceProvider = GetServiceProvider();
 
         var telemetryClient = serviceProvider.GetRequiredService<TelemetryClient>();
-        var logger = serviceProvider.GetRequiredService<ILoggerFactory>();
+        var loggerFactory = serviceProvider.GetRequiredService<ILoggerFactory>();
 
         using var meterListener = new MeterListener();
         using var activityListener = new ActivityListener();
 
         ConfigureMetering(meterListener, telemetryClient);
         ConfigureTracing(activityListener, telemetryClient);
 
-        var kernel = GetKernel(logger);
-        var planner = GetSequentialPlanner(kernel, logger);
+        var kernel = GetKernel(loggerFactory);
+        var planner = GetSequentialPlanner(kernel, loggerFactory);
 
         try
         {
@@ -92,7 +92,7 @@ private static void ConfigureApplicationInsightsTelemetry(ServiceCollection serv
 
         services.AddLogging(loggingBuilder =>
         {
-            loggingBuilder.AddFilter<ApplicationInsightsLoggerProvider>(typeof(Program).FullName, LogLevel);
+            loggingBuilder.AddFilter<ApplicationInsightsLoggerProvider>(logLevel => logLevel == LogLevel);
             loggingBuilder.SetMinimumLevel(LogLevel);
         });
 

dotnet/samples/KernelSyntaxExamples/Example43_GetModelResult.cs

@@ -35,13 +35,13 @@ public static async Task RunAsync()
         var textResult = await myFunction.InvokeAsync("Sci-fi",
             settings: new CompleteRequestSettings { ResultsPerPrompt = 3, MaxTokens = 500, Temperature = 1, TopP = 0.5 });
         Console.WriteLine(textResult);
-        Console.WriteLine(textResult.ModelResults.Select(result => result.GetOpenAITextResult()).AsJson());
+        Console.WriteLine(textResult.ModelResults.Select(result => result.GetOpenAIChatResult()).AsJson());
         Console.WriteLine();
 
         // Using the Kernel RunAsync
         textResult = await kernel.RunAsync("sorry I forgot your birthday", myFunction);
         Console.WriteLine(textResult);
-        Console.WriteLine(textResult.ModelResults.LastOrDefault()?.GetOpenAITextResult()?.Usage.AsJson());
+        Console.WriteLine(textResult.ModelResults.LastOrDefault()?.GetOpenAIChatResult()?.Usage.AsJson());
         Console.WriteLine();
 
         // Using Chat Completion directly

dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading;
@@ -46,6 +47,35 @@ private protected ClientBase(ILoggerFactory? loggerFactory = null)
     /// </summary>
     private protected ILogger Logger { get; set; }
 
+    /// <summary>
+    /// Instance of <see cref="Meter"/> for metrics.
+    /// </summary>
+    private static Meter s_meter = new(typeof(ClientBase).Assembly.GetName().Name);
+
+    /// <summary>
+    /// Instance of <see cref="Counter{T}"/> to keep track of the number of prompt tokens used.
+    /// </summary>
+    private static Counter<int> s_promptTokensCounter =
+        s_meter.CreateCounter<int>(
+            name: "SK.Connectors.OpenAI.PromptTokens",
+            description: "Number of prompt tokens used");
+
+    /// <summary>
+    /// Instance of <see cref="Counter{T}"/> to keep track of the number of completion tokens used.
+    /// </summary>
+    private static Counter<int> s_completionTokensCounter =
+        s_meter.CreateCounter<int>(
+            name: "SK.Connectors.OpenAI.CompletionTokens",
+            description: "Number of completion tokens used");
+
+    /// <summary>
+    /// Instance of <see cref="Counter{T}"/> to keep track of the total number of tokens used.
+    /// </summary>
+    private static Counter<int> s_totalTokensCounter =
+        s_meter.CreateCounter<int>(
+            name: "SK.Connectors.OpenAI.TotalTokens",
+            description: "Total number of tokens used");
+
     /// <summary>
     /// Creates completions for the prompt and settings.
     /// </summary>
@@ -78,6 +108,8 @@ private protected async Task<IReadOnlyList<ITextResult>> InternalGetTextResultsA
             throw new SKException("Text completions not found");
         }
 
+        this.CaptureUsageDetails(responseData.Usage);
+
         return responseData.Choices.Select(choice => new TextResult(responseData, choice)).ToList();
     }
 
@@ -168,12 +200,16 @@ private protected async Task<IReadOnlyList<IChatResult>> InternalGetChatResultsA
             throw new SKException("Chat completions null response");
         }
 
-        if (response.Value.Choices.Count == 0)
+        var responseData = response.Value;
+
+        if (responseData.Choices.Count == 0)
         {
             throw new SKException("Chat completions not found");
         }
 
-        return response.Value.Choices.Select(chatChoice => new ChatResult(response.Value, chatChoice)).ToList();
+        this.CaptureUsageDetails(responseData.Usage);
+
+        return responseData.Choices.Select(chatChoice => new ChatResult(responseData, chatChoice)).ToList();
     }
 
     /// <summary>
@@ -444,4 +480,19 @@ private static async Task<T> RunRequestAsync<T>(Func<Task<T>> request)
                 $"Something went wrong: {e.Message}", e);
         }
     }
+
+    /// <summary>
+    /// Captures usage details, including token information.
+    /// </summary>
+    /// <param name="usage">Instance of <see cref="CompletionsUsage"/> with usage details.</param>
+    private void CaptureUsageDetails(CompletionsUsage usage)
+    {
+        this.Logger.LogInformation(
+            "Prompt tokens: {PromptTokens}. Completion tokens: {CompletionTokens}. Total tokens: {TotalTokens}.",
+            usage.PromptTokens, usage.CompletionTokens, usage.TotalTokens);
+
+        s_promptTokensCounter.Add(usage.PromptTokens);
+        s_completionTokensCounter.Add(usage.CompletionTokens);
+        s_totalTokensCounter.Add(usage.TotalTokens);
+    }
 }