Skip to content

Commit

Permalink
Various improvements for Semantic Kernel
Browse files Browse the repository at this point in the history
  • Loading branch information
@drawbyperpetual
drawbyperpetual committed Oct 15, 2024
1 parent 623c02f commit 7036e1d
Show file tree
Hide file tree
Showing 16 changed files with 150 additions and 154 deletions.
29 changes: 0 additions & 29 deletions src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,34 +508,6 @@
</Flyout>
</FlyoutBase.AttachedFlyout>
</TextBox>

<!--<StackPanel
Margin="0,0,4,0"
HorizontalAlignment="Right"
VerticalAlignment="Stretch"
Orientation="Horizontal">-->
<!--<Button
x:Name="RecallBtn"
x:Uid="RecallButtonAutomation"
Width="32"
Height="32"
Padding="0"
Command="{x:Bind RecallCommand}"
Content="{ui:FontIcon Glyph=&#xE81C;,
FontSize=12}"
Foreground="{ThemeResource TextFillColorSecondaryBrush}"
Style="{StaticResource SubtleButtonStyle}"
TabIndex="2"
Visibility="Collapsed">
<ToolTipService.ToolTip>
<TextBlock x:Uid="RecallBtnToolTip" TextWrapping="WrapWholeWords" />
</ToolTipService.ToolTip>
<animations:Implicit.Animations>
<animations:TranslationAnimation Duration="0:0:1" />
<animations:ScaleAnimation Duration="0:0:1" />
<animations:OffsetAnimation Duration="0:0:1" />
</animations:Implicit.Animations>
</Button>-->
<Grid
Width="32"
Height="32"
Expand Down Expand Up @@ -662,7 +634,6 @@
<VisualState.Setters>
<Setter Target="Loader.IsLoading" Value="True" />
<Setter Target="InputTxtBox.IsEnabled" Value="False" />
<!--<Setter Target="RecallBtn.IsEnabled" Value="False" />-->
<Setter Target="SendBtn.IsEnabled" Value="False" />
<Setter Target="DisclaimerPresenter.Visibility" Value="Collapsed" />
<Setter Target="LoadingText.Visibility" Value="Visible" />
Expand Down
31 changes: 6 additions & 25 deletions src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
// The Microsoft Corporation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.ComponentModel;
using System.Threading.Tasks;

using AdvancedPaste.Helpers;
using AdvancedPaste.Models;
using AdvancedPaste.ViewModels;
using CommunityToolkit.Mvvm.Input;
Expand Down Expand Up @@ -40,7 +40,7 @@ public string PlaceholderText

public object Footer
{
get => (object)GetValue(FooterProperty);
get => GetValue(FooterProperty);
set => SetValue(FooterProperty, value);
}

Expand All @@ -50,7 +50,7 @@ public PromptBox()

ViewModel = App.GetService<OptionsViewModel>();
ViewModel.PropertyChanged += ViewModel_PropertyChanged;
ViewModel.CustomActionActivated += ViewModel_CustomActionActivated;
ViewModel.PreviewRequested += ViewModel_PreviewRequested;
}

private void ViewModel_PropertyChanged(object sender, PropertyChangedEventArgs e)
Expand All @@ -62,15 +62,12 @@ private void ViewModel_PropertyChanged(object sender, PropertyChangedEventArgs e
}
}

private void ViewModel_CustomActionActivated(object sender, CustomActionActivatedEventArgs e)
private void ViewModel_PreviewRequested(object sender, EventArgs e)
{
Logger.LogTrace();

if (!e.PasteResult)
{
PreviewGrid.Width = InputTxtBox.ActualWidth;
PreviewFlyout.ShowAt(InputTxtBox);
}
PreviewGrid.Width = InputTxtBox.ActualWidth;
PreviewFlyout.ShowAt(InputTxtBox);
}

private void Grid_Loaded(object sender, RoutedEventArgs e)
Expand All @@ -81,22 +78,6 @@ private void Grid_Loaded(object sender, RoutedEventArgs e)
[RelayCommand]
private async Task GenerateCustomAsync() => await ViewModel.GenerateCustomFunctionAsync(PasteActionSource.PromptBox);

[RelayCommand]
private void Recall()
{
Logger.LogTrace();

InputTxtBox.IsEnabled = true;

var lastQuery = ViewModel.RecallPreviousCustomQuery();
if (lastQuery != null)
{
InputTxtBox.Text = lastQuery.Query;
}

ClipboardHelper.SetClipboardTextContent(lastQuery.ClipboardData);
}

private async void InputTxtBox_KeyDown(object sender, Microsoft.UI.Xaml.Input.KeyRoutedEventArgs e)
{
if (e.Key == Windows.System.VirtualKey.Enter && InputTxtBox.Text.Length > 0 && ViewModel.IsCustomAIEnabled)
Expand Down
2 changes: 2 additions & 0 deletions src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ internal static async Task<bool> HasDataAsync(DataPackageView clipboardData)
return availableFormats == ClipboardFormat.Text ? !string.IsNullOrEmpty(await clipboardData.GetTextAsync()) : availableFormats != ClipboardFormat.None;
}

internal static async Task<string> GetTextOrNullAsync(DataPackageView clipboardData) => clipboardData.Contains(StandardDataFormats.Text) ? await clipboardData.GetTextAsync() : null;

internal static async Task TryCopyPasteDataPackageAsync(DataPackage dataPackage, Action onCopied)
{
Logger.LogTrace();
Expand Down
1 change: 0 additions & 1 deletion src/modules/AdvancedPaste/AdvancedPaste/Helpers/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,5 @@ namespace AdvancedPaste.Helpers
internal static class Constants
{
internal static readonly string AdvancedPasteModuleName = "AdvancedPaste";
internal static readonly string LastQueryJsonFileName = "lastQuery.json";
}
}
1 change: 1 addition & 0 deletions src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public static async Task<DataPackage> TransformAsync(PasteFormats format, DataPa
PasteFormats.PasteAsPngFile => await ToPngFileAsync(clipboardData),
PasteFormats.PasteAsHtmlFile => await ToHtmlFileAsync(clipboardData),
PasteFormats.KernelQuery => throw new ArgumentException($"Unsupported format {format}", nameof(format)),
PasteFormats.CustomTextTransformation => throw new ArgumentException($"Unsupported format {format}", nameof(format)),
_ => throw new ArgumentException($"Unknown value {format}", nameof(format)),
};
}
Expand Down
14 changes: 0 additions & 14 deletions src/modules/AdvancedPaste/AdvancedPaste/Models/CustomActionActivatedEventArgs.cs
View file

This file was deleted.

10 changes: 10 additions & 0 deletions src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,19 @@ public sealed class PasteFormatMetadataAttribute : Attribute

public bool RequiresAIService { get; init; }

public bool IsPreviewable { get; init; }

public ClipboardFormat SupportedClipboardFormats { get; init; }

public string IPCKey { get; init; }

/// <summary>
/// Gets a description of the action that should be exposed to Semantic Kernel, or <see langword="null"/> if it should not be exposed.
/// </summary>
public string KernelFunctionDescription { get; init; }

/// <summary>
/// Gets a value indicating whether the action is internal to the system and should not be exposed to the user.
/// </summary>
public bool IsInternalAction { get; init; }
}
18 changes: 17 additions & 1 deletion src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
// See the LICENSE file in the project root for more information.

using Microsoft.PowerToys.Settings.UI.Library;
using Windows.Devices.Radios;

namespace AdvancedPaste.Models;

Expand All @@ -14,6 +13,7 @@ public enum PasteFormats
ResourceId = "PasteAsPlainText",
IconGlyph = "\uE8E9",
RequiresAIService = false,
IsPreviewable = false,
SupportedClipboardFormats = ClipboardFormat.Text)]
PlainText,

Expand All @@ -22,6 +22,7 @@ public enum PasteFormats
ResourceId = "PasteAsMarkdown",
IconGlyph = "\ue8a5",
RequiresAIService = false,
IsPreviewable = false,
SupportedClipboardFormats = ClipboardFormat.Text,
KernelFunctionDescription = "Takes clipboard text and formats it as markdown text.")]
Markdown,
Expand All @@ -31,6 +32,7 @@ public enum PasteFormats
ResourceId = "PasteAsJson",
IconGlyph = "\uE943",
RequiresAIService = false,
IsPreviewable = false,
SupportedClipboardFormats = ClipboardFormat.Text,
KernelFunctionDescription = "Takes clipboard text and formats it as JSON text.")]
Json,
Expand All @@ -40,6 +42,7 @@ public enum PasteFormats
ResourceId = "ImageToText",
IconGlyph = "\uE91B",
RequiresAIService = false,
IsPreviewable = true,
SupportedClipboardFormats = ClipboardFormat.Image,
IPCKey = AdvancedPasteAdditionalActions.PropertyNames.ImageToText,
KernelFunctionDescription = "Takes an image in the clipboard and extracts all text from it using OCR.")]
Expand All @@ -50,6 +53,7 @@ public enum PasteFormats
ResourceId = "PasteAsTxtFile",
IconGlyph = "\uE8D2",
RequiresAIService = false,
IsPreviewable = false,
SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html,
IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsTxtFile,
KernelFunctionDescription = "Takes text or HTML data in the clipboard and transforms it to a TXT file.")]
Expand All @@ -60,6 +64,7 @@ public enum PasteFormats
ResourceId = "PasteAsPngFile",
IconGlyph = "\uE8B9",
RequiresAIService = false,
IsPreviewable = false,
SupportedClipboardFormats = ClipboardFormat.Image,
IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsPngFile,
KernelFunctionDescription = "Takes an image in the clipboard and transforms it to a PNG file.")]
Expand All @@ -70,6 +75,7 @@ public enum PasteFormats
ResourceId = "PasteAsHtmlFile",
IconGlyph = "\uF6FA",
RequiresAIService = false,
IsPreviewable = false,
SupportedClipboardFormats = ClipboardFormat.Html,
IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsHtmlFile,
KernelFunctionDescription = "Takes HTML data in the clipboard and transforms it to an HTML file.")]
Expand All @@ -79,6 +85,16 @@ public enum PasteFormats
IsCoreAction = false,
IconGlyph = "\uE945",
RequiresAIService = true,
IsPreviewable = true,
SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html | ClipboardFormat.Audio | ClipboardFormat.Image)]
KernelQuery,

[PasteFormatMetadata(
IsCoreAction = false,
RequiresAIService = true,
IsPreviewable = true,
SupportedClipboardFormats = ClipboardFormat.Text,
KernelFunctionDescription = "Takes input instructions and transforms clipboard text (not TXT files) with these input instructions, putting the result back on the clipboard. This uses AI to accomplish the task.",
IsInternalAction = true)]
CustomTextTransformation,
}
2 changes: 1 addition & 1 deletion src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ namespace AdvancedPaste.Services;

public interface ICustomTextTransformService
{
Task<string> TransformStringAsync(string inputInstructions, string inputString);
Task<string> TransformTextAsync(string inputInstructions, string inputText);
}
2 changes: 1 addition & 1 deletion src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ namespace AdvancedPaste.Services;

public interface IKernelService
{
Task<DataPackage> GetCompletionAsync(string inputInstructions, DataPackageView clipboardData);
Task<DataPackage> TransformClipboardAsync(string inputInstructions, DataPackageView clipboardData);
}
21 changes: 15 additions & 6 deletions src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Text.Json;
using System.Threading.Tasks;
using AdvancedPaste.Helpers;
using AdvancedPaste.Models;
using AdvancedPaste.Telemetry;
using Azure;
using Azure.AI.OpenAI;
using ManagedCommon;
Expand Down Expand Up @@ -43,14 +45,14 @@ private async Task<Completions> GetAICompletionAsync(string systemInstructions,
return response;
}

public async Task<string> TransformStringAsync(string inputInstructions, string inputString)
public async Task<string> TransformTextAsync(string inputInstructions, string inputText)
{
if (string.IsNullOrWhiteSpace(inputInstructions))
{
return string.Empty;
}

if (string.IsNullOrWhiteSpace(inputString))
if (string.IsNullOrWhiteSpace(inputText))
{
Logger.LogWarning("Clipboard has no usable text data");
return string.Empty;
Expand All @@ -65,21 +67,28 @@ public async Task<string> TransformStringAsync(string inputInstructions, string
{inputInstructions}
Clipboard Content:
{inputString}
{inputText}
Output:
";

try
{
var response = await GetAICompletionAsync(systemInstructions, userMessage);
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(response.Usage.PromptTokens, response.Usage.CompletionTokens, ModelName));

var usage = response.Usage;
AdvancedPasteGenerateCustomFormatEvent telemetryEvent = new(usage.PromptTokens, usage.CompletionTokens, ModelName);
PowerToysTelemetry.Log.WriteEvent(telemetryEvent);

var logEvent = new { telemetryEvent.PromptTokens, telemetryEvent.CompletionTokens, telemetryEvent.ModelName };
Logger.LogDebug($"{nameof(TransformTextAsync)} complete; {JsonSerializer.Serialize(logEvent)}");

return response.Choices[0].Text;
}
catch (Exception ex)
{
Logger.LogError($"{nameof(GetAICompletionAsync)} failed", ex);
PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomErrorEvent(ex.Message));
Logger.LogError($"{nameof(TransformTextAsync)} failed", ex);
PowerToysTelemetry.Log.WriteEvent(new AdvancedPasteGenerateCustomErrorEvent(ex.Message));

throw new PasteActionException(ErrorHelpers.TranslateErrorText((ex as RequestFailedException)?.Status ?? -1), ex);
}
Expand Down
17 changes: 17 additions & 0 deletions src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using AdvancedPaste.Helpers;
using AdvancedPaste.Models;
using Microsoft.SemanticKernel;
using Windows.ApplicationModel.DataTransfer;

Expand All @@ -14,6 +16,7 @@ internal static class KernelExtensions
{
private const string DataPackageKey = "DataPackage";
private const string LastErrorKey = "LastError";
private const string ActionChainKey = "ActionChain";

internal static DataPackageView GetDataPackageView(this Kernel kernel)
{
Expand All @@ -40,4 +43,18 @@ internal static async Task<string> GetDataFormatsAsync(this Kernel kernel)
internal static Exception GetLastError(this Kernel kernel) => kernel.Data.TryGetValue(LastErrorKey, out object obj) ? obj as Exception : null;

internal static void SetLastError(this Kernel kernel, Exception error) => kernel.Data[LastErrorKey] = error;

internal static List<PasteFormats> GetActionChain(this Kernel kernel)
{
if (kernel.Data.TryGetValue(ActionChainKey, out var actionChainObj))
{
return (List<PasteFormats>)actionChainObj;
}
else
{
List<PasteFormats> actionChain = [];
kernel.Data[ActionChainKey] = actionChain;
return actionChain;
}
}
}
Loading

1 comment on commit 7036e1d

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@check-spelling-bot Report

🔴 Please review

See the 📜action log or 📝 job summary for details.

Unrecognized words (1)

Previewable

Previously acknowledged words that are now absent applayout appsfolder buildtask cswinrt directshow DOPUS GBarm netcore nugets QDir SYSTEMSETTINGS SYSTEMWOW telem TOTALCMD USEPOSITION USESIZE winappdriver xplorer 🫥
To accept these unrecognized words as correct and remove the previously acknowledged and now absent words, you could run the following commands

... in a clone of the [email protected]:microsoft/PowerToys.git repository
on the dev/ani/advanced-paste-semantic-kernel branch (ℹ️ how do I use this?):

curl -s -S -L 'https://raw.githubusercontent.com/check-spelling/check-spelling/v0.0.22/apply.pl' |
perl - 'https://github.com/microsoft/PowerToys/actions/runs/11337137960/attempts/1'
Available 📚 dictionaries could cover words (expected and unrecognized) not in the 📘 dictionary

This includes both expected items (1933) from .github/actions/spell-check/expect.txt and unrecognized words (1)

Dictionary Entries Covers Uniquely
cspell:r/src/r.txt 543 1 1
cspell:cpp/src/people.txt 23 1
cspell:cpp/src/ecosystem.txt 51 1

Consider adding them (in .github/workflows/spelling2.yml) for uses: check-spelling/[email protected] in its with:

      with:
        extra_dictionaries:
          cspell:r/src/r.txt
          cspell:cpp/src/people.txt
          cspell:cpp/src/ecosystem.txt

To stop checking additional dictionaries, add (in .github/workflows/spelling2.yml) for uses: check-spelling/[email protected] in its with:

check_extra_dictionaries: ''
Errors (2)

See the 📜action log or 📝 job summary for details.

❌ Errors Count
❌ ignored-expect-variant 1
ℹ️ non-alpha-in-dictionary 1

See ❌ Event descriptions for more information.

If the flagged items are 🤯 false positives

If items relate to a ...

  • binary file (or some other file you wouldn't want to check at all).

    Please add a file path to the excludes.txt file matching the containing file.

    File paths are Perl 5 Regular Expressions - you can test yours before committing to verify it will match your files.

    ^ refers to the file's path from the root of the repository, so ^README\.md$ would exclude README.md (on whichever branch you're using).

  • well-formed pattern.

    If you can write a pattern that would match it,
    try adding it to the patterns.txt file.

    Patterns are Perl 5 Regular Expressions - you can test yours before committing to verify it will match your lines.

    Note that patterns can't match multiline strings.

Please sign in to comment.