#459 IME UI does not follow the cursor in Windows Terminal

src/cascadia/TerminalControl/TSFInputControl.h

@@ -0,0 +1,101 @@
+#pragma once
+#include "TSFInputControl.g.h"
+#include "CursorPositionEventArgs.g.h"
+#include "FontInfoEventArgs.g.h"
+#include "cppwinrt_utils.h"
+
+namespace winrt::Microsoft::Terminal::TerminalControl::implementation
+{
+    struct CursorPositionEventArgs :
+        public CursorPositionEventArgsT<CursorPositionEventArgs>
+    {
+    public:
+        CursorPositionEventArgs(){};
+
+        Windows::Foundation::Point CurrentPosition()
+        {
+            return _curPos;
+        }
+
+        void CurrentPosition(Windows::Foundation::Point newPosition)
+        {
+            _curPos = newPosition;
+        }
+
+    private:
+        Windows::Foundation::Point _curPos;
+    };
+
+    struct FontInfoEventArgs :
+        public FontInfoEventArgsT<FontInfoEventArgs>
+    {
+    public:
+        FontInfoEventArgs(){};
+
+        Windows::Foundation::Size FontSize()
+        {
+            return _curSize;
+        }
+
+        void FontSize(Windows::Foundation::Size newSize)
+        {
+            _curSize = newSize;
+        }
+
+        winrt::hstring FontFace()
+        {
+            return _fontFace;
+        }
+
+        void FontFace(winrt::hstring newFontFace)
+        {
+            _fontFace = newFontFace;
+        }
+
+    private:
+        winrt::hstring _fontFace;
+        Windows::Foundation::Size _curSize;
+    };
+
+    struct TSFInputControl : TSFInputControlT<TSFInputControl>
+    {
+    public:
+        TSFInputControl();
+
+        void NotifyFocusEnter();
+        void NotifyFocusLeave();
+
+        static void OnCompositionChanged(Windows::UI::Xaml::DependencyObject const&, Windows::UI::Xaml::DependencyPropertyChangedEventArgs const&);
+
+        // -------------------------------- WinRT Events ---------------------------------
+        DECLARE_EVENT_WITH_TYPED_EVENT_HANDLER(CurrentCursorPosition, _currentCursorPositionHandlers, TerminalControl::TSFInputControl, TerminalControl::CursorPositionEventArgs);
+        DECLARE_EVENT_WITH_TYPED_EVENT_HANDLER(CurrentFontInfo, _currentFontInfoHandlers, TerminalControl::TSFInputControl, TerminalControl::FontInfoEventArgs);
+        DECLARE_EVENT(CompositionCompleted, _compositionCompletedHandlers, TerminalControl::CompositionCompletedEventArgs);
+
+    private:
+        void _layoutRequestedHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextLayoutRequestedEventArgs const& args);
+        void _compositionStartedHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextCompositionStartedEventArgs const& args);
+        void _compositionCompletedHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextCompositionCompletedEventArgs const& args);
+        void _focusRemovedHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::Foundation::IInspectable const& object);
+        void _selectionRequestedHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextSelectionRequestedEventArgs const& args);
+        void _textRequestedHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextTextRequestedEventArgs const& args);
+        void _selectionUpdatingHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextSelectionUpdatingEventArgs const& args);
+        void _textUpdatingHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextTextUpdatingEventArgs const& args);
+        void _formatUpdatingHandler(winrt::Windows::UI::Text::Core::CoreTextEditContext sender, winrt::Windows::UI::Text::Core::CoreTextFormatUpdatingEventArgs const& args);
+
+        Windows::UI::Xaml::Controls::Canvas _canvas;
+        Windows::UI::Xaml::Controls::TextBlock _textBlock;
+
+        Windows::UI::Text::Core::CoreTextEditContext _editContext;
+
+        std::wstring _inputBuffer;
+
+        void _Create();
+    };
+}
+namespace winrt::Microsoft::Terminal::TerminalControl::factory_implementation
+{
+    struct TSFInputControl : TSFInputControlT<TSFInputControl, implementation::TSFInputControl>
+    {
+    };
+}

src/cascadia/TerminalControl/TSFInputControl.idl

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+namespace Microsoft.Terminal.TerminalControl
+{
+    delegate void CompositionCompletedEventArgs(String text);
+
+    runtimeclass CursorPositionEventArgs
+    {
+        Windows.Foundation.Point CurrentPosition { get; set; };
+    }
+
+    runtimeclass FontInfoEventArgs
+    {
+        String FontFace { get; set; };
+        Windows.Foundation.Size FontSize { get; set; };
+    }
+
+    [default_interface]
+    runtimeclass TSFInputControl : Windows.UI.Xaml.Controls.UserControl
+    {
+        TSFInputControl();
+
+        event CompositionCompletedEventArgs CompositionCompleted;
+        event Windows.Foundation.TypedEventHandler<TSFInputControl, CursorPositionEventArgs> CurrentCursorPosition;
+        event Windows.Foundation.TypedEventHandler<TSFInputControl, FontInfoEventArgs> CurrentFontInfo;
+
+        void NotifyFocusEnter();
+        void NotifyFocusLeave();
+    }
+}

src/cascadia/TerminalControl/TermControl.cpp

@@ -68,12 +68,22 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         _touchAnchor{ std::nullopt },
         _cursorTimer{},
         _lastMouseClick{},
-        _lastMouseClickPos{}
+        _lastMouseClickPos{},
+        _tsfInputControl{ nullptr }
     {
         _EnsureStaticInitialization();
         _Create();
     }
 
+    inline winrt::Windows::UI::Color ColorRefToColor(const COLORREF& colorref)
+    {
+        winrt::Windows::UI::Color color;
+        color.R = GetRValue(colorref);
+        color.G = GetGValue(colorref);
+        color.B = GetBValue(colorref);
+        return color;
+    }
+
     void TermControl::_Create()
     {
         Controls::Grid container;
@@ -101,6 +111,12 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         _scrollBar.LargeChange(4);
         _scrollBar.Visibility(Visibility::Visible);
 
+        _tsfInputControl = TSFInputControl();
+        _tsfInputControl.CompositionCompleted({ this, &TermControl::_CompositionCompleted });
+        _tsfInputControl.CurrentCursorPosition({ this, &TermControl::_CurrentCursorPositionHandler });
+        _tsfInputControl.CurrentFontInfo({ this, &TermControl::_FontInfoHandler });
+        container.Children().Append(_tsfInputControl);
+
         // Create the SwapChainPanel that will display our content
         Controls::SwapChainPanel swapChainPanel;
 
@@ -193,6 +209,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
                 auto lock = _terminal->LockForWriting();
                 _DoResize(width, height);
             }
+
+            // set TSF Foreground
+            Media::SolidColorBrush foregroundBrush{};
+            foregroundBrush.Color(ColorRefToColor(_settings.DefaultForeground()));
+            _tsfInputControl.Foreground(foregroundBrush);
         });
     }
 
@@ -245,6 +266,12 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         //      The Codepage is additionally not actually used by the DX engine at all.
         _actualFont = { fontFace, 0, 10, { 0, fontHeight }, CP_UTF8, false };
         _desiredFont = { _actualFont };
+
+        // set TSF Foreground
+        Media::SolidColorBrush foregroundBrush{};
+        foregroundBrush.Color(ColorRefToColor(_settings.DefaultForeground()));
+        _tsfInputControl.Foreground(foregroundBrush);
+        _tsfInputControl.Margin(newMargin);
     }
 
     // Method Description:
@@ -1188,6 +1215,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         }
         _focused = true;
 
+        if (_tsfInputControl != nullptr)
+        {
+            _tsfInputControl.NotifyFocusEnter();
+        }
+
         if (_cursorTimer.has_value())
         {
             _cursorTimer.value().Start();
@@ -1206,6 +1238,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         }
         _focused = false;
 
+        if (_tsfInputControl != nullptr)
+        {
+            _tsfInputControl.NotifyFocusLeave();
+        }
+
         if (_cursorTimer.has_value())
         {
             _cursorTimer.value().Stop();
@@ -1866,6 +1903,45 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         return terminalPosition;
     }
 
+    // Method Description:
+    // - Composition Completion handler for the TSFInputControl that
+    //   handles writing text out to TerminalConnection
+    // Arguments:
+    // - text: the text to write to TerminalConnection
+    // Return Value:
+    // - <none>
+    void TermControl::_CompositionCompleted(winrt::hstring text)
+    {
+        _connection.WriteInput(text);
+    }
+
+    // Method Description:
+    // - CurrentCursorPosition handler for the TSFInputControl that
+    //   handles returning current cursor position.
+    // Arguments:
+    // - eventArgs: event for storing the current cursor position
+    // Return Value:
+    // - <none>
+    void TermControl::_CurrentCursorPositionHandler(const IInspectable& /*sender*/, const CursorPositionEventArgs& eventArgs)
+    {
+        const COORD cursorPos = _terminal->GetCursorPosition();
+        Windows::Foundation::Point p = { gsl::narrow<float>(cursorPos.X), gsl::narrow<float>(cursorPos.Y) };
+        eventArgs.CurrentPosition(p);
+    }
+
+    // Method Description:
+    // - FontInfo handler for the TSFInputControl that
+    //   handles returning current font information
+    // Arguments:
+    // - eventArgs: event for storing the current font information
+    // Return Value:
+    // - <none>
+    void TermControl::_FontInfoHandler(const IInspectable& /*sender*/, const FontInfoEventArgs& eventArgs)
+    {
+        eventArgs.FontSize(CharacterDimensions());
+        eventArgs.FontFace(_actualFont.GetFaceName());
+    }
+
     // Method Description:
     // - Returns the number of clicks that occurred (double and triple click support)
     // Arguments:

src/cascadia/TerminalControl/TermControl.h

@@ -11,7 +11,7 @@
 #include "../../renderer/base/Renderer.hpp"
 #include "../../renderer/dx/DxRenderer.hpp"
 #include "../../cascadia/TerminalCore/Terminal.hpp"
-#include "../../cascadia/inc/cppwinrt_utils.h"
+#include "cppwinrt_utils.h"
 
 namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 {
@@ -101,6 +101,8 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         Windows::UI::Xaml::Controls::Image _bgImageLayer;
         Windows::UI::Xaml::Controls::SwapChainPanel _swapChainPanel;
         Windows::UI::Xaml::Controls::Primitives::ScrollBar _scrollBar;
+        TSFInputControl _tsfInputControl;
+
         event_token _connectionOutputEventToken;
 
         std::unique_ptr<::Microsoft::Terminal::Core::Terminal> _terminal;
@@ -196,6 +198,11 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         const COORD _GetTerminalPosition(winrt::Windows::Foundation::Point cursorPosition);
         const unsigned int _NumberOfClicks(winrt::Windows::Foundation::Point clickPos, Timestamp clickTime);
         double _GetAutoScrollSpeed(double cursorDistanceFromBorder) const;
+
+        // TSFInputControl Handlers
+        void _CompositionCompleted(winrt::hstring text);
+        void _CurrentCursorPositionHandler(const IInspectable& /*sender*/, const CursorPositionEventArgs& eventArgs);
+        void _FontInfoHandler(const IInspectable& /*sender*/, const FontInfoEventArgs& eventArgs);
     };
 }
 

src/cascadia/TerminalControl/TerminalControl.vcxproj

@@ -31,6 +31,9 @@
     <ClInclude Include="TermControlAutomationPeer.h">
       <DependentUpon>TermControlAutomationPeer.idl</DependentUpon>
     </ClInclude>
+    <ClInclude Include="TSFInputControl.h">
+      <DependentUpon>TSFInputControl.idl</DependentUpon>
+    </ClInclude>
     <ClInclude Include="UiaTextRange.hpp" />
     <ClInclude Include="XamlUiaTextRange.h" />
   </ItemGroup>
@@ -43,6 +46,9 @@
     <ClCompile Include="TermControl.cpp">
       <DependentUpon>TermControl.idl</DependentUpon>
     </ClCompile>
+    <ClCompile Include="TSFInputControl.cpp">
+      <DependentUpon>TSFInputControl.idl</DependentUpon>
+    </ClCompile>
     <ClCompile Include="$(GeneratedFilesDir)module.g.cpp" />
     <ClCompile Include="TermControlAutomationPeer.cpp">
       <DependentUpon>TermControlAutomationPeer.idl</DependentUpon>
@@ -53,6 +59,7 @@
   <ItemGroup>
     <Midl Include="TermControl.idl" />
     <Midl Include="TermControlAutomationPeer.idl" />
+    <Midl Include="TSFInputControl.idl" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
@@ -84,7 +91,7 @@
       <AdditionalDependencies>dwrite.lib;dxgi.lib;d2d1.lib;d3d11.lib;shcore.lib;winmm.lib;pathcch.lib;propsys.lib;uiautomationcore.lib;Shlwapi.lib;ntdll.lib;user32.lib;kernel32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <ClCompile>
-      <AdditionalIncludeDirectories>$(OpenConsoleDir)src\types\inc;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(OpenConsoleDir)src\cascadia\inc;$(OpenConsoleDir)src\types\inc;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
   </ItemDefinitionGroup>
 </Project>

src/cascadia/TerminalControl/pch.h

@@ -23,15 +23,19 @@
 #include <unknwn.h>
 #include <winrt/Windows.Foundation.h>
 #include <winrt/Windows.Foundation.Collections.h>
+#include <winrt/Windows.Graphics.Display.h>
 #include <winrt/windows.ui.core.h>
 #include <winrt/Windows.ui.input.h>
 #include <winrt/Windows.UI.Xaml.h>
 #include <winrt/Windows.UI.Xaml.Automation.Peers.h>
+#include <winrt/Windows.UI.Text.Core.h>
 #include <winrt/Windows.UI.Xaml.Controls.h>
 #include <winrt/Windows.UI.Xaml.Controls.Primitives.h>
-#include <winrt/Windows.ui.xaml.media.h>
-#include <winrt/Windows.ui.xaml.media.imaging.h>
-#include <winrt/Windows.ui.xaml.input.h>
+#include <winrt/Windows.UI.Xaml.Data.h>
+#include <winrt/Windows.UI.Xaml.Media.h>
+#include <winrt/Windows.UI.Xaml.Media.Imaging.h>
+#include <winrt/Windows.UI.Xaml.Input.h>
+#include <winrt/Windows.UI.Xaml.Interop.h>
 
 #include <windows.ui.xaml.media.dxinterop.h>