Draw the cursor underneath text, and above the background (#6337)

## Summary of the Pull Request

![textAboveCursor003](https://user-images.githubusercontent.com/18356694/83681722-67a24d00-a5a8-11ea-8d9b-2d294065e4e4.gif)

This is the plan that @miniksa suggested to me. Instead of trying to do lots of work in all the renderers to do backgrounds as one pass, and foregrounds as another, we can localize this change to basically just the DX renderer.
1. First, we give the DX engine a "heads up" on where the cursor is going to be drawn during the frame, in `PrepareRenderInfo`.
  - This function is left unimplemented in the other render engines.
2. While printing runs of text, the DX renderer will try to paint the cursor in `CustomTextRenderer::DrawGlyphRun` INSTEAD of `DxEngine::PaintCursor`. This lets us weave the cursor background between the text background and the text.

## References

* #6151 was a spec in this general area. I should probably go back and update it, and we should probably approve that first.
* #6193 is also right up in this mess

## PR Checklist
* [x] Closes #1203
* [x] I work here
* [ ] Tests added/passed
* [n/a] Requires documentation to be updated

## Detailed Description of the Pull Request / Additional comments

* This is essentially `"cursorTextColor": "textForeground"` from #6151.
* A follow up work item is needed to add support for the current behavior, (`"cursorTextColor": null`), and hooking up that setting to the renderer.

(cherry picked from commit 1fcd957)

src/host/ut_host/VtRendererTests.cpp

@@ -1492,7 +1492,7 @@ void VtRendererTest::TestCursorVisibility()
 
     VERIFY_ARE_NOT_EQUAL(origin, engine->_lastText);
 
-    IRenderEngine::CursorOptions options{};
+    CursorOptions options{};
     options.coordCursor = origin;
 
     // Frame 1: Paint the cursor at the home position. At the end of the frame,

src/interactivity/onecore/BgfxEngine.cpp

@@ -179,7 +179,7 @@ BgfxEngine::BgfxEngine(PVOID SharedViewBase, LONG DisplayHeight, LONG DisplayWid
     return S_OK;
 }
 
-[[nodiscard]] HRESULT BgfxEngine::PaintCursor(const IRenderEngine::CursorOptions& options) noexcept
+[[nodiscard]] HRESULT BgfxEngine::PaintCursor(const CursorOptions& options) noexcept
 {
     // TODO: MSFT: 11448021 - Modify BGFX to support rendering full-width
     // characters and a full-width cursor.

src/renderer/base/RenderEngineBase.cpp

@@ -35,3 +35,8 @@ HRESULT RenderEngineBase::UpdateTitle(const std::wstring& newTitle) noexcept
     }
     return hr;
 }
+
+HRESULT RenderEngineBase::PrepareRenderInfo(const RenderFrameInfo& /*info*/) noexcept
+{
+    return S_FALSE;
+}

src/renderer/base/renderer.cpp

@@ -139,6 +139,9 @@ try
     // B. Perform Scroll Operations
     RETURN_IF_FAILED(_PerformScrolling(pEngine));
 
+    // C. Prepare the engine with additional information before we start drawing.
+    RETURN_IF_FAILED(_PrepareRenderInfo(pEngine));
+
     // 1. Paint Background
     RETURN_IF_FAILED(_PaintBackground(pEngine));
 
@@ -840,12 +843,16 @@ void Renderer::_PaintBufferOutputGridLineHelper(_In_ IRenderEngine* const pEngin
 }
 
 // Routine Description:
-// - Paint helper to draw the cursor within the buffer.
+// - Retrieve information about the cursor, and pack it into a CursorOptions
+//   which the render engine can use for painting the cursor.
+// - If the cursor is "off", or the cursor is out of bounds of the viewport,
+//   this will return nullopt (indicating the cursor shouldn't be painted this
+//   frame)
 // Arguments:
 // - <none>
 // Return Value:
-// - <none>
-void Renderer::_PaintCursor(_In_ IRenderEngine* const pEngine)
+// - nullopt if the cursor is off or out-of-frame, otherwise a CursorOptions
+[[nodiscard]] std::optional<CursorOptions> Renderer::_GetCursorInfo()
 {
     if (_pData->IsCursorVisible())
     {
@@ -867,7 +874,7 @@ void Renderer::_PaintCursor(_In_ IRenderEngine* const pEngine)
             bool useColor = cursorColor != INVALID_COLOR;
 
             // Build up the cursor parameters including position, color, and drawing options
-            IRenderEngine::CursorOptions options;
+            CursorOptions options;
             options.coordCursor = coordCursor;
             options.ulCursorHeightPercent = _pData->GetCursorHeight();
             options.cursorPixelWidth = _pData->GetCursorPixelWidth();
@@ -877,10 +884,43 @@ void Renderer::_PaintCursor(_In_ IRenderEngine* const pEngine)
             options.cursorColor = cursorColor;
             options.isOn = _pData->IsCursorOn();
 
-            // Draw it within the viewport
-            LOG_IF_FAILED(pEngine->PaintCursor(options));
+            return { options };
         }
     }
+    return std::nullopt;
+}
+
+// Routine Description:
+// - Paint helper to draw the cursor within the buffer.
+// Arguments:
+// - engine - The render engine that we're targeting.
+// Return Value:
+// - <none>
+void Renderer::_PaintCursor(_In_ IRenderEngine* const pEngine)
+{
+    const auto cursorInfo = _GetCursorInfo();
+    if (cursorInfo.has_value())
+    {
+        LOG_IF_FAILED(pEngine->PaintCursor(cursorInfo.value()));
+    }
+}
+
+// Routine Description:
+// - Retrieves info from the render data to prepare the engine with, before the
+//   frame is drawn. Some renderers might want to use this information to affect
+//   later drawing decisions.
+//   * Namely, the DX renderer uses this to know the cursor position and state
+//     before PaintCursor is called, so it can draw the cursor underneath the
+//     text.
+// Arguments:
+// - engine - The render engine that we're targeting.
+// Return Value:
+// - S_OK if the engine prepared successfully, or a relevant error via HRESULT.
+[[nodiscard]] HRESULT Renderer::_PrepareRenderInfo(_In_ IRenderEngine* const pEngine)
+{
+    RenderFrameInfo info;
+    info.cursorInfo = _GetCursorInfo();
+    return pEngine->PrepareRenderInfo(info);
 }
 
 // Routine Description:

src/renderer/base/renderer.hpp

@@ -127,6 +127,9 @@ namespace Microsoft::Console::Render
 
         [[nodiscard]] HRESULT _PaintTitle(IRenderEngine* const pEngine);
 
+        [[nodiscard]] std::optional<CursorOptions> _GetCursorInfo();
+        [[nodiscard]] HRESULT _PrepareRenderInfo(_In_ IRenderEngine* const pEngine);
+
         // Helper functions to diagnose issues with painting and layout.
         // These are only actually effective/on in Debug builds when the flag is set using an attached debugger.
         bool _fDebug = false;

src/renderer/dx/CustomTextRenderer.cpp

@@ -5,6 +5,8 @@
 
 #include "CustomTextRenderer.h"
 
+#include "../../inc/DefaultSettings.h"
+
 #include <wrl.h>
 #include <wrl/client.h>
 #include <VersionHelpers.h>
@@ -220,6 +222,151 @@ using namespace Microsoft::Console::Render;
                               clientDrawingEffect);
 }
 
+// Function Description:
+// - Attempt to draw the cursor. If the cursor isn't visible or on, this
+//   function will do nothing. If the cursor isn't within the bounds of the
+//   current run of text, then this function will do nothing.
+// - This function will get called twice during a run, once before the text is
+//   drawn (underneath the text), and again after the text is drawn (above the
+//   text). Depending on if the cursor wants to be drawn above or below the
+//   text, this function will do nothing for the first/second pass
+//   (respectively).
+// Arguments:
+// - d2dContext - Pointer to the current D2D drawing context
+// - textRunBounds - The bounds of the current run of text.
+// - drawingContext - Pointer to structure of information required to draw
+// - firstPass - true if we're being called before the text is drawn, false afterwards.
+// Return Value:
+// - S_FALSE if we did nothing, S_OK if we successfully painted, otherwise an appropriate HRESULT
+[[nodiscard]] HRESULT _drawCursor(gsl::not_null<ID2D1DeviceContext*> d2dContext,
+                                  D2D1_RECT_F textRunBounds,
+                                  const DrawingContext& drawingContext,
+                                  const bool firstPass)
+try
+{
+    if (!drawingContext.cursorInfo.has_value())
+    {
+        return S_FALSE;
+    }
+
+    const auto& options = drawingContext.cursorInfo.value();
+
+    // if the cursor is off, do nothing - it should not be visible.
+    if (!options.isOn)
+    {
+        return S_FALSE;
+    }
+
+    // TODO GH#6338: Add support for `"cursorTextColor": null` for letting the
+    // cursor draw on top again.
+
+    // Only draw the filled box in the first pass. All the other cursors should
+    // be drawn in the second pass.
+    //           | type==FullBox |
+    // firstPass |   T   |   F   |
+    //    T      | draw  |  skip |
+    //    F      | skip  |  draw |
+    if ((options.cursorType == CursorType::FullBox) != firstPass)
+    {
+        return S_FALSE;
+    }
+
+    const til::size glyphSize{ til::math::flooring,
+                               drawingContext.cellSize.width,
+                               drawingContext.cellSize.height };
+
+    // Create rectangular block representing where the cursor can fill.
+    D2D1_RECT_F rect = til::rectangle{ til::point{ options.coordCursor } }.scale_up(glyphSize);
+
+    // If we're double-width, make it one extra glyph wider
+    if (options.fIsDoubleWidth)
+    {
+        rect.right += glyphSize.width();
+    }
+
+    // If the cursor isn't within the bounds of this current run of text, do nothing.
+    if (rect.top > textRunBounds.bottom ||
+        rect.bottom <= textRunBounds.top ||
+        rect.left > textRunBounds.right ||
+        rect.right <= textRunBounds.left)
+    {
+        return S_FALSE;
+    }
+
+    CursorPaintType paintType = CursorPaintType::Fill;
+
+    switch (options.cursorType)
+    {
+    case CursorType::Legacy:
+    {
+        // Enforce min/max cursor height
+        ULONG ulHeight = std::clamp(options.ulCursorHeightPercent, MinCursorHeightPercent, MaxCursorHeightPercent);
+        ulHeight = (glyphSize.height<ULONG>() * ulHeight) / 100;
+        rect.top = rect.bottom - ulHeight;
+        break;
+    }
+    case CursorType::VerticalBar:
+    {
+        // It can't be wider than one cell or we'll have problems in invalidation, so restrict here.
+        // It's either the left + the proposed width from the ease of access setting, or
+        // it's the right edge of the block cursor as a maximum.
+        rect.right = std::min(rect.right, rect.left + options.cursorPixelWidth);
+        break;
+    }
+    case CursorType::Underscore:
+    {
+        rect.top = rect.bottom - 1;
+        break;
+    }
+    case CursorType::EmptyBox:
+    {
+        paintType = CursorPaintType::Outline;
+        break;
+    }
+    case CursorType::FullBox:
+    {
+        break;
+    }
+    default:
+        return E_NOTIMPL;
+    }
+
+    Microsoft::WRL::ComPtr<ID2D1SolidColorBrush> brush;
+
+    if (options.fUseColor)
+    {
+        // Make sure to make the cursor opaque
+        RETURN_IF_FAILED(d2dContext->CreateSolidColorBrush(til::color{ OPACITY_OPAQUE | options.cursorColor },
+                                                           &brush));
+    }
+
+    switch (paintType)
+    {
+    case CursorPaintType::Fill:
+    {
+        d2dContext->FillRectangle(rect, brush.Get());
+        break;
+    }
+    case CursorPaintType::Outline:
+    {
+        // DrawRectangle in straddles physical pixels in an attempt to draw a line
+        // between them. To avoid this, bump the rectangle around by half the stroke width.
+        rect.top += 0.5f;
+        rect.left += 0.5f;
+        rect.bottom -= 0.5f;
+        rect.right -= 0.5f;
+
+        d2dContext->DrawRectangle(rect, brush.Get());
+        break;
+    }
+    default:
+        return E_NOTIMPL;
+    }
+
+    return S_OK;
+}
+CATCH_RETURN()
+
 // Routine Description:
 // - Implementation of IDWriteTextRenderer::DrawInlineObject
 // - Passes drawing control from the outer layout down into the context of an embedded object
@@ -292,6 +439,8 @@ using namespace Microsoft::Console::Render;
 
     d2dContext->FillRectangle(rect, drawingContext->backgroundBrush);
 
+    RETURN_IF_FAILED(_drawCursor(d2dContext.Get(), rect, *drawingContext, true));
+
     // GH#5098: If we're rendering with cleartype text, we need to always render
     // onto an opaque background. If our background _isn't_ opaque, then we need
     // to use grayscale AA for this run of text.
@@ -479,6 +628,9 @@ using namespace Microsoft::Console::Render;
                                             drawingContext->foregroundBrush,
                                             clientDrawingEffect));
     }
+
+    RETURN_IF_FAILED(_drawCursor(d2dContext.Get(), rect, *drawingContext, false));
+
     return S_OK;
 }
 #pragma endregion

src/renderer/dx/CustomTextRenderer.h

@@ -5,6 +5,7 @@
 
 #include <wrl/implements.h>
 #include "BoxDrawingEffect.h"
+#include "../../renderer/inc/CursorOptions.h"
 
 namespace Microsoft::Console::Render
 {
@@ -17,6 +18,7 @@ namespace Microsoft::Console::Render
                        IDWriteFactory* dwriteFactory,
                        const DWRITE_LINE_SPACING spacing,
                        const D2D_SIZE_F cellSize,
+                       const std::optional<CursorOptions>& cursorInfo,
                        const D2D1_DRAW_TEXT_OPTIONS options = D2D1_DRAW_TEXT_OPTIONS_NONE) noexcept
         {
             this->renderTarget = renderTarget;
@@ -26,6 +28,7 @@ namespace Microsoft::Console::Render
             this->dwriteFactory = dwriteFactory;
             this->spacing = spacing;
             this->cellSize = cellSize;
+            this->cursorInfo = cursorInfo;
             this->options = options;
         }
 
@@ -36,9 +39,20 @@ namespace Microsoft::Console::Render
         IDWriteFactory* dwriteFactory;
         DWRITE_LINE_SPACING spacing;
         D2D_SIZE_F cellSize;
+        std::optional<CursorOptions> cursorInfo;
         D2D1_DRAW_TEXT_OPTIONS options;
     };
 
+    // Helper to choose which Direct2D method to use when drawing the cursor rectangle
+    enum class CursorPaintType
+    {
+        Fill,
+        Outline
+    };
+
+    constexpr const ULONG MinCursorHeightPercent = 25;
+    constexpr const ULONG MaxCursorHeightPercent = 100;
+
     class CustomTextRenderer : public ::Microsoft::WRL::RuntimeClass<::Microsoft::WRL::RuntimeClassFlags<::Microsoft::WRL::ClassicCom | ::Microsoft::WRL::InhibitFtmBase>, IDWriteTextRenderer>
     {
     public: