From 0e75e52c8c01190de26a56a225145a368fa46e9e Mon Sep 17 00:00:00 2001
From: Max Rydahl Andersen <max@xam.dk>
Date: Thu, 19 Sep 2019 00:30:01 +0200
Subject: [PATCH] fix: use native api to enable windows ansi support

Why:

 * cmd.exe and powershell does not enable ansi without native tweaking.

This change addreses the need by:

 * use Graalvm native support to call out to native setConsoleMode()

Fixes #4095
---
 .../quarkus/runtime/logging/ConsoleAPI.java   | 49 +++++++++++++++++++
 .../runtime/logging/LoggingSetupRecorder.java |  9 +++-
 .../logging/WindowsConsoleDirectives.java     | 38 ++++++++++++++
 3 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 core/runtime/src/main/java/io/quarkus/runtime/logging/ConsoleAPI.java
 create mode 100644 core/runtime/src/main/java/io/quarkus/runtime/logging/WindowsConsoleDirectives.java

diff --git a/core/runtime/src/main/java/io/quarkus/runtime/logging/ConsoleAPI.java b/core/runtime/src/main/java/io/quarkus/runtime/logging/ConsoleAPI.java
new file mode 100644
index 0000000000000..33bb938d77ec3
--- /dev/null
+++ b/core/runtime/src/main/java/io/quarkus/runtime/logging/ConsoleAPI.java
@@ -0,0 +1,49 @@
+package io.quarkus.runtime.logging;
+
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
+import org.graalvm.nativeimage.c.CContext;
+import org.graalvm.nativeimage.c.function.CFunction;
+import org.graalvm.nativeimage.c.type.CIntPointer;
+import org.graalvm.word.Pointer;
+
+@CContext(WindowsConsoleDirectives.class)
+@Platforms(Platform.WINDOWS.class)
+public class ConsoleAPI {
+
+    static final int STD_INPUT_HANDLE = -10;
+    static final int STD_OUTPUT_HANDLE = -11;
+    static final int STD_ERROR_HANDLE = -12;
+
+    static final int ENABLE_PROCESSED_INPUT = 0x0001;
+    static final int ENABLE_LINE_INPUT = 0x0002;
+    static final int ENABLE_ECHO_INPUT = 0x0004;
+    static final int ENABLE_WINDOW_INPUT = 0x0008;
+    static final int ENABLE_MOUSE_INPUT = 0x0010;
+    static final int ENABLE_INSERT_MODE = 0x0020;
+    static final int ENABLE_QUICK_EDIT_MODE = 0x0040;
+    static final int ENABLE_EXTENDED_FLAGS = 0x0080;
+
+    // HANDLE WINAPI GetStdHandle(
+    // __in DWORD nStdHandle
+    // );
+    @CFunction
+    public static native Pointer GetStdHandle(int nStdHandle);
+
+    // BOOL WINAPI SetConsoleMode(
+    //   _In_  HANDLE hConsoleHandle,
+    //   _In_  DWORD dwMode);
+    @CFunction
+    public static native int SetConsoleMode(Pointer hConsoleHandle, int dwMode);
+
+    // BOOL WINAPI GetConsoleMode(
+    //   _In_   HANDLE hConsoleHandle,
+    //   _Out_  LPDWORD lpMode);
+    @CFunction
+    public static native void GetConsoleMode(Pointer hConsoleHandle, CIntPointer dwMode);
+
+    /**
+     * kernel32 = ctypes.windll.kernel32
+     * kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)
+     **/
+}
\ No newline at end of file
diff --git a/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java b/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java
index 31d523acfb59e..ad141d19d1881 100644
--- a/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java
+++ b/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java
@@ -15,6 +15,7 @@
 import java.util.logging.Level;
 
 import org.graalvm.nativeimage.ImageInfo;
+import org.graalvm.word.Pointer;
 import org.jboss.logmanager.EmbeddedConfigurator;
 import org.jboss.logmanager.LogContext;
 import org.jboss.logmanager.Logger;
@@ -107,7 +108,13 @@ private boolean hasColorSupport() {
                 // setConsoleMode.
                 // For now we turn it off to not generate noisy output for most
                 // users.
-                return false;
+                if (ImageInfo.inImageRuntimeCode()) {
+                    Pointer getStdHandle = ConsoleAPI.GetStdHandle(ConsoleAPI.STD_OUTPUT_HANDLE);
+                    ConsoleAPI.SetConsoleMode(getStdHandle, 7);
+                    return true;
+                } else {
+                    return false;
+                }
             } else {
                 // Must be on some Unix variant or ANSI-enabled windows terminal...
                 return true;
diff --git a/core/runtime/src/main/java/io/quarkus/runtime/logging/WindowsConsoleDirectives.java b/core/runtime/src/main/java/io/quarkus/runtime/logging/WindowsConsoleDirectives.java
new file mode 100644
index 0000000000000..9d71af0ac0397
--- /dev/null
+++ b/core/runtime/src/main/java/io/quarkus/runtime/logging/WindowsConsoleDirectives.java
@@ -0,0 +1,38 @@
+package io.quarkus.runtime.logging;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
+import org.graalvm.nativeimage.c.CContext;
+
+import com.oracle.svm.core.util.VMError;
+
+@Platforms(Platform.WINDOWS.class)
+public class WindowsConsoleDirectives implements CContext.Directives {
+
+    private static final String[] windowsLibs = new String[] { "<windows.h>" };
+
+    @Override
+    public boolean isInConfiguration() {
+        return Platform.includedIn(Platform.WINDOWS.class);
+    }
+
+    @Override
+    public List<String> getHeaderFiles() {
+        if (Platform.includedIn(Platform.WINDOWS.class)) {
+            List<String> result = new ArrayList<>(Arrays.asList(windowsLibs));
+            return result;
+        } else {
+            throw VMError.shouldNotReachHere("Unsupported OS");
+        }
+    }
+
+    @Override
+    public List<String> getMacroDefinitions() {
+        return Arrays.asList("_WIN64");
+    }
+
+}
\ No newline at end of file