win32: use native ANSI sequence processing, if possible

Windows 10 version 1511 (also known as Anniversary Update), according to https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences introduced native support for ANSI sequence processing. This allows using colors from the entire 24-bit color range. All we need to do is test whether the console's "virtual processing support" can be enabled. If it can, we do not even need to start the `console_thread` to handle ANSI sequences. Signed-off-by: Johannes Schindelin <[email protected]>
dscho · Nov 22, 2023 · 60107fb · 60107fb
1 parent 4b968f3
commit 60107fb
Showing 1 changed file with 32 additions and 0 deletions.
diff --git a/compat/winansi.c b/compat/winansi.c
@@ -594,6 +594,35 @@ static void detect_msys_tty(int fd)
 
 #endif
 
+static HANDLE stdout_handle;
+static DWORD stdout_console_mode;
+
+static void reset_stdout_console_mode(void)
+{
+	SetConsoleMode(stdout_handle, stdout_console_mode);
+}
+
+static int enable_virtual_processing(void)
+{
+	stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
+	if (stdout_handle == INVALID_HANDLE_VALUE)
+		return 0;
+
+	if (!GetConsoleMode(stdout_handle, &stdout_console_mode))
+		return 0;
+
+	if (stdout_console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+		return 1;
+
+	if (!SetConsoleMode(stdout_handle,
+			    stdout_console_mode |
+			    ENABLE_VIRTUAL_TERMINAL_PROCESSING))
+		return 0;
+
+	atexit(reset_stdout_console_mode);
+	return 1;
+}
+
 /*
  * Wrapper for isatty().  Most calls in the main git code
  * call isatty(1 or 2) to see if the instance is interactive
@@ -632,6 +661,9 @@ void winansi_init(void)
 		return;
 	}
 
+	if (enable_virtual_processing())
+		return;
+
 	/* create a named pipe to communicate with the console thread */
 	if (swprintf(name, ARRAY_SIZE(name) - 1, L"\\\\.\\pipe\\winansi%lu",
 		     GetCurrentProcessId()) < 0)