Merge branch 'FarGroup:master' into master

far/changelog

@@ -1,3 +1,25 @@
+--------------------------------------------------------------------------------
+drkns 2024-12-02 20:39:53+00:00 - build 6398
+
+1. Use DECRQM 2027 to check grapheme clusters state (thanks to DHowett).
+
+--------------------------------------------------------------------------------
+drkns 2024-12-01 23:24:48+00:00 - build 6397
+
+1. Experimental: FMSG_KEEPBACKGROUND is a no-op.
+
+2. Correction of 6396.
+
+--------------------------------------------------------------------------------
+drkns 2024-11-30 23:34:14+00:00 - build 6396
+
+1. Warn the user when the terminal uses grapheme clusters.
+
+--------------------------------------------------------------------------------
+skipik 2024-11-26 17:23:11+03:00 - build 6395
+
+1. SQLite v3.47.1.
+
 --------------------------------------------------------------------------------
 drkns 2024-11-22 19:07:44+00:00 - build 6394
 

far/char_width.cpp

@@ -39,6 +39,7 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // Internal:
 #include "console.hpp"
 #include "locale.hpp"
+#include "log.hpp"
 
 // Platform:
 
@@ -552,4 +553,22 @@ namespace char_width
 		static const auto Result = console.GetWidthPreciseExpensive(U'𝖆');
 		return Result > 1;
 	}
+
+	bool is_grapheme_clusters_on()
+	{
+		static const auto Result = []
+		{
+			if (const auto IsOn = console.is_grapheme_clusters_on(); IsOn)
+			{
+				LOGDEBUG(L"Grapheme clusters (VT): {}"sv, *IsOn);
+				return *IsOn;
+			}
+
+			const auto IsOn = console.GetWidthPreciseExpensive(L"à"sv) == 1;
+			LOGDEBUG(L"Grapheme clusters (heuristics): {}"sv, IsOn);
+			return IsOn;
+		}();
+
+		return Result;
+	}
 }

far/char_width.hpp

@@ -63,6 +63,9 @@ namespace char_width
 
 	[[nodiscard]]
 	bool is_half_width_surrogate_broken();
+
+	[[nodiscard]]
+	bool is_grapheme_clusters_on();
 }
 
 #endif // CHAR_WIDTH_HPP_D66C86AC_3415_4FD1_89DA_0AB843FFEEB8

far/config.cpp

@@ -364,6 +364,8 @@ void Options::InterfaceSettings()
 	Builder.AddEditField(strTitleAddons, 47);
 	Builder.AddOKCancel();
 
+	const auto PreviousFullWidthAwareRendering = FullWidthAwareRendering.Get();
+
 	if (Builder.ShowDialog())
 	{
 		if (CMOpt.CopyTimeRule)
@@ -378,6 +380,20 @@ void Options::InterfaceSettings()
 		Panels->SetScreenPosition();
 		// $ 10.07.2001 SKV ! надо это делать, иначе если кейбар спрятали, будет полный рамс.
 		Panels->Redraw();
+
+		if (!PreviousFullWidthAwareRendering && FullWidthAwareRendering && char_width::is_grapheme_clusters_on())
+		{
+			Message(FMSG_WARNING, msg(lng::MWarning),
+			{
+				msg(lng::MConfigFullWidthAwareRenderingGraphemeClustersDected1),
+				msg(lng::MConfigFullWidthAwareRenderingGraphemeClustersDected2),
+				msg(lng::MConfigFullWidthAwareRenderingGraphemeClustersDected3),
+				msg(lng::MConfigFullWidthAwareRenderingGraphemeClustersDected4),
+			},
+			{
+				lng::MOk
+			});
+		}
 	}
 }
 

far/console.cpp

@@ -631,15 +631,15 @@ namespace console_detail
 
 		std::optional<size_t>
 			FirstTokenPrefixPos,
-			FirstTokenSuffixPos,
-			SecondTokenPrefixPos,
-			SecondTokenSuffixPos;
+			FirstTokenSuffixPos;
 
 		const auto
 			TokenPrefix = CSI "?"sv,
 			TokenSuffix = L"c"sv;
 
-		while (!SecondTokenSuffixPos)
+		size_t DA_ResponseSize{};
+
+		for (;;)
 		{
 			wchar_t ResponseBuffer[8192];
 			size_t ResponseSize;
@@ -655,21 +655,19 @@ namespace console_detail
 
 			if (FirstTokenPrefixPos && !FirstTokenSuffixPos)
 				if (const auto Pos = Response.find(TokenSuffix, *FirstTokenPrefixPos + TokenPrefix.size()); Pos != Response.npos)
+				{
 					FirstTokenSuffixPos = Pos;
+					DA_ResponseSize = Pos + TokenSuffix.size();
+				}
 
-			if (FirstTokenSuffixPos && !SecondTokenPrefixPos)
-				if (const auto Pos = Response.find(TokenPrefix, *FirstTokenSuffixPos + TokenSuffix.size()); Pos != Response.npos)
-					SecondTokenPrefixPos = Pos;
-
-			if (SecondTokenPrefixPos && !SecondTokenSuffixPos)
-				if (const auto Pos = Response.find(TokenSuffix, *SecondTokenPrefixPos + TokenPrefix.size()); Pos != Response.npos)
-					SecondTokenSuffixPos = Pos;
+			if (DA_ResponseSize && Response.size() >= DA_ResponseSize * 2)
+			{
+				if (const auto DA_Response = string_view(Response).substr(*FirstTokenPrefixPos, DA_ResponseSize); Response.ends_with(DA_Response))
+					break;
+			}
 		}
 
-		Response.resize(*SecondTokenPrefixPos);
-		Response.erase(0, *FirstTokenSuffixPos + TokenSuffix.size());
-
-		return Response;
+		return Response.substr(DA_ResponseSize, Response.size() - DA_ResponseSize * 2);
 	}
 
 	console::console():
@@ -2906,7 +2904,7 @@ namespace console_detail
 		return ExternalConsole.Imports.pWriteOutput.operator bool();
 	}
 
-	size_t console::GetWidthPreciseExpensive(char32_t const Codepoint)
+	size_t console::GetWidthPreciseExpensive(string_view const Str)
 	{
 		// It ain't stupid if it works
 
@@ -2944,21 +2942,26 @@ namespace console_detail
 		}
 
 		DWORD Written;
-		const auto Pair = encoding::utf16::to_surrogate(Codepoint);
-		const std::array Chars{ Pair.first, Pair.second };
-		if (!WriteConsole(m_WidthTestScreen.native_handle(), Chars.data(), Pair.second? 2 : 1, &Written, {}))
+		if (!WriteConsole(m_WidthTestScreen.native_handle(), Str.data(), static_cast<DWORD>(Str.size()), &Written, {}))
 		{
 			LOGWARNING(L"WriteConsole(): {}"sv, os::last_error());
-			return 1;
+			return Str.size();
 		}
 
 		CONSOLE_SCREEN_BUFFER_INFO Info;
 		if (!get_console_screen_buffer_info(m_WidthTestScreen.native_handle(), &Info))
-			return 1;
+			return Str.size();
 
 		return Info.dwCursorPosition.X;
 	}
 
+	size_t console::GetWidthPreciseExpensive(char32_t const Codepoint)
+	{
+		const auto Pair = encoding::utf16::to_surrogate(Codepoint);
+		const std::array Chars{ Pair.first, Pair.second };
+		return GetWidthPreciseExpensive({ Chars.data(), Pair.second? 2uz : 1uz });
+	}
+
 	void console::ClearWideCache()
 	{
 		m_WidthTestScreen = {};
@@ -3095,6 +3098,49 @@ namespace console_detail
 		send_vt_command(far::format(OSC("9001;CmdNotFound;{}"), Command));
 	}
 
+	std::optional<bool> console::is_grapheme_clusters_on() const
+	{
+		try
+		{
+#define DECRQM_REQUEST "?2027"
+
+			const auto ResponseData = query_vt(CSI DECRQM_REQUEST "$p"sv);
+			if (ResponseData.empty())
+			{
+				LOGWARNING(L"DECRQM 2027 query is not supported"sv);
+				return {};
+			}
+
+			const auto
+				Prefix = CSI DECRQM_REQUEST ";"sv,
+				Suffix = L"$y"sv;
+
+#undef DECRQM_REQUEST
+
+			const auto give_up = [&]
+			{
+				throw far_exception(far::format(L"Incorrect response: {}"sv, ResponseData), false);
+			};
+
+			if (ResponseData.size() != Prefix.size() + 1 + Suffix.size() || !ResponseData.starts_with(Prefix) || !ResponseData.ends_with(Suffix))
+				give_up();
+
+			switch (ResponseData[Prefix.size()])
+			{
+			case L'3': return true;
+			case L'4': return false;
+			default:
+				give_up();
+				std::unreachable();
+			}
+		}
+		catch (far_exception const& e)
+		{
+			LOGERROR(L"{}"sv, e);
+			return {};
+		}
+	}
+
 	bool console::GetCursorRealPosition(point& Position) const
 	{
 		CONSOLE_SCREEN_BUFFER_INFO ConsoleScreenBufferInfo;

far/console.hpp

@@ -189,6 +189,8 @@ namespace console_detail
 		[[nodiscard]]
 		bool ExternalRendererLoaded() const;
 
+		[[nodiscard]]
+		size_t GetWidthPreciseExpensive(string_view Str);
 		[[nodiscard]]
 		size_t GetWidthPreciseExpensive(char32_t Codepoint);
 		void ClearWideCache();
@@ -212,6 +214,9 @@ namespace console_detail
 		void command_finished(int ExitCode) const;
 		void command_not_found(string_view Command) const;
 
+		[[nodiscard]]
+		std::optional<bool> is_grapheme_clusters_on() const;
+
 		[[nodiscard]]
 		short GetDelta() const;
 

far/farlang.templ.m4

@@ -2871,6 +2871,62 @@ upd:"Рендерінг з урахуванням повної ширини"
 "Візуалізацыя з улікам поўнай шырыні"
 upd:"Fullwidth-aware rendering"
 
+MConfigFullWidthAwareRenderingGraphemeClustersDected1
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+"Wydaje się, że Twój terminal obsługuje klastry grapheme Unicode."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+upd:"It appears that your terminal supports Unicode grapheme clusters."
+
+MConfigFullWidthAwareRenderingGraphemeClustersDected2
+upd:"Unfortunately, Far does not support them yet."
+"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+"Niestety, Far jeszcze ich nie obsługuje."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+upd:"Unfortunately, Far does not support them yet."
+
+MConfigFullWidthAwareRenderingGraphemeClustersDected3
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+"Dla zachowania najlepszych wrażeń, zalecamy przełączenie na renderowanie oparte na punktach kodowych."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+upd:"For best experience it is recommended to switch to codepoint-based rendering."
+
+MConfigFullWidthAwareRenderingGraphemeClustersDected4
+upd:"Consult the documentation of your terminal for advice on how to do this."
+"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+"Sprawdź w dokumentacji terminala, aby dowiedzieć się jak to zrobić."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+upd:"Consult the documentation of your terminal for advice on how to do this."
+
 MConfigClearType
 "Перер&исовка с поддержкой ClearType (может быть медленной)"
 "Cl&earType-friendly redraw (can be slow)"

far/message.cpp

@@ -449,12 +449,9 @@ static message_result MessageImpl(
 	// *** Без Диалога! ***
 	HideCursor();
 
-	if (!(Flags & MSG_KEEPBACKGROUND))
-	{
-		SetScreen(Position, L' ', colors::PaletteColorToFarColor((Flags & MSG_WARNING)? COL_WARNDIALOGTEXT : COL_DIALOGTEXT));
-		DropShadow(Position);
-		Box({ Position.left + 3, Position.top + 1, Position.right - 3, Position.bottom - 1 }, colors::PaletteColorToFarColor((Flags & MSG_WARNING)? COL_WARNDIALOGBOX : COL_DIALOGBOX), DOUBLE_BOX);
-	}
+	SetScreen(Position, L' ', colors::PaletteColorToFarColor((Flags & MSG_WARNING)? COL_WARNDIALOGTEXT : COL_DIALOGTEXT));
+	DropShadow(Position);
+	Box({ Position.left + 3, Position.top + 1, Position.right - 3, Position.bottom - 1 }, colors::PaletteColorToFarColor((Flags & MSG_WARNING)? COL_WARNDIALOGBOX : COL_DIALOGBOX), DOUBLE_BOX);
 
 	SetColor((Flags & MSG_WARNING)?COL_WARNDIALOGTEXT:COL_DIALOGTEXT);
 

far/message.hpp

@@ -52,7 +52,6 @@ enum class lng;
 enum
 {
 	MSG_WARNING        = 0_bit,
-	MSG_KEEPBACKGROUND = 2_bit,
 	MSG_LEFTALIGN      = 3_bit,
 
 	MSG_KILLSAVESCREEN = 28_bit,

far/plugapi.cpp

@@ -1224,7 +1224,6 @@ intptr_t WINAPI apiMessageFn(const UUID* PluginId, const UUID* Id, unsigned long
 
 		const DWORD InternalFlags =
 			((Flags & FMSG_WARNING)? MSG_WARNING : 0) |
-			((Flags & FMSG_KEEPBACKGROUND)? MSG_KEEPBACKGROUND : 0) |
 			((Flags & FMSG_LEFTALIGN)? MSG_LEFTALIGN : 0);
 
 		return static_cast<intptr_t>(Message(