From d499f1cd815dc5bdbdbc8b067e5ab20189297125 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Thu, 20 Apr 2023 11:06:53 +0200 Subject: [PATCH 01/10] Implementation. --- docs/design/features/hybrid-globalization.md | 14 ++++ .../Interop/Browser/Interop.CompareInfo.cs | 6 ++ .../CompareInfo/CompareInfoTests.IsPrefix.cs | 54 +++++++----- .../CompareInfo/CompareInfoTests.IsSuffix.cs | 57 +++++++------ .../tests/CompareInfo/CompareInfoTestsBase.cs | 1 + .../tests/Hybrid/Hybrid.WASM.Tests.csproj | 2 + .../src/Resources/Strings.resx | 9 ++ .../Globalization/CompareInfo.WebAssembly.cs | 75 +++++++++++++++-- .../src/System/Globalization/CompareInfo.cs | 22 ++++- src/mono/sample/wasm/browser-bench/String.cs | 83 +++++++++++++++++-- src/mono/wasm/runtime/corebindings.c | 4 + src/mono/wasm/runtime/es6/dotnet.es6.lib.js | 2 + src/mono/wasm/runtime/exports-linker.ts | 4 +- .../net6-legacy/hybrid-globalization.ts | 72 +++++++++++++++- 14 files changed, 338 insertions(+), 67 deletions(-) diff --git a/docs/design/features/hybrid-globalization.md b/docs/design/features/hybrid-globalization.md index b7e74af942007..d830a19f27f52 100644 --- a/docs/design/features/hybrid-globalization.md +++ b/docs/design/features/hybrid-globalization.md @@ -181,3 +181,17 @@ hiraganaBig.localeCompare(katakanaSmall, "en-US", { sensitivity: "base" }) // 0; `IgnoreKanaType | IgnoreWidth | IgnoreSymbols | IgnoreNonSpace` `IgnoreKanaType | IgnoreWidth | IgnoreSymbols | IgnoreNonSpace | IgnoreCase` + + +**String starts with / ends with** + +Affected public APIs: +- CompareInfo.IsPrefix +- CompareInfo.IsSuffix +- String.StartsWith +- String.EndsWith + +Web API does not expose locale-sensitive endsWith/startsWith function. As a workaround, both strings get normalized and weightless characters are removed. Resulting strings are cut to the same length and comparison is performed. This approach results in the same compare option limitations as described under **String comparison**. Because we are normalizing strings to be able to cut them, we cannot calculate the match length on the original strings. Methods that calculate this information throw PlatformNotSupported exception: + +- [CompareInfo.IsPrefix](https://learn.microsoft.com/en-us/dotnet/api/system.globalization.compareinfo.isprefix?view=net-8.0#system-globalization-compareinfo-isprefix(system-readonlyspan((system-char))-system-readonlyspan((system-char))-system-globalization-compareoptions-system-int32@)) +- [CompareInfo.IsSuffix](https://learn.microsoft.com/en-us/dotnet/api/system.globalization.compareinfo.issuffix?view=net-8.0#system-globalization-compareinfo-issuffix(system-readonlyspan((system-char))-system-readonlyspan((system-char))-system-globalization-compareoptions-system-int32@)) diff --git a/src/libraries/Common/src/Interop/Browser/Interop.CompareInfo.cs b/src/libraries/Common/src/Interop/Browser/Interop.CompareInfo.cs index 693b908a0c691..5294d846ad7e8 100644 --- a/src/libraries/Common/src/Interop/Browser/Interop.CompareInfo.cs +++ b/src/libraries/Common/src/Interop/Browser/Interop.CompareInfo.cs @@ -9,5 +9,11 @@ internal static unsafe partial class JsGlobalization { [MethodImplAttribute(MethodImplOptions.InternalCall)] internal static extern unsafe int CompareString(out string exceptionMessage, in string culture, char* str1, int str1Len, char* str2, int str2Len, global::System.Globalization.CompareOptions options); + + [MethodImplAttribute(MethodImplOptions.InternalCall)] + internal static extern unsafe bool StartsWith(out string exceptionMessage, in string culture, char* str1, int str1Len, char* str2, int str2Len, global::System.Globalization.CompareOptions options); + + [MethodImplAttribute(MethodImplOptions.InternalCall)] + internal static extern unsafe bool EndsWith(out string exceptionMessage, in string culture, char* str1, int str1Len, char* str2, int str2Len, global::System.Globalization.CompareOptions options); } } diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs index 2b20169674939..89557f8067e2d 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs @@ -7,14 +7,8 @@ namespace System.Globalization.Tests { - public class CompareInfoIsPrefixTests + public class CompareInfoIsPrefixTests : CompareInfoTestsBase { - private static CompareInfo s_invariantCompare = CultureInfo.InvariantCulture.CompareInfo; - private static CompareInfo s_germanCompare = new CultureInfo("de-DE").CompareInfo; - private static CompareInfo s_hungarianCompare = new CultureInfo("hu-HU").CompareInfo; - private static CompareInfo s_turkishCompare = new CultureInfo("tr-TR").CompareInfo; - private static CompareInfo s_frenchCompare = new CultureInfo("fr-FR").CompareInfo; - public static IEnumerable IsPrefix_TestData() { // Empty strings @@ -31,7 +25,8 @@ public static IEnumerable IsPrefix_TestData() yield return new object[] { s_invariantCompare, "dzsdzsfoobar", "ddzsf", CompareOptions.Ordinal, false, 0 }; yield return new object[] { s_hungarianCompare, "dzsdzsfoobar", "ddzsf", CompareOptions.Ordinal, false, 0 }; yield return new object[] { s_invariantCompare, "dz", "d", CompareOptions.None, true, 1 }; - yield return new object[] { s_hungarianCompare, "dz", "d", CompareOptions.None, false, 0 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + yield return new object[] { s_hungarianCompare, "dz", "d", CompareOptions.None, false, 0 }; yield return new object[] { s_hungarianCompare, "dz", "d", CompareOptions.Ordinal, true, 1 }; // Turkish @@ -56,7 +51,7 @@ public static IEnumerable IsPrefix_TestData() yield return new object[] { s_invariantCompare, "\u00C0nimal", "a\u0300", CompareOptions.Ordinal, false, 0 }; yield return new object[] { s_invariantCompare, "\u00C0nimal", "a\u0300", CompareOptions.OrdinalIgnoreCase, false, 0 }; yield return new object[] { s_invariantCompare, "FooBar", "Foo\u0400Bar", CompareOptions.Ordinal, false, 0 }; - yield return new object[] { s_invariantCompare, "FooBA\u0300R", "FooB\u00C0R", CompareOptions.IgnoreNonSpace, true, 7 }; + yield return new object[] { s_invariantCompare, "FooBA\u0300R", "FooB\u00C0R", supportedIgnoreNonSpaceOption, true, 7 }; yield return new object[] { s_invariantCompare, "o\u0308", "o", CompareOptions.None, false, 0 }; yield return new object[] { s_invariantCompare, "o\u0308", "o", CompareOptions.Ordinal, true, 1 }; yield return new object[] { s_invariantCompare, "o\u0000\u0308", "o", CompareOptions.None, true, 1 }; @@ -76,16 +71,20 @@ public static IEnumerable IsPrefix_TestData() yield return new object[] { s_invariantCompare, "\uD800\uD800", "\uD800\uD800", CompareOptions.None, true, 2 }; // Ignore symbols - yield return new object[] { s_invariantCompare, "Test's can be interesting", "Tests", CompareOptions.IgnoreSymbols, true, 6 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) // only a few symbols are ignored + yield return new object[] { s_invariantCompare, "Test's can be interesting", "Tests", CompareOptions.IgnoreSymbols, true, 6 }; yield return new object[] { s_invariantCompare, "Test's can be interesting", "Tests", CompareOptions.None, false, 0 }; // Platform differences bool useNls = PlatformDetection.IsNlsGlobalization; - if (useNls) + if (useNls || PlatformDetection.IsHybridGlobalizationOnBrowser) { - yield return new object[] { s_hungarianCompare, "dzsdzsfoobar", "ddzsf", CompareOptions.None, true, 7 }; - yield return new object[] { s_invariantCompare, "''Tests", "Tests", CompareOptions.IgnoreSymbols, true, 7 }; - yield return new object[] { s_frenchCompare, "\u0153", "oe", CompareOptions.None, true, 1 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + yield return new object[] { s_hungarianCompare, "dzsdzsfoobar", "ddzsf", CompareOptions.None, true, 7 }; + yield return new object[] { s_invariantCompare, "''Tests", "Tests", CompareOptions.IgnoreSymbols, true, 7 }; + yield return new object[] { s_frenchCompare, "\u0153", "oe", CompareOptions.None, true, 1 }; + } yield return new object[] { s_invariantCompare, "\uD800\uDC00", "\uD800", CompareOptions.None, true, 1 }; yield return new object[] { s_invariantCompare, "\uD800\uDC00", "\uD800", CompareOptions.IgnoreCase, true, 1 }; } @@ -106,12 +105,18 @@ public static IEnumerable IsPrefix_TestData() } // Prefixes where matched length does not equal value string length - yield return new object[] { s_invariantCompare, "dzxyz", "\u01F3", CompareOptions.IgnoreNonSpace, true, 2 }; - yield return new object[] { s_invariantCompare, "\u01F3xyz", "dz", CompareOptions.IgnoreNonSpace, true, 1 }; - yield return new object[] { s_germanCompare, "Strasse xyz", "stra\u00DFe", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, true, 7 }; - yield return new object[] { s_germanCompare, "Strasse xyz", "xtra\u00DFe", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, false, 0 }; - yield return new object[] { s_germanCompare, "stra\u00DFe xyz", "Strasse", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, true, 6 }; - yield return new object[] { s_germanCompare, "stra\u00DFe xyz", "Xtrasse", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, false, 0 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + yield return new object[] { s_invariantCompare, "dzxyz", "\u01F3", supportedIgnoreNonSpaceOption, true, 2 }; + yield return new object[] { s_invariantCompare, "\u01F3xyz", "dz", supportedIgnoreNonSpaceOption, true, 1 }; + } + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + yield return new object[] { s_germanCompare, "Strasse xyz", "stra\u00DFe", supportedIgnoreCaseIgnoreNonSpaceOptions, true, 7 }; + yield return new object[] { s_germanCompare, "stra\u00DFe xyz", "Strasse", supportedIgnoreCaseIgnoreNonSpaceOptions, true, 6 }; + } + yield return new object[] { s_germanCompare, "Strasse xyz", "xtra\u00DFe", supportedIgnoreCaseIgnoreNonSpaceOptions, false, 0 }; + yield return new object[] { s_germanCompare, "stra\u00DFe xyz", "Xtrasse", supportedIgnoreCaseIgnoreNonSpaceOptions, false, 0 }; } [Theory] @@ -140,8 +145,11 @@ public void IsPrefix(CompareInfo compareInfo, string source, string value, Compa valueBoundedMemory.MakeReadonly(); Assert.Equal(expected, compareInfo.IsPrefix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options)); - Assert.Equal(expected, compareInfo.IsPrefix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options, out int actualMatchLength)); - Assert.Equal(expectedMatchLength, actualMatchLength); + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + Assert.Equal(expected, compareInfo.IsPrefix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options, out int actualMatchLength)); + Assert.Equal(expectedMatchLength, actualMatchLength); + } } [Fact] @@ -150,7 +158,7 @@ public void IsPrefix_UnassignedUnicode() bool result = PlatformDetection.IsNlsGlobalization ? true : false; int expectedMatchLength = (result) ? 6 : 0; IsPrefix(s_invariantCompare, "FooBar", "Foo\uFFFFBar", CompareOptions.None, result, expectedMatchLength); - IsPrefix(s_invariantCompare, "FooBar", "Foo\uFFFFBar", CompareOptions.IgnoreNonSpace, result, expectedMatchLength); + IsPrefix(s_invariantCompare, "FooBar", "Foo\uFFFFBar", supportedIgnoreNonSpaceOption, result, expectedMatchLength); } [Fact] diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs index edac5882b797c..cd5cc17aa3589 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs @@ -7,15 +7,8 @@ namespace System.Globalization.Tests { - public class CompareInfoIsSuffixTests + public class CompareInfoIsSuffixTests : CompareInfoTestsBase { - private static CompareInfo s_invariantCompare = CultureInfo.InvariantCulture.CompareInfo; - private static CompareInfo s_germanCompare = new CultureInfo("de-DE").CompareInfo; - private static CompareInfo s_hungarianCompare = new CultureInfo("hu-HU").CompareInfo; - private static CompareInfo s_turkishCompare = new CultureInfo("tr-TR").CompareInfo; - private static CompareInfo s_frenchCompare = new CultureInfo("fr-FR").CompareInfo; - private static CompareInfo s_slovakCompare = new CultureInfo("sk-SK").CompareInfo; - public static IEnumerable IsSuffix_TestData() { // Empty strings @@ -32,12 +25,16 @@ public static IEnumerable IsSuffix_TestData() yield return new object[] { s_invariantCompare, "foobardzsdzs", "rddzs", CompareOptions.None, false, 0 }; yield return new object[] { s_invariantCompare, "foobardzsdzs", "rddzs", CompareOptions.Ordinal, false, 0 }; yield return new object[] { s_invariantCompare, "dz", "z", CompareOptions.None, true, 1 }; - yield return new object[] { s_hungarianCompare, "dz", "z", CompareOptions.None, false, 0 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + yield return new object[] { s_hungarianCompare, "dz", "z", CompareOptions.None, false, 0 }; yield return new object[] { s_hungarianCompare, "dz", "z", CompareOptions.Ordinal, true, 1 }; // Slovak - yield return new object[] { s_slovakCompare, "ch", "h", CompareOptions.None, false, 0 }; - yield return new object[] { s_slovakCompare, "velmi chora", "hora", CompareOptions.None, false, 0 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + yield return new object[] { s_slovakCompare, "ch", "h", CompareOptions.None, false, 0 }; + yield return new object[] { s_slovakCompare, "velmi chora", "hora", CompareOptions.None, false, 0 }; + } yield return new object[] { s_slovakCompare, "chh", "H", CompareOptions.IgnoreCase, true, 1 }; // Turkish @@ -62,7 +59,7 @@ public static IEnumerable IsSuffix_TestData() yield return new object[] { s_invariantCompare, "Exhibit \u00C0", "a\u0300", CompareOptions.Ordinal, false, 0 }; yield return new object[] { s_invariantCompare, "Exhibit \u00C0", "a\u0300", CompareOptions.OrdinalIgnoreCase, false, 0 }; yield return new object[] { s_invariantCompare, "FooBar", "Foo\u0400Bar", CompareOptions.Ordinal, false, 0 }; - yield return new object[] { s_invariantCompare, "FooBA\u0300R", "FooB\u00C0R", CompareOptions.IgnoreNonSpace, true, 7 }; + yield return new object[] { s_invariantCompare, "FooBA\u0300R", "FooB\u00C0R", supportedIgnoreNonSpaceOption, true, 7 }; yield return new object[] { s_invariantCompare, "o\u0308", "o", CompareOptions.None, false, 0 }; yield return new object[] { s_invariantCompare, "o\u0308", "o", CompareOptions.Ordinal, false, 0 }; yield return new object[] { s_invariantCompare, "o\u0308o", "o", CompareOptions.None, true, 1 }; @@ -83,7 +80,8 @@ public static IEnumerable IsSuffix_TestData() yield return new object[] { s_invariantCompare, "\uD800\uD800", "\uD800\uD800", CompareOptions.None, true, 2 }; // Ignore symbols - yield return new object[] { s_invariantCompare, "More Test's", "Tests", CompareOptions.IgnoreSymbols, true, 6 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + yield return new object[] { s_invariantCompare, "More Test's", "Tests", CompareOptions.IgnoreSymbols, true, 6 }; yield return new object[] { s_invariantCompare, "More Test's", "Tests", CompareOptions.None, false, 0 }; // NULL character @@ -91,10 +89,13 @@ public static IEnumerable IsSuffix_TestData() yield return new object[] { s_invariantCompare, "a\u0000b", "b\u0000b", CompareOptions.None, false, 0 }; // Platform differences - if (PlatformDetection.IsNlsGlobalization) + if (PlatformDetection.IsNlsGlobalization || PlatformDetection.IsHybridGlobalizationOnBrowser) { - yield return new object[] { s_hungarianCompare, "foobardzsdzs", "rddzs", CompareOptions.None, true, 7 }; - yield return new object[] { s_frenchCompare, "\u0153", "oe", CompareOptions.None, true, 1 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + yield return new object[] { s_hungarianCompare, "foobardzsdzs", "rddzs", CompareOptions.None, true, 7 }; + yield return new object[] { s_frenchCompare, "\u0153", "oe", CompareOptions.None, true, 1 }; + } yield return new object[] { s_invariantCompare, "\uD800\uDC00", "\uDC00", CompareOptions.None, true, 1 }; yield return new object[] { s_invariantCompare, "\uD800\uDC00", "\uDC00", CompareOptions.IgnoreCase, true, 1 }; } else @@ -106,12 +107,15 @@ public static IEnumerable IsSuffix_TestData() } // Suffixes where matched length does not equal value string length - yield return new object[] { s_invariantCompare, "xyzdz", "\u01F3", CompareOptions.IgnoreNonSpace, true, 2 }; - yield return new object[] { s_invariantCompare, "xyz\u01F3", "dz", CompareOptions.IgnoreNonSpace, true, 1 }; - yield return new object[] { s_germanCompare, "xyz Strasse", "stra\u00DFe", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, true, 7 }; - yield return new object[] { s_germanCompare, "xyz Strasse", "xtra\u00DFe", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, false, 0 }; - yield return new object[] { s_germanCompare, "xyz stra\u00DFe", "Strasse", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, true, 6 }; - yield return new object[] { s_germanCompare, "xyz stra\u00DFe", "Xtrasse", CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace, false, 0 }; + yield return new object[] { s_germanCompare, "xyz Strasse", "xtra\u00DFe", supportedIgnoreCaseIgnoreNonSpaceOptions, false, 0 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + yield return new object[] { s_invariantCompare, "xyzdz", "\u01F3", supportedIgnoreNonSpaceOption, true, 2 }; + yield return new object[] { s_invariantCompare, "xyz\u01F3", "dz", supportedIgnoreNonSpaceOption, true, 1 }; + yield return new object[] { s_germanCompare, "xyz stra\u00DFe", "Strasse", supportedIgnoreCaseIgnoreNonSpaceOptions, true, 6 }; + yield return new object[] { s_germanCompare, "xyz Strasse", "stra\u00DFe", supportedIgnoreCaseIgnoreNonSpaceOptions, true, 7 }; + } + yield return new object[] { s_germanCompare, "xyz stra\u00DFe", "Xtrasse", supportedIgnoreCaseIgnoreNonSpaceOptions, false, 0 }; } [Theory] @@ -140,8 +144,11 @@ public void IsSuffix(CompareInfo compareInfo, string source, string value, Compa valueBoundedMemory.MakeReadonly(); Assert.Equal(expected, compareInfo.IsSuffix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options)); - Assert.Equal(expected, compareInfo.IsSuffix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options, out int actualMatchLength)); - Assert.Equal(expectedMatchLength, actualMatchLength); + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { + Assert.Equal(expected, compareInfo.IsSuffix(sourceBoundedMemory.Span, valueBoundedMemory.Span, options, out int actualMatchLength)); + Assert.Equal(expectedMatchLength, actualMatchLength); + } } [Fact] @@ -151,7 +158,7 @@ public void IsSuffix_UnassignedUnicode() int expectedMatchLength = (result) ? 6 : 0; IsSuffix(s_invariantCompare, "FooBar", "Foo\uFFFFBar", CompareOptions.None, result, expectedMatchLength); - IsSuffix(s_invariantCompare, "FooBar", "Foo\uFFFFBar", CompareOptions.IgnoreNonSpace, result, expectedMatchLength); + IsSuffix(s_invariantCompare, "FooBar", "Foo\uFFFFBar", supportedIgnoreNonSpaceOption, result, expectedMatchLength); } [Fact] diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTestsBase.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTestsBase.cs index 617c13e1555ee..01983c6865145 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTestsBase.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTestsBase.cs @@ -28,6 +28,7 @@ public class CompareInfoTestsBase protected static CompareInfo s_turkishCompare = new CultureInfo("tr-TR").CompareInfo; protected static CompareInfo s_japaneseCompare = new CultureInfo("ja-JP").CompareInfo; protected static CompareInfo s_slovakCompare = new CultureInfo("sk-SK").CompareInfo; + protected static CompareInfo s_frenchCompare = new CultureInfo("fr-FR").CompareInfo; protected static CompareOptions supportedIgnoreNonSpaceOption = PlatformDetection.IsHybridGlobalizationOnBrowser ? CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreKanaType : diff --git a/src/libraries/System.Globalization/tests/Hybrid/Hybrid.WASM.Tests.csproj b/src/libraries/System.Globalization/tests/Hybrid/Hybrid.WASM.Tests.csproj index a65d8448c4d70..b8a19486362e3 100644 --- a/src/libraries/System.Globalization/tests/Hybrid/Hybrid.WASM.Tests.csproj +++ b/src/libraries/System.Globalization/tests/Hybrid/Hybrid.WASM.Tests.csproj @@ -10,5 +10,7 @@ + + diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index 6865608cb14aa..abe518d97e4bd 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -4043,6 +4043,15 @@ AssemblyName.GetAssemblyName() is not supported on this platform. + + CompareOptions = {0} are not supported when HybridGlobalization=true. Disable it to load larger ICU bundle, then use this option. + + + CompareOptions = {0} are not supported for culture = {1} when HybridGlobalization=true. Disable it to load larger ICU bundle, then use this option. + + + Match length calculation is not supported when HybridGlobalization=true. Disable it to load larger ICU bundle, then use this function. + Arrays with non-zero lower bounds are not supported. diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs index 3ca001a9573bd..ef1b26f5bef7d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs @@ -8,13 +8,36 @@ namespace System.Globalization { public partial class CompareInfo { - private unsafe int JsCompareString(ReadOnlySpan string1, ReadOnlySpan string2, CompareOptions options) + private static void AssertHybridOnWasm(CompareOptions options) { Debug.Assert(!GlobalizationMode.Invariant); Debug.Assert(!GlobalizationMode.UseNls); Debug.Assert(GlobalizationMode.Hybrid); Debug.Assert((options & (CompareOptions.Ordinal | CompareOptions.OrdinalIgnoreCase)) == 0); + } + + private static void AssertComparisonSupported(CompareOptions options, string cultureName) + { + if (CompareOptionsNotSupported(options)) + throw new PlatformNotSupportedException(GetPNSE(options)); + + if (CompareOptionsNotSupportedForCulture(options, cultureName)) + throw new PlatformNotSupportedException(GetPNSEForCulture(options, cultureName)); + } + + private static void AssertIndexingSupported(CompareOptions options, string cultureName) + { + if (IndexingOptionsNotSupported(options) || CompareOptionsNotSupported(options)) + throw new PlatformNotSupportedException(GetPNSE(options)); + + if (CompareOptionsNotSupportedForCulture(options, cultureName)) + throw new PlatformNotSupportedException(GetPNSEForCulture(options, cultureName)); + } + private unsafe int JsCompareString(ReadOnlySpan string1, ReadOnlySpan string2, CompareOptions options) + { + AssertHybridOnWasm(options); + AssertComparisonSupported(options, m_name); if (CompareOptionsNotSupported(options)) throw new PlatformNotSupportedException(GetPNSE(options)); @@ -38,14 +61,55 @@ private unsafe int JsCompareString(ReadOnlySpan string1, ReadOnlySpan source, ReadOnlySpan prefix, CompareOptions options) + { + AssertHybridOnWasm(options); + Debug.Assert(!prefix.IsEmpty); + string cultureName = m_name; + + string exceptionMessage; + bool result; + fixed (char* pSource = &MemoryMarshal.GetReference(source)) + fixed (char* pPrefix = &MemoryMarshal.GetReference(prefix)) + { + result = Interop.JsGlobalization.StartsWith(out exceptionMessage, cultureName, pSource, source.Length, pPrefix, prefix.Length, options); + } + + if (!string.IsNullOrEmpty(exceptionMessage)) + throw new Exception(exceptionMessage); + + return result; + } + + private unsafe bool JsEndsWith(ReadOnlySpan source, ReadOnlySpan prefix, CompareOptions options) + { + AssertHybridOnWasm(options); + Debug.Assert(!prefix.IsEmpty); + string cultureName = m_name; + + string exceptionMessage; + bool result; + fixed (char* pSource = &MemoryMarshal.GetReference(source)) + fixed (char* pPrefix = &MemoryMarshal.GetReference(prefix)) + { + result = Interop.JsGlobalization.EndsWith(out exceptionMessage, cultureName, pSource, source.Length, pPrefix, prefix.Length, options); + } + + if (!string.IsNullOrEmpty(exceptionMessage)) + throw new Exception(exceptionMessage); + + return result; + } + + private static bool IndexingOptionsNotSupported(CompareOptions options) => + (options & CompareOptions.IgnoreSymbols) == CompareOptions.IgnoreSymbols; + private static bool CompareOptionsNotSupported(CompareOptions options) => (options & CompareOptions.IgnoreWidth) == CompareOptions.IgnoreWidth || ((options & CompareOptions.IgnoreNonSpace) == CompareOptions.IgnoreNonSpace && (options & CompareOptions.IgnoreKanaType) != CompareOptions.IgnoreKanaType); - private static string GetPNSE(CompareOptions options) => - $"CompareOptions = {options} are not supported when HybridGlobalization=true. Disable it to load larger ICU bundle, then use this option."; - + SR.Format(SR.PlatformNotSupported_HybridGlobalizationWithCompareOptions, options); private static bool CompareOptionsNotSupportedForCulture(CompareOptions options, string cultureName) => (options == CompareOptions.IgnoreKanaType && @@ -53,8 +117,7 @@ private static bool CompareOptionsNotSupportedForCulture(CompareOptions options, (options == CompareOptions.None && (cultureName.Split('-')[0] == "ja")); - private static string GetPNSEForCulture(CompareOptions options, string cultureName) => - $"CompareOptions = {options} are not supported for culture = {cultureName} when HybridGlobalization=true. Disable it to load larger ICU bundle, then use this option."; + SR.Format(SR.PlatformNotSupported_HybridGlobalizationWithCompareOptions, options, cultureName); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.cs index 3a3a5ba955375..f205901e6d4ba 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.cs @@ -612,7 +612,12 @@ public unsafe bool IsPrefix(ReadOnlySpan source, ReadOnlySpan prefix else { // Linguistic comparison requested and we don't need to special-case any args. - +#if TARGET_BROWSER + if (GlobalizationMode.Hybrid) + { + throw new PlatformNotSupportedException(SR.PlatformNotSupported_HybridGlobalizationWithMatchLength); + } +#endif int tempMatchLength = 0; matched = StartsWithCore(source, prefix, options, &tempMatchLength); matchLength = tempMatchLength; @@ -624,6 +629,10 @@ public unsafe bool IsPrefix(ReadOnlySpan source, ReadOnlySpan prefix private unsafe bool StartsWithCore(ReadOnlySpan source, ReadOnlySpan prefix, CompareOptions options, int* matchLengthPtr) => GlobalizationMode.UseNls ? NlsStartsWith(source, prefix, options, matchLengthPtr) : +#if TARGET_BROWSER + GlobalizationMode.Hybrid ? + JsStartsWith(source, prefix, options) : +#endif IcuStartsWith(source, prefix, options, matchLengthPtr); public bool IsPrefix(string source, string prefix) @@ -750,7 +759,12 @@ public unsafe bool IsSuffix(ReadOnlySpan source, ReadOnlySpan suffix else { // Linguistic comparison requested and we don't need to special-case any args. - +#if TARGET_BROWSER + if (GlobalizationMode.Hybrid) + { + throw new PlatformNotSupportedException(SR.PlatformNotSupported_HybridGlobalizationWithMatchLength); + } +#endif int tempMatchLength = 0; matched = EndsWithCore(source, suffix, options, &tempMatchLength); matchLength = tempMatchLength; @@ -767,6 +781,10 @@ public bool IsSuffix(string source, string suffix) private unsafe bool EndsWithCore(ReadOnlySpan source, ReadOnlySpan suffix, CompareOptions options, int* matchLengthPtr) => GlobalizationMode.UseNls ? NlsEndsWith(source, suffix, options, matchLengthPtr) : +#if TARGET_BROWSER + GlobalizationMode.Hybrid ? + JsEndsWith(source, suffix, options) : +#endif IcuEndsWith(source, suffix, options, matchLengthPtr); /// diff --git a/src/mono/sample/wasm/browser-bench/String.cs b/src/mono/sample/wasm/browser-bench/String.cs index 608801c5177b0..16cc2c6c9faa6 100644 --- a/src/mono/sample/wasm/browser-bench/String.cs +++ b/src/mono/sample/wasm/browser-bench/String.cs @@ -23,7 +23,11 @@ public StringTask() new TextInfoToTitleCase(), new StringCompareMeasurement(), new StringEqualsMeasurement(), - new CompareInfoMeasurement(), + new CompareInfoCompareMeasurement(), + new CompareInfoStartsWithMeasurement(), + new CompareInfoEndsWithMeasurement(), + new StringStartsWithMeasurement(), + new StringEndsWithMeasurement(), }; } @@ -124,16 +128,21 @@ public class TextInfoToTitleCase : TextInfoMeasurement public override void RunStep() => textInfo.ToTitleCase(str); } - public class StringsCompare : StringMeasurement + public abstract class StringsCompare : StringMeasurement { - protected string str2; + protected string strDifferentSuffix; + protected string strDifferentPrefix; public void InitializeStringsForComparison() { InitializeString(); - // worst case: strings may differ only with the last char + // worst case: strings may differ only with the last/first char + char originalLastChar = data[len-1]; data[len-1] = (char)random.Next(0x80); - str2 = new string(data); + strDifferentSuffix = new string(data); + data[len-1] = originalLastChar; + data[0] = (char)random.Next(0x80); + strDifferentPrefix = new string(data); } public override string Name => "Strings Compare Base"; } @@ -149,7 +158,7 @@ public override Task BeforeBatch() return Task.CompletedTask; } public override string Name => "String Compare"; - public override void RunStep() => string.Compare(str, str2, cultureInfo, CompareOptions.None); + public override void RunStep() => string.Compare(str, strDifferentSuffix, cultureInfo, CompareOptions.None); } public class StringEqualsMeasurement : StringsCompare @@ -160,10 +169,10 @@ public override Task BeforeBatch() return Task.CompletedTask; } public override string Name => "String Equals"; - public override void RunStep() => string.Equals(str, str2, StringComparison.InvariantCulture); + public override void RunStep() => string.Equals(str, strDifferentSuffix, StringComparison.InvariantCulture); } - public class CompareInfoMeasurement : StringsCompare + public class CompareInfoCompareMeasurement : StringsCompare { protected CompareInfo compareInfo; @@ -174,7 +183,63 @@ public override Task BeforeBatch() return Task.CompletedTask; } public override string Name => "CompareInfo Compare"; - public override void RunStep() => compareInfo.Compare(str, str2); + public override void RunStep() => compareInfo.Compare(str, strDifferentSuffix); + } + + public class CompareInfoStartsWithMeasurement : StringsCompare + { + protected CompareInfo compareInfo; + + public override Task BeforeBatch() + { + compareInfo = new CultureInfo("hy-AM").CompareInfo; + InitializeStringsForComparison(); + return Task.CompletedTask; + } + public override string Name => "CompareInfo IsPrefix"; + public override void RunStep() => compareInfo.IsPrefix(str, strDifferentSuffix); + } + + public class CompareInfoEndsWithMeasurement : StringsCompare + { + protected CompareInfo compareInfo; + + public override Task BeforeBatch() + { + compareInfo = new CultureInfo("it-IT").CompareInfo; + InitializeStringsForComparison(); + return Task.CompletedTask; + } + public override string Name => "CompareInfo IsSuffix"; + public override void RunStep() => compareInfo.IsSuffix(str, strDifferentPrefix); + } + + public class StringStartsWithMeasurement : StringsCompare + { + protected CultureInfo cultureInfo; + + public override Task BeforeBatch() + { + cultureInfo = new CultureInfo("bs-BA"); + InitializeStringsForComparison(); + return Task.CompletedTask; + } + public override string Name => "String StartsWith"; + public override void RunStep() => str.StartsWith(strDifferentSuffix, false, cultureInfo); + } + + public class StringEndsWithMeasurement : StringsCompare + { + protected CultureInfo cultureInfo; + + public override Task BeforeBatch() + { + cultureInfo = new CultureInfo("nb-NO"); + InitializeStringsForComparison(); + return Task.CompletedTask; + } + public override string Name => "String EndsWith"; + public override void RunStep() => str.EndsWith(strDifferentPrefix, false, cultureInfo); } } } diff --git a/src/mono/wasm/runtime/corebindings.c b/src/mono/wasm/runtime/corebindings.c index ded8b022d7ddf..0b196d087ea8b 100644 --- a/src/mono/wasm/runtime/corebindings.c +++ b/src/mono/wasm/runtime/corebindings.c @@ -46,6 +46,8 @@ extern void* mono_wasm_invoke_js_blazor (MonoString **exceptionMessage, void *ca extern void mono_wasm_change_case_invariant(MonoString **exceptionMessage, const uint16_t* src, int32_t srcLength, uint16_t* dst, int32_t dstLength, mono_bool bToUpper); extern void mono_wasm_change_case(MonoString **exceptionMessage, MonoString **culture, const uint16_t* src, int32_t srcLength, uint16_t* dst, int32_t dstLength, mono_bool bToUpper); extern int mono_wasm_compare_string(MonoString **exceptionMessage, MonoString **culture, const uint16_t* str1, int32_t str1Length, const uint16_t* str2, int32_t str2Length, int32_t options); +extern mono_bool mono_wasm_starts_with(MonoString **exceptionMessage, MonoString **culture, const uint16_t* str1, int32_t str1Length, const uint16_t* str2, int32_t str2Length, int32_t options); +extern mono_bool mono_wasm_ends_with(MonoString **exceptionMessage, MonoString **culture, const uint16_t* str1, int32_t str1Length, const uint16_t* str2, int32_t str2Length, int32_t options); void bindings_initialize_internals (void) { @@ -77,4 +79,6 @@ void bindings_initialize_internals (void) mono_add_internal_call ("Interop/JsGlobalization::ChangeCaseInvariant", mono_wasm_change_case_invariant); mono_add_internal_call ("Interop/JsGlobalization::ChangeCase", mono_wasm_change_case); mono_add_internal_call ("Interop/JsGlobalization::CompareString", mono_wasm_compare_string); + mono_add_internal_call ("Interop/JsGlobalization::StartsWith", mono_wasm_starts_with); + mono_add_internal_call ("Interop/JsGlobalization::EndsWith", mono_wasm_ends_with); } diff --git a/src/mono/wasm/runtime/es6/dotnet.es6.lib.js b/src/mono/wasm/runtime/es6/dotnet.es6.lib.js index e6139d3537519..04a068e86b3c0 100644 --- a/src/mono/wasm/runtime/es6/dotnet.es6.lib.js +++ b/src/mono/wasm/runtime/es6/dotnet.es6.lib.js @@ -101,6 +101,8 @@ let linked_functions = [ "mono_wasm_change_case_invariant", "mono_wasm_change_case", "mono_wasm_compare_string", + "mono_wasm_starts_with", + "mono_wasm_ends_with", "icudt68_dat", ]; diff --git a/src/mono/wasm/runtime/exports-linker.ts b/src/mono/wasm/runtime/exports-linker.ts index ae0305551a42c..b3957a43397a6 100644 --- a/src/mono/wasm/runtime/exports-linker.ts +++ b/src/mono/wasm/runtime/exports-linker.ts @@ -26,7 +26,7 @@ import { mono_wasm_invoke_js_blazor, mono_wasm_invoke_js_with_args_ref, mono_wasm_get_object_property_ref, mono_wasm_set_object_property_ref, mono_wasm_get_by_index_ref, mono_wasm_set_by_index_ref, mono_wasm_get_global_object_ref } from "./net6-legacy/method-calls"; -import { mono_wasm_change_case, mono_wasm_change_case_invariant, mono_wasm_compare_string } from "./net6-legacy/hybrid-globalization"; +import { mono_wasm_change_case, mono_wasm_change_case_invariant, mono_wasm_compare_string, mono_wasm_ends_with, mono_wasm_starts_with } from "./net6-legacy/hybrid-globalization"; // the methods would be visible to EMCC linker // --- keep in sync with dotnet.cjs.lib.js --- @@ -97,6 +97,8 @@ export function export_linker(): any { mono_wasm_change_case_invariant, mono_wasm_change_case, mono_wasm_compare_string, + mono_wasm_starts_with, + mono_wasm_ends_with, // threading exports, if threading is enabled ...mono_wasm_threads_exports, diff --git a/src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts b/src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts index ae3ee2ac648f6..84dea8f1daa49 100644 --- a/src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts +++ b/src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts @@ -5,7 +5,7 @@ import { Module } from "../imports"; import { mono_wasm_new_external_root } from "../roots"; import {MonoString, MonoStringRef } from "../types"; import { Int32Ptr } from "../types/emscripten"; -import { conv_string_root, js_string_to_mono_string_root } from "../strings"; +import { StringDecoder, conv_string_root, js_string_to_mono_string_root } from "../strings"; import { setU16 } from "../memory"; export function mono_wasm_change_case_invariant(exceptionMessage: Int32Ptr, src: number, srcLength: number, dst: number, dstLength: number, toUpper: number) : void{ @@ -84,6 +84,76 @@ export function pass_exception_details(ex: any, exceptionMessage: Int32Ptr){ exceptionRoot.release(); } +export function mono_wasm_starts_with(exceptionMessage: Int32Ptr, culture: MonoStringRef, str1: number, str1Length: number, str2: number, str2Length: number, options: number): number{ + const cultureRoot = mono_wasm_new_external_root(culture); + try{ + const cultureName = conv_string_root(cultureRoot); + const sd = new StringDecoder(); + const prefix = get_clean_string(sd, str2, str2Length); + // no need to look for an empty string + if (prefix.length == 0) + return 1; // true + + const source = get_clean_string(sd, str1, str1Length); + if (source.length < prefix.length) + return 0; //false + const sourceOfPrefixLength = source.slice(0, prefix.length); + + const casePicker = (options & 0x1f); + const locale = cultureName ? cultureName : undefined; + const result = compare_strings(sourceOfPrefixLength, prefix, locale, casePicker); + if (result == -2) + throw new Error("$Invalid comparison option."); + return result === 0 ? 1 : 0; // equals ? true : false + } + catch (ex: any) { + pass_exception_details(ex, exceptionMessage); + return -1; + } + finally { + cultureRoot.release(); + } +} + +export function mono_wasm_ends_with(exceptionMessage: Int32Ptr, culture: MonoStringRef, str1: number, str1Length: number, str2: number, str2Length: number, options: number): number{ + const cultureRoot = mono_wasm_new_external_root(culture); + try{ + const cultureName = conv_string_root(cultureRoot); + const sd = new StringDecoder(); + const suffix = get_clean_string(sd, str2, str2Length); + if (suffix.length == 0) + return 1; // true + + const source = get_clean_string(sd, str1, str1Length); + const diff = source.length - suffix.length; + if (diff < 0) + return 0; //false + const sourceOfSuffixLength = source.slice(diff, source.length); + + const casePicker = (options & 0x1f); + const locale = cultureName ? cultureName : undefined; + const result = compare_strings(sourceOfSuffixLength, suffix, locale, casePicker); + if (result == -2) + throw new Error("$Invalid comparison option."); + return result === 0 ? 1 : 0; // equals ? true : false + } + catch (ex: any) { + pass_exception_details(ex, exceptionMessage); + return -1; + } + finally { + cultureRoot.release(); + } +} + +function get_clean_string(sd: StringDecoder, strPtr: number, strLen: number) +{ + const str = sd.decode(strPtr, (strPtr + 2*strLen)); + const nStr = str.normalize(); + // could we skip zero-width chars here? + return nStr.replace(/[\u200B-\u200D\uFEFF\0]/g, ""); +} + export function compare_strings(string1: string, string2: string, locale: string | undefined, casePicker: number) : number{ switch (casePicker) { From f2aea00ff732673b16a7fced836d65f6f462c0de Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Thu, 20 Apr 2023 11:21:47 +0200 Subject: [PATCH 02/10] HG does not belong to legacy code. --- src/mono/wasm/runtime/exports-linker.ts | 2 +- .../{net6-legacy => }/hybrid-globalization.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) rename src/mono/wasm/runtime/{net6-legacy => }/hybrid-globalization.ts (97%) diff --git a/src/mono/wasm/runtime/exports-linker.ts b/src/mono/wasm/runtime/exports-linker.ts index b3957a43397a6..c76463b1a3bc0 100644 --- a/src/mono/wasm/runtime/exports-linker.ts +++ b/src/mono/wasm/runtime/exports-linker.ts @@ -26,7 +26,7 @@ import { mono_wasm_invoke_js_blazor, mono_wasm_invoke_js_with_args_ref, mono_wasm_get_object_property_ref, mono_wasm_set_object_property_ref, mono_wasm_get_by_index_ref, mono_wasm_set_by_index_ref, mono_wasm_get_global_object_ref } from "./net6-legacy/method-calls"; -import { mono_wasm_change_case, mono_wasm_change_case_invariant, mono_wasm_compare_string, mono_wasm_ends_with, mono_wasm_starts_with } from "./net6-legacy/hybrid-globalization"; +import { mono_wasm_change_case, mono_wasm_change_case_invariant, mono_wasm_compare_string, mono_wasm_ends_with, mono_wasm_starts_with } from "./hybrid-globalization"; // the methods would be visible to EMCC linker // --- keep in sync with dotnet.cjs.lib.js --- diff --git a/src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts b/src/mono/wasm/runtime/hybrid-globalization.ts similarity index 97% rename from src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts rename to src/mono/wasm/runtime/hybrid-globalization.ts index 84dea8f1daa49..1239349177d5b 100644 --- a/src/mono/wasm/runtime/net6-legacy/hybrid-globalization.ts +++ b/src/mono/wasm/runtime/hybrid-globalization.ts @@ -1,12 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -import { Module } from "../imports"; -import { mono_wasm_new_external_root } from "../roots"; -import {MonoString, MonoStringRef } from "../types"; -import { Int32Ptr } from "../types/emscripten"; -import { StringDecoder, conv_string_root, js_string_to_mono_string_root } from "../strings"; -import { setU16 } from "../memory"; +import { Module } from "./imports"; +import { mono_wasm_new_external_root } from "./roots"; +import {MonoString, MonoStringRef } from "./types"; +import { Int32Ptr } from "./types/emscripten"; +import { StringDecoder, conv_string_root, js_string_to_mono_string_root } from "./strings"; +import { setU16 } from "./memory"; export function mono_wasm_change_case_invariant(exceptionMessage: Int32Ptr, src: number, srcLength: number, dst: number, dstLength: number, toUpper: number) : void{ try{ From 58a7930e6ee47637bf849dd504da5dcbde2472f0 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Thu, 20 Apr 2023 11:43:30 +0200 Subject: [PATCH 03/10] No need to create new instance when existing one is exported. --- src/mono/wasm/runtime/hybrid-globalization.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/mono/wasm/runtime/hybrid-globalization.ts b/src/mono/wasm/runtime/hybrid-globalization.ts index 1239349177d5b..f13edb2751ec4 100644 --- a/src/mono/wasm/runtime/hybrid-globalization.ts +++ b/src/mono/wasm/runtime/hybrid-globalization.ts @@ -5,7 +5,7 @@ import { Module } from "./imports"; import { mono_wasm_new_external_root } from "./roots"; import {MonoString, MonoStringRef } from "./types"; import { Int32Ptr } from "./types/emscripten"; -import { StringDecoder, conv_string_root, js_string_to_mono_string_root } from "./strings"; +import { conv_string_root, js_string_to_mono_string_root, string_decoder } from "./strings"; import { setU16 } from "./memory"; export function mono_wasm_change_case_invariant(exceptionMessage: Int32Ptr, src: number, srcLength: number, dst: number, dstLength: number, toUpper: number) : void{ @@ -88,13 +88,12 @@ export function mono_wasm_starts_with(exceptionMessage: Int32Ptr, culture: MonoS const cultureRoot = mono_wasm_new_external_root(culture); try{ const cultureName = conv_string_root(cultureRoot); - const sd = new StringDecoder(); - const prefix = get_clean_string(sd, str2, str2Length); + const prefix = get_clean_string(str2, str2Length); // no need to look for an empty string if (prefix.length == 0) return 1; // true - const source = get_clean_string(sd, str1, str1Length); + const source = get_clean_string(str1, str1Length); if (source.length < prefix.length) return 0; //false const sourceOfPrefixLength = source.slice(0, prefix.length); @@ -119,12 +118,11 @@ export function mono_wasm_ends_with(exceptionMessage: Int32Ptr, culture: MonoStr const cultureRoot = mono_wasm_new_external_root(culture); try{ const cultureName = conv_string_root(cultureRoot); - const sd = new StringDecoder(); - const suffix = get_clean_string(sd, str2, str2Length); + const suffix = get_clean_string(str2, str2Length); if (suffix.length == 0) return 1; // true - const source = get_clean_string(sd, str1, str1Length); + const source = get_clean_string(str1, str1Length); const diff = source.length - suffix.length; if (diff < 0) return 0; //false @@ -146,9 +144,9 @@ export function mono_wasm_ends_with(exceptionMessage: Int32Ptr, culture: MonoStr } } -function get_clean_string(sd: StringDecoder, strPtr: number, strLen: number) +function get_clean_string(strPtr: number, strLen: number) { - const str = sd.decode(strPtr, (strPtr + 2*strLen)); + const str = string_decoder.decode(strPtr, (strPtr + 2*strLen)); const nStr = str.normalize(); // could we skip zero-width chars here? return nStr.replace(/[\u200B-\u200D\uFEFF\0]/g, ""); From c80bf05714534cbcdfb6918c1f3e946df9674649 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Thu, 20 Apr 2023 14:50:12 +0000 Subject: [PATCH 04/10] TextEncoder's behavior varies between hosts. --- src/mono/wasm/runtime/hybrid-globalization.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mono/wasm/runtime/hybrid-globalization.ts b/src/mono/wasm/runtime/hybrid-globalization.ts index f13edb2751ec4..e886e53e7e8bc 100644 --- a/src/mono/wasm/runtime/hybrid-globalization.ts +++ b/src/mono/wasm/runtime/hybrid-globalization.ts @@ -97,7 +97,7 @@ export function mono_wasm_starts_with(exceptionMessage: Int32Ptr, culture: MonoS if (source.length < prefix.length) return 0; //false const sourceOfPrefixLength = source.slice(0, prefix.length); - + const casePicker = (options & 0x1f); const locale = cultureName ? cultureName : undefined; const result = compare_strings(sourceOfPrefixLength, prefix, locale, casePicker); @@ -146,9 +146,8 @@ export function mono_wasm_ends_with(exceptionMessage: Int32Ptr, culture: MonoStr function get_clean_string(strPtr: number, strLen: number) { - const str = string_decoder.decode(strPtr, (strPtr + 2*strLen)); + const str = get_utf16_string(strPtr, strLen); const nStr = str.normalize(); - // could we skip zero-width chars here? return nStr.replace(/[\u200B-\u200D\uFEFF\0]/g, ""); } From edc10fe0bc73f7cb1408cb07419a6de1b3e94cf6 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Thu, 20 Apr 2023 16:52:23 +0200 Subject: [PATCH 05/10] Nit --- src/mono/wasm/runtime/hybrid-globalization.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/wasm/runtime/hybrid-globalization.ts b/src/mono/wasm/runtime/hybrid-globalization.ts index e886e53e7e8bc..dd77e5234cc93 100644 --- a/src/mono/wasm/runtime/hybrid-globalization.ts +++ b/src/mono/wasm/runtime/hybrid-globalization.ts @@ -5,7 +5,7 @@ import { Module } from "./imports"; import { mono_wasm_new_external_root } from "./roots"; import {MonoString, MonoStringRef } from "./types"; import { Int32Ptr } from "./types/emscripten"; -import { conv_string_root, js_string_to_mono_string_root, string_decoder } from "./strings"; +import { conv_string_root, js_string_to_mono_string_root } from "./strings"; import { setU16 } from "./memory"; export function mono_wasm_change_case_invariant(exceptionMessage: Int32Ptr, src: number, srcLength: number, dst: number, dstLength: number, toUpper: number) : void{ @@ -97,7 +97,7 @@ export function mono_wasm_starts_with(exceptionMessage: Int32Ptr, culture: MonoS if (source.length < prefix.length) return 0; //false const sourceOfPrefixLength = source.slice(0, prefix.length); - + const casePicker = (options & 0x1f); const locale = cultureName ? cultureName : undefined; const result = compare_strings(sourceOfPrefixLength, prefix, locale, casePicker); From fe0244c302148a03d542914e4f797c31d335b3f2 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Fri, 21 Apr 2023 08:17:50 +0200 Subject: [PATCH 06/10] Cutting prevents us from using IgnoreSymbols. --- docs/design/features/hybrid-globalization.md | 5 ++++- .../tests/CompareInfo/CompareInfoTests.IsPrefix.cs | 6 ++++-- .../tests/CompareInfo/CompareInfoTests.IsSuffix.cs | 4 +++- .../src/System/Globalization/CompareInfo.WebAssembly.cs | 2 ++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/design/features/hybrid-globalization.md b/docs/design/features/hybrid-globalization.md index d830a19f27f52..629d68ef3cc84 100644 --- a/docs/design/features/hybrid-globalization.md +++ b/docs/design/features/hybrid-globalization.md @@ -191,7 +191,10 @@ Affected public APIs: - String.StartsWith - String.EndsWith -Web API does not expose locale-sensitive endsWith/startsWith function. As a workaround, both strings get normalized and weightless characters are removed. Resulting strings are cut to the same length and comparison is performed. This approach results in the same compare option limitations as described under **String comparison**. Because we are normalizing strings to be able to cut them, we cannot calculate the match length on the original strings. Methods that calculate this information throw PlatformNotSupported exception: +Web API does not expose locale-sensitive endsWith/startsWith function. As a workaround, both strings get normalized and weightless characters are removed. Resulting strings are cut to the same length and comparison is performed. This approach, beyond having the same compare option limitations as described under **String comparison**, has additional limitations connected with the workaround used. Because we are normalizing strings to be able to cut them, we cannot calculate the match length on the original strings. Methods that calculate this information throw PlatformNotSupported exception: - [CompareInfo.IsPrefix](https://learn.microsoft.com/en-us/dotnet/api/system.globalization.compareinfo.isprefix?view=net-8.0#system-globalization-compareinfo-isprefix(system-readonlyspan((system-char))-system-readonlyspan((system-char))-system-globalization-compareoptions-system-int32@)) - [CompareInfo.IsSuffix](https://learn.microsoft.com/en-us/dotnet/api/system.globalization.compareinfo.issuffix?view=net-8.0#system-globalization-compareinfo-issuffix(system-readonlyspan((system-char))-system-readonlyspan((system-char))-system-globalization-compareoptions-system-int32@)) + +- `IgnoreSymbols` +Only comparisons that do not skip character types are allowed. E.g. `IgnoreSymbols` skips symbol-chars in comparison/indexing. All `CompareOptions` combinations that include `IgnoreSymbols` throw `PlatformNotSupportedException`. diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs index 89557f8067e2d..c1031137f894b 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs @@ -71,9 +71,11 @@ public static IEnumerable IsPrefix_TestData() yield return new object[] { s_invariantCompare, "\uD800\uD800", "\uD800\uD800", CompareOptions.None, true, 2 }; // Ignore symbols - if (!PlatformDetection.IsHybridGlobalizationOnBrowser) // only a few symbols are ignored + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { yield return new object[] { s_invariantCompare, "Test's can be interesting", "Tests", CompareOptions.IgnoreSymbols, true, 6 }; - yield return new object[] { s_invariantCompare, "Test's can be interesting", "Tests", CompareOptions.None, false, 0 }; + yield return new object[] { s_invariantCompare, "Test's can be interesting", "Tests", CompareOptions.None, false, 0 }; + } // Platform differences bool useNls = PlatformDetection.IsNlsGlobalization; diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs index cd5cc17aa3589..76ff91a086475 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs @@ -81,8 +81,10 @@ public static IEnumerable IsSuffix_TestData() // Ignore symbols if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + { yield return new object[] { s_invariantCompare, "More Test's", "Tests", CompareOptions.IgnoreSymbols, true, 6 }; - yield return new object[] { s_invariantCompare, "More Test's", "Tests", CompareOptions.None, false, 0 }; + yield return new object[] { s_invariantCompare, "More Test's", "Tests", CompareOptions.None, false, 0 }; + } // NULL character yield return new object[] { s_invariantCompare, "a\u0000b", "a\u0000b", CompareOptions.None, true, 3 }; diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs index ef1b26f5bef7d..1e2846fa7a9ed 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs @@ -65,6 +65,7 @@ private unsafe bool JsStartsWith(ReadOnlySpan source, ReadOnlySpan p { AssertHybridOnWasm(options); Debug.Assert(!prefix.IsEmpty); + IndexingOptionsNotSupported(options); string cultureName = m_name; string exceptionMessage; @@ -85,6 +86,7 @@ private unsafe bool JsEndsWith(ReadOnlySpan source, ReadOnlySpan pre { AssertHybridOnWasm(options); Debug.Assert(!prefix.IsEmpty); + IndexingOptionsNotSupported(options); string cultureName = m_name; string exceptionMessage; From d66035b8ae33e625b64971b750902448711216a1 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Fri, 21 Apr 2023 08:38:39 +0200 Subject: [PATCH 07/10] Fixed asserts. --- .../System/Globalization/CompareInfo.WebAssembly.cs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs index 1e2846fa7a9ed..425be3e34bfb3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs @@ -39,14 +39,6 @@ private unsafe int JsCompareString(ReadOnlySpan string1, ReadOnlySpan source, ReadOnlySpan p { AssertHybridOnWasm(options); Debug.Assert(!prefix.IsEmpty); - IndexingOptionsNotSupported(options); + AssertIndexingSupported(options); string cultureName = m_name; string exceptionMessage; @@ -86,7 +78,7 @@ private unsafe bool JsEndsWith(ReadOnlySpan source, ReadOnlySpan pre { AssertHybridOnWasm(options); Debug.Assert(!prefix.IsEmpty); - IndexingOptionsNotSupported(options); + AssertIndexingSupported(options); string cultureName = m_name; string exceptionMessage; From c6a973309085a8910d7bf7e52b87abbbf28fba63 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz Date: Fri, 21 Apr 2023 08:40:36 +0200 Subject: [PATCH 08/10] Fix. --- .../src/System/Globalization/CompareInfo.WebAssembly.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs index 425be3e34bfb3..474935f9ace31 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CompareInfo.WebAssembly.cs @@ -37,7 +37,8 @@ private static void AssertIndexingSupported(CompareOptions options, string cultu private unsafe int JsCompareString(ReadOnlySpan string1, ReadOnlySpan string2, CompareOptions options) { AssertHybridOnWasm(options); - AssertComparisonSupported(options, m_name); + string cultureName = m_name; + AssertComparisonSupported(options, cultureName); string exceptionMessage; int cmpResult; @@ -57,8 +58,8 @@ private unsafe bool JsStartsWith(ReadOnlySpan source, ReadOnlySpan p { AssertHybridOnWasm(options); Debug.Assert(!prefix.IsEmpty); - AssertIndexingSupported(options); string cultureName = m_name; + AssertIndexingSupported(options, cultureName); string exceptionMessage; bool result; @@ -78,8 +79,8 @@ private unsafe bool JsEndsWith(ReadOnlySpan source, ReadOnlySpan pre { AssertHybridOnWasm(options); Debug.Assert(!prefix.IsEmpty); - AssertIndexingSupported(options); string cultureName = m_name; + AssertIndexingSupported(options, cultureName); string exceptionMessage; bool result; From 92ecb753e01caa11508047d858d57a83223f4b6c Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Fri, 21 Apr 2023 12:55:42 +0000 Subject: [PATCH 09/10] Match platform with behavior. --- .../tests/CompareInfo/CompareInfoTests.IsPrefix.cs | 9 ++++++--- src/mono/wasm/runtime/hybrid-globalization.ts | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs index c1031137f894b..a79d910c2de80 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs @@ -78,8 +78,10 @@ public static IEnumerable IsPrefix_TestData() } // Platform differences - bool useNls = PlatformDetection.IsNlsGlobalization; - if (useNls || PlatformDetection.IsHybridGlobalizationOnBrowser) + // in HybridGlobalization on Browser we use TextEncoder that is not supported for v8 and the manual decoding works like NLS + bool behavesLikeNls = PlatformDetection.IsNlsGlobalization || + (PlatformDetection.IsHybridGlobalizationOnBrowser && !PlatformDetection.IsBrowserDomSupportedOrNodeJS); + if (behavesLikeNls) { if (!PlatformDetection.IsHybridGlobalizationOnBrowser) { @@ -93,7 +95,8 @@ public static IEnumerable IsPrefix_TestData() else { yield return new object[] { s_hungarianCompare, "dzsdzsfoobar", "ddzsf", CompareOptions.None, false, 0 }; - yield return new object[] { s_invariantCompare, "''Tests", "Tests", CompareOptions.IgnoreSymbols, false, 0 }; + if (!PlatformDetection.IsHybridGlobalizationOnBrowser) + yield return new object[] { s_invariantCompare, "''Tests", "Tests", CompareOptions.IgnoreSymbols, false, 0 }; yield return new object[] { s_frenchCompare, "\u0153", "oe", CompareOptions.None, false, 0 }; yield return new object[] { s_invariantCompare, "\uD800\uDC00", "\uD800", CompareOptions.None, false, 0 }; yield return new object[] { s_invariantCompare, "\uD800\uDC00", "\uD800", CompareOptions.IgnoreCase, false, 0 }; diff --git a/src/mono/wasm/runtime/hybrid-globalization.ts b/src/mono/wasm/runtime/hybrid-globalization.ts index dd77e5234cc93..b18c0181d3b5f 100644 --- a/src/mono/wasm/runtime/hybrid-globalization.ts +++ b/src/mono/wasm/runtime/hybrid-globalization.ts @@ -5,7 +5,7 @@ import { Module } from "./imports"; import { mono_wasm_new_external_root } from "./roots"; import {MonoString, MonoStringRef } from "./types"; import { Int32Ptr } from "./types/emscripten"; -import { conv_string_root, js_string_to_mono_string_root } from "./strings"; +import { conv_string_root, js_string_to_mono_string_root, string_decoder } from "./strings"; import { setU16 } from "./memory"; export function mono_wasm_change_case_invariant(exceptionMessage: Int32Ptr, src: number, srcLength: number, dst: number, dstLength: number, toUpper: number) : void{ @@ -69,7 +69,7 @@ export function mono_wasm_compare_string(exceptionMessage: Int32Ptr, culture: Mo } } -export function get_utf16_string(ptr: number, length: number): string{ +function get_utf16_string(ptr: number, length: number): string{ const view = new Uint16Array(Module.HEAPU16.buffer, ptr, length); let string = ""; for (let i = 0; i < length; i++) @@ -146,7 +146,7 @@ export function mono_wasm_ends_with(exceptionMessage: Int32Ptr, culture: MonoStr function get_clean_string(strPtr: number, strLen: number) { - const str = get_utf16_string(strPtr, strLen); + const str = string_decoder.decode(strPtr, (strPtr + 2*strLen)); const nStr = str.normalize(); return nStr.replace(/[\u200B-\u200D\uFEFF\0]/g, ""); } From 880e4222488932fb350049445f678f114c7883a5 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Fri, 21 Apr 2023 13:31:49 +0000 Subject: [PATCH 10/10] Missing changes to prev commit. --- .../tests/CompareInfo/CompareInfoTests.IsPrefix.cs | 2 +- .../tests/CompareInfo/CompareInfoTests.IsSuffix.cs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs index a79d910c2de80..02b959e0950d0 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsPrefix.cs @@ -104,7 +104,7 @@ public static IEnumerable IsPrefix_TestData() // ICU bugs // UInt16 overflow: https://unicode-org.atlassian.net/browse/ICU-20832 fixed in https://github.com/unicode-org/icu/pull/840 (ICU 65) - if (useNls || PlatformDetection.ICUVersion.Major >= 65) + if (PlatformDetection.IsNlsGlobalization || PlatformDetection.ICUVersion.Major >= 65) { yield return new object[] { s_frenchCompare, "b", new string('a', UInt16.MaxValue + 1), CompareOptions.None, false, 0 }; } diff --git a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs index 76ff91a086475..e5d8a10527c05 100644 --- a/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs +++ b/src/libraries/System.Globalization/tests/CompareInfo/CompareInfoTests.IsSuffix.cs @@ -91,7 +91,10 @@ public static IEnumerable IsSuffix_TestData() yield return new object[] { s_invariantCompare, "a\u0000b", "b\u0000b", CompareOptions.None, false, 0 }; // Platform differences - if (PlatformDetection.IsNlsGlobalization || PlatformDetection.IsHybridGlobalizationOnBrowser) + // in HybridGlobalization on Browser we use TextEncoder that is not supported for v8 and the manual decoding works like NLS + bool behavesLikeNls = PlatformDetection.IsNlsGlobalization || + (PlatformDetection.IsHybridGlobalizationOnBrowser && !PlatformDetection.IsBrowserDomSupportedOrNodeJS); + if (behavesLikeNls) { if (!PlatformDetection.IsHybridGlobalizationOnBrowser) {