From 904bb85e233167540f3b9b835b9b7a4807b4b0e2 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Thu, 20 Jan 2022 15:42:13 -0800 Subject: [PATCH 01/14] Adding vectorized path for Span.Reverse that uses SSSE3 and AVX2 where possible --- .../src/System/MemoryExtensions.cs | 29 ++++++++++ .../src/System/SpanHelpers.Byte.cs | 54 +++++++++++++++++++ .../src/System/SpanHelpers.Char.cs | 38 +++++++++++++ 3 files changed, 121 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 3e0ff6e492a5d..bbc858a78067c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -6,6 +6,8 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace System { @@ -1548,6 +1550,14 @@ public static void Reverse(this Span span) return; } + if (RuntimeHelpers.IsBitwiseEquatable()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + SpanHelpers.ReverseByteRef(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); + return; + } + } ref T first = ref MemoryMarshal.GetReference(span); ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, span.Length), -1); do @@ -1560,6 +1570,25 @@ public static void Reverse(this Span span) } while (Unsafe.IsAddressLessThan(ref first, ref last)); } + + // private static readonly short[] ReverseMaskAvx2Short = + // { + // 0, 1, 2, 3, 4, 5, 6, 7, // first 128-bit lane + // 0, 1, 2, 3, 4, 5, 6, 7, // second 128-bit lane + // }; + + // private static readonly int[] ReverseMaskAvx2Int32 = + // { + // 0, 1, 2, 3, // first 128-bit lane + // 0, 1, 2, 3 // second 128-bit lane + // }; + + // private static readonly long[] ReverseMaskAvx2Int64 = + // { + // 0, 1, // first 128-bit lane + // 0, 1 // second 128-bit lane + // }; + /// /// Creates a new span over the target array. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index e8fca14efa52a..8762e70369630 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2238,5 +2238,59 @@ private static uint FindFirstMatchedLane(Vector128 compareResult) // Find the first lane that is set inside compareResult. return (uint)BitOperations.TrailingZeroCount(selectedLanes) >> 2; } + + public static void ReverseByteRef(ref byte buf, nint length) + { + ref byte first = ref buf; + ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -1); + int numBytesWritten = 0; + if (Avx2.IsSupported && Vector256.Count * 2 <= length) + { + Vector256 ReverseMask = Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // first 128-bit lane + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); // second 128-bit lane + last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -Vector256.Count); + do + { + Vector256 tempFirst = Unsafe.As>(ref first); + Vector256 tempLast = Unsafe.As>(ref last); + tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); + tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector256.Count); + last = ref Unsafe.Add(ref last, -Vector256.Count); + numBytesWritten += Vector256.Count * 2; + } while ((length - numBytesWritten) >= Vector256.Count * 2); + } + else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + { + Vector128 ReverseMask = Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -Vector128.Count); + do + { + Vector128 tempFirst = Unsafe.As>(ref first); + Vector128 tempLast = Unsafe.As>(ref last); + tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); + tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector128.Count); + last = ref Unsafe.Add(ref last, -Vector128.Count); + numBytesWritten += Vector128.Count * 2; + } while ((length - numBytesWritten) >= Vector128.Count * 2); + } + last = ref Unsafe.Add(ref first, (length - numBytesWritten) - 1); + while ((length - numBytesWritten) > 1) + { + byte temp = first; + first = last; + last = temp; + first = ref Unsafe.Add(ref first, 1); + last = ref Unsafe.Add(ref last, -1); + numBytesWritten += 2; + } + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index a662fe73e1ca4..5a370af3a39ba 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2015,5 +2015,43 @@ private static int FindFirstMatchedLane(Vector128 compareResult) return BitOperations.TrailingZeroCount(selectedLanes) >> 3; } + + public static void ReverseCharRef(ref char buf, nint length) + { + nint numBytes = length * sizeof(char); + ref byte first = ref Unsafe.As(ref buf); + ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(char)); + if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + { + Vector256 ReverseMask = Vector256.Create( + (byte)14, (byte)15, (byte)12, (byte)13, (byte)10, (byte)11, (byte)8, (byte)9, (byte)6, (byte)7, (byte)4, (byte)5, (byte)2, (byte)3, (byte)0, (byte)1, // first 128-bit lane + (byte)14, (byte)15, (byte)12, (byte)13, (byte)10, (byte)11, (byte)8, (byte)9, (byte)6, (byte)7, (byte)4, (byte)5, (byte)2, (byte)3, (byte)0, (byte)1); // second 128-bit lane + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + do + { + Vector256 tempFirst = Unsafe.As>(ref first); + Vector256 tempLast = Unsafe.As>(ref last); + tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); + tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector256.Count); + last = ref Unsafe.Add(ref last, -Vector256.Count); + } while (Unsafe.IsAddressLessThan(ref first, ref last) && + (int)Unsafe.ByteOffset(ref first, ref last) > Vector256.Count * 2); + } + ref char firstChar = ref Unsafe.As(ref first); + ref char lastChar = ref Unsafe.As(ref last); + while (Unsafe.IsAddressLessThan(ref firstChar, ref lastChar)) + { + char temp = firstChar; + firstChar = lastChar; + lastChar = temp; + firstChar = ref Unsafe.Add(ref firstChar, 1); + lastChar = ref Unsafe.Add(ref lastChar, -1); + } + } } } From 088771f9f10560c4c2dfff81972aa0c7960b299c Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Thu, 20 Jan 2022 21:01:10 -0800 Subject: [PATCH 02/14] Added vectorized paths for Span.Reverse for primitive types that are the same size as char, int, or long that use AVX2 or SSSE3 where possible --- .../src/System/MemoryExtensions.cs | 37 +++--- .../src/System/SpanHelpers.Char.cs | 30 ++++- .../src/System/SpanHelpers.cs | 120 +++++++++++++++++- 3 files changed, 161 insertions(+), 26 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index bbc858a78067c..a50c0f2c6b2ef 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1550,14 +1550,30 @@ public static void Reverse(this Span span) return; } - if (RuntimeHelpers.IsBitwiseEquatable()) + if (typeof(T).IsValueType) { if (Unsafe.SizeOf() == sizeof(byte)) { SpanHelpers.ReverseByteRef(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); return; } + else if (Unsafe.SizeOf() == sizeof(char)) + { + SpanHelpers.ReverseCharRef(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); + return; + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + SpanHelpers.ReverseInt32Ref(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); + return; + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + SpanHelpers.ReverseInt64Ref(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); + return; + } } + ref T first = ref MemoryMarshal.GetReference(span); ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, span.Length), -1); do @@ -1570,25 +1586,6 @@ public static void Reverse(this Span span) } while (Unsafe.IsAddressLessThan(ref first, ref last)); } - - // private static readonly short[] ReverseMaskAvx2Short = - // { - // 0, 1, 2, 3, 4, 5, 6, 7, // first 128-bit lane - // 0, 1, 2, 3, 4, 5, 6, 7, // second 128-bit lane - // }; - - // private static readonly int[] ReverseMaskAvx2Int32 = - // { - // 0, 1, 2, 3, // first 128-bit lane - // 0, 1, 2, 3 // second 128-bit lane - // }; - - // private static readonly long[] ReverseMaskAvx2Int64 = - // { - // 0, 1, // first 128-bit lane - // 0, 1 // second 128-bit lane - // }; - /// /// Creates a new span over the target array. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 5a370af3a39ba..8f364ad295fff 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2019,13 +2019,14 @@ private static int FindFirstMatchedLane(Vector128 compareResult) public static void ReverseCharRef(ref char buf, nint length) { nint numBytes = length * sizeof(char); + int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(char)); if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { Vector256 ReverseMask = Vector256.Create( - (byte)14, (byte)15, (byte)12, (byte)13, (byte)10, (byte)11, (byte)8, (byte)9, (byte)6, (byte)7, (byte)4, (byte)5, (byte)2, (byte)3, (byte)0, (byte)1, // first 128-bit lane - (byte)14, (byte)15, (byte)12, (byte)13, (byte)10, (byte)11, (byte)8, (byte)9, (byte)6, (byte)7, (byte)4, (byte)5, (byte)2, (byte)3, (byte)0, (byte)1); // second 128-bit lane + (byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, // first 128-bit lane + 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); // second 128-bit lane last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); do { @@ -2039,18 +2040,37 @@ public static void ReverseCharRef(ref char buf, nint length) Unsafe.As>(ref last) = tempFirst; first = ref Unsafe.Add(ref first, Vector256.Count); last = ref Unsafe.Add(ref last, -Vector256.Count); - } while (Unsafe.IsAddressLessThan(ref first, ref last) && - (int)Unsafe.ByteOffset(ref first, ref last) > Vector256.Count * 2); + numBytesWritten += Vector256.Count * 2; + } while (numBytes - numBytesWritten >= Vector256.Count * 2); } + else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + { + Vector128 ReverseMask = Vector128.Create((byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + do + { + Vector128 tempFirst = Unsafe.As>(ref first); + Vector128 tempLast = Unsafe.As>(ref last); + tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); + tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector128.Count); + last = ref Unsafe.Add(ref last, -Vector128.Count); + numBytesWritten += Vector128.Count * 2; + } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + } + last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(char)); ref char firstChar = ref Unsafe.As(ref first); ref char lastChar = ref Unsafe.As(ref last); - while (Unsafe.IsAddressLessThan(ref firstChar, ref lastChar)) + while (numBytes - numBytesWritten > sizeof(char)) { char temp = firstChar; firstChar = lastChar; lastChar = temp; firstChar = ref Unsafe.Add(ref firstChar, 1); lastChar = ref Unsafe.Add(ref lastChar, -1); + numBytesWritten += sizeof(char) * 2; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index fba4f5cdbebcd..57a20eeb5b4e8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using Internal.Runtime.CompilerServices; namespace System { @@ -403,5 +405,121 @@ public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLe // Write only element. ip = default; } + + public static void ReverseInt32Ref(ref int buf, nint length) + { + nint numBytes = length * sizeof(int); + int numBytesWritten = 0; + ref byte first = ref Unsafe.As(ref buf); + ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(int)); + if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + { + Vector256 ReverseMask = Vector256.Create( + (byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, // first 128-bit lane + 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); // second 128-bit lane + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + do + { + Vector256 tempFirst = Unsafe.As>(ref first); + Vector256 tempLast = Unsafe.As>(ref last); + tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); + tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector256.Count); + last = ref Unsafe.Add(ref last, -Vector256.Count); + numBytesWritten += Vector256.Count * 2; + } while (numBytes - numBytesWritten >= Vector256.Count * 2); + } + else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + { + Vector128 ReverseMask = Vector128.Create((byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + do + { + Vector128 tempFirst = Unsafe.As>(ref first); + Vector128 tempLast = Unsafe.As>(ref last); + tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); + tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector128.Count); + last = ref Unsafe.Add(ref last, -Vector128.Count); + numBytesWritten += Vector128.Count * 2; + } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + } + last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(int)); + ref int firstInt = ref Unsafe.As(ref first); + ref int lastInt = ref Unsafe.As(ref last); + while (numBytes - numBytesWritten > sizeof(int)) + { + int temp = firstInt; + firstInt = lastInt; + lastInt = temp; + firstInt = ref Unsafe.Add(ref firstInt, 1); + lastInt = ref Unsafe.Add(ref lastInt, -1); + numBytesWritten += sizeof(int) * 2; + } + } + + public static void ReverseInt64Ref(ref long buf, nint length) + { + nint numBytes = length * sizeof(long); + int numBytesWritten = 0; + ref byte first = ref Unsafe.As(ref buf); + ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(long)); + if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + { + Vector256 ReverseMask = Vector256.Create( + (byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // first 128-bit lane + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); // second 128-bit lane + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + do + { + Vector256 tempFirst = Unsafe.As>(ref first); + Vector256 tempLast = Unsafe.As>(ref last); + tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); + tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector256.Count); + last = ref Unsafe.Add(ref last, -Vector256.Count); + numBytesWritten += Vector256.Count * 2; + } while (numBytes - numBytesWritten >= Vector256.Count * 2); + } + else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + { + Vector128 ReverseMask = Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + do + { + Vector128 tempFirst = Unsafe.As>(ref first); + Vector128 tempLast = Unsafe.As>(ref last); + tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); + tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; + first = ref Unsafe.Add(ref first, Vector128.Count); + last = ref Unsafe.Add(ref last, -Vector128.Count); + numBytesWritten += Vector128.Count * 2; + } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + } + last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(long)); + ref long firstLong = ref Unsafe.As(ref first); + ref long lastLong = ref Unsafe.As(ref last); + while (numBytes - numBytesWritten > sizeof(long)) + { + long temp = firstLong; + firstLong = lastLong; + lastLong = temp; + firstLong = ref Unsafe.Add(ref firstLong, 1); + lastLong = ref Unsafe.Add(ref lastLong, -1); + numBytesWritten += sizeof(long) * 2; + } + } } } From fd882e995b33bfd97f757edc2d00a357e04d3684 Mon Sep 17 00:00:00 2001 From: Alex Covington <68252706+alexcovington@users.noreply.github.com> Date: Fri, 28 Jan 2022 08:19:52 -0800 Subject: [PATCH 03/14] Apply suggestions from code review Co-authored-by: Theodore Tsirpanis --- .../System.Private.CoreLib/src/System/MemoryExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index a50c0f2c6b2ef..0855dc8cf8a78 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1550,7 +1550,7 @@ public static void Reverse(this Span span) return; } - if (typeof(T).IsValueType) + if (!RuntimeHelpers.IsReferenceOrContainsReferences()) { if (Unsafe.SizeOf() == sizeof(byte)) { From 74deff772f31bded1b070b2373e65142a2a0e535 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Fri, 28 Jan 2022 15:59:52 -0800 Subject: [PATCH 04/14] Added vectorized paths for Span.Reverse to Array.Reverse. Added explicit inlining and moved generic fallbacks into their own private methods. --- .../src/System/Array.cs | 31 +++++++++++++++++++ .../src/System/MemoryExtensions.cs | 5 +++ 2 files changed, 36 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs index cb42dbd6159a7..0db51986f498b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Array.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs @@ -1637,6 +1637,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)) // located at index length - i - 1, where length is the // length of the array. // + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(Array array) { if (array == null) @@ -1718,6 +1719,7 @@ public static void Reverse(Array array, int index, int length) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(T[] array) { if (array == null) @@ -1725,6 +1727,7 @@ public static void Reverse(T[] array) Reverse(array, 0, array.Length); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(T[] array, int index, int length) { if (array == null) @@ -1739,6 +1742,34 @@ public static void Reverse(T[] array, int index, int length) if (length <= 1) return; + if (!RuntimeHelpers.IsReferenceOrContainsReferences()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + SpanHelpers.ReverseByteRef(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); + return; + } + else if (Unsafe.SizeOf() == sizeof(char)) + { + SpanHelpers.ReverseCharRef(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); + return; + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + SpanHelpers.ReverseInt32Ref(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); + return; + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + SpanHelpers.ReverseInt64Ref(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); + return; + } + } + ReverseInner(array, index, length); + } + + private static void ReverseInner(T[] array, int index, int length) + { ref T first = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index); ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -1); do diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 0855dc8cf8a78..2a59cd83122a3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1543,6 +1543,7 @@ ref MemoryMarshal.GetReference(value), /// /// Reverses the sequence of the elements in the entire span. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(this Span span) { if (span.Length <= 1) @@ -1573,7 +1574,11 @@ public static void Reverse(this Span span) return; } } + ReverseInner(span); + } + private static void ReverseInner(this Span span) + { ref T first = ref MemoryMarshal.GetReference(span); ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, span.Length), -1); do From 289978960ba0240175d910d186b0cd4514dafafb Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Tue, 1 Feb 2022 11:15:33 -0800 Subject: [PATCH 05/14] Consolidate fall back case into single method, use one wrapper for both Span.Reverse and Array.Reverse --- .../src/System/Array.cs | 39 +---------- .../src/System/MemoryExtensions.cs | 39 +---------- .../src/System/SpanHelpers.Byte.cs | 25 ++++---- .../src/System/SpanHelpers.Char.cs | 10 ++- .../src/System/SpanHelpers.cs | 64 +++++++++++++++---- 5 files changed, 70 insertions(+), 107 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs index 0db51986f498b..9899e0e665ffb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Array.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs @@ -1742,44 +1742,7 @@ public static void Reverse(T[] array, int index, int length) if (length <= 1) return; - if (!RuntimeHelpers.IsReferenceOrContainsReferences()) - { - if (Unsafe.SizeOf() == sizeof(byte)) - { - SpanHelpers.ReverseByteRef(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); - return; - } - else if (Unsafe.SizeOf() == sizeof(char)) - { - SpanHelpers.ReverseCharRef(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); - return; - } - else if (Unsafe.SizeOf() == sizeof(int)) - { - SpanHelpers.ReverseInt32Ref(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); - return; - } - else if (Unsafe.SizeOf() == sizeof(long)) - { - SpanHelpers.ReverseInt64Ref(ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)), length); - return; - } - } - ReverseInner(array, index, length); - } - - private static void ReverseInner(T[] array, int index, int length) - { - ref T first = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index); - ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -1); - do - { - T temp = first; - first = last; - last = temp; - first = ref Unsafe.Add(ref first, 1); - last = ref Unsafe.Add(ref last, -1); - } while (Unsafe.IsAddressLessThan(ref first, ref last)); + SpanHelpers.Reverse(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index), (nuint)length); } // Sorts the elements of an array. The sort compares the elements to each diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 2a59cd83122a3..f8f4cb601fc65 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1550,46 +1550,9 @@ public static void Reverse(this Span span) { return; } - - if (!RuntimeHelpers.IsReferenceOrContainsReferences()) - { - if (Unsafe.SizeOf() == sizeof(byte)) - { - SpanHelpers.ReverseByteRef(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); - return; - } - else if (Unsafe.SizeOf() == sizeof(char)) - { - SpanHelpers.ReverseCharRef(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); - return; - } - else if (Unsafe.SizeOf() == sizeof(int)) - { - SpanHelpers.ReverseInt32Ref(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); - return; - } - else if (Unsafe.SizeOf() == sizeof(long)) - { - SpanHelpers.ReverseInt64Ref(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); - return; - } - } - ReverseInner(span); + SpanHelpers.Reverse(ref MemoryMarshal.GetReference(span), (nuint)span.Length); } - private static void ReverseInner(this Span span) - { - ref T first = ref MemoryMarshal.GetReference(span); - ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, span.Length), -1); - do - { - T temp = first; - first = last; - last = temp; - first = ref Unsafe.Add(ref first, 1); - last = ref Unsafe.Add(ref last, -1); - } while (Unsafe.IsAddressLessThan(ref first, ref last)); - } /// /// Creates a new span over the target array. diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 8762e70369630..b1cc3ca8c01d2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2239,16 +2239,17 @@ private static uint FindFirstMatchedLane(Vector128 compareResult) return (uint)BitOperations.TrailingZeroCount(selectedLanes) >> 2; } - public static void ReverseByteRef(ref byte buf, nint length) + public static void ReverseByteRef(ref byte buf, nuint length) { + int numBytes = (int)length; ref byte first = ref buf; - ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -1); + ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -1); int numBytesWritten = 0; - if (Avx2.IsSupported && Vector256.Count * 2 <= length) + if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { Vector256 ReverseMask = Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // first 128-bit lane 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); // second 128-bit lane - last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -Vector256.Count); + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); do { Vector256 tempFirst = Unsafe.As>(ref first); @@ -2262,12 +2263,12 @@ public static void ReverseByteRef(ref byte buf, nint length) first = ref Unsafe.Add(ref first, Vector256.Count); last = ref Unsafe.Add(ref last, -Vector256.Count); numBytesWritten += Vector256.Count * 2; - } while ((length - numBytesWritten) >= Vector256.Count * 2); + } while ((numBytes - numBytesWritten) >= Vector256.Count * 2); } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { Vector128 ReverseMask = Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - last = ref Unsafe.Add(ref Unsafe.Add(ref first, length), -Vector128.Count); + last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); do { Vector128 tempFirst = Unsafe.As>(ref first); @@ -2279,14 +2280,12 @@ public static void ReverseByteRef(ref byte buf, nint length) first = ref Unsafe.Add(ref first, Vector128.Count); last = ref Unsafe.Add(ref last, -Vector128.Count); numBytesWritten += Vector128.Count * 2; - } while ((length - numBytesWritten) >= Vector128.Count * 2); + } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } - last = ref Unsafe.Add(ref first, (length - numBytesWritten) - 1); - while ((length - numBytesWritten) > 1) + last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - 1); + while ((numBytes - numBytesWritten) > 1) { - byte temp = first; - first = last; - last = temp; + (last, first) = (first, last); first = ref Unsafe.Add(ref first, 1); last = ref Unsafe.Add(ref last, -1); numBytesWritten += 2; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 8f364ad295fff..2d2f3280e3691 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2016,9 +2016,9 @@ private static int FindFirstMatchedLane(Vector128 compareResult) return BitOperations.TrailingZeroCount(selectedLanes) >> 3; } - public static void ReverseCharRef(ref char buf, nint length) + public static void ReverseCharRef(ref char buf, nuint length) { - nint numBytes = length * sizeof(char); + nint numBytes = (int)length * sizeof(char); int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(char)); @@ -2043,7 +2043,7 @@ public static void ReverseCharRef(ref char buf, nint length) numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { Vector128 ReverseMask = Vector128.Create((byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); @@ -2065,9 +2065,7 @@ public static void ReverseCharRef(ref char buf, nint length) ref char lastChar = ref Unsafe.As(ref last); while (numBytes - numBytesWritten > sizeof(char)) { - char temp = firstChar; - firstChar = lastChar; - lastChar = temp; + (lastChar, firstChar) = (firstChar, lastChar); firstChar = ref Unsafe.Add(ref firstChar, 1); lastChar = ref Unsafe.Add(ref lastChar, -1); numBytesWritten += sizeof(char) * 2; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index 57a20eeb5b4e8..d16ad9d2e5e22 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; using Internal.Runtime.CompilerServices; @@ -406,9 +408,9 @@ public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLe ip = default; } - public static void ReverseInt32Ref(ref int buf, nint length) + public static void ReverseInt32Ref(ref int buf, nuint length) { - nint numBytes = length * sizeof(int); + int numBytes = (int)length * sizeof(int); int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(int)); @@ -433,7 +435,7 @@ public static void ReverseInt32Ref(ref int buf, nint length) numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { Vector128 ReverseMask = Vector128.Create((byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); @@ -455,18 +457,16 @@ public static void ReverseInt32Ref(ref int buf, nint length) ref int lastInt = ref Unsafe.As(ref last); while (numBytes - numBytesWritten > sizeof(int)) { - int temp = firstInt; - firstInt = lastInt; - lastInt = temp; + (lastInt, firstInt) = (firstInt, lastInt); firstInt = ref Unsafe.Add(ref firstInt, 1); lastInt = ref Unsafe.Add(ref lastInt, -1); numBytesWritten += sizeof(int) * 2; } } - public static void ReverseInt64Ref(ref long buf, nint length) + public static void ReverseInt64Ref(ref long buf, nuint length) { - nint numBytes = length * sizeof(long); + nint numBytes = (int)length * sizeof(long); int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(long)); @@ -491,7 +491,7 @@ public static void ReverseInt64Ref(ref long buf, nint length) numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= length) + else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { Vector128 ReverseMask = Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); @@ -513,13 +513,53 @@ public static void ReverseInt64Ref(ref long buf, nint length) ref long lastLong = ref Unsafe.As(ref last); while (numBytes - numBytesWritten > sizeof(long)) { - long temp = firstLong; - firstLong = lastLong; - lastLong = temp; + (lastLong, firstLong) = (firstLong, lastLong); firstLong = ref Unsafe.Add(ref firstLong, 1); lastLong = ref Unsafe.Add(ref lastLong, -1); numBytesWritten += sizeof(long) * 2; } } + + public static void Reverse(ref T elements, nuint length) + { + Debug.Assert(length > 0); + if (!RuntimeHelpers.IsReferenceOrContainsReferences()) + { + if (Unsafe.SizeOf() == sizeof(byte)) + { + ReverseByteRef(ref Unsafe.As(ref elements), length); + return; + } + else if (Unsafe.SizeOf() == sizeof(char)) + { + ReverseCharRef(ref Unsafe.As(ref elements), length); + return; + } + else if (Unsafe.SizeOf() == sizeof(int)) + { + ReverseInt32Ref(ref Unsafe.As(ref elements), length); + return; + } + else if (Unsafe.SizeOf() == sizeof(long)) + { + ReverseInt64Ref(ref Unsafe.As(ref elements), length); + return; + } + } + ReverseInner(ref elements, length); + } + + private static void ReverseInner(ref T elements, nuint length) + { + Debug.Assert(length > 0); + ref T first = ref elements; + ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, (int)length), -1); + do + { + (last, first) = (first, last); + first = ref Unsafe.Add(ref first, 1); + last = ref Unsafe.Add(ref last, -1); + } while (Unsafe.IsAddressLessThan(ref first, ref last)); + } } } From a6c810195c632f257f9dfaa09d4305aa61bc8c4e Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Mon, 7 Feb 2022 14:26:16 -0800 Subject: [PATCH 06/14] Remove redundant AggressiveInlining, add AggressiveInlining to single wrapper instead --- src/libraries/System.Private.CoreLib/src/System/Array.cs | 5 +---- .../System.Private.CoreLib/src/System/MemoryExtensions.cs | 1 - .../System.Private.CoreLib/src/System/SpanHelpers.cs | 1 + 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs index 9899e0e665ffb..e7341aa738c5f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Array.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs @@ -1637,7 +1637,6 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(array)) // located at index length - i - 1, where length is the // length of the array. // - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(Array array) { if (array == null) @@ -1719,15 +1718,13 @@ public static void Reverse(Array array, int index, int length) } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(T[] array) { if (array == null) ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array); - Reverse(array, 0, array.Length); + SpanHelpers.Reverse(ref MemoryMarshal.GetArrayDataReference(array), (nuint)array.Length); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(T[] array, int index, int length) { if (array == null) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index f8f4cb601fc65..e232a4f19867c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1543,7 +1543,6 @@ ref MemoryMarshal.GetReference(value), /// /// Reverses the sequence of the elements in the entire span. /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(this Span span) { if (span.Length <= 1) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index d16ad9d2e5e22..350aa72ef1f12 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -520,6 +520,7 @@ public static void ReverseInt64Ref(ref long buf, nuint length) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(ref T elements, nuint length) { Debug.Assert(length > 0); From 3aa7cf12ca349d147609757e36a5ed96fdd3a499 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Tue, 8 Feb 2022 08:54:19 -0800 Subject: [PATCH 07/14] Simplify method names, add comments --- .../src/System/SpanHelpers.Byte.cs | 43 +++++++++- .../src/System/SpanHelpers.Char.cs | 37 ++++++++- .../src/System/SpanHelpers.cs | 82 +++++++++++++++++-- 3 files changed, 154 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index b1cc3ca8c01d2..798d354f279de 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2239,7 +2239,7 @@ private static uint FindFirstMatchedLane(Vector128 compareResult) return (uint)BitOperations.TrailingZeroCount(selectedLanes) >> 2; } - public static void ReverseByteRef(ref byte buf, nuint length) + public static void ReverseRef(ref byte buf, nuint length) { int numBytes = (int)length; ref byte first = ref buf; @@ -2252,14 +2252,39 @@ public static void ReverseByteRef(ref byte buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); do { + // Load in values from beginning and end of the array. Vector256 tempFirst = Unsafe.As>(ref first); Vector256 tempLast = Unsafe.As>(ref last); + + // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. + // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: + // +-------------------------------------------------------------------------------+ + // | A1 | B1 | C1 | D1 | E1 | F1 | G1 | H1 | I1 | J1 | K1 | L1 | M1 | N1 | O1 | P1 | + // +-------------------------------------------------------------------------------+ + // | A2 | B2 | C2 | D2 | E2 | F2 | G2 | H2 | I2 | J2 | K2 | L2 | M2 | N2 | O2 | P2 | + // +-------------------------------------------------------------------------------+ + // Shuffle ---> + // +-------------------------------------------------------------------------------+ + // | P1 | O1 | N1 | M1 | L1 | K1 | J1 | I1 | H1 | G1 | F1 | E1 | D1 | C1 | B1 | A1 | + // +-------------------------------------------------------------------------------+ + // | P2 | O2 | N2 | M2 | L2 | K2 | J2 | I2 | H2 | G2 | F2 | E2 | D2 | C2 | B2 | A2 | + // +-------------------------------------------------------------------------------+ + // Permute ---> + // +-------------------------------------------------------------------------------+ + // | P2 | O2 | N2 | M2 | L2 | K2 | J2 | I2 | H2 | G2 | F2 | E2 | D2 | C2 | B2 | A2 | + // +-------------------------------------------------------------------------------+ + // | P1 | O1 | N1 | M1 | L1 | K1 | J1 | I1 | H1 | G1 | F1 | E1 | D1 | C1 | B1 | A1 | + // +-------------------------------------------------------------------------------+ tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); tempLast = Avx2.Shuffle(tempLast, ReverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); last = ref Unsafe.Add(ref last, -Vector256.Count); numBytesWritten += Vector256.Count * 2; @@ -2271,17 +2296,33 @@ public static void ReverseByteRef(ref byte buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); do { + // Load in values from beginning and end of the array. Vector128 tempFirst = Unsafe.As>(ref first); Vector128 tempLast = Unsafe.As>(ref last); + + // Shuffle to reverse each vector: + // +---------------------------------------------------------------+ + // | A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | + // +---------------------------------------------------------------+ + // ---> + // +---------------------------------------------------------------+ + // | P | O | N | M | L | K | J | I | H | G | F | E | D | C | B | A | + // +---------------------------------------------------------------+ tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); last = ref Unsafe.Add(ref last, -Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } + + // Store any remaining values one-by-one last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - 1); while ((numBytes - numBytesWritten) > 1) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 2d2f3280e3691..5cf658d4dbca3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2016,7 +2016,7 @@ private static int FindFirstMatchedLane(Vector128 compareResult) return BitOperations.TrailingZeroCount(selectedLanes) >> 3; } - public static void ReverseCharRef(ref char buf, nuint length) + public static void ReverseRef(ref char buf, nuint length) { nint numBytes = (int)length * sizeof(char); int numBytesWritten = 0; @@ -2030,14 +2030,33 @@ public static void ReverseCharRef(ref char buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); do { + // Load in values from beginning and end of the array. Vector256 tempFirst = Unsafe.As>(ref first); Vector256 tempLast = Unsafe.As>(ref last); + + // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. + // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: + // +---------------------------------------------------------------+ + // | A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | + // +---------------------------------------------------------------+ + // Shuffle ---> + // +---------------------------------------------------------------+ + // | H | G | F | E | D | C | B | A | P | O | N | M | L | K | J | I | + // +---------------------------------------------------------------+ + // Permute ---> + // +---------------------------------------------------------------+ + // | P | O | N | M | L | K | J | I | H | G | F | E | D | C | B | A | + // +---------------------------------------------------------------+ tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); tempLast = Avx2.Shuffle(tempLast, ReverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); last = ref Unsafe.Add(ref last, -Vector256.Count); numBytesWritten += Vector256.Count * 2; @@ -2049,17 +2068,33 @@ public static void ReverseCharRef(ref char buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); do { + // Load in values from beginning and end of the array. Vector128 tempFirst = Unsafe.As>(ref first); Vector128 tempLast = Unsafe.As>(ref last); + + // Shuffle to reverse each vector: + // +-------------------------------+ + // | A | B | C | D | E | F | G | H | + // +-------------------------------+ + // ---> + // +-------------------------------+ + // | H | G | F | E | D | C | B | A | + // +-------------------------------+ tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); last = ref Unsafe.Add(ref last, -Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } + + // Store any remaining values one-by-one last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(char)); ref char firstChar = ref Unsafe.As(ref first); ref char lastChar = ref Unsafe.As(ref last); diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index 350aa72ef1f12..40954ac4f2637 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -408,7 +408,7 @@ public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLe ip = default; } - public static void ReverseInt32Ref(ref int buf, nuint length) + public static void ReverseRef(ref int buf, nuint length) { int numBytes = (int)length * sizeof(int); int numBytesWritten = 0; @@ -422,14 +422,33 @@ public static void ReverseInt32Ref(ref int buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); do { + // Load in values from beginning and end of the array. Vector256 tempFirst = Unsafe.As>(ref first); Vector256 tempLast = Unsafe.As>(ref last); + + // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. + // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: + // +-------------------------------+ + // | A | B | C | D | E | F | G | H | + // +-------------------------------+ + // Shuffle ---> + // +-------------------------------+ + // | D | C | B | A | H | G | F | E | + // +-------------------------------+ + // Permute ---> + // +-------------------------------+ + // | H | G | F | E | D | C | B | A | + // +-------------------------------+ tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); tempLast = Avx2.Shuffle(tempLast, ReverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); last = ref Unsafe.Add(ref last, -Vector256.Count); numBytesWritten += Vector256.Count * 2; @@ -441,17 +460,33 @@ public static void ReverseInt32Ref(ref int buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); do { + // Load in values from beginning and end of the array. Vector128 tempFirst = Unsafe.As>(ref first); Vector128 tempLast = Unsafe.As>(ref last); + + // Shuffle to reverse each vector: + // +---------------+ + // | A | B | C | D | + // +---------------+ + // ---> + // +---------------+ + // | D | C | B | A | + // +---------------+ tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); last = ref Unsafe.Add(ref last, -Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } + + // Store any remaining values one-by-one last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(int)); ref int firstInt = ref Unsafe.As(ref first); ref int lastInt = ref Unsafe.As(ref last); @@ -464,7 +499,7 @@ public static void ReverseInt32Ref(ref int buf, nuint length) } } - public static void ReverseInt64Ref(ref long buf, nuint length) + public static void ReverseRef(ref long buf, nuint length) { nint numBytes = (int)length * sizeof(long); int numBytesWritten = 0; @@ -478,14 +513,33 @@ public static void ReverseInt64Ref(ref long buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); do { + // Load in values from beginning and end of the array. Vector256 tempFirst = Unsafe.As>(ref first); Vector256 tempLast = Unsafe.As>(ref last); + + // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. + // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: + // +---------------+ + // | A | B | C | D | + // +---------------+ + // Shuffle ---> + // +---------------+ + // | B | A | D | C | + // +---------------+ + // Permute ---> + // +---------------+ + // | D | C | B | A | + // +---------------+ tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); tempLast = Avx2.Shuffle(tempLast, ReverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); last = ref Unsafe.Add(ref last, -Vector256.Count); numBytesWritten += Vector256.Count * 2; @@ -497,17 +551,33 @@ public static void ReverseInt64Ref(ref long buf, nuint length) last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); do { + // Load in values from beginning and end of the array. Vector128 tempFirst = Unsafe.As>(ref first); Vector128 tempLast = Unsafe.As>(ref last); + + // Shuffle to reverse each vector: + // +-------+ + // | A | B | + // +-------+ + // ---> + // +-------+ + // | B | A | + // +-------+ tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + + // Store the reversed vectors Unsafe.As>(ref first) = tempLast; Unsafe.As>(ref last) = tempFirst; + + // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); last = ref Unsafe.Add(ref last, -Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } + + // Store any remaining values one-by-one last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(long)); ref long firstLong = ref Unsafe.As(ref first); ref long lastLong = ref Unsafe.As(ref last); @@ -528,22 +598,22 @@ public static void Reverse(ref T elements, nuint length) { if (Unsafe.SizeOf() == sizeof(byte)) { - ReverseByteRef(ref Unsafe.As(ref elements), length); + ReverseRef(ref Unsafe.As(ref elements), length); return; } else if (Unsafe.SizeOf() == sizeof(char)) { - ReverseCharRef(ref Unsafe.As(ref elements), length); + ReverseRef(ref Unsafe.As(ref elements), length); return; } else if (Unsafe.SizeOf() == sizeof(int)) { - ReverseInt32Ref(ref Unsafe.As(ref elements), length); + ReverseRef(ref Unsafe.As(ref elements), length); return; } else if (Unsafe.SizeOf() == sizeof(long)) { - ReverseInt64Ref(ref Unsafe.As(ref elements), length); + ReverseRef(ref Unsafe.As(ref elements), length); return; } } From 86caa86818884f906870d176eda46ca0382ede07 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Tue, 8 Feb 2022 08:56:32 -0800 Subject: [PATCH 08/14] Just overload Reverse --- .../src/System/SpanHelpers.Byte.cs | 2 +- .../src/System/SpanHelpers.Char.cs | 2 +- .../System.Private.CoreLib/src/System/SpanHelpers.cs | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 798d354f279de..03c5509b066cf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2239,7 +2239,7 @@ private static uint FindFirstMatchedLane(Vector128 compareResult) return (uint)BitOperations.TrailingZeroCount(selectedLanes) >> 2; } - public static void ReverseRef(ref byte buf, nuint length) + public static void Reverse(ref byte buf, nuint length) { int numBytes = (int)length; ref byte first = ref buf; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 5cf658d4dbca3..cc34788320125 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2016,7 +2016,7 @@ private static int FindFirstMatchedLane(Vector128 compareResult) return BitOperations.TrailingZeroCount(selectedLanes) >> 3; } - public static void ReverseRef(ref char buf, nuint length) + public static void Reverse(ref char buf, nuint length) { nint numBytes = (int)length * sizeof(char); int numBytesWritten = 0; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index 40954ac4f2637..a4b7f5ba86729 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -408,7 +408,7 @@ public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLe ip = default; } - public static void ReverseRef(ref int buf, nuint length) + public static void Reverse(ref int buf, nuint length) { int numBytes = (int)length * sizeof(int); int numBytesWritten = 0; @@ -499,7 +499,7 @@ public static void ReverseRef(ref int buf, nuint length) } } - public static void ReverseRef(ref long buf, nuint length) + public static void Reverse(ref long buf, nuint length) { nint numBytes = (int)length * sizeof(long); int numBytesWritten = 0; @@ -598,22 +598,22 @@ public static void Reverse(ref T elements, nuint length) { if (Unsafe.SizeOf() == sizeof(byte)) { - ReverseRef(ref Unsafe.As(ref elements), length); + Reverse(ref Unsafe.As(ref elements), length); return; } else if (Unsafe.SizeOf() == sizeof(char)) { - ReverseRef(ref Unsafe.As(ref elements), length); + Reverse(ref Unsafe.As(ref elements), length); return; } else if (Unsafe.SizeOf() == sizeof(int)) { - ReverseRef(ref Unsafe.As(ref elements), length); + Reverse(ref Unsafe.As(ref elements), length); return; } else if (Unsafe.SizeOf() == sizeof(long)) { - ReverseRef(ref Unsafe.As(ref elements), length); + Reverse(ref Unsafe.As(ref elements), length); return; } } From 7c4cd525bc82c5accf965ea8e1acef574c613b77 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Thu, 10 Mar 2022 13:15:23 -0800 Subject: [PATCH 09/14] Use Unsafe.Subtract where it is semantically more intuitive, camelCase for reverseMask variable --- .../src/System/SpanHelpers.Byte.cs | 24 ++++----- .../src/System/SpanHelpers.Char.cs | 24 ++++----- .../src/System/SpanHelpers.cs | 52 +++++++++---------- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 03c5509b066cf..97e67ce095e9b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2243,13 +2243,13 @@ public static void Reverse(ref byte buf, nuint length) { int numBytes = (int)length; ref byte first = ref buf; - ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -1); + ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), 1); int numBytesWritten = 0; if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 ReverseMask = Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // first 128-bit lane + Vector256 reverseMask = Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // first 128-bit lane 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); // second 128-bit lane - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { // Load in values from beginning and end of the array. @@ -2275,9 +2275,9 @@ public static void Reverse(ref byte buf, nuint length) // +-------------------------------------------------------------------------------+ // | P1 | O1 | N1 | M1 | L1 | K1 | J1 | I1 | H1 | G1 | F1 | E1 | D1 | C1 | B1 | A1 | // +-------------------------------------------------------------------------------+ - tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Shuffle(tempFirst, reverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Shuffle(tempLast, reverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors @@ -2286,14 +2286,14 @@ public static void Reverse(ref byte buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Add(ref last, -Vector256.Count); + last = ref Unsafe.Subtract(ref last, Vector256.Count); numBytesWritten += Vector256.Count * 2; } while ((numBytes - numBytesWritten) >= Vector256.Count * 2); } else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { - Vector128 ReverseMask = Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + Vector128 reverseMask = Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); do { // Load in values from beginning and end of the array. @@ -2308,8 +2308,8 @@ public static void Reverse(ref byte buf, nuint length) // +---------------------------------------------------------------+ // | P | O | N | M | L | K | J | I | H | G | F | E | D | C | B | A | // +---------------------------------------------------------------+ - tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); - tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + tempFirst = Ssse3.Shuffle(tempFirst, reverseMask); + tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors Unsafe.As>(ref first) = tempLast; @@ -2317,7 +2317,7 @@ public static void Reverse(ref byte buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Add(ref last, -Vector128.Count); + last = ref Unsafe.Subtract(ref last, Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } @@ -2328,7 +2328,7 @@ public static void Reverse(ref byte buf, nuint length) { (last, first) = (first, last); first = ref Unsafe.Add(ref first, 1); - last = ref Unsafe.Add(ref last, -1); + last = ref Unsafe.Subtract(ref last, 1); numBytesWritten += 2; } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index cc34788320125..7b2989bfac102 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2021,13 +2021,13 @@ public static void Reverse(ref char buf, nuint length) nint numBytes = (int)length * sizeof(char); int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); - ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(char)); + ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(char)); if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 ReverseMask = Vector256.Create( + Vector256 reverseMask = Vector256.Create( (byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, // first 128-bit lane 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); // second 128-bit lane - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { // Load in values from beginning and end of the array. @@ -2047,9 +2047,9 @@ public static void Reverse(ref char buf, nuint length) // +---------------------------------------------------------------+ // | P | O | N | M | L | K | J | I | H | G | F | E | D | C | B | A | // +---------------------------------------------------------------+ - tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Shuffle(tempFirst, reverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Shuffle(tempLast, reverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors @@ -2058,14 +2058,14 @@ public static void Reverse(ref char buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Add(ref last, -Vector256.Count); + last = ref Unsafe.Subtract(ref last, Vector256.Count); numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { - Vector128 ReverseMask = Vector128.Create((byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + Vector128 reverseMask = Vector128.Create((byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); do { // Load in values from beginning and end of the array. @@ -2080,8 +2080,8 @@ public static void Reverse(ref char buf, nuint length) // +-------------------------------+ // | H | G | F | E | D | C | B | A | // +-------------------------------+ - tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); - tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + tempFirst = Ssse3.Shuffle(tempFirst, reverseMask); + tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors Unsafe.As>(ref first) = tempLast; @@ -2089,7 +2089,7 @@ public static void Reverse(ref char buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Add(ref last, -Vector128.Count); + last = ref Unsafe.Subtract(ref last, Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } @@ -2102,7 +2102,7 @@ public static void Reverse(ref char buf, nuint length) { (lastChar, firstChar) = (firstChar, lastChar); firstChar = ref Unsafe.Add(ref firstChar, 1); - lastChar = ref Unsafe.Add(ref lastChar, -1); + lastChar = ref Unsafe.Subtract(ref lastChar, 1); numBytesWritten += sizeof(char) * 2; } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index a4b7f5ba86729..b25f52aaca05f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -413,13 +413,13 @@ public static void Reverse(ref int buf, nuint length) int numBytes = (int)length * sizeof(int); int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); - ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(int)); + ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(int)); if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 ReverseMask = Vector256.Create( + Vector256 reverseMask = Vector256.Create( (byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, // first 128-bit lane 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); // second 128-bit lane - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { // Load in values from beginning and end of the array. @@ -439,9 +439,9 @@ public static void Reverse(ref int buf, nuint length) // +-------------------------------+ // | H | G | F | E | D | C | B | A | // +-------------------------------+ - tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Shuffle(tempFirst, reverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Shuffle(tempLast, reverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors @@ -450,14 +450,14 @@ public static void Reverse(ref int buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Add(ref last, -Vector256.Count); + last = ref Unsafe.Subtract(ref last, Vector256.Count); numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { - Vector128 ReverseMask = Vector128.Create((byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + Vector128 reverseMask = Vector128.Create((byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); do { // Load in values from beginning and end of the array. @@ -472,8 +472,8 @@ public static void Reverse(ref int buf, nuint length) // +---------------+ // | D | C | B | A | // +---------------+ - tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); - tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + tempFirst = Ssse3.Shuffle(tempFirst, reverseMask); + tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors Unsafe.As>(ref first) = tempLast; @@ -481,7 +481,7 @@ public static void Reverse(ref int buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Add(ref last, -Vector128.Count); + last = ref Unsafe.Subtract(ref last, Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } @@ -494,7 +494,7 @@ public static void Reverse(ref int buf, nuint length) { (lastInt, firstInt) = (firstInt, lastInt); firstInt = ref Unsafe.Add(ref firstInt, 1); - lastInt = ref Unsafe.Add(ref lastInt, -1); + lastInt = ref Unsafe.Subtract(ref lastInt, 1); numBytesWritten += sizeof(int) * 2; } } @@ -504,13 +504,13 @@ public static void Reverse(ref long buf, nuint length) nint numBytes = (int)length * sizeof(long); int numBytesWritten = 0; ref byte first = ref Unsafe.As(ref buf); - ref byte last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -sizeof(long)); + ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(long)); if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 ReverseMask = Vector256.Create( + Vector256 reverseMask = Vector256.Create( (byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // first 128-bit lane 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); // second 128-bit lane - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector256.Count); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { // Load in values from beginning and end of the array. @@ -530,9 +530,9 @@ public static void Reverse(ref long buf, nuint length) // +---------------+ // | D | C | B | A | // +---------------+ - tempFirst = Avx2.Shuffle(tempFirst, ReverseMask); + tempFirst = Avx2.Shuffle(tempFirst, reverseMask); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, ReverseMask); + tempLast = Avx2.Shuffle(tempLast, reverseMask); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors @@ -541,14 +541,14 @@ public static void Reverse(ref long buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Add(ref last, -Vector256.Count); + last = ref Unsafe.Subtract(ref last, Vector256.Count); numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) { - Vector128 ReverseMask = Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); - last = ref Unsafe.Add(ref Unsafe.Add(ref first, numBytes), -Vector128.Count); + Vector128 reverseMask = Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); + last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); do { // Load in values from beginning and end of the array. @@ -563,8 +563,8 @@ public static void Reverse(ref long buf, nuint length) // +-------+ // | B | A | // +-------+ - tempFirst = Ssse3.Shuffle(tempFirst, ReverseMask); - tempLast = Ssse3.Shuffle(tempLast, ReverseMask); + tempFirst = Ssse3.Shuffle(tempFirst, reverseMask); + tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors Unsafe.As>(ref first) = tempLast; @@ -572,7 +572,7 @@ public static void Reverse(ref long buf, nuint length) // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Add(ref last, -Vector128.Count); + last = ref Unsafe.Subtract(ref last, Vector128.Count); numBytesWritten += Vector128.Count * 2; } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); } @@ -585,7 +585,7 @@ public static void Reverse(ref long buf, nuint length) { (lastLong, firstLong) = (firstLong, lastLong); firstLong = ref Unsafe.Add(ref firstLong, 1); - lastLong = ref Unsafe.Add(ref lastLong, -1); + lastLong = ref Unsafe.Subtract(ref lastLong, 1); numBytesWritten += sizeof(long) * 2; } } @@ -624,12 +624,12 @@ private static void ReverseInner(ref T elements, nuint length) { Debug.Assert(length > 0); ref T first = ref elements; - ref T last = ref Unsafe.Add(ref Unsafe.Add(ref first, (int)length), -1); + ref T last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, (int)length), 1); do { (last, first) = (first, last); first = ref Unsafe.Add(ref first, 1); - last = ref Unsafe.Add(ref last, -1); + last = ref Unsafe.Subtract(ref last, 1); } while (Unsafe.IsAddressLessThan(ref first, ref last)); } } From 3c3f1403cd41c5e31305878906954e0e6c47c024 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Fri, 18 Mar 2022 15:48:24 -0700 Subject: [PATCH 10/14] Camel case formatting, add condition check for Array.Reverse to avoid reversing empty or single-element array, better shuffle for int and long Reverse using bit control mask instead of vector control mask --- .../src/System/Array.cs | 3 +- .../src/System/SpanHelpers.Byte.cs | 5 +- .../src/System/SpanHelpers.cs | 61 ++++++++----------- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs index e7341aa738c5f..d7eebc759d796 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Array.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs @@ -1722,7 +1722,8 @@ public static void Reverse(T[] array) { if (array == null) ThrowHelper.ThrowArgumentNullException(ExceptionArgument.array); - SpanHelpers.Reverse(ref MemoryMarshal.GetArrayDataReference(array), (nuint)array.Length); + if (array.Length > 1) + SpanHelpers.Reverse(ref MemoryMarshal.GetArrayDataReference(array), (nuint)array.Length); } public static void Reverse(T[] array, int index, int length) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 97e67ce095e9b..dc90558130cf1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2247,8 +2247,9 @@ public static void Reverse(ref byte buf, nuint length) int numBytesWritten = 0; if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 reverseMask = Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // first 128-bit lane - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); // second 128-bit lane + Vector256 reverseMask = Vector256.Create( + (byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, // first 128-bit lane + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); // first 128-bit lane last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index b25f52aaca05f..7393d1f316c83 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -6,7 +6,6 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; -using Internal.Runtime.CompilerServices; namespace System { @@ -416,15 +415,12 @@ public static void Reverse(ref int buf, nuint length) ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(int)); if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 reverseMask = Vector256.Create( - (byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, // first 128-bit lane - 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); // second 128-bit lane last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { // Load in values from beginning and end of the array. - Vector256 tempFirst = Unsafe.As>(ref first); - Vector256 tempLast = Unsafe.As>(ref last); + Vector256 tempFirst = Unsafe.As>(ref first); + Vector256 tempLast = Unsafe.As>(ref last); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: @@ -439,14 +435,14 @@ public static void Reverse(ref int buf, nuint length) // +-------------------------------+ // | H | G | F | E | D | C | B | A | // +-------------------------------+ - tempFirst = Avx2.Shuffle(tempFirst, reverseMask); + tempFirst = Avx2.Shuffle(tempFirst, 0b00011011); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, reverseMask); + tempLast = Avx2.Shuffle(tempLast, 0b00011011); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); @@ -454,15 +450,14 @@ public static void Reverse(ref int buf, nuint length) numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) + else if (Sse2.IsSupported && Vector128.Count * 2 <= numBytes) { - Vector128 reverseMask = Vector128.Create((byte)12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3); last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); do { // Load in values from beginning and end of the array. - Vector128 tempFirst = Unsafe.As>(ref first); - Vector128 tempLast = Unsafe.As>(ref last); + Vector128 tempFirst = Unsafe.As>(ref first); + Vector128 tempLast = Unsafe.As>(ref last); // Shuffle to reverse each vector: // +---------------+ @@ -472,12 +467,12 @@ public static void Reverse(ref int buf, nuint length) // +---------------+ // | D | C | B | A | // +---------------+ - tempFirst = Ssse3.Shuffle(tempFirst, reverseMask); - tempLast = Ssse3.Shuffle(tempLast, reverseMask); + tempFirst = Sse2.Shuffle(tempFirst, 0b00011011); + tempLast = Sse2.Shuffle(tempLast, 0b00011011); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); @@ -507,15 +502,12 @@ public static void Reverse(ref long buf, nuint length) ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(long)); if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) { - Vector256 reverseMask = Vector256.Create( - (byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // first 128-bit lane - 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); // second 128-bit lane last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); do { // Load in values from beginning and end of the array. - Vector256 tempFirst = Unsafe.As>(ref first); - Vector256 tempLast = Unsafe.As>(ref last); + Vector256 tempFirst = Unsafe.As>(ref first); + Vector256 tempLast = Unsafe.As>(ref last); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: @@ -530,14 +522,14 @@ public static void Reverse(ref long buf, nuint length) // +---------------+ // | D | C | B | A | // +---------------+ - tempFirst = Avx2.Shuffle(tempFirst, reverseMask); + tempFirst = Avx2.Shuffle(tempFirst, 0b01001110); tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, reverseMask); + tempLast = Avx2.Shuffle(tempLast, 0b01001110); tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector256.Count); @@ -545,15 +537,14 @@ public static void Reverse(ref long buf, nuint length) numBytesWritten += Vector256.Count * 2; } while (numBytes - numBytesWritten >= Vector256.Count * 2); } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) + else if (Sse2.IsSupported && Vector128.Count * 2 <= numBytes) { - Vector128 reverseMask = Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7); last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); do { // Load in values from beginning and end of the array. - Vector128 tempFirst = Unsafe.As>(ref first); - Vector128 tempLast = Unsafe.As>(ref last); + Vector128 tempFirst = Unsafe.As>(ref first); + Vector128 tempLast = Unsafe.As>(ref last); // Shuffle to reverse each vector: // +-------+ @@ -563,12 +554,12 @@ public static void Reverse(ref long buf, nuint length) // +-------+ // | B | A | // +-------+ - tempFirst = Ssse3.Shuffle(tempFirst, reverseMask); - tempLast = Ssse3.Shuffle(tempLast, reverseMask); + tempFirst = Sse2.Shuffle(tempFirst, 0b01001110); + tempLast = Sse2.Shuffle(tempLast, 0b01001110); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; + Unsafe.As>(ref first) = tempLast; + Unsafe.As>(ref last) = tempFirst; // Adjust the references to point to the next vector first = ref Unsafe.Add(ref first, Vector128.Count); From 94f45cc1a812d8ce574f4a99210fe67a7b83b0dd Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Fri, 25 Mar 2022 16:54:08 -0700 Subject: [PATCH 11/14] Rework loops to use new LoadUnsafe/StoreUnsafe vector APIs. Use Permute4x64 and PermuteVar8x32 for Int32 and Int64 respectively to reduce total operations. --- .../src/System/SpanHelpers.Byte.cs | 66 +++--- .../src/System/SpanHelpers.Char.cs | 73 ++++--- .../src/System/SpanHelpers.cs | 194 ++++++++---------- 3 files changed, 152 insertions(+), 181 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index dc90558130cf1..43d2524580265 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2241,21 +2241,21 @@ private static uint FindFirstMatchedLane(Vector128 compareResult) public static void Reverse(ref byte buf, nuint length) { - int numBytes = (int)length; - ref byte first = ref buf; - ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), 1); - int numBytesWritten = 0; - if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { Vector256 reverseMask = Vector256.Create( (byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, // first 128-bit lane 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); // first 128-bit lane - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + // Load in values from beginning and end of the array. - Vector256 tempFirst = Unsafe.As>(ref first); - Vector256 tempLast = Unsafe.As>(ref last); + Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: @@ -2282,24 +2282,25 @@ public static void Reverse(ref byte buf, nuint length) tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Subtract(ref last, Vector256.Count); - numBytesWritten += Vector256.Count * 2; - } while ((numBytes - numBytesWritten) >= Vector256.Count * 2); + Vector256.StoreUnsafe(tempLast, ref buf, firstOffset); + Vector256.StoreUnsafe(tempFirst, ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) + else if (Sse2.IsSupported && (nuint)Vector128.Count * 2 <= length) { Vector128 reverseMask = Vector128.Create((byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); - do + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + // Load in values from beginning and end of the array. - Vector128 tempFirst = Unsafe.As>(ref first); - Vector128 tempLast = Unsafe.As>(ref last); + Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref buf, lastOffset); // Shuffle to reverse each vector: // +---------------------------------------------------------------+ @@ -2313,24 +2314,19 @@ public static void Reverse(ref byte buf, nuint length) tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Subtract(ref last, Vector128.Count); - numBytesWritten += Vector128.Count * 2; - } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + Vector128.StoreUnsafe(tempLast, ref buf, firstOffset); + Vector128.StoreUnsafe(tempFirst, ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } // Store any remaining values one-by-one - last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - 1); - while ((numBytes - numBytesWritten) > 1) + for (nuint i = 0; i < (length / 2); i++) { + ref byte first = ref Unsafe.Add(ref buf, i); + ref byte last = ref Unsafe.Add(ref buf, length - 1 - i); (last, first) = (first, last); - first = ref Unsafe.Add(ref first, 1); - last = ref Unsafe.Subtract(ref last, 1); - numBytesWritten += 2; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index 7b2989bfac102..e3bd8565bdddd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2018,21 +2018,23 @@ private static int FindFirstMatchedLane(Vector128 compareResult) public static void Reverse(ref char buf, nuint length) { - nint numBytes = (int)length * sizeof(char); - int numBytesWritten = 0; - ref byte first = ref Unsafe.As(ref buf); - ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(char)); - if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + ref byte bufByte = ref Unsafe.As(ref buf); + nuint byteLength = length * sizeof(char); + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { Vector256 reverseMask = Vector256.Create( (byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, // first 128-bit lane 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); // second 128-bit lane - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (byteLength / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { + nuint firstOffset = i * numElements; + nuint lastOffset = byteLength - ((1 + i) * numElements); + // Load in values from beginning and end of the array. - Vector256 tempFirst = Unsafe.As>(ref first); - Vector256 tempLast = Unsafe.As>(ref last); + Vector256 tempFirst = Vector256.LoadUnsafe(ref bufByte, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref bufByte, lastOffset); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: @@ -2053,24 +2055,25 @@ public static void Reverse(ref char buf, nuint length) tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Subtract(ref last, Vector256.Count); - numBytesWritten += Vector256.Count * 2; - } while (numBytes - numBytesWritten >= Vector256.Count * 2); + Vector256.StoreUnsafe(tempLast, ref bufByte, firstOffset); + Vector256.StoreUnsafe(tempFirst, ref bufByte, lastOffset); + } + bufByte = ref Unsafe.Add(ref bufByte, numIters * numElements); + length -= numIters * (nuint)Vector256.Count * 2; } - else if (Ssse3.IsSupported && Vector128.Count * 2 <= numBytes) + else if (Sse2.IsSupported && (nuint)Vector128.Count * 2 <= length) { Vector128 reverseMask = Vector128.Create((byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); - do + nuint numElements = (nuint)Vector128.Count; + nuint numIters = ((length * sizeof(char)) / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { + nuint firstOffset = i * numElements; + nuint lastOffset = byteLength - ((1 + i) * numElements); + // Load in values from beginning and end of the array. - Vector128 tempFirst = Unsafe.As>(ref first); - Vector128 tempLast = Unsafe.As>(ref last); + Vector128 tempFirst = Vector128.LoadUnsafe(ref bufByte, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref bufByte, lastOffset); // Shuffle to reverse each vector: // +-------------------------------+ @@ -2084,26 +2087,20 @@ public static void Reverse(ref char buf, nuint length) tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Subtract(ref last, Vector128.Count); - numBytesWritten += Vector128.Count * 2; - } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + Vector128.StoreUnsafe(tempLast, ref bufByte, firstOffset); + Vector128.StoreUnsafe(tempFirst, ref bufByte, lastOffset); + } + bufByte = ref Unsafe.Add(ref bufByte, numIters * numElements); + length -= numIters * (nuint)Vector128.Count * 2; } // Store any remaining values one-by-one - last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(char)); - ref char firstChar = ref Unsafe.As(ref first); - ref char lastChar = ref Unsafe.As(ref last); - while (numBytes - numBytesWritten > sizeof(char)) + buf = ref Unsafe.As(ref bufByte); + for (nuint i = 0; i < (length / 2); i++) { - (lastChar, firstChar) = (firstChar, lastChar); - firstChar = ref Unsafe.Add(ref firstChar, 1); - lastChar = ref Unsafe.Subtract(ref lastChar, 1); - numBytesWritten += sizeof(char) * 2; + ref char first = ref Unsafe.Add(ref buf, i); + ref char last = ref Unsafe.Add(ref buf, length - 1 - i); + (last, first) = (first, last); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index 7393d1f316c83..58a24ee81c2c2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -409,55 +409,51 @@ public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLe public static void Reverse(ref int buf, nuint length) { - int numBytes = (int)length * sizeof(int); - int numBytesWritten = 0; - ref byte first = ref Unsafe.As(ref buf); - ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(int)); - if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (length / numElements) / 2; + Vector256 reverseMask = Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0); + for (nuint i = 0; i < numIters; i++) { - // Load in values from beginning and end of the array. - Vector256 tempFirst = Unsafe.As>(ref first); - Vector256 tempLast = Unsafe.As>(ref last); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + + // Load the values into vectors + Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: // +-------------------------------+ // | A | B | C | D | E | F | G | H | // +-------------------------------+ - // Shuffle ---> - // +-------------------------------+ - // | D | C | B | A | H | G | F | E | - // +-------------------------------+ - // Permute ---> + // ---> // +-------------------------------+ // | H | G | F | E | D | C | B | A | // +-------------------------------+ - tempFirst = Avx2.Shuffle(tempFirst, 0b00011011); - tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, 0b00011011); - tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); - - // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Subtract(ref last, Vector256.Count); - numBytesWritten += Vector256.Count * 2; - } while (numBytes - numBytesWritten >= Vector256.Count * 2); + tempFirst = Avx2.PermuteVar8x32(tempFirst, reverseMask); + tempLast = Avx2.PermuteVar8x32(tempLast, reverseMask); + + // Store the values into final location + Vector256.StoreUnsafe(tempLast, ref buf, firstOffset); + Vector256.StoreUnsafe(tempFirst, ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - else if (Sse2.IsSupported && Vector128.Count * 2 <= numBytes) + else if (Sse2.IsSupported && (nuint)Vector128.Count * 2 <= length) { - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); - do + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load in values from beginning and end of the array. - Vector128 tempFirst = Unsafe.As>(ref first); - Vector128 tempLast = Unsafe.As>(ref last); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + + // Load the values into vectors + Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref buf, lastOffset); // Shuffle to reverse each vector: // +---------------+ @@ -467,84 +463,72 @@ public static void Reverse(ref int buf, nuint length) // +---------------+ // | D | C | B | A | // +---------------+ - tempFirst = Sse2.Shuffle(tempFirst, 0b00011011); - tempLast = Sse2.Shuffle(tempLast, 0b00011011); - - // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Subtract(ref last, Vector128.Count); - numBytesWritten += Vector128.Count * 2; - } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + tempFirst = Sse2.Shuffle(tempFirst, 0b00_01_10_11); + tempLast = Sse2.Shuffle(tempLast, 0b00_01_10_11); + + // Store the values into final location + Vector128.StoreUnsafe(tempLast, ref buf, firstOffset); + Vector128.StoreUnsafe(tempFirst, ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } // Store any remaining values one-by-one - last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(int)); - ref int firstInt = ref Unsafe.As(ref first); - ref int lastInt = ref Unsafe.As(ref last); - while (numBytes - numBytesWritten > sizeof(int)) + for (nuint i = 0; i < (length / 2); i++) { + ref int firstInt = ref Unsafe.Add(ref buf, i); + ref int lastInt = ref Unsafe.Add(ref buf, length - 1 - i); (lastInt, firstInt) = (firstInt, lastInt); - firstInt = ref Unsafe.Add(ref firstInt, 1); - lastInt = ref Unsafe.Subtract(ref lastInt, 1); - numBytesWritten += sizeof(int) * 2; } } public static void Reverse(ref long buf, nuint length) { - nint numBytes = (int)length * sizeof(long); - int numBytesWritten = 0; - ref byte first = ref Unsafe.As(ref buf); - ref byte last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), sizeof(long)); - if (Avx2.IsSupported && Vector256.Count * 2 <= numBytes) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector256.Count); - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load in values from beginning and end of the array. - Vector256 tempFirst = Unsafe.As>(ref first); - Vector256 tempLast = Unsafe.As>(ref last); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + // Load the values into vectors + Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: // +---------------+ // | A | B | C | D | // +---------------+ - // Shuffle ---> - // +---------------+ - // | B | A | D | C | - // +---------------+ - // Permute ---> + // ---> // +---------------+ // | D | C | B | A | // +---------------+ - tempFirst = Avx2.Shuffle(tempFirst, 0b01001110); - tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); - tempLast = Avx2.Shuffle(tempLast, 0b01001110); - tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); - - // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector256.Count); - last = ref Unsafe.Subtract(ref last, Vector256.Count); - numBytesWritten += Vector256.Count * 2; - } while (numBytes - numBytesWritten >= Vector256.Count * 2); + tempFirst = Avx2.Permute4x64(tempFirst, 0b00_01_10_11); + tempLast = Avx2.Permute4x64(tempLast, 0b00_01_10_11); + + // Store the values into final location + Vector256.StoreUnsafe(tempLast, ref buf, firstOffset); + Vector256.StoreUnsafe(tempFirst, ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - else if (Sse2.IsSupported && Vector128.Count * 2 <= numBytes) + else if (Sse2.IsSupported && (nuint)Vector128.Count * 2 <= length) { - last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, numBytes), Vector128.Count); - do + ref int bufInt = ref Unsafe.As(ref buf); + nuint intLength = length * (sizeof(long) / sizeof(int)); + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (intLength / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load in values from beginning and end of the array. - Vector128 tempFirst = Unsafe.As>(ref first); - Vector128 tempLast = Unsafe.As>(ref last); + nuint firstOffset = i * numElements; + nuint lastOffset = intLength - ((1 + i) * numElements); + // Load the values into vectors + Vector128 tempFirst = Vector128.LoadUnsafe(ref bufInt, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref bufInt, lastOffset); // Shuffle to reverse each vector: // +-------+ @@ -554,30 +538,24 @@ public static void Reverse(ref long buf, nuint length) // +-------+ // | B | A | // +-------+ - tempFirst = Sse2.Shuffle(tempFirst, 0b01001110); - tempLast = Sse2.Shuffle(tempLast, 0b01001110); - - // Store the reversed vectors - Unsafe.As>(ref first) = tempLast; - Unsafe.As>(ref last) = tempFirst; - - // Adjust the references to point to the next vector - first = ref Unsafe.Add(ref first, Vector128.Count); - last = ref Unsafe.Subtract(ref last, Vector128.Count); - numBytesWritten += Vector128.Count * 2; - } while ((numBytes - numBytesWritten) >= Vector128.Count * 2); + tempFirst = Sse2.Shuffle(tempFirst, 0b0100_1110); + tempLast = Sse2.Shuffle(tempLast, 0b0100_1110); + + // Store the values into final location + Vector128.StoreUnsafe(tempLast, ref bufInt, firstOffset); + Vector128.StoreUnsafe(tempFirst, ref bufInt, lastOffset); + } + bufInt = ref Unsafe.Add(ref bufInt, numIters * numElements); + buf = ref Unsafe.As(ref bufInt); + length -= numIters * (nuint)Vector128.Count * 2; } // Store any remaining values one-by-one - last = ref Unsafe.Add(ref first, (numBytes - numBytesWritten) - sizeof(long)); - ref long firstLong = ref Unsafe.As(ref first); - ref long lastLong = ref Unsafe.As(ref last); - while (numBytes - numBytesWritten > sizeof(long)) + for (nuint i = 0; i < (length / 2); i++) { + ref long firstLong = ref Unsafe.Add(ref buf, i); + ref long lastLong = ref Unsafe.Add(ref buf, length - 1 - i); (lastLong, firstLong) = (firstLong, lastLong); - firstLong = ref Unsafe.Add(ref firstLong, 1); - lastLong = ref Unsafe.Subtract(ref lastLong, 1); - numBytesWritten += sizeof(long) * 2; } } From 39e906fe208c31b5fa7acf00e02259874c8c5888 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Mon, 28 Mar 2022 13:04:04 -0700 Subject: [PATCH 12/14] Improve readability of code --- .../src/System/SpanHelpers.Byte.cs | 12 ++++++------ .../src/System/SpanHelpers.Char.cs | 12 ++++++------ .../src/System/SpanHelpers.cs | 16 ++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 43d2524580265..83ca747c6896a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2277,13 +2277,13 @@ public static void Reverse(ref byte buf, nuint length) // | P1 | O1 | N1 | M1 | L1 | K1 | J1 | I1 | H1 | G1 | F1 | E1 | D1 | C1 | B1 | A1 | // +-------------------------------------------------------------------------------+ tempFirst = Avx2.Shuffle(tempFirst, reverseMask); - tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); + tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 0b00_01); tempLast = Avx2.Shuffle(tempLast, reverseMask); - tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + tempLast = Avx2.Permute2x128(tempLast, tempLast, 0b00_01); // Store the reversed vectors - Vector256.StoreUnsafe(tempLast, ref buf, firstOffset); - Vector256.StoreUnsafe(tempFirst, ref buf, lastOffset); + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); } buf = ref Unsafe.Add(ref buf, numIters * numElements); length -= numIters * numElements * 2; @@ -2314,8 +2314,8 @@ public static void Reverse(ref byte buf, nuint length) tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors - Vector128.StoreUnsafe(tempLast, ref buf, firstOffset); - Vector128.StoreUnsafe(tempFirst, ref buf, lastOffset); + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); } buf = ref Unsafe.Add(ref buf, numIters * numElements); length -= numIters * numElements * 2; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index e3bd8565bdddd..24994c6504138 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -2050,13 +2050,13 @@ public static void Reverse(ref char buf, nuint length) // | P | O | N | M | L | K | J | I | H | G | F | E | D | C | B | A | // +---------------------------------------------------------------+ tempFirst = Avx2.Shuffle(tempFirst, reverseMask); - tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 1); + tempFirst = Avx2.Permute2x128(tempFirst, tempFirst, 0b00_01); tempLast = Avx2.Shuffle(tempLast, reverseMask); - tempLast = Avx2.Permute2x128(tempLast, tempLast, 1); + tempLast = Avx2.Permute2x128(tempLast, tempLast, 0b00_01); // Store the reversed vectors - Vector256.StoreUnsafe(tempLast, ref bufByte, firstOffset); - Vector256.StoreUnsafe(tempFirst, ref bufByte, lastOffset); + tempLast.StoreUnsafe(ref bufByte, firstOffset); + tempFirst.StoreUnsafe(ref bufByte, lastOffset); } bufByte = ref Unsafe.Add(ref bufByte, numIters * numElements); length -= numIters * (nuint)Vector256.Count * 2; @@ -2087,8 +2087,8 @@ public static void Reverse(ref char buf, nuint length) tempLast = Ssse3.Shuffle(tempLast, reverseMask); // Store the reversed vectors - Vector128.StoreUnsafe(tempLast, ref bufByte, firstOffset); - Vector128.StoreUnsafe(tempFirst, ref bufByte, lastOffset); + tempLast.StoreUnsafe(ref bufByte, firstOffset); + tempFirst.StoreUnsafe(ref bufByte, lastOffset); } bufByte = ref Unsafe.Add(ref bufByte, numIters * numElements); length -= numIters * (nuint)Vector128.Count * 2; diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index 58a24ee81c2c2..b79d676cceb2a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -436,8 +436,8 @@ public static void Reverse(ref int buf, nuint length) tempLast = Avx2.PermuteVar8x32(tempLast, reverseMask); // Store the values into final location - Vector256.StoreUnsafe(tempLast, ref buf, firstOffset); - Vector256.StoreUnsafe(tempFirst, ref buf, lastOffset); + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); } buf = ref Unsafe.Add(ref buf, numIters * numElements); length -= numIters * numElements * 2; @@ -467,8 +467,8 @@ public static void Reverse(ref int buf, nuint length) tempLast = Sse2.Shuffle(tempLast, 0b00_01_10_11); // Store the values into final location - Vector128.StoreUnsafe(tempLast, ref buf, firstOffset); - Vector128.StoreUnsafe(tempFirst, ref buf, lastOffset); + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); } buf = ref Unsafe.Add(ref buf, numIters * numElements); length -= numIters * numElements * 2; @@ -510,8 +510,8 @@ public static void Reverse(ref long buf, nuint length) tempLast = Avx2.Permute4x64(tempLast, 0b00_01_10_11); // Store the values into final location - Vector256.StoreUnsafe(tempLast, ref buf, firstOffset); - Vector256.StoreUnsafe(tempFirst, ref buf, lastOffset); + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); } buf = ref Unsafe.Add(ref buf, numIters * numElements); length -= numIters * numElements * 2; @@ -542,8 +542,8 @@ public static void Reverse(ref long buf, nuint length) tempLast = Sse2.Shuffle(tempLast, 0b0100_1110); // Store the values into final location - Vector128.StoreUnsafe(tempLast, ref bufInt, firstOffset); - Vector128.StoreUnsafe(tempFirst, ref bufInt, lastOffset); + tempLast.StoreUnsafe(ref bufInt, firstOffset); + tempFirst.StoreUnsafe(ref bufInt, lastOffset); } bufInt = ref Unsafe.Add(ref bufInt, numIters * numElements); buf = ref Unsafe.As(ref bufInt); From 1ed5bef98d9c79b3a995b8e6ef8022612ac1ab39 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Mon, 25 Apr 2022 08:58:16 -0700 Subject: [PATCH 13/14] Fix formatting, fix typos in comments --- .../System.Private.CoreLib/src/System/MemoryExtensions.cs | 6 ++---- .../System.Private.CoreLib/src/System/SpanHelpers.Byte.cs | 2 +- .../System.Private.CoreLib/src/System/SpanHelpers.cs | 6 ++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index e232a4f19867c..2d8ae3f523a43 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1545,14 +1545,12 @@ ref MemoryMarshal.GetReference(value), /// public static void Reverse(this Span span) { - if (span.Length <= 1) + if (span.Length > 1) { - return; + SpanHelpers.Reverse(ref MemoryMarshal.GetReference(span), (nuint)span.Length); } - SpanHelpers.Reverse(ref MemoryMarshal.GetReference(span), (nuint)span.Length); } - /// /// Creates a new span over the target array. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 83ca747c6896a..58bf4c6030486 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -2245,7 +2245,7 @@ public static void Reverse(ref byte buf, nuint length) { Vector256 reverseMask = Vector256.Create( (byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, // first 128-bit lane - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); // first 128-bit lane + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); // second 128-bit lane nuint numElements = (nuint)Vector256.Count; nuint numIters = (length / numElements) / 2; for (nuint i = 0; i < numIters; i++) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index b79d676cceb2a..aa0db3f3a8256 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -423,8 +423,7 @@ public static void Reverse(ref int buf, nuint length) Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); - // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. - // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: + // Permute to reverse each vector: // +-------------------------------+ // | A | B | C | D | E | F | G | H | // +-------------------------------+ @@ -497,8 +496,7 @@ public static void Reverse(ref long buf, nuint length) Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); - // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. - // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: + // Permute to reverse each vector: // +---------------+ // | A | B | C | D | // +---------------+ From 80ae8abd83d30b0b45f1180d4fb0a008382fef55 Mon Sep 17 00:00:00 2001 From: Alex Covington Date: Mon, 25 Apr 2022 10:16:05 -0700 Subject: [PATCH 14/14] Use temporary variable for generic case instead for better IL --- .../System.Private.CoreLib/src/System/SpanHelpers.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index aa0db3f3a8256..07bf18d04e7ee 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -594,7 +594,9 @@ private static void ReverseInner(ref T elements, nuint length) ref T last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, (int)length), 1); do { - (last, first) = (first, last); + T temp = first; + first = last; + last = temp; first = ref Unsafe.Add(ref first, 1); last = ref Unsafe.Subtract(ref last, 1); } while (Unsafe.IsAddressLessThan(ref first, ref last));