diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h
index 21bd50642d4b7..4754e0a294fe8 100644
--- a/src/coreclr/inc/readytorun.h
+++ b/src/coreclr/inc/readytorun.h
@@ -14,11 +14,13 @@
#define READYTORUN_SIGNATURE 0x00525452 // 'RTR'
-// Keep these in sync with src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
-#define READYTORUN_MAJOR_VERSION 0x0008
+// Keep these in sync with
+// src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
+// src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
+#define READYTORUN_MAJOR_VERSION 0x0009
#define READYTORUN_MINOR_VERSION 0x0000
-#define MINIMUM_READYTORUN_MAJOR_VERSION 0x008
+#define MINIMUM_READYTORUN_MAJOR_VERSION 0x009
// R2R Version 2.1 adds the InliningInfo section
// R2R Version 2.2 adds the ProfileDataInfo section
@@ -27,6 +29,7 @@
// R2R Version 6.0 changes managed layout for sequential types with any unmanaged non-blittable fields.
// R2R 6.0 is not backward compatible with 5.x or earlier.
// R2R Version 8.0 Changes the alignment of the Int128 type
+// R2R Version 9.0 adds support for the Vector512 type
struct READYTORUN_CORE_HEADER
{
diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp
index 78357b2918cb5..9603fd7c1bb6b 100644
--- a/src/coreclr/jit/hwintrinsicarm64.cpp
+++ b/src/coreclr/jit/hwintrinsicarm64.cpp
@@ -762,22 +762,25 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
{
assert(sig->numArgs == 2);
- if (varTypeIsFloating(simdBaseType))
+ if (!varTypeIsFloating(simdBaseType))
{
- CORINFO_ARG_LIST_HANDLE arg1 = sig->args;
- CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(arg1);
- var_types argType = TYP_UNKNOWN;
- CORINFO_CLASS_HANDLE argClass = NO_CLASS_HANDLE;
+ // We can't trivially handle division for integral types using SIMD
+ break;
+ }
+
+ CORINFO_ARG_LIST_HANDLE arg1 = sig->args;
+ CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(arg1);
+ var_types argType = TYP_UNKNOWN;
+ CORINFO_CLASS_HANDLE argClass = NO_CLASS_HANDLE;
- argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
- op2 = getArgForHWIntrinsic(argType, argClass);
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
+ op2 = getArgForHWIntrinsic(argType, argClass);
- argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass)));
- op1 = getArgForHWIntrinsic(argType, argClass);
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass)));
+ op1 = getArgForHWIntrinsic(argType, argClass);
- retNode = gtNewSimdBinOpNode(GT_DIV, retType, op1, op2, simdBaseJitType, simdSize,
- /* isSimdAsHWIntrinsic */ false);
- }
+ retNode = gtNewSimdBinOpNode(GT_DIV, retType, op1, op2, simdBaseJitType, simdSize,
+ /* isSimdAsHWIntrinsic */ false);
break;
}
diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp
index 2f79e46fb3585..7c9479bbc5caa 100644
--- a/src/coreclr/jit/hwintrinsicxarch.cpp
+++ b/src/coreclr/jit/hwintrinsicxarch.cpp
@@ -1145,23 +1145,25 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic intrinsic,
{
assert(sig->numArgs == 2);
- if (varTypeIsFloating(simdBaseType))
+ if (!varTypeIsFloating(simdBaseType))
{
- CORINFO_ARG_LIST_HANDLE arg1 = sig->args;
- CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(arg1);
- var_types argType = TYP_UNKNOWN;
- CORINFO_CLASS_HANDLE argClass = NO_CLASS_HANDLE;
+ // We can't trivially handle division for integral types using SIMD
+ break;
+ }
- argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
- op2 = getArgForHWIntrinsic(argType, argClass);
+ CORINFO_ARG_LIST_HANDLE arg1 = sig->args;
+ CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(arg1);
+ var_types argType = TYP_UNKNOWN;
+ CORINFO_CLASS_HANDLE argClass = NO_CLASS_HANDLE;
- argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass)));
- op1 = getArgForHWIntrinsic(argType, argClass);
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
+ op2 = getArgForHWIntrinsic(argType, argClass);
- retNode = gtNewSimdBinOpNode(GT_DIV, retType, op1, op2, simdBaseJitType, simdSize,
- /* isSimdAsHWIntrinsic */ false);
- break;
- }
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass)));
+ op1 = getArgForHWIntrinsic(argType, argClass);
+
+ retNode = gtNewSimdBinOpNode(GT_DIV, retType, op1, op2, simdBaseJitType, simdSize,
+ /* isSimdAsHWIntrinsic */ false);
break;
}
@@ -2061,7 +2063,7 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic intrinsic,
{
assert(sig->numArgs == 2);
- if (varTypeIsByte(simdBaseType) || varTypeIsLong(simdBaseType))
+ if (varTypeIsByte(simdBaseType) || varTypeIsLong(simdBaseType) || (simdBaseType == TYP_DOUBLE))
{
// byte, sbyte, long, and ulong would require more work to support
break;
diff --git a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
index 6f77813cd0614..7d0c985486c9d 100644
--- a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
+++ b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
@@ -4,13 +4,14 @@
//
// Please keep the data structures in this file in sync with the managed version at
// src/Common/src/Internal/Runtime/ModuleHeaders.cs
-//
+// src/coreclr/inc/readytorun.h
+
struct ReadyToRunHeaderConstants
{
static const uint32_t Signature = 0x00525452; // 'RTR'
- static const uint32_t CurrentMajorVersion = 8;
+ static const uint32_t CurrentMajorVersion = 9;
static const uint32_t CurrentMinorVersion = 0;
};
diff --git a/src/coreclr/tools/Common/Compiler/VectorFieldLayoutAlgorithm.cs b/src/coreclr/tools/Common/Compiler/VectorFieldLayoutAlgorithm.cs
index ae5d8c58c2a9c..ac5247190775a 100644
--- a/src/coreclr/tools/Common/Compiler/VectorFieldLayoutAlgorithm.cs
+++ b/src/coreclr/tools/Common/Compiler/VectorFieldLayoutAlgorithm.cs
@@ -44,9 +44,28 @@ public override ComputedInstanceFieldLayout ComputeInstanceLayout(DefType defTyp
alignment = new LayoutInt(16);
}
}
+ else if (name == "Vector256`1")
+ {
+ if (defType.Context.Target.Architecture == TargetArchitecture.ARM)
+ {
+ // No such type exists for the Procedure Call Standard for ARM. We will default
+ // to the same alignment as __m128, which is supported by the ABI.
+ alignment = new LayoutInt(8);
+ }
+ else if (defType.Context.Target.Architecture == TargetArchitecture.ARM64)
+ {
+ // The Procedure Call Standard for ARM 64-bit (with SVE support) defaults to
+ // 16-byte alignment for __m256.
+ alignment = new LayoutInt(16);
+ }
+ else
+ {
+ alignment = new LayoutInt(32);
+ }
+ }
else
{
- Debug.Assert(name == "Vector256`1");
+ Debug.Assert(name == "Vector512`1");
if (defType.Context.Target.Architecture == TargetArchitecture.ARM)
{
@@ -62,7 +81,7 @@ public override ComputedInstanceFieldLayout ComputeInstanceLayout(DefType defTyp
}
else
{
- alignment = new LayoutInt(32);
+ alignment = new LayoutInt(64);
}
}
@@ -106,6 +125,7 @@ public override ValueTypeShapeCharacteristics ComputeValueTypeShapeCharacteristi
8 => ValueTypeShapeCharacteristics.Vector64Aggregate,
16 => ValueTypeShapeCharacteristics.Vector128Aggregate,
32 => ValueTypeShapeCharacteristics.Vector128Aggregate,
+ 64 => ValueTypeShapeCharacteristics.Vector128Aggregate,
_ => ValueTypeShapeCharacteristics.None
};
}
@@ -116,9 +136,10 @@ public static bool IsVectorType(DefType type)
{
return type.IsIntrinsic &&
type.Namespace == "System.Runtime.Intrinsics" &&
- (type.Name == "Vector64`1" ||
- type.Name == "Vector128`1" ||
- type.Name == "Vector256`1");
+ ((type.Name == "Vector64`1") ||
+ (type.Name == "Vector128`1") ||
+ (type.Name == "Vector256`1") ||
+ (type.Name == "Vector512`1"));
}
}
}
diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
index 144bd50f949db..f198538949666 100644
--- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
+++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
@@ -8,13 +8,14 @@ namespace Internal.Runtime
//
// Please keep the data structures in this file in sync with the native version at
// src/coreclr/inc/readytorun.h
+ // src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
//
internal struct ReadyToRunHeaderConstants
{
public const uint Signature = 0x00525452; // 'RTR'
- public const ushort CurrentMajorVersion = 8;
+ public const ushort CurrentMajorVersion = 9;
public const ushort CurrentMinorVersion = 0;
}
#if READYTORUN
diff --git a/src/coreclr/tools/Common/TypeSystem/CodeGen/TargetDetails.CodeGen.cs b/src/coreclr/tools/Common/TypeSystem/CodeGen/TargetDetails.CodeGen.cs
index 353399eb78b7f..d02249f296890 100644
--- a/src/coreclr/tools/Common/TypeSystem/CodeGen/TargetDetails.CodeGen.cs
+++ b/src/coreclr/tools/Common/TypeSystem/CodeGen/TargetDetails.CodeGen.cs
@@ -40,5 +40,10 @@ public enum SimdVectorLength
/// Specifies that native vectors are 256 bit (e.g. AVX on x86).
///
Vector256Bit,
+
+ ///
+ /// Specifies that native vectors are 512 bit (e.g. AVX512 on x86).
+ ///
+ Vector512Bit,
}
}
diff --git a/src/coreclr/tools/Common/TypeSystem/Common/TargetDetails.cs b/src/coreclr/tools/Common/TypeSystem/Common/TargetDetails.cs
index a090cbf1b0542..ef77bfa5d10df 100644
--- a/src/coreclr/tools/Common/TypeSystem/Common/TargetDetails.cs
+++ b/src/coreclr/tools/Common/TypeSystem/Common/TargetDetails.cs
@@ -109,12 +109,12 @@ public int MaximumAlignment
{
if (Architecture == TargetArchitecture.ARM)
{
- // Corresponds to alignment required for __m128 (there's no __m256)
+ // Corresponds to alignment required for __m128 (there's no __m256/__m512)
return 8;
}
else if (Architecture == TargetArchitecture.ARM64)
{
- // Corresponds to alignmet required for __m256
+ // Corresponds to alignmet required for __m128 (there's no __m256/__m512)
return 16;
}
else if (Architecture == TargetArchitecture.LoongArch64)
@@ -122,8 +122,8 @@ public int MaximumAlignment
return 16;
}
- // 256-bit vector is the type with the highest alignment we support
- return 32;
+ // 512-bit vector is the type with the highest alignment we support
+ return 64;
}
}
@@ -136,8 +136,8 @@ public int DefaultPackingSize
{
get
{
- // We use default packing size of 32 irrespective of the platform.
- return 32;
+ // We use default packing size of 64 irrespective of the platform.
+ return 64;
}
}
diff --git a/src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs b/src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs
index 91a9834603764..6402d5e535a2a 100644
--- a/src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs
+++ b/src/coreclr/tools/Common/TypeSystem/Interop/IL/MarshalHelpers.cs
@@ -874,6 +874,7 @@ private static bool IsValidForGenericMarshalling(
// * Vector64: Represents the __m64 ABI primitive which requires currently unimplemented handling
// * Vector128: Represents the __m128 ABI primitive which requires currently unimplemented handling
// * Vector256: Represents the __m256 ABI primitive which requires currently unimplemented handling
+ // * Vector512: Represents the __m512 ABI primitive which requires currently unimplemented handling
// * Vector: Has a variable size (either __m128 or __m256) and isn't readily usable for interop scenarios
return !InteropTypes.IsSystemNullable(type.Context, type)
&& !InteropTypes.IsSystemSpan(type.Context, type)
@@ -881,6 +882,7 @@ private static bool IsValidForGenericMarshalling(
&& !InteropTypes.IsSystemRuntimeIntrinsicsVector64T(type.Context, type)
&& !InteropTypes.IsSystemRuntimeIntrinsicsVector128T(type.Context, type)
&& !InteropTypes.IsSystemRuntimeIntrinsicsVector256T(type.Context, type)
+ && !InteropTypes.IsSystemRuntimeIntrinsicsVector512T(type.Context, type)
&& !InteropTypes.IsSystemNumericsVectorT(type.Context, type);
}
diff --git a/src/coreclr/tools/Common/TypeSystem/Interop/InteropTypes.cs b/src/coreclr/tools/Common/TypeSystem/Interop/InteropTypes.cs
index 8bbdb13630bfe..3beeb5f187a89 100644
--- a/src/coreclr/tools/Common/TypeSystem/Interop/InteropTypes.cs
+++ b/src/coreclr/tools/Common/TypeSystem/Interop/InteropTypes.cs
@@ -145,6 +145,11 @@ public static bool IsSystemRuntimeIntrinsicsVector256T(TypeSystemContext context
return IsCoreNamedType(context, type, "System.Runtime.Intrinsics", "Vector256`1");
}
+ public static bool IsSystemRuntimeIntrinsicsVector512T(TypeSystemContext context, TypeDesc type)
+ {
+ return IsCoreNamedType(context, type, "System.Runtime.Intrinsics", "Vector512`1");
+ }
+
public static bool IsSystemNumericsVectorT(TypeSystemContext context, TypeDesc type)
{
return IsCoreNamedType(context, type, "System.Numerics", "Vector`1");
diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/VectorOfTFieldLayoutAlgorithm.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/VectorOfTFieldLayoutAlgorithm.cs
index e0e175c0dcefe..1a1eef14d5582 100644
--- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/VectorOfTFieldLayoutAlgorithm.cs
+++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/VectorOfTFieldLayoutAlgorithm.cs
@@ -36,6 +36,10 @@ public override ComputedInstanceFieldLayout ComputeInstanceLayout(DefType defTyp
{
instanceFieldSize = new LayoutInt(32);
}
+ else if (targetDetails.MaximumSimdVectorLength == SimdVectorLength.Vector512Bit)
+ {
+ instanceFieldSize = new LayoutInt(64);
+ }
else
{
Debug.Assert(targetDetails.MaximumSimdVectorLength == SimdVectorLength.None);
diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilerContext.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilerContext.cs
index 73dcd33c1b2f9..88a74a9ebc2f0 100644
--- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilerContext.cs
+++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCompilerContext.cs
@@ -52,6 +52,8 @@ public ReadyToRunCompilerContext(TargetDetails details, SharedGenericsMode gener
matchingVectorType = "Vector128`1";
else if (details.MaximumSimdVectorLength == SimdVectorLength.Vector256Bit)
matchingVectorType = "Vector256`1";
+ else if (details.MaximumSimdVectorLength == SimdVectorLength.Vector512Bit)
+ matchingVectorType = "Vector512`1";
// No architecture has completely stable handling of Vector in the abi (Arm64 may change to SVE)
_vectorOfTFieldLayoutAlgorithm = new VectorOfTFieldLayoutAlgorithm(_r2rFieldLayoutAlgorithm, _vectorFieldLayoutAlgorithm, matchingVectorType, bubbleIncludesCorelib);
diff --git a/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/InstanceFieldLayout.cs b/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/InstanceFieldLayout.cs
index 249090e38eae5..a67197c0a1901 100644
--- a/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/InstanceFieldLayout.cs
+++ b/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/InstanceFieldLayout.cs
@@ -230,6 +230,18 @@ public class Class16Align
{
Vector128 vector16Align;
}
+
+ [StructLayout(LayoutKind.Sequential)]
+ public class Class32Align
+ {
+ Vector256 vector32Align;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public class Class64Align
+ {
+ Vector512 vector64Align;
+ }
}
namespace Auto
@@ -358,17 +370,53 @@ public struct int8x16x2
public Vector128 _1;
}
+ [StructLayout(LayoutKind.Auto)]
+ public struct int8x32x2
+ {
+ public Vector256 _0;
+ public Vector256 _1;
+ }
+
+ [StructLayout(LayoutKind.Auto)]
+ public struct int8x64x2
+ {
+ public Vector512 _0;
+ public Vector512 _1;
+ }
+
public struct Wrapper_int8x16x2
{
public int8x16x2 fld;
}
+ public struct Wrapper_int8x32x2
+ {
+ public int8x32x2 fld;
+ }
+
+ public struct Wrapper_int8x64x2
+ {
+ public int8x64x2 fld;
+ }
+
public struct Wrapper_int8x16x2_2
{
public bool fld1;
public int8x16x2 fld2;
}
+ public struct Wrapper_int8x32x2_2
+ {
+ public bool fld1;
+ public int8x32x2 fld2;
+ }
+
+ public struct Wrapper_int8x64x2_2
+ {
+ public bool fld1;
+ public int8x64x2 fld2;
+ }
+
[StructLayout(LayoutKind.Auto)]
public struct StructByte
{
@@ -453,6 +501,18 @@ public class Class16Align
{
Vector128 vector16Align;
}
+
+ [StructLayout(LayoutKind.Sequential)]
+ public class Class32Align
+ {
+ Vector256 vector32Align;
+ }
+
+ [StructLayout(LayoutKind.Sequential)]
+ public class Class64Align
+ {
+ Vector512 vector64Align;
+ }
}
namespace IsByRefLike
diff --git a/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/Platform.cs b/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/Platform.cs
index 2239925645c77..75ade965f9318 100644
--- a/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/Platform.cs
+++ b/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/CoreTestAssembly/Platform.cs
@@ -182,4 +182,22 @@ public readonly struct Vector128
private readonly ulong _00;
private readonly ulong _01;
}
+
+ [Intrinsic]
+ [StructLayout(LayoutKind.Sequential, Size = 32)]
+ public readonly struct Vector256
+ where T : struct
+ {
+ private readonly Vector128 _lower;
+ private readonly Vector128 _upper;
+ }
+
+ [Intrinsic]
+ [StructLayout(LayoutKind.Sequential, Size = 64)]
+ public readonly struct Vector512
+ where T : struct
+ {
+ private readonly Vector256 _lower;
+ private readonly Vector256 _upper;
+ }
}
diff --git a/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/InstanceFieldLayoutTests.cs b/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/InstanceFieldLayoutTests.cs
index e846cbfe31af9..8c49d5ad91b81 100644
--- a/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/InstanceFieldLayoutTests.cs
+++ b/src/coreclr/tools/aot/ILCompiler.TypeSystem.Tests/InstanceFieldLayoutTests.cs
@@ -303,6 +303,50 @@ public void TestSequentialTypeLayoutClass16Align()
}
}
+ [Fact]
+ public void TestSequentialTypeLayoutClass32Align()
+ {
+ MetadataType classType = _testModule.GetType("Sequential", "Class32Align");
+ Assert.Equal(0x28, classType.InstanceByteCount.AsInt);
+ foreach (var f in classType.GetFields())
+ {
+ if (f.IsStatic)
+ continue;
+
+ switch (f.Name)
+ {
+ case "vector32Align":
+ Assert.Equal(0x8, f.Offset.AsInt);
+ break;
+ default:
+ Assert.True(false);
+ break;
+ }
+ }
+ }
+
+ [Fact]
+ public void TestSequentialTypeLayoutClass64Align()
+ {
+ MetadataType classType = _testModule.GetType("Sequential", "Class64Align");
+ Assert.Equal(0x48, classType.InstanceByteCount.AsInt);
+ foreach (var f in classType.GetFields())
+ {
+ if (f.IsStatic)
+ continue;
+
+ switch (f.Name)
+ {
+ case "vector64Align":
+ Assert.Equal(0x8, f.Offset.AsInt);
+ break;
+ default:
+ Assert.True(false);
+ break;
+ }
+ }
+ }
+
[Fact]
public void TestAutoLayoutStruct()
{
@@ -826,6 +870,50 @@ public void TestAutoTypeLayoutClass16Align()
}
}
+ [Fact]
+ public void TestAutoTypeLayoutClass32Align()
+ {
+ MetadataType classType = _testModule.GetType("Auto", "Class32Align");
+ Assert.Equal(0x28, classType.InstanceByteCount.AsInt);
+ foreach (var f in classType.GetFields())
+ {
+ if (f.IsStatic)
+ continue;
+
+ switch (f.Name)
+ {
+ case "vector32Align":
+ Assert.Equal(0x8, f.Offset.AsInt);
+ break;
+ default:
+ Assert.True(false);
+ break;
+ }
+ }
+ }
+
+ [Fact]
+ public void TestAutoTypeLayoutClass64Align()
+ {
+ MetadataType classType = _testModule.GetType("Auto", "Class64Align");
+ Assert.Equal(0x48, classType.InstanceByteCount.AsInt);
+ foreach (var f in classType.GetFields())
+ {
+ if (f.IsStatic)
+ continue;
+
+ switch (f.Name)
+ {
+ case "vector64Align":
+ Assert.Equal(0x8, f.Offset.AsInt);
+ break;
+ default:
+ Assert.True(false);
+ break;
+ }
+ }
+ }
+
[Fact]
public void TestTypeContainsGCPointers()
{
@@ -904,15 +992,47 @@ public void TestWrapperAroundVectorTypes()
Assert.Equal(16, instantiatedType.InstanceFieldAlignment.AsInt);
}
+ {
+ MetadataType type = (MetadataType)_testModule.GetType("System.Runtime.Intrinsics", "Vector256`1");
+ MetadataType instantiatedType = type.MakeInstantiatedType(_context.GetWellKnownType(WellKnownType.Byte));
+ Assert.Equal(32, instantiatedType.InstanceFieldAlignment.AsInt);
+ }
+
+ {
+ MetadataType type = (MetadataType)_testModule.GetType("System.Runtime.Intrinsics", "Vector512`1");
+ MetadataType instantiatedType = type.MakeInstantiatedType(_context.GetWellKnownType(WellKnownType.Byte));
+ Assert.Equal(64, instantiatedType.InstanceFieldAlignment.AsInt);
+ }
+
{
DefType type = _testModule.GetType("Auto", "int8x16x2");
Assert.Equal(16, type.InstanceFieldAlignment.AsInt);
}
+ {
+ DefType type = _testModule.GetType("Auto", "int8x32x2");
+ Assert.Equal(32, type.InstanceFieldAlignment.AsInt);
+ }
+
+ {
+ DefType type = _testModule.GetType("Auto", "int8x64x2");
+ Assert.Equal(64, type.InstanceFieldAlignment.AsInt);
+ }
+
{
DefType type = _testModule.GetType("Auto", "Wrapper_int8x16x2");
Assert.Equal(16, type.InstanceFieldAlignment.AsInt);
}
+
+ {
+ DefType type = _testModule.GetType("Auto", "Wrapper_int8x32x2");
+ Assert.Equal(32, type.InstanceFieldAlignment.AsInt);
+ }
+
+ {
+ DefType type = _testModule.GetType("Auto", "Wrapper_int8x64x2");
+ Assert.Equal(64, type.InstanceFieldAlignment.AsInt);
+ }
}
}
}
diff --git a/src/coreclr/vm/classlayoutinfo.cpp b/src/coreclr/vm/classlayoutinfo.cpp
index a37d7a0652121..c326a3e26e15b 100644
--- a/src/coreclr/vm/classlayoutinfo.cpp
+++ b/src/coreclr/vm/classlayoutinfo.cpp
@@ -162,6 +162,7 @@ namespace
case 8:
case 16:
case 32:
+ case 64:
break;
default:
COMPlusThrowHR(COR_E_INVALIDPROGRAM, BFA_METADATA_CORRUPT);
@@ -979,7 +980,8 @@ EEClassNativeLayoutInfo* EEClassNativeLayoutInfo::CollectNativeLayoutFieldMetada
pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__UINT128)) ||
pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR64T)) ||
pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR128T)) ||
- pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR256T)))
+ pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR256T)) ||
+ pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR512T)))
{
pNativeLayoutInfo->m_alignmentRequirement = pEEClassLayoutInfo->m_ManagedLargestAlignmentRequirementOfAllMembers;
}
diff --git a/src/coreclr/vm/classnames.h b/src/coreclr/vm/classnames.h
index 00a192fd36f39..29ee7c26d5e84 100644
--- a/src/coreclr/vm/classnames.h
+++ b/src/coreclr/vm/classnames.h
@@ -52,6 +52,9 @@
#define g_Vector256ClassName "System.Runtime.Intrinsics.Vector256`1"
#define g_Vector256Name "Vector256`1"
+#define g_Vector512ClassName "System.Runtime.Intrinsics.Vector512`1"
+#define g_Vector512Name "Vector512`1"
+
#define g_EnumeratorToEnumClassName "System.Runtime.InteropServices.CustomMarshalers.EnumeratorToEnumVariantMarshaler"
#define g_ExceptionClassName "System.Exception"
#define g_ExecutionEngineExceptionClassName "System.ExecutionEngineException"
diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h
index 5e1b486c9f2fb..61946c83fb224 100644
--- a/src/coreclr/vm/corelib.h
+++ b/src/coreclr/vm/corelib.h
@@ -484,6 +484,7 @@ DEFINE_METHOD(NATIVELIBRARY, LOADLIBRARYCALLBACKSTUB, LoadLibraryCallback
DEFINE_CLASS(VECTOR64T, Intrinsics, Vector64`1)
DEFINE_CLASS(VECTOR128T, Intrinsics, Vector128`1)
DEFINE_CLASS(VECTOR256T, Intrinsics, Vector256`1)
+DEFINE_CLASS(VECTOR512T, Intrinsics, Vector512`1)
DEFINE_CLASS(VECTORT, Numerics, Vector`1)
diff --git a/src/coreclr/vm/fieldmarshaler.h b/src/coreclr/vm/fieldmarshaler.h
index a5c77ca5bfd11..83b2c79fb4f05 100644
--- a/src/coreclr/vm/fieldmarshaler.h
+++ b/src/coreclr/vm/fieldmarshaler.h
@@ -24,7 +24,7 @@ class MethodTable;
// Currently we set this to the packing size of the largest supported
// fundamental type and let the field marshaller downsize where needed.
//=======================================================================
-#define DEFAULT_PACKING_SIZE 32
+#define DEFAULT_PACKING_SIZE 64
//=======================================================================
// This structure contains information about where a field is placed in a structure, as well as it's size and alignment.
diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp
index 0c97fe4064237..8e79bd6ab8498 100644
--- a/src/coreclr/vm/methodtable.cpp
+++ b/src/coreclr/vm/methodtable.cpp
@@ -2238,8 +2238,8 @@ bool MethodTable::ClassifyEightBytesWithManagedLayout(SystemVStructRegisterPassi
LPCUTF8 namespaceName;
LPCUTF8 className = GetFullyQualifiedNameInfo(&namespaceName);
- if ((strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector128`1") == 0) ||
- (strcmp(className, "Vector64`1") == 0))
+ if ((strcmp(className, "Vector512`1") == 0) || (strcmp(className, "Vector256`1") == 0) ||
+ (strcmp(className, "Vector128`1") == 0) || (strcmp(className, "Vector64`1") == 0))
{
assert(strcmp(namespaceName, "System.Runtime.Intrinsics") == 0);
@@ -2488,8 +2488,8 @@ bool MethodTable::ClassifyEightBytesWithNativeLayout(SystemVStructRegisterPassin
LPCUTF8 namespaceName;
LPCUTF8 className = GetFullyQualifiedNameInfo(&namespaceName);
- if ((strcmp(className, "Vector256`1") == 0) || (strcmp(className, "Vector128`1") == 0) ||
- (strcmp(className, "Vector64`1") == 0))
+ if ((strcmp(className, "Vector512`1") == 0) || (strcmp(className, "Vector256`1") == 0) ||
+ (strcmp(className, "Vector128`1") == 0) || (strcmp(className, "Vector64`1") == 0))
{
assert(strcmp(namespaceName, "System.Runtime.Intrinsics") == 0);
diff --git a/src/coreclr/vm/methodtablebuilder.cpp b/src/coreclr/vm/methodtablebuilder.cpp
index d77b5c5c826d1..fae0605d0b0bd 100644
--- a/src/coreclr/vm/methodtablebuilder.cpp
+++ b/src/coreclr/vm/methodtablebuilder.cpp
@@ -9884,6 +9884,22 @@ void MethodTableBuilder::CheckForSystemTypes()
pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16;
#else
pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 32; // sizeof(__m256)
+ #endif // TARGET_ARM elif TARGET_ARM64
+ }
+ else if (strcmp(name, g_Vector512Name) == 0)
+ {
+ #ifdef TARGET_ARM
+ // No such type exists for the Procedure Call Standard for ARM. We will default
+ // to the same alignment as __m128, which is supported by the ABI.
+
+ pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 8;
+ #elif defined(TARGET_ARM64)
+ // The Procedure Call Standard for ARM 64-bit (with SVE support) defaults to
+ // 16-byte alignment for __m256.
+
+ pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16;
+ #else
+ pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 64; // sizeof(__m512)
#endif // TARGET_ARM elif TARGET_ARM64
}
else
diff --git a/src/coreclr/vm/mlinfo.cpp b/src/coreclr/vm/mlinfo.cpp
index a0080a6747534..3f74772e014c4 100644
--- a/src/coreclr/vm/mlinfo.cpp
+++ b/src/coreclr/vm/mlinfo.cpp
@@ -1081,6 +1081,7 @@ namespace
// * Vector64: Represents the __m64 ABI primitive which requires currently unimplemented handling
// * Vector128: Represents the __m128 ABI primitive which requires currently unimplemented handling
// * Vector256: Represents the __m256 ABI primitive which requires currently unimplemented handling
+ // * Vector512: Represents the __m512 ABI primitive which requires currently unimplemented handling
// * Vector: Has a variable size (either __m128 or __m256) and isn't readily usable for interop scenarios
return !pMT->HasSameTypeDefAs(g_pNullableClass)
&& !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__SPAN))
@@ -1088,6 +1089,7 @@ namespace
&& !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR64T))
&& !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR128T))
&& !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR256T))
+ && !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTOR512T))
&& !pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTORT));
}
diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems
index 9d3d95c4e2631..bb540ae696aac 100644
--- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems
+++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems
@@ -957,6 +957,9 @@
+
+
+
diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs
index 541f64416fca0..b8e80993de38d 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128_1.cs
@@ -198,15 +198,10 @@ public T this[int index]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128 operator /(Vector128 left, T right)
{
- Unsafe.SkipInit(out Vector128 result);
-
- for (int index = 0; index < Count; index++)
- {
- T value = Scalar.Divide(left.GetElementUnsafe(index), right);
- result.SetElementUnsafe(index, value);
- }
-
- return result;
+ return Vector128.Create(
+ left._lower / right,
+ left._upper / right
+ );
}
/// Compares two vectors to determine if all elements are equal.
@@ -258,15 +253,10 @@ public T this[int index]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128 operator <<(Vector128 value, int shiftCount)
{
- Unsafe.SkipInit(out Vector128 result);
-
- for (int index = 0; index < Count; index++)
- {
- T element = Scalar.ShiftLeft(value.GetElementUnsafe(index), shiftCount);
- result.SetElementUnsafe(index, element);
- }
-
- return result;
+ return Vector128.Create(
+ value._lower << shiftCount,
+ value._upper << shiftCount
+ );
}
/// Multiplies two vectors to compute their element-wise product.
@@ -330,15 +320,10 @@ public T this[int index]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128 operator >>(Vector128 value, int shiftCount)
{
- Unsafe.SkipInit(out Vector128 result);
-
- for (int index = 0; index < Count; index++)
- {
- T element = Scalar.ShiftRightArithmetic(value.GetElementUnsafe(index), shiftCount);
- result.SetElementUnsafe(index, element);
- }
-
- return result;
+ return Vector128.Create(
+ value._lower >> shiftCount,
+ value._upper >> shiftCount
+ );
}
/// Subtracts two vectors to compute their difference.
@@ -390,15 +375,10 @@ public static Vector128 operator >>(Vector128 value, int shiftCount)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128 operator >>>(Vector128 value, int shiftCount)
{
- Unsafe.SkipInit(out Vector128 result);
-
- for (int index = 0; index < Count; index++)
- {
- T element = Scalar.ShiftRightLogical(value.GetElementUnsafe(index), shiftCount);
- result.SetElementUnsafe(index, element);
- }
-
- return result;
+ return Vector128.Create(
+ value._lower >>> shiftCount,
+ value._upper >>> shiftCount
+ );
}
/// Determines whether the specified object is equal to the current instance.
diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs
index 14b8e5bdfd1b0..212133baf28b3 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs
@@ -236,7 +236,7 @@ public static Vector256 AsUInt32(this Vector256 vector)
public static Vector256 AsUInt64(this Vector256 vector)
where T : struct => vector.As();
- /// Reinterprets a as a new .
+ /// Reinterprets a as a new .
/// The type of the elements in the vector.
/// The vector to reinterpret.
/// reinterpreted as a new .
@@ -254,10 +254,10 @@ public static Vector256 AsVector256(this Vector value)
return result;
}
- /// Reinterprets a as a new .
+ /// Reinterprets a as a new .
/// The type of the elements in the vector.
/// The vector to reinterpret.
- /// reinterpreted as a new .
+ /// reinterpreted as a new .
/// The type of () is not supported.
[Intrinsic]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -796,7 +796,8 @@ public static Vector256 Create(ReadOnlySpan values)
/// On x86, this method corresponds to __m256i _mm256_setr_epi8
[Intrinsic]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static Vector256 Create(byte e0, byte e1, byte e2, byte e3, byte e4, byte e5, byte e6, byte e7, byte e8, byte e9, byte e10, byte e11, byte e12, byte e13, byte e14, byte e15, byte e16, byte e17, byte e18, byte e19, byte e20, byte e21, byte e22, byte e23, byte e24, byte e25, byte e26, byte e27, byte e28, byte e29, byte e30, byte e31)
+ public static Vector256 Create(byte e0, byte e1, byte e2, byte e3, byte e4, byte e5, byte e6, byte e7, byte e8, byte e9, byte e10, byte e11, byte e12, byte e13, byte e14, byte e15,
+ byte e16, byte e17, byte e18, byte e19, byte e20, byte e21, byte e22, byte e23, byte e24, byte e25, byte e26, byte e27, byte e28, byte e29, byte e30, byte e31)
{
return Create(
Vector128.Create(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15),
@@ -926,7 +927,8 @@ public static Vector256 Create(long e0, long e1, long e2, long e3)
[Intrinsic]
[CLSCompliant(false)]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static Vector256 Create(sbyte e0, sbyte e1, sbyte e2, sbyte e3, sbyte e4, sbyte e5, sbyte e6, sbyte e7, sbyte e8, sbyte e9, sbyte e10, sbyte e11, sbyte e12, sbyte e13, sbyte e14, sbyte e15, sbyte e16, sbyte e17, sbyte e18, sbyte e19, sbyte e20, sbyte e21, sbyte e22, sbyte e23, sbyte e24, sbyte e25, sbyte e26, sbyte e27, sbyte e28, sbyte e29, sbyte e30, sbyte e31)
+ public static Vector256 Create(sbyte e0, sbyte e1, sbyte e2, sbyte e3, sbyte e4, sbyte e5, sbyte e6, sbyte e7, sbyte e8, sbyte e9, sbyte e10, sbyte e11, sbyte e12, sbyte e13, sbyte e14, sbyte e15,
+ sbyte e16, sbyte e17, sbyte e18, sbyte e19, sbyte e20, sbyte e21, sbyte e22, sbyte e23, sbyte e24, sbyte e25, sbyte e26, sbyte e27, sbyte e28, sbyte e29, sbyte e30, sbyte e31)
{
return Create(
Vector128.Create(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15),
@@ -2743,6 +2745,41 @@ public static T ToScalar(this Vector256 vector)
return vector.GetElementUnsafe(0);
}
+ /// Converts the given vector to a new with the lower 256-bits set to the value of the given vector and the upper 256-bits initialized to zero.
+ /// The type of the input vector.
+ /// The vector to extend.
+ /// A new with the lower 256-bits set to the value of and the upper 256-bits initialized to zero.
+ /// The type of () is not supported.
+ [Intrinsic]
+ public static Vector512 ToVector512(this Vector256 vector)
+ where T : struct
+ {
+ ThrowHelper.ThrowForUnsupportedIntrinsicsVector256BaseType();
+
+ Vector512 result = default;
+ result.SetLowerUnsafe(vector);
+ return result;
+ }
+
+ /// Converts the given vector to a new with the lower 256-bits set to the value of the given vector and the upper 256-bits left uninitialized.
+ /// The type of the input vector.
+ /// The vector to extend.
+ /// A new with the lower 256-bits set to the value of and the upper 256-bits left uninitialized.
+ /// The type of () is not supported.
+ [Intrinsic]
+ public static unsafe Vector512 ToVector512Unsafe(this Vector256 vector)
+ where T : struct
+ {
+ ThrowHelper.ThrowForUnsupportedIntrinsicsVector256BaseType();
+
+ // This relies on us stripping the "init" flag from the ".locals"
+ // declaration to let the upper bits be uninitialized.
+
+ Unsafe.SkipInit(out Vector512 result);
+ result.SetLowerUnsafe(vector);
+ return result;
+ }
+
/// Tries to copy a to a given span.
/// The type of the input vector.
/// The vector to copy.
@@ -2808,7 +2845,7 @@ public static bool TryCopyTo(this Vector256 vector, Span destination)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static (Vector256 Lower, Vector256 Upper) Widen(Vector256 source) => (WidenLower(source), WidenUpper(source));
- /// Widens the lower half of a into a .
+ /// Widens the lower half of a into a .
/// The vector whose elements are to be widened.
/// A vector that contain the widened lower half of .
[Intrinsic]
@@ -2824,7 +2861,7 @@ public static Vector256 WidenLower(Vector256 source)
);
}
- /// Widens the lower half of a into a .
+ /// Widens the lower half of a into a .
/// The vector whose elements are to be widened.
/// A vector that contain the widened lower half of .
[Intrinsic]
@@ -2839,7 +2876,7 @@ public static Vector256 WidenLower(Vector256 source)
);
}
- /// Widens the lower half of a into a .
+ /// Widens the lower half of a into a .
/// The vector whose elements are to be widened.
/// A vector that contain the widened lower half of .
[Intrinsic]
@@ -2854,7 +2891,7 @@ public static Vector256 WidenLower(Vector256 source)
);
}
- /// Widens the lower half of a into a .
+ /// Widens the lower half of a into a .
/// The vector whose elements are to be widened.
/// A vector that contain the widened lower half of .
[Intrinsic]
@@ -2869,7 +2906,7 @@ public static Vector256 WidenLower(Vector256 source)
Vector128.WidenUpper(lower)
);
}
- /// Widens the lower half of a into a .
+ /// Widens the lower half of a into a .
/// The vector whose elements are to be widened.
/// A vector that contain the widened lower half of .
[Intrinsic]
@@ -2884,7 +2921,7 @@ public static Vector256 WidenLower(Vector256 source)
);
}
- /// Widens the lower half of a into a .
+ /// Widens the lower half of a into a .
/// The vector whose elements are to be widened.
/// A vector that contain the widened lower half of .
[Intrinsic]
diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs
index caa8fe308a2ea..56e19827b31b4 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256_1.cs
@@ -4,7 +4,6 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
-using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
@@ -197,15 +196,10 @@ public T this[int index]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256 operator /(Vector256 left, T right)
{
- Unsafe.SkipInit(out Vector256 result);
-
- for (int index = 0; index < Count; index++)
- {
- T value = Scalar.Divide(left.GetElementUnsafe(index), right);
- result.SetElementUnsafe(index, value);
- }
-
- return result;
+ return Vector256.Create(
+ left._lower / right,
+ left._upper / right
+ );
}
/// Compares two vectors to determine if all elements are equal.
@@ -257,15 +251,10 @@ public T this[int index]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256 operator <<(Vector256 value, int shiftCount)
{
- Unsafe.SkipInit(out Vector256 result);
-
- for (int index = 0; index < Count; index++)
- {
- T element = Scalar.ShiftLeft(value.GetElementUnsafe(index), shiftCount);
- result.SetElementUnsafe(index, element);
- }
-
- return result;
+ return Vector256.Create(
+ value._lower << shiftCount,
+ value._upper << shiftCount
+ );
}
/// Multiplies two vectors to compute their element-wise product.
@@ -329,15 +318,10 @@ public T this[int index]
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256 operator >>(Vector256 value, int shiftCount)
{
- Unsafe.SkipInit(out Vector256 result);
-
- for (int index = 0; index < Count; index++)
- {
- T element = Scalar.ShiftRightArithmetic(value.GetElementUnsafe(index), shiftCount);
- result.SetElementUnsafe(index, element);
- }
-
- return result;
+ return Vector256.Create(
+ value._lower >> shiftCount,
+ value._upper >> shiftCount
+ );
}
/// Subtracts two vectors to compute their difference.
@@ -389,15 +373,10 @@ public static Vector256 operator >>(Vector256 value, int shiftCount)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256 operator >>>(Vector256 value, int shiftCount)
{
- Unsafe.SkipInit(out Vector256 result);
-
- for (int index = 0; index < Count; index++)
- {
- T element = Scalar.ShiftRightLogical(value.GetElementUnsafe(index), shiftCount);
- result.SetElementUnsafe(index, element);
- }
-
- return result;
+ return Vector256.Create(
+ value._lower >>> shiftCount,
+ value._upper >>> shiftCount
+ );
}
/// Determines whether the specified object is equal to the current instance.
diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs
new file mode 100644
index 0000000000000..9b7be1d777fb6
--- /dev/null
+++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs
@@ -0,0 +1,3167 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+
+namespace System.Runtime.Intrinsics
+{
+ // We mark certain methods with AggressiveInlining to ensure that the JIT will
+ // inline them. The JIT would otherwise not inline the method since it, at the
+ // point it tries to determine inline profability, currently cannot determine
+ // that most of the code-paths will be optimized away as "dead code".
+ //
+ // We then manually inline cases (such as certain intrinsic code-paths) that
+ // will generate code small enough to make the AgressiveInlining profitable. The
+ // other cases (such as the software fallback) are placed in their own method.
+ // This ensures we get good codegen for the "fast-path" and allows the JIT to
+ // determine inline profitability of the other paths as it would normally.
+
+ // Many of the instance methods were moved to be extension methods as it results
+ // in overall better codegen. This is because instance methods require the C# compiler
+ // to generate extra locals as the `this` parameter has to be passed by reference.
+ // Having them be extension methods means that the `this` parameter can be passed by
+ // value instead, thus reducing the number of locals and helping prevent us from hitting
+ // the internal inlining limits of the JIT.
+
+ /// Provides a collection of static methods for creating, manipulating, and otherwise operting on 512-bit vectors.
+ public static unsafe class Vector512
+ {
+ internal const int Size = 64;
+
+#if TARGET_ARM
+ internal const int Alignment = 8;
+#elif TARGET_ARM64
+ internal const int Alignment = 16;
+#else
+ internal const int Alignment = 64;
+#endif
+
+ /// Gets a value that indicates whether 512-bit vector operations are subject to hardware acceleration through JIT intrinsic support.
+ /// if 512-bit vector operations are subject to hardware acceleration; otherwise, .
+ /// 512-bit vector operations are subject to hardware acceleration on systems that support Single Instruction, Multiple Data (SIMD) instructions for 512-bit vectors and the RyuJIT just-in-time compiler is used to compile managed code.
+ public static bool IsHardwareAccelerated
+ {
+ [Intrinsic]
+ get => IsHardwareAccelerated;
+ }
+
+ /// Computes the absolute value of each element in a vector.
+ /// The type of the elements in the vector.
+ /// The vector that will have its absolute value computed.
+ /// A vector whose elements are the absolute value of the elements in .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Abs(Vector512 vector)
+ where T : struct
+ {
+ return Create(
+ Vector256.Abs(vector._lower),
+ Vector256.Abs(vector._upper)
+ );
+ }
+
+ /// Adds two vectors to compute their sum.
+ /// The type of the elements in the vector.
+ /// The vector to add with .
+ /// The vector to add with .
+ /// The sum of and .
+ /// The type of and () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Add(Vector512 left, Vector512 right)
+ where T : struct => left + right;
+
+ /// Computes the bitwise-and of a given vector and the ones complement of another vector.
+ /// The type of the elements in the vector.
+ /// The vector to bitwise-and with .
+ /// The vector to that is ones-complemented before being bitwise-and with .
+ /// The bitwise-and of and the ones-complement of .
+ /// The type of and () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AndNot(Vector512 left, Vector512 right)
+ where T : struct
+ {
+ return Create(
+ Vector256.AndNot(left._lower, right._lower),
+ Vector256.AndNot(left._upper, right._upper)
+ );
+ }
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the input vector.
+ /// The type of the elements in the output vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () or the type of the target () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 As(this Vector512 vector)
+ where TFrom : struct
+ where TTo : struct
+ {
+ ThrowHelper.ThrowForUnsupportedIntrinsicsVector512BaseType();
+ ThrowHelper.ThrowForUnsupportedIntrinsicsVector512BaseType();
+
+ return Unsafe.As, Vector512>(ref vector);
+ }
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsByte(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsDouble(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsInt16(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsInt32(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsInt64(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsNInt(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsNUInt(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsSByte(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsSingle(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsUInt16(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsUInt32(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsUInt64(this Vector512 vector)
+ where T : struct => vector.As();
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 AsVector512(this Vector value)
+ where T : struct
+ {
+ Debug.Assert(Vector512.Count >= Vector.Count);
+ ThrowHelper.ThrowForUnsupportedIntrinsicsVector512BaseType();
+
+ Vector512 result = default;
+ Unsafe.WriteUnaligned(ref Unsafe.As, byte>(ref result), value);
+ return result;
+ }
+
+ /// Reinterprets a as a new .
+ /// The type of the elements in the vector.
+ /// The vector to reinterpret.
+ /// reinterpreted as a new .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector AsVector(this Vector512 value)
+ where T : struct
+ {
+ Debug.Assert(Vector512.Count >= Vector.Count);
+ ThrowHelper.ThrowForUnsupportedIntrinsicsVector512BaseType();
+
+ ref byte address = ref Unsafe.As, byte>(ref value);
+ return Unsafe.ReadUnaligned>(ref address);
+ }
+
+ /// Computes the bitwise-and of two vectors.
+ /// The type of the elements in the vector.
+ /// The vector to bitwise-and with .
+ /// The vector to bitwise-and with .
+ /// The bitwise-and of and .
+ /// The type of and () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 BitwiseAnd(Vector512 left, Vector512 right)
+ where T : struct => left & right;
+
+ /// Computes the bitwise-or of two vectors.
+ /// The type of the elements in the vector.
+ /// The vector to bitwise-or with .
+ /// The vector to bitwise-or with .
+ /// The bitwise-or of and .
+ /// The type of and () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 BitwiseOr(Vector512 left, Vector512 right)
+ where T : struct => left | right;
+
+ /// Computes the ceiling of each element in a vector.
+ /// The vector that will have its ceiling computed.
+ /// A vector whose elements are the ceiling of the elements in .
+ ///
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Ceiling(Vector512 vector)
+ {
+ return Create(
+ Vector256.Ceiling(vector._lower),
+ Vector256.Ceiling(vector._upper)
+ );
+ }
+
+ /// Computes the ceiling of each element in a vector.
+ /// The vector that will have its ceiling computed.
+ /// A vector whose elements are the ceiling of the elements in .
+ ///
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Ceiling(Vector512 vector)
+ {
+ return Create(
+ Vector256.Ceiling(vector._lower),
+ Vector256.Ceiling(vector._upper)
+ );
+ }
+
+ /// Conditionally selects a value from two vectors on a bitwise basis.
+ /// The type of the elements in the vector.
+ /// The mask that is used to select a value from or .
+ /// The vector that is selected when the corresponding bit in is one.
+ /// The vector that is selected when the corresponding bit in is zero.
+ /// A vector whose bits come from or based on the value of .
+ /// The type of , , and () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConditionalSelect(Vector512 condition, Vector512 left, Vector512 right)
+ where T : struct
+ {
+ return Create(
+ Vector256.ConditionalSelect(condition._lower, left._lower, right._lower),
+ Vector256.ConditionalSelect(condition._upper, left._upper, right._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToDouble(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToDouble(vector._lower),
+ Vector256.ConvertToDouble(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToDouble(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToDouble(vector._lower),
+ Vector256.ConvertToDouble(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToInt32(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToInt32(vector._lower),
+ Vector256.ConvertToInt32(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToInt64(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToInt64(vector._lower),
+ Vector256.ConvertToInt64(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToSingle(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToSingle(vector._lower),
+ Vector256.ConvertToSingle(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToSingle(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToSingle(vector._lower),
+ Vector256.ConvertToSingle(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToUInt32(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToUInt32(vector._lower),
+ Vector256.ConvertToUInt32(vector._upper)
+ );
+ }
+
+ /// Converts a to a .
+ /// The vector to convert.
+ /// The converted vector.
+ [Intrinsic]
+ [CLSCompliant(false)]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 ConvertToUInt64(Vector512 vector)
+ {
+ return Create(
+ Vector256.ConvertToUInt64(vector._lower),
+ Vector256.ConvertToUInt64(vector._upper)
+ );
+ }
+
+ /// Copies a to a given array.
+ /// The type of the elements in the vector.
+ /// The vector to be copied.
+ /// The array to which is copied.
+ /// The length of is less than .
+ /// The type of and () is not supported.
+ /// is null.
+ public static void CopyTo(this Vector512 vector, T[] destination)
+ where T : struct
+ {
+ // We explicitly don't check for `null` because historically this has thrown `NullReferenceException` for perf reasons
+
+ if (destination.Length < Vector512.Count)
+ {
+ ThrowHelper.ThrowArgumentException_DestinationTooShort();
+ }
+
+ ref byte address = ref Unsafe.As(ref MemoryMarshal.GetArrayDataReference(destination));
+ Unsafe.WriteUnaligned(ref address, vector);
+ }
+
+ /// Copies a to a given array starting at the specified index.
+ /// The type of the elements in the vector.
+ /// The vector to be copied.
+ /// The array to which is copied.
+ /// The starting index of which will be copied to.
+ /// The length of is less than .
+ /// is negative or greater than the length of .
+ /// The type of and () is not supported.
+ /// is null.
+ public static void CopyTo(this Vector512 vector, T[] destination, int startIndex)
+ where T : struct
+ {
+ // We explicitly don't check for `null` because historically this has thrown `NullReferenceException` for perf reasons
+
+ if ((uint)startIndex >= (uint)destination.Length)
+ {
+ ThrowHelper.ThrowStartIndexArgumentOutOfRange_ArgumentOutOfRange_IndexMustBeLess();
+ }
+
+ if ((destination.Length - startIndex) < Vector512.Count)
+ {
+ ThrowHelper.ThrowArgumentException_DestinationTooShort();
+ }
+
+ ref byte address = ref Unsafe.As(ref MemoryMarshal.GetArrayDataReference(destination));
+ Unsafe.WriteUnaligned(ref Unsafe.Add(ref address, startIndex), vector);
+ }
+
+ /// Copies a to a given span.
+ /// The type of the elements in the vector.
+ /// The vector to be copied.
+ /// The span to which the is copied.
+ /// The length of is less than .
+ /// The type of and () is not supported.
+ public static void CopyTo(this Vector512 vector, Span destination)
+ where T : struct
+ {
+ if ((uint)destination.Length < (uint)Vector512.Count)
+ {
+ ThrowHelper.ThrowArgumentException_DestinationTooShort();
+ }
+
+ ref byte address = ref Unsafe.As(ref MemoryMarshal.GetReference(destination));
+ Unsafe.WriteUnaligned(ref address, vector);
+ }
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The type of the elements in the vector.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ /// The type of () is not supported.
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(T value)
+ where T : struct
+ {
+ Vector256 vector = Vector256.Create(value);
+ return Create(vector, vector);
+ }
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ /// On x86, this method corresponds to __m512i _mm512_set1_epi8
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(byte value) => Create(value);
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ /// On x86, this method corresponds to __m512d _mm512_set1_pd
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(double value) => Create(value);
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ /// On x86, this method corresponds to __m512i _mm512_set1_epi16
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(short value) => Create(value);
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ /// On x86, this method corresponds to __m512i _mm512_set1_epi32
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(int value) => Create(value);
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ /// On x86, this method corresponds to __m512i _mm512_set1_epi64x
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(long value) => Create(value);
+
+ /// Creates a new instance with all elements initialized to the specified value.
+ /// The value that all elements will be initialized to.
+ /// A new with all elements initialized to .
+ [Intrinsic]
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static Vector512 Create(nint value) => Create(value);
+
+ ///