diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 98d6bb788d450..10670fcc928a0 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -26860,6 +26860,10 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_Sve_Load2xVectorAndUnzip: case NI_Sve_Load3xVectorAndUnzip: case NI_Sve_Load4xVectorAndUnzip: + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: addr = Op(2); break; #endif // TARGET_ARM64 diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 095f31246d0c6..54329c60dc6ca 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -448,6 +448,14 @@ void HWIntrinsicInfo::lookupImmBounds( } break; + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: + immLowerBound = (int)SVE_PRFOP_PLDL1KEEP; + immUpperBound = (int)SVE_PRFOP_CONST15; + break; + default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 23b09c0cef751..54d359f6566de 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -1489,6 +1489,33 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: + { + assert(hasImmediateOperand); + assert(HWIntrinsicInfo::HasEnumOperand(intrin.id)); + if (intrin.op3->IsCnsIntOrI()) + { + GetEmitter()->emitIns_PRFOP_R_R_I(ins, emitSize, + (insSvePrfop)intrin.op3->AsIntConCommon()->IconValue(), op1Reg, + op2Reg, 0); + } + else + { + assert(!intrin.op3->isContainedIntOrIImmed()); + + HWIntrinsicImmOpHelper helper(this, intrin.op3, node); + for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + { + const insSvePrfop prfop = (insSvePrfop)helper.ImmValue(); + GetEmitter()->emitIns_PRFOP_R_R_I(ins, emitSize, prfop, op1Reg, op2Reg, 0); + } + } + break; + } + case NI_Vector64_ToVector128: GetEmitter()->emitIns_Mov(ins, emitSize, targetReg, op1Reg, /* canSkip */ false); break; diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index e047fc64447c1..6821e3c3e1f95 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -145,6 +145,10 @@ HARDWARE_INTRINSIC(Sve, Negate, HARDWARE_INTRINSIC(Sve, Or, -1, -1, false, {INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_sve_orr, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, OrAcross, -1, -1, false, {INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_sve_orv, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, PopCount, -1, -1, false, {INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt, INS_sve_cnt}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) +HARDWARE_INTRINSIC(Sve, PrefetchBytes, -1, 3, false, {INS_invalid, INS_sve_prfb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) +HARDWARE_INTRINSIC(Sve, PrefetchInt16, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) +HARDWARE_INTRINSIC(Sve, PrefetchInt32, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) +HARDWARE_INTRINSIC(Sve, PrefetchInt64, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, true, {INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, ReverseElement32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revw, INS_sve_revw, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index cebda711b24f6..436c770065d3b 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -3194,6 +3194,10 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x2: case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x3: case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x4: + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: assert(hasImmediateOperand); assert(varTypeIsIntegral(intrin.op3)); if (intrin.op3->IsCnsIntOrI()) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index ea444c5cea320..fc8fec26e4192 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1447,6 +1447,10 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x2: case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x3: case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x4: + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: needBranchTargetReg = !intrin.op3->isContainedIntOrIImmed(); break; @@ -1966,28 +1970,40 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou (argNum == lowVectorOperandNum) ? lowVectorCandidates : RBM_NONE); } } - else if (intrin.id == NI_Sve_StoreAndZip) - { - srcCount += BuildAddrUses(intrin.op2); - } else { - SingleTypeRegSet candidates = lowVectorOperandNum == 2 ? lowVectorCandidates : RBM_NONE; + switch (intrin.id) + { + case NI_Sve_StoreAndZip: + case NI_Sve_PrefetchBytes: + case NI_Sve_PrefetchInt16: + case NI_Sve_PrefetchInt32: + case NI_Sve_PrefetchInt64: + assert(intrinsicTree->OperIsMemoryLoadOrStore()); + srcCount += BuildAddrUses(intrin.op2); + break; - if (intrin.op2->gtType == TYP_MASK) - { - assert(lowVectorOperandNum != 2); - candidates = RBM_ALLMASK.GetPredicateRegSet(); - } + default: + { + SingleTypeRegSet candidates = lowVectorOperandNum == 2 ? lowVectorCandidates : RBM_NONE; - if (forceOp2DelayFree) - { - srcCount += BuildDelayFreeUses(intrin.op2, nullptr, candidates); - } - else - { - srcCount += isRMW ? BuildDelayFreeUses(intrin.op2, intrin.op1, candidates) - : BuildOperandUses(intrin.op2, candidates); + if (intrin.op2->gtType == TYP_MASK) + { + assert(lowVectorOperandNum != 2); + candidates = RBM_ALLMASK.GetPredicateRegSet(); + } + + if (forceOp2DelayFree) + { + srcCount += BuildDelayFreeUses(intrin.op2, nullptr, candidates); + } + else + { + srcCount += isRMW ? BuildDelayFreeUses(intrin.op2, intrin.op1, candidates) + : BuildOperandUses(intrin.op2, candidates); + } + } + break; } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs index 868300bf14aca..95013fa99049e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs @@ -92,4 +92,67 @@ public enum SveMaskPattern : byte /// All = 31 // All available (implicitly a multiple of two). } + + public enum SvePrefetchType : byte + { + /// + /// PLDL1KEEP + /// + LoadL1Temporal = 0, + + /// + /// PLDL1STRM + /// + LoadL1NonTemporal = 1, + + /// + /// PLDL2KEEP + /// + LoadL2Temporal = 2, + + /// + /// PLDL2STRM + /// + LoadL2NonTemporal = 3, + + /// + /// PLDL3KEEP + /// + LoadL3Temporal = 4, + + /// + /// PLDL3STRM + /// + LoadL3NonTemporal = 5, + + /// + /// PSTL1KEEP + /// + StoreL1Temporal = 8, + + /// + /// PSTL1STRM + /// + StoreL1NonTemporal = 9, + + /// + /// PSTL2KEEP + /// + StoreL2Temporal = 10, + + /// + /// PSTL2STRM + /// + StoreL2NonTemporal = 11, + + /// + /// PSTL3KEEP + /// + StoreL3Temporal = 12, + + /// + /// PSTL3STRM + /// + StoreL3NonTemporal = 13 + }; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index bbce99bcf0371..a1c005072db18 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -3238,6 +3238,30 @@ internal Arm64() { } /// public static unsafe Vector PopCount(Vector value) { throw new PlatformNotSupportedException(); } + /// + /// void svprfb(svbool_t pg, const void *base, enum svprfop op) + /// PRFB op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchBytes(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw new PlatformNotSupportedException(); } + + /// + /// void svprfh(svbool_t pg, const void *base, enum svprfop op) + /// PRFH op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchInt16(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw new PlatformNotSupportedException(); } + + /// + /// void svprfw(svbool_t pg, const void *base, enum svprfop op) + /// PRFW op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchInt32(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw new PlatformNotSupportedException(); } + + /// + /// void svprfd(svbool_t pg, const void *base, enum svprfop op) + /// PRFD op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchInt64(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw new PlatformNotSupportedException(); } + /// Reverse all elements diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 8295f37ba4083..caf1d81398217 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -3294,6 +3294,29 @@ internal Arm64() { } /// public static unsafe Vector PopCount(Vector value) => PopCount(value); + /// + /// void svprfb(svbool_t pg, const void *base, enum svprfop op) + /// PRFB op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchBytes(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) => PrefetchBytes(mask, address, prefetchType); + + /// + /// void svprfh(svbool_t pg, const void *base, enum svprfop op) + /// PRFH op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchInt16(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) => PrefetchInt16(mask, address, prefetchType); + + /// + /// void svprfw(svbool_t pg, const void *base, enum svprfop op) + /// PRFW op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchInt32(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) => PrefetchInt32(mask, address, prefetchType); + + /// + /// void svprfd(svbool_t pg, const void *base, enum svprfop op) + /// PRFD op, Pg, [Xbase, #0, MUL VL] + /// + public static unsafe void PrefetchInt64(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) => PrefetchInt64(mask, address, prefetchType); /// Reverse all elements diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index b5d78df0f8b8d..bedb402a7534b 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4627,6 +4627,11 @@ internal Arm64() { } public static System.Numerics.Vector PopCount(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector PopCount(System.Numerics.Vector value) { throw null; } + public static unsafe void PrefetchBytes(System.Numerics.Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; } + public static unsafe void PrefetchInt16(System.Numerics.Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; } + public static unsafe void PrefetchInt32(System.Numerics.Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; } + public static unsafe void PrefetchInt64(System.Numerics.Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; } + public static System.Numerics.Vector ReverseElement(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ReverseElement(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ReverseElement(System.Numerics.Vector value) { throw null; } @@ -4959,6 +4964,22 @@ public enum SveMaskPattern : byte LargestMultipleOf3 = 30, // The largest multiple of 3. All = 31 // All available (implicitly a multiple of two). }; + + public enum SvePrefetchType : byte + { + LoadL1Temporal = 0, + LoadL1NonTemporal = 1, + LoadL2Temporal = 2, + LoadL2NonTemporal = 3, + LoadL3Temporal = 4, + LoadL3NonTemporal = 5, + StoreL1Temporal = 8, + StoreL1NonTemporal = 9, + StoreL2Temporal = 10, + StoreL2NonTemporal = 11, + StoreL3Temporal = 12, + StoreL3NonTemporal = 13 + }; } namespace System.Runtime.Intrinsics.X86 { diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index e1a3d2294618d..ba132701e739c 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3406,6 +3406,11 @@ ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_PopCount_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "PopCount", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "Helpers.BitCount(firstOp[i]) != result[i]", ["GetIterResult"] = "Helpers.BitCount(leftOp[i])"}), ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "Sve_PopCount_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "PopCount", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "Helpers.BitCount(firstOp[i]) != result[i]", ["GetIterResult"] = "Helpers.BitCount(leftOp[i])"}), + ("SvePrefetchTest.template", new Dictionary { ["TestName"] = "Sve_PrefetchBytes", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "PrefetchBytes", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidPrefetch"] = "SvePrefetchType.LoadL1Temporal", ["InvalidPrefetch"] = "(SvePrefetchType)100"}), + ("SvePrefetchTest.template", new Dictionary { ["TestName"] = "Sve_PrefetchInt16", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "PrefetchInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidPrefetch"] = "SvePrefetchType.LoadL2NonTemporal", ["InvalidPrefetch"] = "(SvePrefetchType)18"}), + ("SvePrefetchTest.template", new Dictionary { ["TestName"] = "Sve_PrefetchInt32", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "PrefetchInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidPrefetch"] = "SvePrefetchType.StoreL3Temporal", ["InvalidPrefetch"] = "(SvePrefetchType)20"}), + ("SvePrefetchTest.template", new Dictionary { ["TestName"] = "Sve_PrefetchInt64", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "PrefetchInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidPrefetch"] = "SvePrefetchType.StoreL1NonTemporal", ["InvalidPrefetch"] = "(SvePrefetchType)87"}), + ("ScalarImm2UnOpTest.template", new Dictionary {["TestName"] = "Sve_SaturatingDecrementBy16BitElementCount_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "SaturatingDecrementBy16BitElementCount", ["RetBaseType"] = "Int32", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Byte", ["Op3BaseType"] = "SveMaskPattern", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["Imm"] = "(Byte)2", ["Imm2"] = "SveMaskPattern.LargestPowerOf2", ["InvalidImm"] = "(Byte)0", ["InvalidImm2"] = "(SveMaskPattern)35", ["ValidateResult"] = "isUnexpectedResult = (result != Helpers.SubtractSaturate((int)data, (int)(imm1 * Helpers.NumberOfElementsInVectorInt16(imm2))));",}), ("ScalarImm2UnOpTest.template", new Dictionary {["TestName"] = "Sve_SaturatingDecrementBy16BitElementCount_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "SaturatingDecrementBy16BitElementCount", ["RetBaseType"] = "Int64", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Byte", ["Op3BaseType"] = "SveMaskPattern", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["Imm"] = "(Byte)12", ["Imm2"] = "SveMaskPattern.VectorCount1", ["InvalidImm"] = "(Byte)19", ["InvalidImm2"] = "(SveMaskPattern)37", ["ValidateResult"] = "isUnexpectedResult = (result != Helpers.SubtractSaturate((long)data, (long)(imm1 * Helpers.NumberOfElementsInVectorInt16(imm2))));",}), ("ScalarImm2UnOpTest.template", new Dictionary {["TestName"] = "Sve_SaturatingDecrementBy16BitElementCount_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "SaturatingDecrementBy16BitElementCount", ["RetBaseType"] = "UInt32", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "Byte", ["Op3BaseType"] = "SveMaskPattern", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["Imm"] = "(Byte)5", ["Imm2"] = "SveMaskPattern.VectorCount2", ["InvalidImm"] = "(Byte)25", ["InvalidImm2"] = "(SveMaskPattern)46", ["ValidateResult"] = "isUnexpectedResult = (result != Helpers.SubtractSaturate((uint)data, (uint)(imm1 * Helpers.NumberOfElementsInVectorInt16(imm2))));",}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SvePrefetchTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SvePrefetchTest.template new file mode 100644 index 0000000000000..8f6bee390f76d --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SvePrefetchTest.template @@ -0,0 +1,225 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in src\tests\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TestName}Test(); + if (test.IsSupported) + { + // Validates basic functionality works + test.RunBasicScenario(); + test.RunBasicScenario_Invalid(); + test.RunBasicScenario_FalseMask(); + test.RunBasicScenario_GenValid(); + test.RunBasicScenario_GenInvalid(); + + // Validates calling via reflection works + test.RunReflectionScenario(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public static class SvePrefetchTypeGenerator{TestName} + { + static Random m_rand = new Random(); + + [MethodImpl(MethodImplOptions.NoInlining)] + public static SvePrefetchType GetValid() + { + return (SvePrefetchType)m_rand.Next(0, 15); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public static SvePrefetchType GetInvalid() + { + return (SvePrefetchType)m_rand.Next(16, 100); + } + } + + public sealed unsafe class {TestName}Test + { + private struct DataTable + { + private byte[] inArray; + private byte[] outArray1; + private byte[] outArray2; + + private GCHandle inHandle; + private GCHandle outHandle1; + private GCHandle outHandle2; + + private ulong alignment; + + public DataTable({Op1BaseType}[] outArray1, {Op1BaseType}[] outArray2, {Op1BaseType}[] inArray, int alignment) + { + int sizeOfInArray = inArray.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfOutArray1 = outArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfOutArray2 = outArray2.Length * Unsafe.SizeOf<{Op1BaseType}>(); + if ((alignment != 64 && alignment != 8) || (alignment * 2) < sizeOfInArray || (alignment * 2) < sizeOfOutArray1 || (alignment * 2) < sizeOfOutArray2) + { + throw new ArgumentException("Invalid value of alignment"); + } + + this.inArray = new byte[alignment * 2 * 2]; + this.outArray1 = new byte[alignment * 2]; + this.outArray2 = new byte[alignment * 2]; + + this.inHandle = GCHandle.Alloc(this.inArray, GCHandleType.Pinned); + this.outHandle1 = GCHandle.Alloc(this.outArray1, GCHandleType.Pinned); + this.outHandle2 = GCHandle.Alloc(this.outArray2, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArrayPtr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray[0]), (uint)sizeOfInArray); + } + + public void* inArrayPtr => Align((byte*)(inHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArray1Ptr => Align((byte*)(outHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArray2Ptr => Align((byte*)(outHandle2.AddrOfPinnedObject().ToPointer()), alignment); + + + public void Dispose() + { + inHandle.Free(); + outHandle1.Free(); + outHandle2.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int OpElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int DestElementCount = OpElementCount * 2; + + private static {Op1BaseType}[] _data = new {Op1BaseType}[DestElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op1VectorType}<{Op1BaseType}> _fld2; + + private DataTable _dataTable; + + public {TestName}Test() + { + Succeeded = true; + for (var i = 0; i < DestElementCount; i++) { _data[i] = {NextValueOp2}; } + _dataTable = new DataTable(new {Op1BaseType}[OpElementCount], new {Op1BaseType}[OpElementCount], _data, LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + {Isa}.{Method}(loadMask, ({Op1BaseType}*)(_dataTable.inArrayPtr), {ValidPrefetch}); + } + + public void RunBasicScenario_Invalid() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Invalid)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + try + { + {Isa}.{Method}(loadMask, ({Op1BaseType}*)(_dataTable.inArrayPtr), {InvalidPrefetch}); + Succeeded = false; + } + catch (ArgumentOutOfRangeException) + { + } + } + + public void RunBasicScenario_FalseMask() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_FalseMask)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateFalseMask{Op1BaseType}(); + + {Isa}.{Method}(loadMask, ({Op1BaseType}*)(_dataTable.inArrayPtr), {ValidPrefetch}); + } + + public void RunBasicScenario_GenValid() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_GenValid)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + {Isa}.{Method}(loadMask, ({Op1BaseType}*)(_dataTable.inArrayPtr), SvePrefetchTypeGenerator{TestName}.GetValid()); + } + + public void RunBasicScenario_GenInvalid() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_GenInvalid)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + try + { + {Isa}.{Method}(loadMask, ({Op1BaseType}*)(_dataTable.inArrayPtr), SvePrefetchTypeGenerator{TestName}.GetInvalid()); + Succeeded = false; + } + catch (ArgumentOutOfRangeException) + { + } + } + + public void RunReflectionScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof(Vector<{Op1BaseType}>), typeof(void*), typeof(SvePrefetchType) }) + .Invoke(null, new object[] { + loadMask, + Pointer.Box(_dataTable.inArrayPtr, typeof({Op1BaseType}*)), + {ValidPrefetch} + }); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + } + } +} \ No newline at end of file