From 5f6619e865e36d5507664f91e823dc9b8af51fbb Mon Sep 17 00:00:00 2001 From: Clinton Ingram Date: Sat, 26 Oct 2024 16:17:04 -0700 Subject: [PATCH] enable EVEX embedded broadcast in more places --- src/coreclr/jit/lower.h | 2 +- src/coreclr/jit/lowerxarch.cpp | 172 +++++++++------------------------ 2 files changed, 45 insertions(+), 129 deletions(-) diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index c5c771167025e..d024e80aa7345 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -566,7 +566,7 @@ class Lowering final : public Phase #if defined(FEATURE_HW_INTRINSICS) // Tries to make 'childNode' contained or regOptional in the 'parentNode' - void TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, GenTree* childNode) const; + void TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, GenTree* childNode); #endif // Checks and makes 'childNode' contained in the 'parentNode' diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 1da40d1dc453b..833746babb082 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -10155,6 +10155,34 @@ void Lowering::TryCompressConstVecData(GenTreeStoreInd* node) LowerNode(broadcast); } +//------------------------------------------------------------------------ +// TryMakeSrcContainedOrRegOptional: Tries to make "childNode" a contained or regOptional node +// +// Arguments: +// parentNode - The hardware intrinsic node which is the parent of 'childNode' +// childNode - The node to check if it can be contained by 'parentNode' +// +void Lowering::TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, GenTree* childNode) +{ + bool supportsRegOptional = false; + + if (IsContainableHWIntrinsicOp(parentNode, childNode, &supportsRegOptional)) + { + if (childNode->IsCnsVec() && parentNode->OperIsEmbBroadcastCompatible() && comp->canUseEvexEncoding()) + { + TryFoldCnsVecForEmbeddedBroadcast(parentNode, childNode->AsVecCon()); + } + else + { + MakeSrcContained(parentNode, childNode); + } + } + else if (supportsRegOptional) + { + MakeSrcRegOptional(parentNode, childNode); + } +} + //---------------------------------------------------------------------------------------------- // ContainCheckHWIntrinsicAddr: Perform containment analysis for an address operand of a hardware // intrinsic node. @@ -10446,16 +10474,8 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } assert(!node->OperIsMemoryLoad()); - bool supportsRegOptional = false; - if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional)) - { - MakeSrcContained(node, op1); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op1); - } + TryMakeSrcContainedOrRegOptional(node, op1); break; } @@ -10557,7 +10577,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { // We don't currently have any IMM intrinsics which are also commutative assert(!isCommutative); - bool supportsRegOptional = false; switch (intrinsicId) { @@ -10601,14 +10620,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) if (!HWIntrinsicInfo::isImmOp(intrinsicId, op2)) { - if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op2); - } + TryMakeSrcContainedOrRegOptional(node, op2); } break; } @@ -10620,16 +10632,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) // byte and sbyte are: pshufb ymm1, ymm2, ymm3/m256 assert(!isCommutative); - bool supportsRegOptional = false; - - if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op2); - } + TryMakeSrcContainedOrRegOptional(node, op2); break; } FALLTHROUGH; @@ -10656,14 +10659,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional)) - { - MakeSrcContained(node, op1); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op1); - } + TryMakeSrcContainedOrRegOptional(node, op1); break; } @@ -10690,22 +10686,11 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional)) - { - MakeSrcContained(node, op1); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op1); - } + TryMakeSrcContainedOrRegOptional(node, op1); } - else if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) + else { - MakeSrcRegOptional(node, op2); + TryMakeSrcContainedOrRegOptional(node, op2); } break; } @@ -10728,14 +10713,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional)) - { - MakeSrcContained(node, op1); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op1); - } + TryMakeSrcContainedOrRegOptional(node, op1); break; } @@ -10754,14 +10732,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional)) - { - MakeSrcContained(node, op1); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op1); - } + TryMakeSrcContainedOrRegOptional(node, op1); break; } @@ -11139,22 +11110,13 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } else { - bool supportsRegOptional = false; - switch (intrinsicId) { case NI_SSE41_BlendVariable: case NI_AVX_BlendVariable: case NI_AVX2_BlendVariable: { - if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op2); - } + TryMakeSrcContainedOrRegOptional(node, op2); break; } @@ -11224,28 +11186,14 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } } - if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op2); - } + TryMakeSrcContainedOrRegOptional(node, op2); break; } case NI_AVXVNNI_MultiplyWideningAndAdd: case NI_AVXVNNI_MultiplyWideningAndAddSaturate: { - if (IsContainableHWIntrinsicOp(node, op3, &supportsRegOptional)) - { - MakeSrcContained(node, op3); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op3); - } + TryMakeSrcContainedOrRegOptional(node, op3); break; } @@ -11308,14 +11256,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_X86Base_X64_DivRem: { // DIV only allows divisor (op3) in memory - if (IsContainableHWIntrinsicOp(node, op3, &supportsRegOptional)) - { - MakeSrcContained(node, op3); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op3); - } + TryMakeSrcContainedOrRegOptional(node, op3); break; } @@ -11331,8 +11272,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case HW_Category_IMM: { - bool supportsRegOptional = false; - switch (intrinsicId) { case NI_SSE_Shuffle: @@ -11396,14 +11335,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op2); - } + TryMakeSrcContainedOrRegOptional(node, op2); break; } @@ -11474,14 +11406,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } } - if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional)) - { - MakeSrcContained(node, op2); - } - else if (supportsRegOptional) - { - op2->SetRegOptional(); - } + TryMakeSrcContainedOrRegOptional(node, op2); break; } @@ -11516,8 +11441,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { case HW_Category_IMM: { - bool supportsRegOptional = false; - switch (intrinsicId) { case NI_AVX512F_Fixup: @@ -11532,14 +11455,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - if (IsContainableHWIntrinsicOp(node, op3, &supportsRegOptional)) - { - MakeSrcContained(node, op3); - } - else if (supportsRegOptional) - { - MakeSrcRegOptional(node, op3); - } + TryMakeSrcContainedOrRegOptional(node, op3); if (!node->isRMWHWIntrinsic(comp)) {