Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Enable EVEX embedded broadcast in more places #109258

Merged
merged 1 commit into from
Nov 5, 2024
Merged

JIT: Enable EVEX embedded broadcast in more places #109258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/coreclr/jit/lower.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,7 @@ class Lowering final : public Phase

#if defined(FEATURE_HW_INTRINSICS)
// Tries to make 'childNode' contained or regOptional in the 'parentNode'
void TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, GenTree* childNode) const;
void TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, GenTree* childNode);
#endif

// Checks and makes 'childNode' contained in the 'parentNode'
Expand Down
172 changes: 44 additions & 128 deletions src/coreclr/jit/lowerxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10155,6 +10155,34 @@ void Lowering::TryCompressConstVecData(GenTreeStoreInd* node)
LowerNode(broadcast);
}

//------------------------------------------------------------------------
// TryMakeSrcContainedOrRegOptional: Tries to make "childNode" a contained or regOptional node
//
// Arguments:
// parentNode - The hardware intrinsic node which is the parent of 'childNode'
// childNode - The node to check if it can be contained by 'parentNode'
//
void Lowering::TryMakeSrcContainedOrRegOptional(GenTreeHWIntrinsic* parentNode, GenTree* childNode)
{
bool supportsRegOptional = false;

if (IsContainableHWIntrinsicOp(parentNode, childNode, &supportsRegOptional))
{
if (childNode->IsCnsVec() && parentNode->OperIsEmbBroadcastCompatible() && comp->canUseEvexEncoding())
{
TryFoldCnsVecForEmbeddedBroadcast(parentNode, childNode->AsVecCon());
}
else
{
MakeSrcContained(parentNode, childNode);
}
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(parentNode, childNode);
}
}

//----------------------------------------------------------------------------------------------
// ContainCheckHWIntrinsicAddr: Perform containment analysis for an address operand of a hardware
// intrinsic node.
Expand Down Expand Up @@ -10446,16 +10474,8 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
}

assert(!node->OperIsMemoryLoad());
bool supportsRegOptional = false;

if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional))
{
MakeSrcContained(node, op1);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op1);
}
TryMakeSrcContainedOrRegOptional(node, op1);
break;
}

Expand Down Expand Up @@ -10557,7 +10577,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
{
// We don't currently have any IMM intrinsics which are also commutative
assert(!isCommutative);
bool supportsRegOptional = false;

switch (intrinsicId)
{
Expand Down Expand Up @@ -10601,14 +10620,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)

if (!HWIntrinsicInfo::isImmOp(intrinsicId, op2))
{
if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op2);
}
TryMakeSrcContainedOrRegOptional(node, op2);
}
break;
}
Expand All @@ -10620,16 +10632,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
// byte and sbyte are: pshufb ymm1, ymm2, ymm3/m256
assert(!isCommutative);

bool supportsRegOptional = false;

if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op2);
}
TryMakeSrcContainedOrRegOptional(node, op2);
break;
}
FALLTHROUGH;
Expand All @@ -10656,14 +10659,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional))
{
MakeSrcContained(node, op1);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op1);
}
TryMakeSrcContainedOrRegOptional(node, op1);
break;
}

Expand All @@ -10690,22 +10686,11 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional))
{
MakeSrcContained(node, op1);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op1);
}
TryMakeSrcContainedOrRegOptional(node, op1);
}
else if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
else
{
MakeSrcRegOptional(node, op2);
TryMakeSrcContainedOrRegOptional(node, op2);
}
break;
}
Expand All @@ -10728,14 +10713,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional))
{
MakeSrcContained(node, op1);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op1);
}
TryMakeSrcContainedOrRegOptional(node, op1);
break;
}

Expand All @@ -10754,14 +10732,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

if (IsContainableHWIntrinsicOp(node, op1, &supportsRegOptional))
{
MakeSrcContained(node, op1);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op1);
}
TryMakeSrcContainedOrRegOptional(node, op1);
break;
}

Expand Down Expand Up @@ -11139,22 +11110,13 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
}
else
{
bool supportsRegOptional = false;

switch (intrinsicId)
{
case NI_SSE41_BlendVariable:
case NI_AVX_BlendVariable:
case NI_AVX2_BlendVariable:
{
if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op2);
}
TryMakeSrcContainedOrRegOptional(node, op2);
break;
}

Expand Down Expand Up @@ -11224,28 +11186,14 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
}
}

if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op2);
}
TryMakeSrcContainedOrRegOptional(node, op2);
break;
}

case NI_AVXVNNI_MultiplyWideningAndAdd:
case NI_AVXVNNI_MultiplyWideningAndAddSaturate:
{
if (IsContainableHWIntrinsicOp(node, op3, &supportsRegOptional))
{
MakeSrcContained(node, op3);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op3);
}
TryMakeSrcContainedOrRegOptional(node, op3);
break;
}

Expand Down Expand Up @@ -11308,14 +11256,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
case NI_X86Base_X64_DivRem:
{
// DIV only allows divisor (op3) in memory
if (IsContainableHWIntrinsicOp(node, op3, &supportsRegOptional))
{
MakeSrcContained(node, op3);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op3);
}
TryMakeSrcContainedOrRegOptional(node, op3);
break;
}

Expand All @@ -11331,8 +11272,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)

case HW_Category_IMM:
{
bool supportsRegOptional = false;

switch (intrinsicId)
{
case NI_SSE_Shuffle:
Expand Down Expand Up @@ -11396,14 +11335,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op2);
}
TryMakeSrcContainedOrRegOptional(node, op2);
break;
}

Expand Down Expand Up @@ -11474,14 +11406,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
}
}

if (IsContainableHWIntrinsicOp(node, op2, &supportsRegOptional))
{
MakeSrcContained(node, op2);
}
else if (supportsRegOptional)
{
op2->SetRegOptional();
}
TryMakeSrcContainedOrRegOptional(node, op2);
break;
}

Expand Down Expand Up @@ -11516,8 +11441,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
{
case HW_Category_IMM:
{
bool supportsRegOptional = false;

switch (intrinsicId)
{
case NI_AVX512F_Fixup:
Expand All @@ -11532,14 +11455,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

if (IsContainableHWIntrinsicOp(node, op3, &supportsRegOptional))
{
MakeSrcContained(node, op3);
}
else if (supportsRegOptional)
{
MakeSrcRegOptional(node, op3);
}
TryMakeSrcContainedOrRegOptional(node, op3);

if (!node->isRMWHWIntrinsic(comp))
{
Expand Down
Loading