From 135fec006e727a31763271984cd712f1659ccbd3 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Wed, 24 Jan 2024 22:25:54 -0800 Subject: [PATCH] Allow TernaryLogic to optimize down to BlendVariableMask where appropriate (#97468) * Allow TernaryLogic to optimize down to BlendVariableMask where appropriate * Ensure the BlendVariableMask node has the parameters in the right order * Don't unnecessarily modify morph.cpp * Ensure we remove `condition` not `op1` --- src/coreclr/jit/hwintrinsic.cpp | 60 ++++++------- src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerxarch.cpp | 145 +++++++++++++++++++++++++++++++- 3 files changed, 175 insertions(+), 31 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 872cdd4dec985..f771a9ec978e2 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -228,11 +228,11 @@ const TernaryLogicInfo& TernaryLogicInfo::lookup(uint8_t control) /* A?andBC:xorBC */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* xnorAandBC */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* andCB */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* B?C:norAC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?C:norAC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* A?andBC:C */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* B?C:!A */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?C:!A */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* A?andBC:B */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* C?B:!A */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?B:!A */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* A?andBC:orBC */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* nandAnandBC */ { TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* andAxnorBC */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::And, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, @@ -246,80 +246,80 @@ const TernaryLogicInfo& TernaryLogicInfo::lookup(uint8_t control) /* A?xnorBC:andBC */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::And, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* xnorCB */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* A?xnorBC:C */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* B?C:nandAC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?C:nandAC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* A?xnorBC:B */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* C?B:nandBA */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?B:nandBA */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* A?xnorBC:orBC */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* nandAxorBC */ { TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* andCA */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AC, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* A?C:norBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?C:norBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* B?andAC:C */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, - /* A?C:!B */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?C:!B */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* B?xnorAC:andAC */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::And, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* xnorCA */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* A?C:xorBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* A?C:nandBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?C:xorBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?C:nandBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* andCorAB */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::And, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* xnorCorBA */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* C */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orCnorBA */ { TernaryLogicOperKind::Nor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Or, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* A?C:B */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?C:B */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* C?orBA:!A */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, - /* A?C:orBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?C:orBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* orC!A */ { TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Or, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* B?andAC:A */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, - /* C?A:!B */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?A:!B */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* B?andAC:orAC */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* nandBnandAC */ { TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* B?xnorAC:A */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, - /* C?A:nandBA */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?A:nandBA */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* B?xnorAC:orAC */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* nandBxorAC */ { TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* B?C:A */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?C:A */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* C?orBA:!B */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, - /* B?C:orAC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?C:orAC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* orC!B */ { TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Or, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* C?orBA:xorBA */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* C?orBA:nandBA */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* orCxorBA */ { TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Or, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orCnandBA */ { TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Or, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* andBA */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AB, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* A?B:norBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Nor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?B:norBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Nor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* C?xnorBA:andBA */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::And, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* xnorBA */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* C?andBA:B */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, - /* A?B:!C */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* A?B:xorBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* A?B:nandBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?B:!C */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?B:xorBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?B:nandBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* andBorAC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::And, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* xnorBorAC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* A?B:C */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?B:C */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* B?orAC:!A */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* B */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orBnorAC */ { TernaryLogicOperKind::Nor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* A?B:orBC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, + /* A?B:orBC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* orB!A */ { TernaryLogicOperKind::Not, TernaryLogicUseFlags::A, TernaryLogicOperKind::Or, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* C?andBA:A */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, - /* B?A:!C */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, - /* B?A:xorAC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, - /* B?A:nandAC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?A:!C */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?A:xorAC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?A:nandAC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* C?andBA:orBA */ { TernaryLogicOperKind::And, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* nandCnandBA */ { TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* C?xnorBA:orBA */ { TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* nandCxorBA */ { TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::C, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* C?B:A */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?B:A */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* B?orAC:!C */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* B?orAC:xorAC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* B?orAC:nandAC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, - /* C?B:orBA */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?B:orBA */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* orB!C */ { TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Or, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orBxorAC */ { TernaryLogicOperKind::Xor, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orBnandAC */ { TernaryLogicOperKind::Nand, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::B, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* andAorBC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::And, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* xnorAorBC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Xnor, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* B?A:C */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?A:C */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Select, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* A?orBC:!B */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, - /* C?A:B */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?A:B */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Select, TernaryLogicUseFlags::B, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* A?orBC:!C */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* A?orBC:xorBC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, /* A?orBC:nandBC */ { TernaryLogicOperKind::Or, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A }, @@ -333,9 +333,9 @@ const TernaryLogicInfo& TernaryLogicInfo::lookup(uint8_t control) /* nandAnorBC */ { TernaryLogicOperKind::Nor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Nand, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* A */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orAnorBC */ { TernaryLogicOperKind::Nor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* B?A:orAC */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, + /* B?A:orAC */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AC, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::B }, /* orA!B */ { TernaryLogicOperKind::Not, TernaryLogicUseFlags::B, TernaryLogicOperKind::Or, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, - /* C?A:orBA */ { TernaryLogicOperKind::Cond, TernaryLogicUseFlags::A, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, + /* C?A:orBA */ { TernaryLogicOperKind::Select, TernaryLogicUseFlags::A, TernaryLogicOperKind::Or, TernaryLogicUseFlags::AB, TernaryLogicOperKind::Cond, TernaryLogicUseFlags::C }, /* orA!C */ { TernaryLogicOperKind::Not, TernaryLogicUseFlags::C, TernaryLogicOperKind::Or, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orAxorBC */ { TernaryLogicOperKind::Xor, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, /* orAnandBC */ { TernaryLogicOperKind::Nand, TernaryLogicUseFlags::BC, TernaryLogicOperKind::Or, TernaryLogicUseFlags::A, TernaryLogicOperKind::None, TernaryLogicUseFlags::None }, diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 3f668c0f2d472..e687f133e206f 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -376,6 +376,7 @@ class Lowering final : public Phase GenTree* LowerHWIntrinsicToScalar(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicGetElement(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* node); + GenTree* LowerHWIntrinsicTernaryLogic(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicWithElement(GenTreeHWIntrinsic* node); GenTree* TryLowerAndOpToResetLowestSetBit(GenTreeOp* andNode); GenTree* TryLowerAndOpToExtractLowestSetBit(GenTreeOp* andNode); diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0ff7658c2470f..a149f71e9b4ae 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -1133,7 +1133,6 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { case NI_Vector128_ConditionalSelect: case NI_Vector256_ConditionalSelect: - case NI_Vector512_ConditionalSelect: { return LowerHWIntrinsicCndSel(node); } @@ -1834,6 +1833,12 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_AVX512F_TernaryLogic: + case NI_AVX512F_VL_TernaryLogic: + { + return LowerHWIntrinsicTernaryLogic(node); + } + default: break; } @@ -2689,6 +2694,144 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* node) return LowerNode(tmp4); } +//---------------------------------------------------------------------------------------------- +// Lowering::LowerHWIntrinsicCndSel: Lowers an AVX512 TernaryLogic call +// +// Arguments: +// node - The hardware intrinsic node. +// +GenTree* Lowering::LowerHWIntrinsicTernaryLogic(GenTreeHWIntrinsic* node) +{ + assert(comp->compIsaSupportedDebugOnly(InstructionSet_AVX512F_VL)); + + var_types simdType = node->gtType; + CorInfoType simdBaseJitType = node->GetSimdBaseJitType(); + var_types simdBaseType = node->GetSimdBaseType(); + unsigned simdSize = node->GetSimdSize(); + + assert(varTypeIsSIMD(simdType)); + assert(varTypeIsArithmetic(simdBaseType)); + assert(simdSize != 0); + + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + GenTree* op3 = node->Op(3); + GenTree* op4 = node->Op(4); + + if (op4->IsCnsIntOrI()) + { + uint8_t control = static_cast(op4->AsIntConCommon()->IconValue()); + + switch (control) + { + case 0xAC: // A ? C : B; (C & A) | (B & ~A) + case 0xB8: // B ? C : A; (C & B) | (A & ~B) + case 0xD8: // C ? B : A; (B & C) | (A & ~C) + case 0xCA: // A ? B : C; (B & A) | (C & ~A) + case 0xE2: // B ? A : C; (A & B) | (C & ~B) + case 0xE4: // C ? A : B; (A & C) | (B & ~C) + { + // For the operations that work as a conditional select, we want + // to try and optimize it to use BlendVariableMask when the condition + // is already a TYP_MASK + + const TernaryLogicInfo& ternLogInfo = TernaryLogicInfo::lookup(control); + + assert(ternLogInfo.oper1 == TernaryLogicOperKind::Select); + assert(ternLogInfo.oper2 == TernaryLogicOperKind::Select); + assert(ternLogInfo.oper3 == TernaryLogicOperKind::Cond); + + GenTree* condition = nullptr; + GenTree* selectTrue = nullptr; + GenTree* selectFalse = nullptr; + + if (ternLogInfo.oper1Use == TernaryLogicUseFlags::A) + { + selectTrue = op1; + + if (ternLogInfo.oper2Use == TernaryLogicUseFlags::B) + { + assert(ternLogInfo.oper3Use == TernaryLogicUseFlags::C); + + selectFalse = op2; + condition = op3; + } + else + { + assert(ternLogInfo.oper2Use == TernaryLogicUseFlags::C); + assert(ternLogInfo.oper3Use == TernaryLogicUseFlags::B); + + selectFalse = op3; + condition = op2; + } + } + else if (ternLogInfo.oper1Use == TernaryLogicUseFlags::B) + { + selectTrue = op2; + + if (ternLogInfo.oper2Use == TernaryLogicUseFlags::A) + { + assert(ternLogInfo.oper3Use == TernaryLogicUseFlags::C); + + selectFalse = op1; + condition = op3; + } + else + { + assert(ternLogInfo.oper2Use == TernaryLogicUseFlags::C); + assert(ternLogInfo.oper3Use == TernaryLogicUseFlags::A); + + selectFalse = op3; + condition = op1; + } + } + else + { + assert(ternLogInfo.oper1Use == TernaryLogicUseFlags::C); + + selectTrue = op3; + + if (ternLogInfo.oper2Use == TernaryLogicUseFlags::A) + { + assert(ternLogInfo.oper3Use == TernaryLogicUseFlags::B); + + selectFalse = op1; + condition = op2; + } + else + { + assert(ternLogInfo.oper2Use == TernaryLogicUseFlags::B); + assert(ternLogInfo.oper3Use == TernaryLogicUseFlags::A); + + selectFalse = op2; + condition = op1; + } + } + + if (!condition->OperIsHWIntrinsic(NI_AVX512F_ConvertMaskToVector)) + { + break; + } + + node->ResetHWIntrinsicId(NI_AVX512F_BlendVariableMask, comp, selectFalse, selectTrue, + condition->AsHWIntrinsic()->Op(1)); + + BlockRange().Remove(condition); + BlockRange().Remove(op4); + break; + } + + default: + { + break; + } + } + } + + ContainCheckHWIntrinsic(node); + return node->gtNext; +} + //---------------------------------------------------------------------------------------------- // Lowering::LowerHWIntrinsicCreate: Lowers a Vector128 or Vector256 or Vector512 Create call //