Skip to content

Commit

Permalink
[VPlan] Funnel
Browse files Browse the repository at this point in the history
  • Loading branch information
@fhahn
fhahn committed Dec 27, 2024
1 parent 407dbc1 commit af48fcc
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 69 deletions.
5 changes: 2 additions & 3 deletions llvm/lib/Transforms/Vectorize/VPlan.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3844,7 +3844,6 @@ class VPlan {
/// VPlan is destroyed.
SmallVector<VPBlockBase *> CreatedBlocks;

public:
/// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
/// wrapping the original header of the scalar loop.
VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
Expand All @@ -3861,8 +3860,8 @@ class VPlan {
VPlan(Loop *L);

VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
setEntry(new VPBasicBlock("preheader"));
ScalarHeader = VPIRBasicBlock::fromBasicBlock(ScalarHeaderBB);
setEntry(createVPBasicBlock("preheader"));
ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
TripCount = TC;
}

Expand Down
6 changes: 2 additions & 4 deletions llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,12 +357,10 @@ void PlainCFGBuilder::buildPlainCFG() {
BB2VPBB[TheLoop->getHeader()] = VectorHeaderVPBB;
VectorHeaderVPBB->clearSuccessors();
VectorLatchVPBB->clearPredecessors();
if (TheLoop->getHeader() != TheLoop->getLoopLatch()) {
if (TheLoop->getHeader() != TheLoop->getLoopLatch())
BB2VPBB[TheLoop->getLoopLatch()] = VectorLatchVPBB;
} else {
else
TheRegion->setExiting(VectorHeaderVPBB);
delete VectorLatchVPBB;
}

// 1. Scan the body of the loop in a topological order to visit each basic
// block after having visited its predecessor basic blocks. Create a VPBB for
Expand Down
35 changes: 18 additions & 17 deletions llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ TEST_F(VPDominatorTreeTest, DominanceNoRegionsTest) {
// }
VPlan &Plan = getPlan();
VPBasicBlock *VPBB0 = Plan.getEntry();
VPBasicBlock *VPBB1 = new VPBasicBlock("VPBB1");
VPBasicBlock *VPBB2 = new VPBasicBlock("VPBB2");
VPBasicBlock *VPBB3 = new VPBasicBlock("VPBB3");
VPBasicBlock *VPBB4 = new VPBasicBlock("VPBB4");
VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("VPBB1");
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("VPBB2");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("VPBB3");
VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("VPBB4");
VPRegionBlock *R1 = new VPRegionBlock(VPBB1, VPBB4);
VPBB2->setParent(R1);
VPBB3->setParent(R1);
Expand Down Expand Up @@ -96,10 +96,10 @@ TEST_F(VPDominatorTreeTest, DominanceRegionsTest) {
//
VPlan &Plan = getPlan();
VPBasicBlock *VPBB0 = Plan.getEntry();
VPBasicBlock *R1BB1 = new VPBasicBlock();
VPBasicBlock *R1BB2 = new VPBasicBlock();
VPBasicBlock *R1BB3 = new VPBasicBlock();
VPBasicBlock *R1BB4 = new VPBasicBlock();
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB3 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB4 = Plan.createVPBasicBlock("");
VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB4, "R1");
R1BB2->setParent(R1);
R1BB3->setParent(R1);
Expand All @@ -111,8 +111,8 @@ TEST_F(VPDominatorTreeTest, DominanceRegionsTest) {
// Cycle.
VPBlockUtils::connectBlocks(R1BB3, R1BB3);

VPBasicBlock *R2BB1 = new VPBasicBlock();
VPBasicBlock *R2BB2 = new VPBasicBlock();
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB2, "R2");
VPBlockUtils::connectBlocks(R2BB1, R2BB2);
VPBlockUtils::connectBlocks(R1, R2);
Expand Down Expand Up @@ -170,14 +170,14 @@ TEST_F(VPDominatorTreeTest, DominanceRegionsTest) {
// VPBB2
//
VPlan &Plan = getPlan();
VPBasicBlock *R1BB1 = new VPBasicBlock("R1BB1");
VPBasicBlock *R1BB2 = new VPBasicBlock("R1BB2");
VPBasicBlock *R1BB3 = new VPBasicBlock("R1BB3");
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("R1BB1");
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("R1BB2");
VPBasicBlock *R1BB3 = Plan.createVPBasicBlock("R1BB3");
VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB3, "R1");

VPBasicBlock *R2BB1 = new VPBasicBlock("R2BB1");
VPBasicBlock *R2BB2 = new VPBasicBlock("R2BB2");
VPBasicBlock *R2BB3 = new VPBasicBlock("R2BB3");
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("");
VPBasicBlock *R2BB3 = Plan.createVPBasicBlock("");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB3, "R2");
R2BB2->setParent(R2);
VPBlockUtils::connectBlocks(R2BB1, R2BB2);
Expand All @@ -193,7 +193,8 @@ TEST_F(VPDominatorTreeTest, DominanceRegionsTest) {

VPBasicBlock *VPBB1 = Plan.getEntry();
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBasicBlock *VPBB2 = new VPBasicBlock("VPBB2");
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""
"VPBB2");
VPBlockUtils::connectBlocks(R1, VPBB2);

VPBlockUtils::connectBlocks(VPBB2, Plan.getScalarHeader());
Expand Down
77 changes: 41 additions & 36 deletions llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,9 +245,9 @@ TEST_F(VPBasicBlockTest, getPlan) {
{
VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB3 = new VPBasicBlock();
VPBasicBlock *VPBB4 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");

// VPBB1
// / \
Expand All @@ -270,8 +270,8 @@ TEST_F(VPBasicBlockTest, getPlan) {
VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
// VPBasicBlock is the entry into the VPlan, followed by a region.
VPBasicBlock *R1BB1 = new VPBasicBlock();
VPBasicBlock *R1BB2 = new VPBasicBlock();
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("");
VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB2, "R1");
VPBlockUtils::connectBlocks(R1BB1, R1BB2);

Expand All @@ -287,21 +287,21 @@ TEST_F(VPBasicBlockTest, getPlan) {

{
VPlan &Plan = getPlan();
VPBasicBlock *R1BB1 = new VPBasicBlock();
VPBasicBlock *R1BB2 = new VPBasicBlock();
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("");
VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB2, "R1");
VPBlockUtils::connectBlocks(R1BB1, R1BB2);

VPBasicBlock *R2BB1 = new VPBasicBlock();
VPBasicBlock *R2BB2 = new VPBasicBlock();
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB2, "R2");
VPBlockUtils::connectBlocks(R2BB1, R2BB2);

VPBasicBlock *VPBB1 = Plan.getEntry();
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(VPBB1, R2);

VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBlockUtils::connectBlocks(R1, VPBB2);
VPBlockUtils::connectBlocks(R2, VPBB2);

Expand Down Expand Up @@ -329,9 +329,9 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
//
VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB3 = new VPBasicBlock();
VPBasicBlock *VPBB4 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");

VPBlockUtils::connectBlocks(VPBB1, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, VPBB3);
Expand Down Expand Up @@ -368,10 +368,10 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
//
VPlan &Plan = getPlan();
VPBasicBlock *VPBB0 = Plan.getEntry();
VPBasicBlock *R1BB1 = new VPBasicBlock();
VPBasicBlock *R1BB2 = new VPBasicBlock();
VPBasicBlock *R1BB3 = new VPBasicBlock();
VPBasicBlock *R1BB4 = new VPBasicBlock();
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB3 = Plan.createVPBasicBlock("");
VPBasicBlock *R1BB4 = Plan.createVPBasicBlock("");
VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB4, "R1");
R1BB2->setParent(R1);
R1BB3->setParent(R1);
Expand All @@ -383,8 +383,8 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
// Cycle.
VPBlockUtils::connectBlocks(R1BB3, R1BB3);

VPBasicBlock *R2BB1 = new VPBasicBlock();
VPBasicBlock *R2BB2 = new VPBasicBlock();
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("");
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB2, "R2");
VPBlockUtils::connectBlocks(R2BB1, R2BB2);
VPBlockUtils::connectBlocks(R1, R2);
Expand Down Expand Up @@ -467,14 +467,17 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
// VPBB2
//
VPlan &Plan = getPlan();
VPBasicBlock *R1BB1 = new VPBasicBlock("R1BB1");
VPBasicBlock *R1BB2 = new VPBasicBlock("R1BB2");
VPBasicBlock *R1BB3 = new VPBasicBlock("R1BB3");
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("R1BB1");
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("R1BB2");
VPBasicBlock *R1BB3 = Plan.createVPBasicBlock("R1BB3");
VPRegionBlock *R1 = new VPRegionBlock(R1BB1, R1BB3, "R1");

VPBasicBlock *R2BB1 = new VPBasicBlock("R2BB1");
VPBasicBlock *R2BB2 = new VPBasicBlock("R2BB2");
VPBasicBlock *R2BB3 = new VPBasicBlock("R2BB3");
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock(""
"R2BB1");
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock(""
"R2BB2");
VPBasicBlock *R2BB3 = Plan.createVPBasicBlock(""
"R2BB3");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB3, "R2");
R2BB2->setParent(R2);
VPBlockUtils::connectBlocks(R2BB1, R2BB2);
Expand All @@ -490,7 +493,8 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {

VPBasicBlock *VPBB1 = Plan.getEntry();
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBasicBlock *VPBB2 = new VPBasicBlock("VPBB2");
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock(""
"VPBB2");
VPBlockUtils::connectBlocks(R1, VPBB2);

// Depth-first.
Expand Down Expand Up @@ -538,8 +542,8 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
// }
//
VPlan &Plan = getPlan();
VPBasicBlock *R2BB1 = new VPBasicBlock("R2BB1");
VPBasicBlock *R2BB2 = new VPBasicBlock("R2BB2");
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("R2BB1");
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("R2BB2");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R2BB2, "R2");
VPBlockUtils::connectBlocks(R2BB1, R2BB2);

Expand Down Expand Up @@ -592,10 +596,11 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
// VPBB2
//
VPlan &Plan = getPlan();
VPBasicBlock *R3BB1 = new VPBasicBlock("R3BB1");
VPBasicBlock *R3BB1 = Plan.createVPBasicBlock("R3BB1");
VPRegionBlock *R3 = new VPRegionBlock(R3BB1, R3BB1, "R3");

VPBasicBlock *R2BB1 = new VPBasicBlock("R2BB1");
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock(""
"R2BB1");
VPRegionBlock *R2 = new VPRegionBlock(R2BB1, R3, "R2");
R3->setParent(R2);
VPBlockUtils::connectBlocks(R2BB1, R3);
Expand All @@ -604,7 +609,7 @@ TEST_F(VPBasicBlockTest, TraversingIteratorTest) {
R2->setParent(R1);

VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock("VPBB2");
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("VPBB2");
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, VPBB2);

Expand Down Expand Up @@ -674,15 +679,15 @@ TEST_F(VPBasicBlockTest, print) {
VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1});
VPInstruction *I3 = new VPInstruction(Instruction::Br, {I1, I2});

VPBasicBlock *VPBB1 = new VPBasicBlock();
VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("");
VPBB1->appendRecipe(I1);
VPBB1->appendRecipe(I2);
VPBB1->appendRecipe(I3);
VPBB1->setName("bb1");

VPInstruction *I4 = new VPInstruction(Instruction::Mul, {I2, I1});
VPInstruction *I5 = new VPInstruction(Instruction::Ret, {I4});
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBB2->appendRecipe(I4);
VPBB2->appendRecipe(I5);
VPBB2->setName("bb2");
Expand Down Expand Up @@ -783,7 +788,7 @@ TEST_F(VPBasicBlockTest, printPlanWithVFsAndUFs) {
VPBB0->appendRecipe(TC);

VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
VPBasicBlock *VPBB1 = new VPBasicBlock();
VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("");
VPBB1->appendRecipe(I1);
VPBB1->setName("bb1");

Expand Down Expand Up @@ -1238,7 +1243,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
TEST_F(VPRecipeTest, dumpRecipeInPlan) {
VPlan &Plan = getPlan();
VPBasicBlock *VPBB0 = Plan.getEntry();
VPBasicBlock *VPBB1 = new VPBasicBlock();
VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("");
VPBlockUtils::connectBlocks(VPBB1, Plan.getScalarHeader());
VPBlockUtils::connectBlocks(VPBB0, VPBB1);

Expand Down Expand Up @@ -1307,7 +1312,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) {
TEST_F(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
VPlan &Plan = getPlan();
VPBasicBlock *VPBB0 = Plan.getEntry();
VPBasicBlock *VPBB1 = new VPBasicBlock();
VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("");
VPBlockUtils::connectBlocks(VPBB1, Plan.getScalarHeader());
VPBlockUtils::connectBlocks(VPBB0, VPBB1);

Expand Down
18 changes: 9 additions & 9 deletions llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
VPBB1->appendRecipe(UseI);
VPBB1->appendRecipe(DefI);

VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPRegionBlock *R1 = new VPRegionBlock(VPBB2, VPBB2, "R1");
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
Expand All @@ -51,7 +51,7 @@ TEST_F(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
new VPInstruction(VPInstruction::BranchOnCond, {CanIV});

VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");

VPBB1->appendRecipe(UseI);
VPBB2->appendRecipe(CanIV);
Expand Down Expand Up @@ -85,9 +85,9 @@ TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {

VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB3 = new VPBasicBlock();
VPBasicBlock *VPBB4 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");

VPBB1->appendRecipe(I1);
VPBB2->appendRecipe(CanIV);
Expand Down Expand Up @@ -125,7 +125,7 @@ TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) {

VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");

VPBB1->appendRecipe(I1);
VPBB1->appendRecipe(BranchOnCond2);
Expand Down Expand Up @@ -158,8 +158,8 @@ TEST_F(VPVerifierTest, DuplicateSuccessorsInsideRegion) {

VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB3 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");

VPBB1->appendRecipe(I1);
VPBB2->appendRecipe(CanIV);
Expand Down Expand Up @@ -187,7 +187,7 @@ TEST_F(VPVerifierTest, DuplicateSuccessorsInsideRegion) {
TEST_F(VPVerifierTest, BlockOutsideRegionWithParent) {
VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = new VPBasicBlock();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");

VPInstruction *DefI = new VPInstruction(Instruction::Add, {});
VPInstruction *BranchOnCond =
Expand Down

0 comments on commit af48fcc

Please sign in to comment.