From 1b89761c30d273ae665b6da4b2ede0675373f2b1 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 11 Oct 2024 10:04:40 +0100 Subject: [PATCH 01/23] [VPlan] Hook IR blocks into VPlan during skeleton creation (NFC) As a first step to move towards modeling the full skeleton in VPlan, start by wrapping IR blocks created during legacy skeleton creation in VPIRBasicBlocks and hook them into the VPlan. This means the skeleton CFG is represented in VPlan, just before execute. This allows moving parts of skeleton creation into recipes in the VPBBs gradually. Note that this allows retiring some manual DT updates, as this will be handled automatically during VPlan execution. --- .../Transforms/Vectorize/LoopVectorize.cpp | 76 +++++++++++++------ llvm/lib/Transforms/Vectorize/VPlan.cpp | 67 +++++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 51 ++++--------- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 ++-- .../Transforms/Vectorize/VPlanTransforms.cpp | 2 + llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 - llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 2 +- .../Transforms/Vectorize/VPlanVerifier.cpp | 9 --- .../Transforms/Vectorize/VPDomTreeTest.cpp | 9 ++- .../Transforms/Vectorize/VPlanTest.cpp | 38 +++++++--- .../Vectorize/VPlanVerifierTest.cpp | 18 +++-- 11 files changed, 167 insertions(+), 123 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 1c64bd2982d764..2dbee4331e238f 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2426,6 +2426,26 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { return VectorTripCount; } +static void connectScalarPreheaderInVPlan(VPlan &Plan) { + VPBlockBase *VectorPH = Plan.getVectorPreheader(); + VPBlockBase *ScalarPH = Plan.getScalarPreheader(); + VPBlockBase *PredVPB = VectorPH->getSinglePredecessor(); + VPBlockUtils::disconnectBlocks(Plan.getEntry(), VectorPH); + VPBlockUtils::connectBlocks(PredVPB, ScalarPH); + VPBlockUtils::connectBlocks(PredVPB, VectorPH); +} + +static void connectCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { + VPBlockBase *ScalarPH = Plan.getScalarPreheader(); + VPBlockBase *VectorPH = Plan.getVectorPreheader(); + VPBlockBase *PredVPB = VectorPH->getSinglePredecessor(); + VPBlockUtils::disconnectBlocks(PredVPB, VectorPH); + VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB); + VPBlockUtils::connectBlocks(PredVPB, CheckVPIRBB); + VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH); + VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH); +} + void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { Value *Count = getTripCount(); // Reuse existing vector loop preheader for TC checks. @@ -2511,13 +2531,14 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { "TC check is expected to dominate Bypass"); // Update dominator for Bypass & LoopExit (if needed). - DT->changeImmediateDominator(Bypass, TCCheckBlock); BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters); if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); LoopBypassBlocks.push_back(TCCheckBlock); + + connectScalarPreheaderInVPlan(Plan); } BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { @@ -2534,6 +2555,8 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { "Should already be a bypass block due to iteration count check"); LoopBypassBlocks.push_back(SCEVCheckBlock); AddedSafetyChecks = true; + + connectCheckBlockInVPlan(Plan, SCEVCheckBlock); return SCEVCheckBlock; } @@ -2570,6 +2593,7 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) { AddedSafetyChecks = true; + connectCheckBlockInVPlan(Plan, MemCheckBlock); return MemCheckBlock; } @@ -7648,10 +7672,10 @@ DenseMap LoopVectorizationPlanner::executePlan( // 0. Generate SCEV-dependent code into the preheader, including TripCount, // before making any changes to the CFG. - if (!BestVPlan.getPreheader()->empty()) { + if (!BestVPlan.getEntry()->empty()) { State.CFG.PrevBB = OrigLoop->getLoopPreheader(); State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator()); - BestVPlan.getPreheader()->execute(&State); + BestVPlan.getEntry()->execute(&State); } if (!ILV.getTripCount()) ILV.setTripCount(State.get(BestVPlan.getTripCount(), VPLane(0))); @@ -7859,8 +7883,6 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, DT->getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"); - // Update dominator for Bypass. - DT->changeImmediateDominator(Bypass, TCCheckBlock); LoopBypassBlocks.push_back(TCCheckBlock); // Save the trip count so we don't have to regenerate it in the @@ -7875,6 +7897,12 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); + VPBlockBase *VectorPH = Plan.getVectorPreheader(); + VPBlockBase *PredVPB = VectorPH->getSinglePredecessor(); + if (PredVPB->getNumSuccessors() == 1) + connectScalarPreheaderInVPlan(Plan); + else + connectCheckBlockInVPlan(Plan, TCCheckBlock); return TCCheckBlock; } @@ -7905,32 +7933,19 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton( EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith( VecEpilogueIterationCountCheck, LoopVectorPreHeader); - DT->changeImmediateDominator(LoopVectorPreHeader, - EPI.MainLoopIterationCountCheck); - EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith( VecEpilogueIterationCountCheck, LoopScalarPreHeader); if (EPI.SCEVSafetyCheck) EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith( VecEpilogueIterationCountCheck, LoopScalarPreHeader); - if (EPI.MemSafetyCheck) + if (EPI.MemSafetyCheck) { EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith( VecEpilogueIterationCountCheck, LoopScalarPreHeader); - - DT->changeImmediateDominator( - VecEpilogueIterationCountCheck, - VecEpilogueIterationCountCheck->getSinglePredecessor()); + } DT->changeImmediateDominator(LoopScalarPreHeader, EPI.EpilogueIterationCountCheck); - if (!Cost->requiresScalarEpilogue(EPI.EpilogueVF.isVector())) - // If there is an epilogue which must run, there's no edge from the - // middle block to exit blocks and thus no need to update the immediate - // dominator of the exit blocks. - DT->changeImmediateDominator(LoopExitBlock, - EPI.EpilogueIterationCountCheck); - // Keep track of bypass blocks, as they feed start values to the induction and // reduction phis in the scalar loop preheader. if (EPI.SCEVSafetyCheck) @@ -8033,6 +8048,20 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( } ReplaceInstWithInst(Insert->getTerminator(), &BI); LoopBypassBlocks.push_back(Insert); + + // A new entry block has been created for the epilogue VPlan. Hook it in. + VPIRBasicBlock *NewEntry = VPIRBasicBlock::fromBasicBlock(Insert); + VPBasicBlock *OldEntry = Plan.getEntry(); + VPBlockUtils::reassociateBlocks(OldEntry, NewEntry); + Plan.setEntry(NewEntry); + for (auto &R : make_early_inc_range(*NewEntry)) { + auto *VPIR = dyn_cast(&R); + if (!VPIR || !isa(VPIR->getInstruction())) + break; + VPIR->eraseFromParent(); + } + + connectScalarPreheaderInVPlan(Plan); return Insert; } @@ -10256,7 +10285,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { // should be removed once induction resume value creation is done // directly in VPlan. EpilogILV.setTripCount(MainILV.getTripCount()); - for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) { + for (auto &R : make_early_inc_range(*BestEpiPlan.getEntry())) { auto *ExpandR = dyn_cast(&R); if (!ExpandR) continue; @@ -10316,8 +10345,6 @@ bool LoopVectorizePass::processLoop(Loop *L) { cast(&R)->setStartValue(StartVal); } - assert(DT->verify(DominatorTree::VerificationLevel::Fast) && - "DT not preserved correctly"); LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT, true, &ExpandedSCEVs); ++LoopsEpilogueVectorized; @@ -10345,6 +10372,9 @@ bool LoopVectorizePass::processLoop(Loop *L) { checkMixedPrecision(L, ORE); } + assert(DT->verify(DominatorTree::VerificationLevel::Fast) && + "DT not preserved correctly"); + std::optional RemainderLoopID = makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll, LLVMLoopVectorizeFollowupEpilogue}); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 54aae122877b98..7de8830bd866ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -170,9 +170,8 @@ VPBasicBlock *VPBlockBase::getEntryBasicBlock() { } void VPBlockBase::setPlan(VPlan *ParentPlan) { - assert( - (ParentPlan->getEntry() == this || ParentPlan->getPreheader() == this) && - "Can only set plan on its entry or preheader block."); + assert(ParentPlan->getEntry() == this && + "Can only set plan on its entry or preheader block."); Plan = ParentPlan; } @@ -463,7 +462,6 @@ void VPIRBasicBlock::execute(VPTransformState *State) { (getNumSuccessors() == 0 || isa(IRBB->getTerminator())) && "other blocks must be terminated by a branch"); } - for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) { VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock(); BasicBlock *PredBB = State->CFG.VPBB2IRBB[PredVPBB]; @@ -851,9 +849,6 @@ VPlan::~VPlan() { Block->dropAllReferences(&DummyValue); VPBlockBase::deleteCFG(Entry); - - Preheader->dropAllReferences(&DummyValue); - delete Preheader; } for (VPValue *VPV : VPLiveInsToFree) delete VPV; @@ -876,9 +871,10 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy, VPIRBasicBlock *Entry = VPIRBasicBlock::fromBasicBlock(TheLoop->getLoopPreheader()); VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph"); + VPBlockUtils::connectBlocks(Entry, VecPreheader); VPIRBasicBlock *ScalarHeader = VPIRBasicBlock::fromBasicBlock(TheLoop->getHeader()); - auto Plan = std::make_unique(Entry, VecPreheader, ScalarHeader); + auto Plan = std::make_unique(Entry, ScalarHeader); // Create SCEV and VPValue for the trip count. @@ -1021,8 +1017,9 @@ void VPlan::execute(VPTransformState *State) { BasicBlock *VectorPreHeader = State->CFG.PrevBB; State->Builder.SetInsertPoint(VectorPreHeader->getTerminator()); - // Disconnect VectorPreHeader from ExitBB in both the CFG and DT. - cast(VectorPreHeader->getTerminator())->setSuccessor(0, nullptr); + replaceVPBBWithIRVPBB( + cast(getVectorLoopRegion()->getSinglePredecessor()), + VectorPreHeader); State->CFG.DTU.applyUpdates( {{DominatorTree::Delete, VectorPreHeader, State->CFG.ExitBB}}); @@ -1049,8 +1046,10 @@ void VPlan::execute(VPTransformState *State) { State->CFG.DTU.applyUpdates( {{DominatorTree::Delete, ScalarPh, ScalarPh->getSingleSuccessor()}}); + ReversePostOrderTraversal> RPOT( + Entry); // Generate code in the loop pre-header and body. - for (VPBlockBase *Block : vp_depth_first_shallow(Entry)) + for (VPBlockBase *Block : make_range(RPOT.begin(), RPOT.end())) Block->execute(State); VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock(); @@ -1101,9 +1100,6 @@ void VPlan::execute(VPTransformState *State) { } State->CFG.DTU.flush(); - assert(State->CFG.DTU.getDomTree().verify( - DominatorTree::VerificationLevel::Fast) && - "DT not preserved correctly"); } InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) { @@ -1156,12 +1152,10 @@ void VPlan::print(raw_ostream &O) const { printLiveIns(O); - if (!getPreheader()->empty()) { - O << "\n"; - getPreheader()->print(O, "", SlotTracker); - } + ReversePostOrderTraversal> + RPOT(getEntry()); - for (const VPBlockBase *Block : vp_depth_first_shallow(getEntry())) { + for (const VPBlockBase *Block : RPOT) { O << '\n'; Block->print(O, "", SlotTracker); } @@ -1192,6 +1186,20 @@ std::string VPlan::getName() const { return Out; } +VPRegionBlock *VPlan::getVectorLoopRegion() { + for (VPBlockBase *B : vp_depth_first_shallow(getEntry())) + if (auto *R = dyn_cast(B)) + return R; + return nullptr; +} + +const VPRegionBlock *VPlan::getVectorLoopRegion() const { + for (const VPBlockBase *B : vp_depth_first_shallow(getEntry())) + if (auto *R = dyn_cast(B)) + return R; + return nullptr; +} + LLVM_DUMP_METHOD void VPlan::printDOT(raw_ostream &O) const { VPlanPrinter Printer(O, *this); @@ -1242,7 +1250,6 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry, VPlan *VPlan::duplicate() { // Clone blocks. - VPBasicBlock *NewPreheader = Preheader->clone(); const auto &[NewEntry, __] = cloneFrom(Entry); BasicBlock *ScalarHeaderIRBB = getScalarHeader()->getIRBasicBlock(); @@ -1252,8 +1259,7 @@ VPlan *VPlan::duplicate() { return VPIRBB && VPIRBB->getIRBasicBlock() == ScalarHeaderIRBB; })); // Create VPlan, clone live-ins and remap operands in the cloned blocks. - auto *NewPlan = - new VPlan(NewPreheader, cast(NewEntry), NewScalarHeader); + auto *NewPlan = new VPlan(cast(NewEntry), NewScalarHeader); DenseMap Old2NewVPValues; for (VPValue *OldLiveIn : VPLiveInsToFree) { Old2NewVPValues[OldLiveIn] = @@ -1273,7 +1279,6 @@ VPlan *VPlan::duplicate() { // else NewTripCount will be created and inserted into Old2NewVPValues when // TripCount is cloned. In any case NewPlan->TripCount is updated below. - remapOperands(Preheader, NewPreheader, Old2NewVPValues); remapOperands(Entry, NewEntry, Old2NewVPValues); // Initialize remaining fields of cloned VPlan. @@ -1287,6 +1292,19 @@ VPlan *VPlan::duplicate() { return NewPlan; } +VPBasicBlock *VPlan::getScalarPreheader() { + auto *MiddleVPBB = + cast(getVectorLoopRegion()->getSingleSuccessor()); + if (MiddleVPBB->getNumSuccessors() == 2) { + // Order is strict: first is the exit block, second is the scalar preheader. + return cast(MiddleVPBB->getSuccessors()[1]); + } + if (auto *IRVPBB = dyn_cast(MiddleVPBB->getSingleSuccessor())) + return IRVPBB; + + return nullptr; +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) Twine VPlanPrinter::getUID(const VPBlockBase *Block) { @@ -1325,8 +1343,6 @@ void VPlanPrinter::dump() { OS << "edge [fontname=Courier, fontsize=30]\n"; OS << "compound=true\n"; - dumpBlock(Plan.getPreheader()); - for (const VPBlockBase *Block : vp_depth_first_shallow(Plan.getEntry())) dumpBlock(Block); @@ -1587,7 +1603,6 @@ void VPSlotTracker::assignNames(const VPlan &Plan) { assignName(Plan.BackedgeTakenCount); for (VPValue *LI : Plan.VPLiveInsToFree) assignName(LI); - assignNames(Plan.getPreheader()); ReversePostOrderTraversal> RPOT(VPBlockDeepTraversalWrapper(Plan.getEntry())); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 035b9e66fd062a..c9393958521295 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3703,11 +3703,6 @@ class VPlan { /// preheader of the vector loop. VPBasicBlock *Entry; - /// VPBasicBlock corresponding to the original preheader. Used to place - /// VPExpandSCEV recipes for expressions used during skeleton creation and the - /// rest of VPlan execution. - VPBasicBlock *Preheader; - /// VPIRBasicBlock wrapping the header of the original scalar loop. VPIRBasicBlock *ScalarHeader; @@ -3752,36 +3747,27 @@ class VPlan { DenseMap SCEVToExpansion; public: - /// Construct a VPlan with original preheader \p Preheader, trip count \p TC, - /// \p Entry to the plan and with \p ScalarHeader wrapping the original header - /// of the scalar loop. At the moment, \p Preheader and \p Entry need to be - /// disconnected, as the bypass blocks between them are not yet modeled in - /// VPlan. - VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry, - VPIRBasicBlock *ScalarHeader) - : VPlan(Preheader, Entry, ScalarHeader) { + /// Construct a VPlan with \p Entry entering the plan and trip count \p TC. + VPlan(VPBasicBlock *Entry, VPValue *TC, VPIRBasicBlock *ScalarHeader) + : VPlan(Entry, ScalarHeader) { TripCount = TC; } - /// Construct a VPlan with original preheader \p Preheader, \p Entry to - /// the plan and with \p ScalarHeader wrapping the original header of the - /// scalar loop. At the moment, \p Preheader and \p Entry need to be - /// disconnected, as the bypass blocks between them are not yet modeled in - /// VPlan. - VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry, - VPIRBasicBlock *ScalarHeader) - : Entry(Entry), Preheader(Preheader), ScalarHeader(ScalarHeader) { + /// Construct a VPlan with \p Entry to the plan. + VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader) + : Entry(Entry), ScalarHeader(ScalarHeader) { Entry->setPlan(this); - Preheader->setPlan(this); - assert(Preheader->getNumSuccessors() == 0 && - Preheader->getNumPredecessors() == 0 && - "preheader must be disconnected"); assert(ScalarHeader->getNumSuccessors() == 0 && "scalar header must be a leaf node"); } ~VPlan(); + void setEntry(VPBasicBlock *VPBB) { + Entry = VPBB; + VPBB->setPlan(this); + } + /// Create initial VPlan, having an "entry" VPBasicBlock (wrapping /// original scalar pre-header ) which contains SCEV expansions that need /// to happen before the CFG is modified; a VPBasicBlock for the vector @@ -3932,12 +3918,9 @@ class VPlan { #endif /// Returns the VPRegionBlock of the vector loop. - VPRegionBlock *getVectorLoopRegion() { - return cast(getEntry()->getSingleSuccessor()); - } - const VPRegionBlock *getVectorLoopRegion() const { - return cast(getEntry()->getSingleSuccessor()); - } + VPRegionBlock *getVectorLoopRegion(); + + const VPRegionBlock *getVectorLoopRegion() const; /// Returns the preheader of the vector loop region. VPBasicBlock *getVectorPreheader() { @@ -3963,13 +3946,11 @@ class VPlan { SCEVToExpansion[S] = V; } - /// \return The block corresponding to the original preheader. - VPBasicBlock *getPreheader() { return Preheader; } - const VPBasicBlock *getPreheader() const { return Preheader; } - /// Clone the current VPlan, update all VPValues of the new VPlan and cloned /// recipes to refer to the clones, and return it. VPlan *duplicate(); + + VPBasicBlock *getScalarPreheader(); }; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 6254ea15191819..39e06736cfb320 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -627,11 +627,11 @@ Value *VPInstruction::generate(VPTransformState &State) { Builder.CreatePHI(IncomingFromOtherPreds->getType(), 2, Name); BasicBlock *VPlanPred = State.CFG - .VPBB2IRBB[cast(getParent()->getSinglePredecessor())]; + .VPBB2IRBB[cast(getParent()->getPredecessors()[0])]; NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred); for (auto *OtherPred : predecessors(Builder.GetInsertBlock())) { - assert(OtherPred != VPlanPred && - "VPlan predecessors should not be connected yet"); + if (OtherPred == VPlanPred) + continue; NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred); } return NewPhi; @@ -3235,13 +3235,17 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, void VPExpandSCEVRecipe::execute(VPTransformState &State) { assert(!State.Lane && "cannot be used in per-lane"); + if (State.ExpandedSCEVs.contains(Expr)) { + State.Builder.SetInsertPoint(State.CFG.VPBB2IRBB[getParent()]); + return; + } + const DataLayout &DL = State.CFG.PrevBB->getDataLayout(); SCEVExpander Exp(SE, DL, "induction"); - Value *Res = Exp.expandCodeFor(Expr, Expr->getType(), &*State.Builder.GetInsertPoint()); - assert(!State.ExpandedSCEVs.contains(Expr) && - "Same SCEV expanded multiple times"); + /* assert(!State.ExpandedSCEVs.contains(Expr) &&*/ + /*"Same SCEV expanded multiple times");*/ State.ExpandedSCEVs[Expr] = Res; State.set(this, Res, VPLane(0)); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a4f0df17f5832f..33ef10dbefc507 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -381,6 +381,8 @@ static bool mergeBlocksIntoPredecessors(VPlan &Plan) { dyn_cast_or_null(VPBB->getSinglePredecessor()); if (!PredVPBB || PredVPBB->getNumSuccessors() != 1) continue; + if (isa(PredVPBB)) + continue; WorkList.push_back(VPBB); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index dd005682203b75..ae53774bff98db 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -412,8 +412,6 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) { UnrollState Unroller(Plan, UF, Ctx); - Unroller.unrollBlock(Plan.getPreheader()); - // Iterate over all blocks in the plan starting from Entry, and unroll // recipes inside them. This includes the vector preheader and middle blocks, // which may set up or post-process per-part values. diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 4621c28b051298..e40af3e2e3d30a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -34,7 +34,7 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, Expanded = Plan.getOrAddLiveIn(E->getValue()); else { Expanded = new VPExpandSCEVRecipe(Expr, SE); - Plan.getPreheader()->appendRecipe(Expanded->getDefiningRecipe()); + Plan.getEntry()->appendRecipe(Expanded->getDefiningRecipe()); } Plan.addSCEVExpansion(Expr, Expanded); return Expanded; diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 8bdb3133243582..fc6b61491f8b5e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -195,15 +195,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { RecipeNumbering[&R] = Cnt++; for (const VPRecipeBase &R : *VPBB) { - if (isa(&R) ^ isa(VPBB)) { - errs() << "VPIRInstructions "; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - R.dump(); - errs() << " "; -#endif - errs() << "not in a VPIRBasicBlock!\n"; - return false; - } for (const VPValue *V : R.definedValues()) { for (const VPUser *U : V->users()) { auto *UI = dyn_cast(U); diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp index 86182ccae0b55b..604f87b634bba6 100644 --- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp @@ -44,8 +44,9 @@ TEST(VPDominatorTreeTest, DominanceNoRegionsTest) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB0); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); VPDominatorTree VPDT; VPDT.recalculate(Plan); @@ -124,8 +125,9 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB0); VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); VPDominatorTree VPDT; VPDT.recalculate(Plan); @@ -206,8 +208,9 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); VPDominatorTree VPDT; VPDT.recalculate(Plan); diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 4a8615cc086b08..81f1707aaf7fba 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -259,8 +259,9 @@ TEST(VPBasicBlockTest, getPlan) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); EXPECT_EQ(&Plan, VPBB1->getPlan()); EXPECT_EQ(&Plan, VPBB2->getPlan()); @@ -281,8 +282,9 @@ TEST(VPBasicBlockTest, getPlan) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); EXPECT_EQ(&Plan, VPBB1->getPlan()); EXPECT_EQ(&Plan, R1->getPlan()); @@ -313,8 +315,9 @@ TEST(VPBasicBlockTest, getPlan) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); EXPECT_EQ(&Plan, VPBB1->getPlan()); EXPECT_EQ(&Plan, R1->getPlan()); @@ -359,8 +362,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); } { @@ -461,8 +465,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB0); VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); } { @@ -546,8 +551,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); } { @@ -596,8 +602,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); } { @@ -690,8 +697,9 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); } delete ScalarHeader; } @@ -734,7 +742,8 @@ TEST(VPBasicBlockTest, print) { auto *ScalarHeader = BasicBlock::Create(C, ""); // FIXME: This looks wrong. auto ScalarHeaderVPBB = std::make_unique(ScalarHeader); - VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB.get()); + VPBlockUtils::connectBlocks(VPBB0, VPBB1); + VPlan Plan(VPBB0, TC, ScalarHeaderVPBB.get()); std::string FullDump; raw_string_ostream OS(FullDump); Plan.printDOT(OS); @@ -820,8 +829,9 @@ TEST(VPBasicBlockTest, printPlanWithVFsAndUFs) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPBB0, VPBB1); VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB); - VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPBB0, TC, ScalarHeaderVPBB); Plan.setName("TestPlan"); Plan.addVF(ElementCount::getFixed(4)); @@ -1291,11 +1301,13 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { TEST(VPRecipeTest, dumpRecipeInPlan) { VPBasicBlock *VPBB0 = new VPBasicBlock("preheader"); VPBasicBlock *VPBB1 = new VPBasicBlock(); + VPBlockUtils::connectBlocks(VPBB0, VPBB1); + LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB); - VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPBB0, ScalarHeaderVPBB); IntegerType *Int32 = IntegerType::get(C, 32); auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32), @@ -1363,11 +1375,13 @@ TEST(VPRecipeTest, dumpRecipeInPlan) { TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) { VPBasicBlock *VPBB0 = new VPBasicBlock("preheader"); VPBasicBlock *VPBB1 = new VPBasicBlock(); + VPBlockUtils::connectBlocks(VPBB0, VPBB1); + LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB); - VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPBB0, ScalarHeaderVPBB); IntegerType *Int32 = IntegerType::get(C, 32); auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32), diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index edb3f8a2952942..2c7566c973c4e1 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -32,8 +32,9 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefSameBB) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -69,8 +70,9 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -116,8 +118,9 @@ TEST(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) { auto TC = std::make_unique(); auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -157,8 +160,9 @@ TEST(VPVerifierTest, DuplicateSuccessorsOutsideRegion) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -199,8 +203,9 @@ TEST(VPVerifierTest, DuplicateSuccessorsInsideRegion) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -233,8 +238,9 @@ TEST(VPVerifierTest, BlockOutsideRegionWithParent) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); + VPBlockUtils::connectBlocks(VPPH, R1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); From b87cf14c957352373feda487be5523d6e0d729d5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 7 Nov 2024 13:13:15 +0000 Subject: [PATCH 02/23] !fixup address comments, thanks! --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 65b11ed551fe2b..d6d8b28e96815c 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2429,6 +2429,10 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { return VectorTripCount; } +/// Helper to connect both the vector and scalar preheaders to the vector +/// preheader's predecessor. This is used when adjusting \p Plan during skeleton +/// creation, i.e. adjusting the plan after introducing an initial runtime +/// check. static void connectScalarPreheaderInVPlan(VPlan &Plan) { VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *ScalarPH = Plan.getScalarPreheader(); From 599e690f42637f6955f1d338b77229e869766d3f Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 7 Nov 2024 21:40:24 +0000 Subject: [PATCH 03/23] !fixup address latest comments, thanks! --- .../Transforms/Vectorize/LoopVectorize.cpp | 34 +++++++++---------- llvm/lib/Transforms/Vectorize/VPlan.h | 18 +++++++--- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d6d8b28e96815c..f1fbaf88ca6a50 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2429,28 +2429,29 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { return VectorTripCount; } -/// Helper to connect both the vector and scalar preheaders to the vector -/// preheader's predecessor. This is used when adjusting \p Plan during skeleton +/// Helper to connect both the vector and scalar preheaders to the Plan's +/// entry. This is used when adjusting \p Plan during skeleton /// creation, i.e. adjusting the plan after introducing an initial runtime /// check. static void connectScalarPreheaderInVPlan(VPlan &Plan) { VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *ScalarPH = Plan.getScalarPreheader(); - VPBlockBase *PredVPB = VectorPH->getSinglePredecessor(); - VPBlockUtils::disconnectBlocks(Plan.getEntry(), VectorPH); - VPBlockUtils::connectBlocks(PredVPB, ScalarPH); - VPBlockUtils::connectBlocks(PredVPB, VectorPH); + VPBlockBase *PredVPB = Plan.getEntry(); + VPBlockUtils::connectBlocks(PredVPB, ScalarPH, -1, 0); + VPBlockUtils::connectBlocks(PredVPB, VectorPH, 0, -1); } -static void connectCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { +/// Introduces a new VPIRBasicBlock for \p CheckIRBB to \p Plan between the +/// vector preheader and its predecessor, also connecting to the scalar +/// preheader. +static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { VPBlockBase *ScalarPH = Plan.getScalarPreheader(); VPBlockBase *VectorPH = Plan.getVectorPreheader(); - VPBlockBase *PredVPB = VectorPH->getSinglePredecessor(); - VPBlockUtils::disconnectBlocks(PredVPB, VectorPH); + VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor(); VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB); - VPBlockUtils::connectBlocks(PredVPB, CheckVPIRBB); + VPBlockUtils::connectBlocks(PreVectorPH, CheckVPIRBB, -1, 1); VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH); - VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH); + VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH, 0, -1); } void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { @@ -2537,7 +2538,6 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { DT->getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"); - // Update dominator for Bypass & LoopExit (if needed). BranchInst &BI = *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters); if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) @@ -2545,6 +2545,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); LoopBypassBlocks.push_back(TCCheckBlock); + // TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here. connectScalarPreheaderInVPlan(Plan); } @@ -2563,7 +2564,7 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { LoopBypassBlocks.push_back(SCEVCheckBlock); AddedSafetyChecks = true; - connectCheckBlockInVPlan(Plan, SCEVCheckBlock); + introduceCheckBlockInVPlan(Plan, SCEVCheckBlock); return SCEVCheckBlock; } @@ -2600,7 +2601,7 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) { AddedSafetyChecks = true; - connectCheckBlockInVPlan(Plan, MemCheckBlock); + introduceCheckBlockInVPlan(Plan, MemCheckBlock); return MemCheckBlock; } @@ -7909,7 +7910,7 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, if (PredVPB->getNumSuccessors() == 1) connectScalarPreheaderInVPlan(Plan); else - connectCheckBlockInVPlan(Plan, TCCheckBlock); + introduceCheckBlockInVPlan(Plan, TCCheckBlock); return TCCheckBlock; } @@ -7946,10 +7947,9 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton( if (EPI.SCEVSafetyCheck) EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith( VecEpilogueIterationCountCheck, LoopScalarPreHeader); - if (EPI.MemSafetyCheck) { + if (EPI.MemSafetyCheck) EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith( VecEpilogueIterationCountCheck, LoopScalarPreHeader); - } DT->changeImmediateDominator(LoopScalarPreHeader, EPI.EpilogueIterationCountCheck); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 5414b30db595fa..46ba6328f0e1a4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1410,7 +1410,7 @@ class VPIRInstruction : public VPRecipeBase { InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override; - Instruction &getInstruction() { return I; } + Instruction &getInstruction() const { return I; } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. @@ -4098,13 +4098,21 @@ class VPBlockUtils { /// the successors of \p From and \p From to the predecessors of \p To. Both /// VPBlockBases must have the same parent, which can be null. Both /// VPBlockBases can be already connected to other VPBlockBases. - static void connectBlocks(VPBlockBase *From, VPBlockBase *To) { + static void connectBlocks(VPBlockBase *From, VPBlockBase *To, + unsigned PredIdx = -1u, unsigned SuccIdx = -1u) { assert((From->getParent() == To->getParent()) && "Can't connect two block with different parents"); - assert(From->getNumSuccessors() < 2 && + assert((SuccIdx != -1u || From->getNumSuccessors() < 2) && "Blocks can't have more than two successors."); - From->appendSuccessor(To); - To->appendPredecessor(From); + if (SuccIdx == -1u) + From->appendSuccessor(To); + else + From->getSuccessors()[SuccIdx] = To; + + if (PredIdx == -1u) + To->appendPredecessor(From); + else + To->getPredecessors()[PredIdx] = From; } /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To From 1a77b551357b630e65b8fcc8cba996cde7cb1505 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 9 Nov 2024 11:24:29 +0000 Subject: [PATCH 04/23] [VPlan] Add PredIdx and SuccIdx arguments to connectBlocks (NFC). --- llvm/lib/Transforms/Vectorize/VPlan.h | 16 ++++++++++++---- .../lib/Transforms/Vectorize/VPlanTransforms.cpp | 5 ++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 18f5f13073aa63..2a4bcd30758468 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4117,13 +4117,21 @@ class VPBlockUtils { /// the successors of \p From and \p From to the predecessors of \p To. Both /// VPBlockBases must have the same parent, which can be null. Both /// VPBlockBases can be already connected to other VPBlockBases. - static void connectBlocks(VPBlockBase *From, VPBlockBase *To) { + static void connectBlocks(VPBlockBase *From, VPBlockBase *To, + unsigned PredIdx = -1u, unsigned SuccIdx = -1u) { assert((From->getParent() == To->getParent()) && "Can't connect two block with different parents"); - assert(From->getNumSuccessors() < 2 && + assert((SuccIdx != -1u || From->getNumSuccessors() < 2) && "Blocks can't have more than two successors."); - From->appendSuccessor(To); - To->appendPredecessor(From); + if (SuccIdx == -1u) + From->appendSuccessor(To); + else + From->getSuccessors()[SuccIdx] = To; + + if (PredIdx == -1u) + To->appendPredecessor(From); + else + To->getPredecessors()[PredIdx] = From; } /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index ea8845eaa75d4d..70163aacb19d9d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -360,9 +360,8 @@ static void addReplicateRegions(VPlan &Plan) { // Record predicated instructions for above packing optimizations. VPBlockBase *Region = createReplicateRegion(RepR, Plan); Region->setParent(CurrentBlock->getParent()); - VPBlockUtils::disconnectBlocks(CurrentBlock, SplitBlock); - VPBlockUtils::connectBlocks(CurrentBlock, Region); - VPBlockUtils::connectBlocks(Region, SplitBlock); + VPBlockUtils::connectBlocks(CurrentBlock, Region, -1, 0); + VPBlockUtils::connectBlocks(Region, SplitBlock, 0, -1); } } From a2137b4dea6380320668cec61f5ef9474bb0cbbd Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 9 Nov 2024 13:46:48 +0000 Subject: [PATCH 05/23] [VPlan] Add insertOnEdge (NFC). --- llvm/lib/Transforms/Vectorize/VPlan.h | 15 +++++++++++++++ llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 +-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 2a4bcd30758468..4038e72829088f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4173,6 +4173,21 @@ class VPBlockUtils { return cast(&Block); }); } + + static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, + VPBlockBase *BlockPtr) { + unsigned SuccIdx = + std::distance(From->getSuccessors().begin(), + find_if(From->getSuccessors(), + [To](VPBlockBase *Succ) { return To == Succ; })); + unsigned PredIx = + std::distance(To->getPredecessors().begin(), + find_if(To->getPredecessors(), [From](VPBlockBase *Pred) { + return From == Pred; + })); + VPBlockUtils::connectBlocks(From, BlockPtr, -1, SuccIdx); + VPBlockUtils::connectBlocks(BlockPtr, To, PredIx, -1); + } }; class VPInterleavedAccessInfo { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 70163aacb19d9d..b9ab8a8fe60107 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -360,8 +360,7 @@ static void addReplicateRegions(VPlan &Plan) { // Record predicated instructions for above packing optimizations. VPBlockBase *Region = createReplicateRegion(RepR, Plan); Region->setParent(CurrentBlock->getParent()); - VPBlockUtils::connectBlocks(CurrentBlock, Region, -1, 0); - VPBlockUtils::connectBlocks(Region, SplitBlock, 0, -1); + VPBlockUtils::insertOnEdge(CurrentBlock, SplitBlock, Region); } } From be2c3a658f081bba28c1d6f490c149e0d16c772c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 9 Nov 2024 14:04:40 +0000 Subject: [PATCH 06/23] !fixup use insertOnEdge. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 +-- llvm/unittests/Transforms/Vectorize/VPlanTest.cpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 05d48685afffff..a8ab3321aa15ef 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2449,9 +2449,8 @@ static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor(); VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB); - VPBlockUtils::connectBlocks(PreVectorPH, CheckVPIRBB, -1, 1); VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH); - VPBlockUtils::connectBlocks(CheckVPIRBB, VectorPH, 0, -1); + VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckVPIRBB); } void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index b4e2b298196b61..13357738eea6d2 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -743,7 +743,7 @@ TEST(VPBasicBlockTest, print) { auto * ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); VPBlockUtils::connectBlocks(VPBB0, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB); + VPlan Plan(VPBB0, TC, ScalarHeaderVPBB); std::string FullDump; raw_string_ostream OS(FullDump); Plan.printDOT(OS); From 466b3931c842b19ec1473146bb89f13725db4ca9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 9 Nov 2024 19:15:33 +0000 Subject: [PATCH 07/23] [VPlan] Add insertOnEdge to VPBlockUtils (NFC). Add a new helper to insert a new VPBlockBase on an edge between 2 blocks. Suggested in https://github.com/llvm/llvm-project/pull/114292 and also useful for some existing code. --- llvm/lib/Transforms/Vectorize/VPlan.h | 18 ++++++++++++++++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 0770ff032aa6a0..abfe97b4ab55b6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4177,6 +4177,24 @@ class VPBlockUtils { return cast(&Block); }); } + + /// Inserts \p BlockPtr on the edge between \p From and \p To. That is, update + /// \p From's successor to \p To to point to \p BlockPtr and \p To's + /// predecessor from \p From to \p BlockPtr. \p From and \p To are added to \p + /// BlockPtr's predecessors and successors respectively. There must be a + /// single edge between \p From and \p To. + static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, + VPBlockBase *BlockPtr) { + auto &Successors = From->getSuccessors(); + auto &Predecessors = To->getPredecessors(); + assert(count(Successors, To) == 1 && count(Predecessors, From) == 1 && + "must have single between From and To"); + unsigned SuccIdx = std::distance(Successors.begin(), find(Successors, To)); + unsigned PredIx = + std::distance(Predecessors.begin(), find(Predecessors, From)); + VPBlockUtils::connectBlocks(From, BlockPtr, -1, SuccIdx); + VPBlockUtils::connectBlocks(BlockPtr, To, PredIx, -1); + } }; class VPInterleavedAccessInfo { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 70163aacb19d9d..b9ab8a8fe60107 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -360,8 +360,7 @@ static void addReplicateRegions(VPlan &Plan) { // Record predicated instructions for above packing optimizations. VPBlockBase *Region = createReplicateRegion(RepR, Plan); Region->setParent(CurrentBlock->getParent()); - VPBlockUtils::connectBlocks(CurrentBlock, Region, -1, 0); - VPBlockUtils::connectBlocks(Region, SplitBlock, 0, -1); + VPBlockUtils::insertOnEdge(CurrentBlock, SplitBlock, Region); } } From 79964516f2c520e7fef255743252e971078f16d2 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 9 Nov 2024 19:21:49 +0000 Subject: [PATCH 08/23] !fixup cleanup after merge. --- .../Transforms/Vectorize/LoopVectorize.cpp | 5 - llvm/lib/Transforms/Vectorize/VPlan.cpp | 5 + llvm/lib/Transforms/Vectorize/VPlan.h | 2 - .../AArch64/sve-tail-folding-forced.ll | 2 +- .../LoopVectorize/AArch64/sve-widen-gep.ll | 3 + .../AArch64/sve2-histcnt-vplan.ll | 18 ++- .../AArch64/synthesize-mask-for-call.ll | 54 ++++--- .../widen-call-with-intrinsic-or-libfunc.ll | 18 ++- .../PowerPC/vplan-force-tail-with-evl.ll | 6 + .../RISCV/riscv-vector-reverse.ll | 136 ++--------------- .../RISCV/vplan-vp-intrinsics-reduction.ll | 37 ++--- .../RISCV/vplan-vp-select-intrinsics.ll | 23 ++- .../first-order-recurrence-chains-vplan.ll | 36 +++-- ...-order-recurrence-sink-replicate-region.ll | 55 ++++--- .../Transforms/LoopVectorize/icmp-uniforms.ll | 3 + .../interleave-and-scalarize-only.ll | 19 ++- .../LoopVectorize/vplan-dot-printing.ll | 3 +- .../LoopVectorize/vplan-iv-transforms.ll | 9 +- .../LoopVectorize/vplan-predicate-switch.ll | 28 ++-- .../vplan-printing-before-execute.ll | 38 +++-- .../vplan-printing-outer-loop.ll | 9 +- .../LoopVectorize/vplan-printing.ll | 144 +++++++++++------- .../vplan-sink-scalars-and-merge-vf1.ll | 9 +- .../vplan-sink-scalars-and-merge.ll | 44 ++++-- .../vplan-unused-interleave-group.ll | 3 + 25 files changed, 376 insertions(+), 333 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index a8ab3321aa15ef..bfb4fc0d3506da 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7671,11 +7671,6 @@ DenseMap LoopVectorizationPlanner::executePlan( OrigLoop->getHeader()->getContext()); VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE); - LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << BestVF - << ", UF=" << BestUF << '\n'); - BestVPlan.setName("Final VPlan"); - LLVM_DEBUG(BestVPlan.dump()); - // Perform the actual loop transformation. VPTransformState State(BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index f4f19f51a9036b..31436c070a314f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1023,6 +1023,11 @@ void VPlan::execute(VPTransformState *State) { replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh); replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB); + LLVM_DEBUG(dbgs() << "Executing best plan with VF=" <VF + << ", UF=" << getUF()<< '\n'); + setName("Final VPlan"); + LLVM_DEBUG(dump()); + // Disconnect the middle block from its single successor (the scalar loop // header) in both the CFG and DT. The branch will be recreated during VPlan // execution. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c69a3be6229307..d84ec63c6e3e6b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4104,8 +4104,6 @@ class VPBlockUtils { unsigned PredIdx = -1u, unsigned SuccIdx = -1u) { assert((From->getParent() == To->getParent()) && "Can't connect two block with different parents"); - assert((PredIdx != -1u || To->getNumPredecessors() < 2) && - "Blocks can't have more than two predecessors."); if (SuccIdx == -1u) From->appendSuccessor(To); else diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll index 0b3f28e8db5c4d..928b86d5c70b1f 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -15,7 +15,7 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-EMPTY: ; VPLANS-NEXT: ir-bb: ; VPLANS-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 umax %n) -; VPLANS-NEXT: No successors +; VPLANS-NEXT: Successor(s): vector.ph ; VPLANS-EMPTY: ; VPLANS-NEXT: vector.ph: ; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]> diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 6ec9eb849dd52a..cd1c32fb56fa7b 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -13,6 +13,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%N> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll index 6257d3325f9796..3ef99ff496a687 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll @@ -16,6 +16,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count ; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -41,9 +44,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]] ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -51,6 +51,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ;; Check that the vectorized plan contains a histogram recipe instead. @@ -59,6 +62,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count ; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -83,9 +89,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]] ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -93,6 +96,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll index af1c63bc405a38..8ac46fe7687d24 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -14,6 +14,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -39,9 +42,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -49,6 +49,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -56,6 +59,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -81,9 +87,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -91,6 +94,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ;; If we have a masked variant at one VF and an unmasked variant at a different @@ -103,6 +109,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -128,9 +137,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -138,6 +144,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -145,6 +154,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -170,9 +182,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -180,6 +189,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ;; If we have two variants at different VFs, neither of which are masked, we @@ -191,6 +203,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -216,9 +231,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -226,6 +238,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -233,6 +248,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -258,9 +276,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -268,6 +283,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } define void @test_v4_v4m(ptr noalias %a, ptr readonly %b) #3 { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index afc2fd5a049ad1..648f6e874abbe0 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -12,6 +12,9 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -37,9 +40,6 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -47,6 +47,9 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -54,6 +57,9 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -79,9 +85,6 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -89,6 +92,9 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; ; diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll index a0696b3204dbd4..cd1d18aad8361d 100644 --- a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll +++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll @@ -14,6 +14,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<%N> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -77,6 +80,9 @@ define void @safe_dep(ptr %p) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<512> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index a38835f5613fd8..ec23fd9b83392b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -62,7 +62,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %0 = zext i32 %n to i64 ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -92,9 +92,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -103,6 +100,9 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] @@ -139,68 +139,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Loop cost is 32 ; CHECK-NEXT: LV: IC is 1 ; CHECK-NEXT: LV: VF is vscale x 4 -; CHECK-NEXT: LV: Not Interleaving. -; CHECK-NEXT: LV: Interleaving is not beneficial. -; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in -; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop -; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1 -; CHECK-NEXT: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' { -; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF -; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %0 = zext i32 %n to i64 -; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) -; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop -; CHECK-EMPTY: -; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]> -; CHECK-NEXT: WIDEN ir<%13> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%add9> = add ir<%13>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]> -; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> -; CHECK-NEXT: Successor(s): ir-bb, scalar.ph -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: Successor(s): ir-bb -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] -; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] -; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: LV: Loop does not require scalar epilogue -; CHECK-NEXT: LV: Loop does not require scalar epilogue -; CHECK-NEXT: LV: Interleaving disabled by the pass manager -; CHECK-NEXT: LV: Loop does not require scalar epilogue -; CHECK-NEXT: LV: Vectorizing: innermost loop. -; CHECK-EMPTY: ; entry: %cmp7 = icmp sgt i32 %n, 0 @@ -281,7 +219,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %0 = zext i32 %n to i64 ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -311,9 +249,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -322,6 +257,9 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] @@ -362,63 +300,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Interleaving is not beneficial. ; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in ; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop -; CHECK-NEXT: Executing best plan with VF=vscale x 4, UF=1 -; CHECK-NEXT: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' { -; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF -; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF -; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %0 = zext i32 %n to i64 -; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) -; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: Successor(s): vector loop -; CHECK-EMPTY: -; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> -; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, vp<[[VF]]> -; CHECK-NEXT: WIDEN ir<%13> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%13>, ir<1.000000e+00> -; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, vp<[[VF]]> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%conv1> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]> -; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> -; CHECK-NEXT: Successor(s): ir-bb, scalar.ph -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: scalar.ph: -; CHECK-NEXT: Successor(s): ir-bb -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] -; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] -; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: LV: Loop does not require scalar epilogue -; CHECK-NEXT: LV: Loop does not require scalar epilogue -; CHECK-NEXT: LV: Interleaving disabled by the pass manager -; CHECK-NEXT: LV: Loop does not require scalar epilogue -; CHECK-NEXT: LV: Vectorizing: innermost loop. +; CHECK: Executing best plan with VF=vscale x 4, UF=1 ; entry: %cmp7 = icmp sgt i32 %n, 0 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll index 414f39d557044a..ac3dfe8307fa2e 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll @@ -29,7 +29,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count ; IF-EVL-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count ; IF-EVL-OUTLOOP-EMPTY: -; IF-EVL-OUTLOOP: vector.ph: +; IF-EVL-OUTLOOP-NEXT: ir-bb: +; IF-EVL-OUTLOOP-NEXT: Successor(s): vector.ph +; IF-EVL-OUTLOOP-EMPTY: +; IF-EVL-OUTLOOP-NEXT: vector.ph: ; IF-EVL-OUTLOOP-NEXT: Successor(s): vector loop ; IF-EVL-OUTLOOP-EMPTY: ; IF-EVL-OUTLOOP-NEXT: vector loop: { @@ -59,10 +62,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-cond ir ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; IF-EVL-OUTLOOP-EMPTY: -; IF-EVL-OUTLOOP-NEXT: ir-bb: -; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) -; IF-EVL-OUTLOOP-NEXT: No successors -; IF-EVL-OUTLOOP-EMPTY: ; IF-EVL-OUTLOOP-NEXT: scalar.ph: ; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb @@ -72,6 +71,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; IF-EVL-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; IF-EVL-OUTLOOP-NEXT: No successors +; IF-EVL-OUTLOOP-EMPTY: +; IF-EVL-OUTLOOP-NEXT: ir-bb: +; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; IF-EVL-OUTLOOP-NEXT: No successors ; IF-EVL-OUTLOOP-NEXT: } ; @@ -109,10 +112,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: EMIT branch-on-cond ir ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; IF-EVL-INLOOP-EMPTY: -; IF-EVL-INLOOP-NEXT: ir-bb: -; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) -; IF-EVL-INLOOP-NEXT: No successors -; IF-EVL-INLOOP-EMPTY: ; IF-EVL-INLOOP-NEXT: scalar.ph: ; IF-EVL-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb @@ -122,6 +121,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; IF-EVL-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; IF-EVL-INLOOP-NEXT: No successors +; IF-EVL-INLOOP-EMPTY: +; IF-EVL-INLOOP-NEXT: ir-bb: +; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; IF-EVL-INLOOP-NEXT: No successors ; IF-EVL-INLOOP-NEXT: } ; @@ -155,10 +158,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]> ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; NO-VP-OUTLOOP-EMPTY: -; NO-VP-OUTLOOP-NEXT: ir-bb: -; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) -; NO-VP-OUTLOOP-NEXT: No successors -; NO-VP-OUTLOOP-EMPTY: ; NO-VP-OUTLOOP-NEXT: scalar.ph: ; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb @@ -168,6 +167,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; NO-VP-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; NO-VP-OUTLOOP-NEXT: No successors +; NO-VP-OUTLOOP-EMPTY: +; NO-VP-OUTLOOP-NEXT: ir-bb: +; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; NO-VP-OUTLOOP-NEXT: No successors ; NO-VP-OUTLOOP-NEXT: } ; @@ -201,10 +204,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]> ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; NO-VP-INLOOP-EMPTY: -; NO-VP-INLOOP-NEXT: ir-bb: -; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) -; NO-VP-INLOOP-NEXT: No successors -; NO-VP-INLOOP-EMPTY: ; NO-VP-INLOOP-NEXT: scalar.ph: ; NO-VP-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb @@ -214,6 +213,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; NO-VP-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; NO-VP-INLOOP-NEXT: No successors +; NO-VP-INLOOP-EMPTY: +; NO-VP-INLOOP-NEXT: ir-bb: +; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; NO-VP-INLOOP-NEXT: No successors ; NO-VP-INLOOP-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll index 6d6cfb5e9d18ed..523cf1bd798b68 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll @@ -7,11 +7,22 @@ define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { - ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF - ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count + ; IF-EVL-NEXT: Live-in ir<%8> = VF * UF + ; IF-EVL-NEXT: Live-in ir<%n.vec> = vector-trip-count ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count - ; IF-EVL: vector.ph: + ; IF-EVL: ir-bb: + ; IF-EVL-NEXT: Successor(s): ir-bb, ir-bb + + ; IF-EVL: ir-bb: + ; IF-EVL-NEXT: IR %4 = call i64 @llvm.vscale.i64() + ; IF-EVL-NEXT: IR %5 = mul i64 %4, 4 + ; IF-EVL-NEXT: IR %6 = sub i64 %5, 1 + ; IF-EVL-NEXT: IR %n.rnd.up = add i64 %N, %6 + ; IF-EVL-NEXT: IR %n.mod.vf = urem i64 %n.rnd.up, %5 + ; IF-EVL-NEXT: IR %n.vec = sub i64 %n.rnd.up, %n.mod.vf + ; IF-EVL-NEXT: IR %7 = call i64 @llvm.vscale.i64() + ; IF-EVL-NEXT: IR %8 = mul i64 %7, 4 ; IF-EVL-NEXT: Successor(s): vector loop ; IF-EVL: vector loop: { @@ -29,15 +40,15 @@ ; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]> ; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]> ; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]> - ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>) + ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%10>, ir<%11>, vp<[[EVL]]>) ; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]> ; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]> - ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]> + ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, ir<%8> + ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, ir<%n.vec> ; IF-EVL-NEXT: No successors ; IF-EVL-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index bcacfb358ec05a..49e5220c46c9c9 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -9,6 +9,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -39,9 +42,6 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22> ; CHECK-NEXT: EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33> @@ -53,6 +53,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -81,6 +84,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -115,9 +121,6 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22> ; CHECK-NEXT: EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33> @@ -131,6 +134,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -165,6 +171,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<4098> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: WIDEN ir<%for.x.next> = mul ir<%x>, ir<2> ; CHECK-NEXT: Successor(s): vector loop @@ -195,9 +204,6 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0> @@ -208,6 +214,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) { ; CHECK-NEXT: IR %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]>) ; CHECK-NEXT: IR %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1) ; CHECK: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -237,6 +246,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<4098> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -269,9 +281,6 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0> @@ -282,6 +291,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) { ; CHECK-NEXT: IR %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]>) ; CHECK-NEXT: IR %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1) ; CHECK: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 8ae538cf63986b..8032decd2271d7 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -15,6 +15,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -77,9 +80,6 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -89,6 +89,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -121,6 +124,9 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 ; CHECK-NEXT: Successor(s): vector loop @@ -164,9 +170,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -176,6 +179,9 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -205,6 +211,9 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 ; CHECK-NEXT: Successor(s): vector loop @@ -234,10 +243,6 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]>) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[RESUME_RED:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<1234> @@ -249,6 +254,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: IR %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]>) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -282,6 +291,9 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -347,9 +359,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -359,6 +368,9 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -397,7 +409,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 smax (1 + (sext i8 %y to i32))) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 @@ -444,9 +456,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -456,6 +465,9 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %C = icmp sgt i32 %iv.next, %recur.next ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -487,6 +499,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<3> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -530,9 +545,6 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -542,6 +554,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: IR %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: vp<[[RESUME_1_P]]>) ; CHECK: IR %ec = icmp ugt i64 %iv, 3 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll index c71ff2b9fa4c47..bc698b6710e99d 100644 --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -43,6 +43,9 @@ for.end: ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<14> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index a71666d8c3167a..bfdadd17188204 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -11,7 +11,7 @@ ; DBG-EMPTY: ; DBG-NEXT: ir-bb: ; DBG-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1000 + (-1 * %start)) -; DBG-NEXT: No successors +; DBG-NEXT: Successor(s): vector.ph ; DBG-EMPTY: ; DBG-NEXT: vector.ph: ; DBG-NEXT: Successor(s): vector loop @@ -72,6 +72,9 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; DBG-NEXT: Live-in ir<1000> = original trip-count ; DBG-EMPTY: +; DBG-NEXT: ir-bb: +; DBG-NEXT: Successor(s): vector.ph +; DBG-EMPTY: ; DBG-NEXT: vector.ph: ; DBG-NEXT: Successor(s): vector loop ; DBG-EMPTY: @@ -112,9 +115,6 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; DBG-NEXT: Successor(s): ir-bb, scalar.ph ; DBG-EMPTY: -; DBG-NEXT: ir-bb: -; DBG-NEXT: No successors -; DBG-EMPTY: ; DBG-NEXT: scalar.ph: ; DBG-NEXT: Successor(s): ir-bb ; DBG-EMPTY: @@ -123,6 +123,9 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: IR %d = phi i1 [ false, %entry ], [ %d.next, %loop.latch ] ; DBG-NEXT: IR %d.next = xor i1 %d, true ; DBG-NEXT: No successors +; DBG-EMPTY: +; DBG-NEXT: ir-bb: +; DBG-NEXT: No successors ; DBG-NEXT: } define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) { @@ -192,7 +195,7 @@ exit: ; DBG-EMPTY: ; DBG-NEXT: ir-bb: ; DBG-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 (1 smax %n) to i64) -; DBG-NEXT: No successors +; DBG-NEXT: Successor(s): vector.ph ; DBG-EMPTY: ; DBG-NEXT: vector.ph: ; DBG-NEXT: SCALAR-CAST vp<[[CAST:%.+]]> = trunc ir<1> to i32 @@ -218,9 +221,6 @@ exit: ; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; DBG-NEXT: Successor(s): ir-bb, scalar.ph ; DBG-EMPTY: -; DBG-NEXT: ir-bb: -; DBG-NEXT: No successors -; DBG-EMPTY: ; DBG-NEXT: scalar.ph: ; DBG-NEXT: EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; DBG-NEXT: Successor(s): ir-bb @@ -230,6 +230,9 @@ exit: ; DBG-NEXT: IR %for = phi i32 [ 0, %entry ], [ %iv.trunc, %loop ] (extra operand: vp<[[RESUME_P]]>) ; DBG: IR %ec = icmp slt i32 %iv.next.trunc, %n ; DBG-NEXT: No successors +; DBG-EMPTY: +; DBG-NEXT: ir-bb: +; DBG-NEXT: No successors ; DBG-NEXT: } define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) { diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index 68dd47537fdfdf..50755f8f05e35a 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -14,8 +14,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: compound=true ; CHECK-NEXT: N0 [label = ; CHECK-NEXT: "ir-bb\:\l" + -; CHECK-NEXT: "No successors\l" +; CHECK-NEXT: "Successor(s): vector.ph\l" ; CHECK-NEXT: ] +; CHECK-NEXT: N0 -> N1 [ label=""] ; CHECK-NEXT: N1 [label = ; CHECK-NEXT: "vector.ph:\l" + ; CHECK-NEXT: "Successor(s): vector loop\l" diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll index 1f815899ed55c2..c342d2f81e979b 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -9,6 +9,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -31,9 +34,6 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -41,6 +41,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ] ; CHECK: IR %iv.next = add i64 %iv, 1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll index 53f5a5658fb68d..f31ecca50a3701 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll @@ -3,15 +3,18 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK: VPlan 'Final VPlan for VF={2},UF={1}' { -; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<2> = VF * UF +; CHECK-NEXT: Live-in ir<%n.vec> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV ((-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %n.mod.vf = urem i64 %0, 2 +; CHECK-NEXT: IR %n.vec = sub i64 %0, %n.mod.vf +; CHECK-NEXT: IR %ind.end = getelementptr i8, ptr %start, i64 %n.vec ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { @@ -76,25 +79,26 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): default.2 ; CHECK-EMPTY: ; CHECK-NEXT: default.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> -; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<2> +; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<%n.vec> ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block +; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: -; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VTC]]> +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<%n.vec> ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_CMP]]> -; CHECK-NEXT: Successor(s): ir-bb, scalar.ph +; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: scalar.ph: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %bc.resume.val = phi ptr [ %ind.end, %middle.block ], [ %start, %entry ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ] +; CHECK-NEXT: IR %ptr.iv = phi ptr [ %bc.resume.val, %scalar.ph ], [ %ptr.iv.next, %loop.latch ] ; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1 ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll index c9612ced3eee01..746e7f7059a6b9 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll @@ -16,7 +16,7 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %and = and i64 %N, 15 ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i4 (trunc i64 %N to i4) to i64) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -42,9 +42,6 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -53,20 +50,27 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: IR %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ] ; CHECK: IR %cmp = icmp eq i64 %iv.next, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; ; CHECK: Executing best plan with VF=8, UF=2 ; CHECK-NEXT: VPlan 'Final VPlan for VF={8},UF={2}' { -; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF -; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<16> = VF * UF +; CHECK-NEXT: Live-in ir<%n.vec> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %and = and i64 %N, 15 ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i4 (trunc i64 %N to i4) to i64) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %n.mod.vf = urem i64 %and, 16 +; CHECK-NEXT: IR %n.vec = sub i64 %and, %n.mod.vf +; CHECK-NEXT: IR %ind.end = sub i64 %and, %n.vec +; CHECK-NEXT: IR %ind.end1 = getelementptr i8, ptr %A, i64 %n.vec ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { @@ -84,26 +88,28 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vp<[[VPTR4:%.+]]> = vector-pointer vp<[[PADD1]]>, ir<1> ; CHECK-NEXT: WIDEN store vp<[[VPTR3]]>, ir<%add> ; CHECK-NEXT: WIDEN store vp<[[VPTR4]]>, ir<%add>.1 -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, ir<16> ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: No successors ; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block +; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: -; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[C:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VTC]]> +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: EMIT vp<[[C:%.+]]> = icmp eq vp<[[TC]]>, ir<%n.vec> ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> -; CHECK-NEXT: Successor(s): ir-bb, scalar.ph +; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: scalar.ph: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %bc.resume.val = phi i64 [ %ind.end, %middle.block ], [ %and, %entry ] +; CHECK-NEXT: IR %bc.resume.val2 = phi ptr [ %ind.end1, %middle.block ], [ %A, %entry ] ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %iv = phi i64 [ %and, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: IR %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ] +; CHECK-NEXT: IR %iv = phi i64 [ %bc.resume.val, %scalar.ph ], [ %iv.next, %loop ] +; CHECK-NEXT: IR %p.src = phi ptr [ %bc.resume.val2, %scalar.ph ], [ %p.src.next, %loop ] ; CHECK: IR %cmp = icmp eq i64 %iv.next, 0 ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll index 50d406d0c04164..2adeb5920cb5b2 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -11,6 +11,9 @@ define void @foo(i64 %n) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<8> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -47,9 +50,6 @@ define void @foo(i64 %n) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -59,6 +59,9 @@ define void @foo(i64 %n) { ; CHECK-NEXT: IR store i64 %outer.iv, ptr %gep.1, align 4 ; CHECK-NEXT: IR %add = add nsw i64 %outer.iv, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } entry: br label %outer.header diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 6bb20a301e0ade..8894b871320bcb 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -13,6 +13,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -38,9 +41,6 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -48,6 +48,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -77,6 +80,9 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -105,9 +111,6 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -115,6 +118,9 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -145,6 +151,9 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -170,10 +179,6 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]>) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00> ; CHECK-NEXT: Successor(s): ir-bb @@ -182,6 +187,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]>) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -208,6 +217,9 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -233,9 +245,6 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00> ; CHECK-NEXT: Successor(s): ir-bb @@ -245,6 +254,9 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: IR %red = phi float [ %red.next, %for.body ], [ 0.000000e+00, %entry ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -275,7 +287,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 smax %n) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -319,9 +331,6 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -329,6 +338,9 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: IR %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ] ; CHECK-NEXT: IR %cmp = icmp ult i64 %i, 5 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -365,6 +377,9 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<256> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -396,9 +411,6 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -406,6 +418,9 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] ; CHECK: IR %cmp = icmp slt i64 %iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -446,6 +461,9 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -475,10 +493,6 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]>) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00> ; CHECK-NEXT: Successor(s): ir-bb @@ -488,6 +502,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: IR %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]>) +; CHECK-NEXT: No successors ; CHECK-NEXT:} entry: @@ -516,6 +534,9 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<128> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -566,9 +587,6 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -576,6 +594,9 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end ] ; CHECK: IR %cmp1 = icmp slt i32 %lsd, 100 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT:} ; entry: @@ -624,7 +645,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: IR %inc = add i64 %div, 1 ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + ((15 + (%y /u 492802768830814060)) /u (1 + (%y /u 492802768830814060)))) ; CHECK-NEXT: EMIT vp<[[EXP_SCEV:%.+]]> = EXPAND SCEV (1 + (%y /u 492802768830814060)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -651,9 +672,6 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -661,6 +679,9 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] ; CHECK: IR %iv.next = add i64 %iv, %inc ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -691,6 +712,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -715,10 +739,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]>) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -726,6 +746,10 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]>) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -752,6 +776,9 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -779,9 +806,6 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -789,6 +813,9 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -818,6 +845,9 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -844,9 +874,6 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -854,6 +881,9 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -882,6 +912,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -929,9 +962,6 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -939,6 +969,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ] ; CHECK: IR %ifcond = fcmp oeq float %ld.value, 5.0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -977,6 +1010,9 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -1003,9 +1039,6 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -1013,6 +1046,9 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -1041,6 +1077,9 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -1082,6 +1121,9 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) { ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -1110,10 +1152,6 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]>) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22> ; CHECK-NEXT: Successor(s): ir-bb @@ -1123,6 +1161,10 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]>) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll index cdeffeff84d03a..a939b1e923a914 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -12,6 +12,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: ir<0> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -49,9 +52,6 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -60,6 +60,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: IR %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 0, %entry ] ; CHECK: IR %tmp5 = trunc i32 %tmp4 to i8 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index 88e7aaccfe2f35..38bd54d8da9843 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -19,7 +19,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -88,7 +88,7 @@ exit: ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -173,7 +173,7 @@ exit: ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -258,6 +258,9 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<11> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0> ; CHECK-NEXT: Successor(s): vector loop @@ -332,7 +335,7 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -430,7 +433,7 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -537,7 +540,7 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -644,7 +647,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -750,7 +753,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -816,7 +819,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (1 + (8 umin %k)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -898,6 +901,9 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<999> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -962,6 +968,9 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<201> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -1033,6 +1042,9 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: @@ -1073,9 +1085,6 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -1089,6 +1098,9 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: IR store i32 %l, ptr %gep.dst, align 16 ; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -1119,7 +1131,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV ((-1 * (ptrtoint ptr %end to i64)) + (ptrtoint ptr %start to i64)) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -1164,9 +1176,6 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -1176,6 +1185,9 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv.next, align 1 ; CHECK-NEXT: IR %c.1 = icmp eq i8 %l, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll index 27d81de260d3b9..9778ef6853a70f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll @@ -13,6 +13,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) { ; CHECK-NEXT: Live-in vp<%1> = vector-trip-count ; CHECK-NEXT: Live-in ir<%length> = original trip-count ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: Successor(s): vector.ph +; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: From 382380aaa9e75b7f3e6819f025122902259d83ee Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 10 Nov 2024 10:05:07 +0000 Subject: [PATCH 09/23] !fixup formatting --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index eb31ff3623f918..960aa68e59d09c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1023,8 +1023,8 @@ void VPlan::execute(VPTransformState *State) { replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh); replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB); - LLVM_DEBUG(dbgs() << "Executing best plan with VF=" <VF - << ", UF=" << getUF()<< '\n'); + LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF + << ", UF=" << getUF() << '\n'); setName("Final VPlan"); LLVM_DEBUG(dump()); From e5b8af3a4c10eb64ce376336176ef00a666bf3ec Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 25 Nov 2024 21:16:49 +0000 Subject: [PATCH 10/23] !fixup address latest comments, thanks! --- .../Transforms/Vectorize/LoopVectorize.cpp | 36 ++++++------- llvm/lib/Transforms/Vectorize/VPlan.cpp | 52 ++++++++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 20 +++++-- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 ++- .../Transforms/Vectorize/VPlanTransforms.cpp | 5 +- .../RISCV/vplan-vp-intrinsics-reduction.ll | 24 ++------- .../Transforms/Vectorize/VPDomTreeTest.cpp | 9 ++-- .../Transforms/Vectorize/VPlanTest.cpp | 38 +++++--------- .../Vectorize/VPlanVerifierTest.cpp | 18 +++---- 9 files changed, 99 insertions(+), 111 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index f16dfdc4abf3b0..5c2ec637e9d9f1 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2432,12 +2432,13 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { /// entry. This is used when adjusting \p Plan during skeleton /// creation, i.e. adjusting the plan after introducing an initial runtime /// check. -static void connectScalarPreheaderInVPlan(VPlan &Plan) { +/// TODO: Connect scalar preheader during initial VPlan construction. +static void connectScalarPreheaderAsBypassInVPlan(VPlan &Plan) { VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *ScalarPH = Plan.getScalarPreheader(); - VPBlockBase *PredVPB = Plan.getEntry(); - VPBlockUtils::connectBlocks(PredVPB, ScalarPH, -1, 0); - VPBlockUtils::connectBlocks(PredVPB, VectorPH, 0, -1); + VPBlockBase *Entry = Plan.getEntry(); + VPBlockUtils::connectBlocks(Entry, ScalarPH); + std::swap(Entry->getSuccessors()[0], Entry->getSuccessors()[1]); } /// Introduces a new VPIRBasicBlock for \p CheckIRBB to \p Plan between the @@ -2447,6 +2448,7 @@ static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { VPBlockBase *ScalarPH = Plan.getScalarPreheader(); VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor(); + assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor"); VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB); VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH); VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckVPIRBB); @@ -2534,7 +2536,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { LoopBypassBlocks.push_back(TCCheckBlock); // TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here. - connectScalarPreheaderInVPlan(Plan); + connectScalarPreheaderAsBypassInVPlan(Plan); } BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { @@ -7712,10 +7714,10 @@ DenseMap LoopVectorizationPlanner::executePlan( // 0. Generate SCEV-dependent code into the preheader, including TripCount, // before making any changes to the CFG. - if (!BestVPlan.getEntry()->empty()) { + if (!BestVPlan.getPreheader()->empty()) { State.CFG.PrevBB = OrigLoop->getLoopPreheader(); State.Builder.SetInsertPoint(OrigLoop->getLoopPreheader()->getTerminator()); - BestVPlan.getEntry()->execute(&State); + BestVPlan.getPreheader()->execute(&State); } if (!ILV.getTripCount()) ILV.setTripCount(State.get(BestVPlan.getTripCount(), VPLane(0))); @@ -7937,10 +7939,11 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); - VPBlockBase *VectorPH = Plan.getVectorPreheader(); - VPBlockBase *PredVPB = VectorPH->getSinglePredecessor(); - if (PredVPB->getNumSuccessors() == 1) - connectScalarPreheaderInVPlan(Plan); + // Connect the vector preheader to the scalar preheader when creating the trip + // count for the epilogue loop. Otherwise there is already a runtime check and + // connection to the scalar preheader, so we introduce it as new check block. + if (ForEpilogue) + connectScalarPreheaderAsBypassInVPlan(Plan); else introduceCheckBlockInVPlan(Plan, TCCheckBlock); return TCCheckBlock; @@ -8093,14 +8096,9 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( VPBasicBlock *OldEntry = Plan.getEntry(); VPBlockUtils::reassociateBlocks(OldEntry, NewEntry); Plan.setEntry(NewEntry); - for (auto &R : make_early_inc_range(*NewEntry)) { - auto *VPIR = dyn_cast(&R); - if (!VPIR || !isa(VPIR->getInstruction())) - break; - VPIR->eraseFromParent(); - } + delete OldEntry; - connectScalarPreheaderInVPlan(Plan); + connectScalarPreheaderAsBypassInVPlan(Plan); return Insert; } @@ -10317,7 +10315,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { // should be removed once induction resume value creation is done // directly in VPlan. EpilogILV.setTripCount(MainILV.getTripCount()); - for (auto &R : make_early_inc_range(*BestEpiPlan.getEntry())) { + for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) { auto *ExpandR = dyn_cast(&R); if (!ExpandR) continue; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index a8a5e7c765a5f9..75ca717b7c1434 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -170,8 +170,7 @@ VPBasicBlock *VPBlockBase::getEntryBasicBlock() { } void VPBlockBase::setPlan(VPlan *ParentPlan) { - assert(ParentPlan->getEntry() == this && - "Can only set plan on its entry or preheader block."); + assert(ParentPlan->getEntry() == this && "Can only set plan on its entry."); Plan = ParentPlan; } @@ -822,6 +821,18 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent, } #endif +VPlan::VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry, + VPIRBasicBlock *ScalarHeader) + : VPlan(Preheader, TC, ScalarHeader) { + VPBlockUtils::connectBlocks(Preheader, Entry); +} + +VPlan::VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry, + VPIRBasicBlock *ScalarHeader) + : VPlan(Preheader, ScalarHeader) { + VPBlockUtils::connectBlocks(Preheader, Entry); +} + VPlan::~VPlan() { if (Entry) { VPValue DummyValue; @@ -851,6 +862,9 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy, VPIRBasicBlock *Entry = VPIRBasicBlock::fromBasicBlock(TheLoop->getLoopPreheader()); VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph"); + // Connect entry only to vector preheader initially. Edges to the scalar + // preheader will be inserted later, during skeleton creation when runtime + // guards are added as needed. VPBlockUtils::connectBlocks(Entry, VecPreheader); VPIRBasicBlock *ScalarHeader = VPIRBasicBlock::fromBasicBlock(TheLoop->getHeader()); @@ -997,20 +1011,21 @@ void VPlan::execute(VPTransformState *State) { BasicBlock *VectorPreHeader = State->CFG.PrevBB; State->Builder.SetInsertPoint(VectorPreHeader->getTerminator()); - replaceVPBBWithIRVPBB( - cast(getVectorLoopRegion()->getSinglePredecessor()), - VectorPreHeader); + // Disconnect VectorPreHeader from ExitBB in both the CFG and DT. + cast(VectorPreHeader->getTerminator())->setSuccessor(0, nullptr); State->CFG.DTU.applyUpdates( {{DominatorTree::Delete, VectorPreHeader, State->CFG.ExitBB}}); - // Replace regular VPBB's for the middle and scalar preheader blocks with - // VPIRBasicBlocks wrapping their IR blocks. The IR blocks are created during - // skeleton creation, so we can only create the VPIRBasicBlocks now during - // VPlan execution rather than earlier during VPlan construction. + // Replace regular VPBB's for the vector preheader, middle and scalar + // preheader blocks with VPIRBasicBlocks wrapping their IR blocks. The IR + // blocks are created during skeleton creation, so we can only create the + // VPIRBasicBlocks now during VPlan execution rather than earlier during VPlan + // construction. + replaceVPBBWithIRVPBB(getVectorPreheader(), VectorPreHeader); BasicBlock *MiddleBB = State->CFG.ExitBB; - VPBasicBlock *MiddleVPBB = getMiddleBlock(); BasicBlock *ScalarPh = MiddleBB->getSingleSuccessor(); replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh); + VPBasicBlock *MiddleVPBB = getMiddleBlock(); replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB); LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF @@ -1033,8 +1048,9 @@ void VPlan::execute(VPTransformState *State) { ReversePostOrderTraversal> RPOT( Entry); - // Generate code in the loop pre-header and body. - for (VPBlockBase *Block : make_range(RPOT.begin(), RPOT.end())) + // Generate code for the VPlan, in parts of the vector skeleton, loop body and + // successor blocks including the middle, exit and scalar preheader blocks. + for (VPBlockBase *Block : RPOT) Block->execute(State); VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock(); @@ -1280,14 +1296,10 @@ VPlan *VPlan::duplicate() { VPBasicBlock *VPlan::getScalarPreheader() { auto *MiddleVPBB = cast(getVectorLoopRegion()->getSingleSuccessor()); - if (MiddleVPBB->getNumSuccessors() == 2) { - // Order is strict: first is the exit block, second is the scalar preheader. - return cast(MiddleVPBB->getSuccessors()[1]); - } - if (auto *IRVPBB = dyn_cast(MiddleVPBB->getSingleSuccessor())) - return IRVPBB; - - return nullptr; + auto *LastSucc = MiddleVPBB->getSuccessors().back(); + // Order is strict: if the last successor is VPIRBasicBlock, it must be the + // single exit. + return isa(LastSucc) ? nullptr : cast(LastSucc); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 8418b42cfec754..26cd0b549bd97c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3715,8 +3715,9 @@ class VPlan { friend class VPlanPrinter; friend class VPSlotTracker; - /// Hold the single entry to the Hierarchical CFG of the VPlan, i.e. the - /// preheader of the vector loop. + /// VPBasicBlock corresponding to the original preheader. Used to place + /// VPExpandSCEV recipes for expressions used during skeleton creation and the + /// rest of VPlan execution. VPBasicBlock *Entry; /// VPIRBasicBlock wrapping the header of the original scalar loop. @@ -3763,12 +3764,21 @@ class VPlan { DenseMap SCEVToExpansion; public: - /// Construct a VPlan with \p Entry entering the plan and trip count \p TC. + /// Construct a VPlan with \p Entry entering the plan, trip count \p TC and + /// with \p ScalarHeader wrapping the original header of the scalar loop. VPlan(VPBasicBlock *Entry, VPValue *TC, VPIRBasicBlock *ScalarHeader) : VPlan(Entry, ScalarHeader) { TripCount = TC; } + /// Constructor variants that take disconnected preheader and entry blocks, + /// connecting them as part of construction. Only used to reduce the need of + /// code changes during transition. + VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry, + VPIRBasicBlock *ScalarHeader); + VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry, + VPIRBasicBlock *ScalarHeader); + /// Construct a VPlan with \p Entry to the plan. VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader) : Entry(Entry), ScalarHeader(ScalarHeader) { @@ -3968,6 +3978,10 @@ class VPlan { SCEVToExpansion[S] = V; } + /// \return The block corresponding to the original preheader. + VPBasicBlock *getPreheader() { return Entry; } + const VPBasicBlock *getPreheader() const { return Entry; } + /// Clone the current VPlan, update all VPValues of the new VPlan and cloned /// recipes to refer to the clones, and return it. VPlan *duplicate(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index cb1f9b4bf346fb..54f3bbde3b9c94 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3241,7 +3241,13 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, void VPExpandSCEVRecipe::execute(VPTransformState &State) { assert(!State.Lane && "cannot be used in per-lane"); if (State.ExpandedSCEVs.contains(Expr)) { + // SCEV Expr has already been expanded, result must already be set. At the + // moment we have to execute the entry block twice (once before skeleton + // creation to get expanded SCEVs used by the skeleton and once during + // regular VPlan execution). State.Builder.SetInsertPoint(State.CFG.VPBB2IRBB[getParent()]); + assert(State.get(this, VPLane(0)) == State.ExpandedSCEVs[Expr] && + "Results must match"); return; } @@ -3249,8 +3255,6 @@ void VPExpandSCEVRecipe::execute(VPTransformState &State) { SCEVExpander Exp(SE, DL, "induction"); Value *Res = Exp.expandCodeFor(Expr, Expr->getType(), &*State.Builder.GetInsertPoint()); - /* assert(!State.ExpandedSCEVs.contains(Expr) &&*/ - /*"Same SCEV expanded multiple times");*/ State.ExpandedSCEVs[Expr] = Res; State.set(this, Res, VPLane(0)); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 33ebff1dee312f..2a1ac4d96201e1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -377,9 +377,8 @@ static bool mergeBlocksIntoPredecessors(VPlan &Plan) { continue; auto *PredVPBB = dyn_cast_or_null(VPBB->getSinglePredecessor()); - if (!PredVPBB || PredVPBB->getNumSuccessors() != 1) - continue; - if (isa(PredVPBB)) + if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 || + isa(PredVPBB)) continue; WorkList.push_back(VPBB); } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll index 199d6a7c858bc5..e7eb5778ffb930 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll @@ -62,10 +62,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-cond ir ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; IF-EVL-OUTLOOP-EMPTY: -; IF-EVL-OUTLOOP-NEXT: ir-bb: -; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; IF-EVL-OUTLOOP-NEXT: No successors -; IF-EVL-OUTLOOP-EMPTY: ; IF-EVL-OUTLOOP-NEXT: scalar.ph: ; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb @@ -77,7 +73,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: No successors ; IF-EVL-OUTLOOP-EMPTY: ; IF-EVL-OUTLOOP-NEXT: ir-bb: -; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) ; IF-EVL-OUTLOOP-NEXT: No successors ; IF-EVL-OUTLOOP-NEXT: } ; @@ -116,10 +112,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: EMIT branch-on-cond ir ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; IF-EVL-INLOOP-EMPTY: -; IF-EVL-INLOOP-NEXT: ir-bb: -; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; IF-EVL-INLOOP-NEXT: No successors -; IF-EVL-INLOOP-EMPTY: ; IF-EVL-INLOOP-NEXT: scalar.ph: ; IF-EVL-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb @@ -131,7 +123,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: No successors ; IF-EVL-INLOOP-EMPTY: ; IF-EVL-INLOOP-NEXT: ir-bb: -; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) ; IF-EVL-INLOOP-NEXT: No successors ; IF-EVL-INLOOP-NEXT: } ; @@ -166,10 +158,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]> ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; NO-VP-OUTLOOP-EMPTY: -; NO-VP-OUTLOOP-NEXT: ir-bb: -; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; NO-VP-OUTLOOP-NEXT: No successors -; NO-VP-OUTLOOP-EMPTY: ; NO-VP-OUTLOOP-NEXT: scalar.ph: ; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb @@ -181,7 +169,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-OUTLOOP-NEXT: No successors ; NO-VP-OUTLOOP-EMPTY: ; NO-VP-OUTLOOP-NEXT: ir-bb: -; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) ; NO-VP-OUTLOOP-NEXT: No successors ; NO-VP-OUTLOOP-NEXT: } ; @@ -216,10 +204,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]> ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; NO-VP-INLOOP-EMPTY: -; NO-VP-INLOOP-NEXT: ir-bb: -; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; NO-VP-INLOOP-NEXT: No successors -; NO-VP-INLOOP-EMPTY: ; NO-VP-INLOOP-NEXT: scalar.ph: ; NO-VP-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb @@ -231,7 +215,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-INLOOP-NEXT: No successors ; NO-VP-INLOOP-EMPTY: ; NO-VP-INLOOP-NEXT: ir-bb: -; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]>) +; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) ; NO-VP-INLOOP-NEXT: No successors ; NO-VP-INLOOP-NEXT: } ; diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp index 604f87b634bba6..86182ccae0b55b 100644 --- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp @@ -44,9 +44,8 @@ TEST(VPDominatorTreeTest, DominanceNoRegionsTest) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB0); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB); VPDominatorTree VPDT; VPDT.recalculate(Plan); @@ -125,9 +124,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB0); VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB); VPDominatorTree VPDT; VPDT.recalculate(Plan); @@ -208,9 +206,8 @@ TEST(VPDominatorTreeTest, DominanceRegionsTest) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); VPDominatorTree VPDT; VPDT.recalculate(Plan); diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 13357738eea6d2..3179cfc676ab67 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -259,9 +259,8 @@ TEST(VPBasicBlockTest, getPlan) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); EXPECT_EQ(&Plan, VPBB1->getPlan()); EXPECT_EQ(&Plan, VPBB2->getPlan()); @@ -282,9 +281,8 @@ TEST(VPBasicBlockTest, getPlan) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); EXPECT_EQ(&Plan, VPBB1->getPlan()); EXPECT_EQ(&Plan, R1->getPlan()); @@ -315,9 +313,8 @@ TEST(VPBasicBlockTest, getPlan) { auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); EXPECT_EQ(&Plan, VPBB1->getPlan()); EXPECT_EQ(&Plan, R1->getPlan()); @@ -362,9 +359,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB4, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); } { @@ -465,9 +461,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB0); VPBlockUtils::connectBlocks(R2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB0, ScalarHeaderVPBB); } { @@ -551,9 +546,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); } { @@ -602,9 +596,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); } { @@ -697,9 +690,8 @@ TEST(VPBasicBlockTest, TraversingIteratorTest) { // Use Plan to properly clean up created blocks. auto TC = std::make_unique(); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); } delete ScalarHeader; } @@ -741,9 +733,8 @@ TEST(VPBasicBlockTest, print) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, "scalar.header"); auto * ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPBB0, VPBB1); VPBlockUtils::connectBlocks(VPBB2, ScalarHeaderVPBB); - VPlan Plan(VPBB0, TC, ScalarHeaderVPBB); + VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB); std::string FullDump; raw_string_ostream OS(FullDump); Plan.printDOT(OS); @@ -834,9 +825,8 @@ TEST(VPBasicBlockTest, printPlanWithVFsAndUFs) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPBB0, VPBB1); VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB); - VPlan Plan(VPBB0, TC, ScalarHeaderVPBB); + VPlan Plan(VPBB0, TC, VPBB1, ScalarHeaderVPBB); Plan.setName("TestPlan"); Plan.addVF(ElementCount::getFixed(4)); @@ -1306,13 +1296,11 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { TEST(VPRecipeTest, dumpRecipeInPlan) { VPBasicBlock *VPBB0 = new VPBasicBlock("preheader"); VPBasicBlock *VPBB1 = new VPBasicBlock(); - VPBlockUtils::connectBlocks(VPBB0, VPBB1); - LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB); - VPlan Plan(VPBB0, ScalarHeaderVPBB); + VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB); IntegerType *Int32 = IntegerType::get(C, 32); auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32), @@ -1380,13 +1368,11 @@ TEST(VPRecipeTest, dumpRecipeInPlan) { TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) { VPBasicBlock *VPBB0 = new VPBasicBlock("preheader"); VPBasicBlock *VPBB1 = new VPBasicBlock(); - VPBlockUtils::connectBlocks(VPBB0, VPBB1); - LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); VPBlockUtils::connectBlocks(VPBB1, ScalarHeaderVPBB); - VPlan Plan(VPBB0, ScalarHeaderVPBB); + VPlan Plan(VPBB0, VPBB1, ScalarHeaderVPBB); IntegerType *Int32 = IntegerType::get(C, 32); auto *AI = BinaryOperator::CreateAdd(PoisonValue::get(Int32), diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index 2c7566c973c4e1..edb3f8a2952942 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -32,9 +32,8 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefSameBB) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -70,9 +69,8 @@ TEST(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -118,9 +116,8 @@ TEST(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) { auto TC = std::make_unique(); auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -160,9 +157,8 @@ TEST(VPVerifierTest, DuplicateSuccessorsOutsideRegion) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -203,9 +199,8 @@ TEST(VPVerifierTest, DuplicateSuccessorsInsideRegion) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, VPBB1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -238,9 +233,8 @@ TEST(VPVerifierTest, BlockOutsideRegionWithParent) { LLVMContext C; auto *ScalarHeader = BasicBlock::Create(C, ""); VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); - VPBlockUtils::connectBlocks(VPPH, R1); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); - VPlan Plan(VPPH, &*TC, ScalarHeaderVPBB); + VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); From 927a66ddbc5ec1ee0f7c4bdf7525c6068300c717 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 25 Nov 2024 21:28:54 +0000 Subject: [PATCH 11/23] !fixup update verifier --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 6cee4851ebb5d6..5c8c3bc7b66de1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -185,6 +185,16 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { RecipeNumbering[&R] = Cnt++; for (const VPRecipeBase &R : *VPBB) { + if (isa(&R) && !isa(VPBB)) { + errs() << "VPIRInstructions "; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + R.dump(); + errs() << " "; +#endif + errs() << "not in a VPIRBasicBlock!\n"; + return false; + } + for (const VPValue *V : R.definedValues()) { for (const VPUser *U : V->users()) { auto *UI = dyn_cast(U); From 3eef6013cb1e11e18215895d5eb2eff7c25a2a66 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 4 Dec 2024 14:43:55 +0000 Subject: [PATCH 12/23] !fixup address comments, thanks! --- .../Transforms/Vectorize/LoopVectorize.cpp | 48 ++++++++----------- llvm/lib/Transforms/Vectorize/VPlan.cpp | 36 +++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 27 ++++++----- 3 files changed, 54 insertions(+), 57 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index cacd49b3f814b5..3ec7c315033d52 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2428,30 +2428,24 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { return VectorTripCount; } -/// Helper to connect both the vector and scalar preheaders to the Plan's -/// entry. This is used when adjusting \p Plan during skeleton -/// creation, i.e. adjusting the plan after introducing an initial runtime -/// check. -/// TODO: Connect scalar preheader during initial VPlan construction. -static void connectScalarPreheaderAsBypassInVPlan(VPlan &Plan) { - VPBlockBase *VectorPH = Plan.getVectorPreheader(); - VPBlockBase *ScalarPH = Plan.getScalarPreheader(); - VPBlockBase *Entry = Plan.getEntry(); - VPBlockUtils::connectBlocks(Entry, ScalarPH); - std::swap(Entry->getSuccessors()[0], Entry->getSuccessors()[1]); -} - /// Introduces a new VPIRBasicBlock for \p CheckIRBB to \p Plan between the -/// vector preheader and its predecessor, also connecting to the scalar -/// preheader. +/// vector preheader and its predecessor, also connecting the new block to the +/// scalar preheader. static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { + VPBlockBase *ScalarPH = Plan.getScalarPreheader(); VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor(); - assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor"); - VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB); - VPBlockUtils::connectBlocks(CheckVPIRBB, ScalarPH); - VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckVPIRBB); + if (PreVectorPH->getNumSuccessors() != 1) { + assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors"); + assert(PreVectorPH->getSuccessors()[0] == ScalarPH && + "Unexpected successor"); + VPIRBasicBlock *CheckVPIRBB = VPIRBasicBlock::fromBasicBlock(CheckIRBB); + VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckVPIRBB); + PreVectorPH = CheckVPIRBB; + } + VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH); + PreVectorPH->swapSuccessors(); } void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { @@ -2536,7 +2530,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { LoopBypassBlocks.push_back(TCCheckBlock); // TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here. - connectScalarPreheaderAsBypassInVPlan(Plan); + introduceCheckBlockInVPlan(Plan, nullptr); } BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { @@ -7919,13 +7913,8 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); - // Connect the vector preheader to the scalar preheader when creating the trip - // count for the epilogue loop. Otherwise there is already a runtime check and - // connection to the scalar preheader, so we introduce it as new check block. - if (ForEpilogue) - connectScalarPreheaderAsBypassInVPlan(Plan); - else - introduceCheckBlockInVPlan(Plan, TCCheckBlock); + // Connect TCCheckblock to the VPlan. + introduceCheckBlockInVPlan(Plan, TCCheckBlock); return TCCheckBlock; } @@ -8071,14 +8060,15 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( ReplaceInstWithInst(Insert->getTerminator(), &BI); LoopBypassBlocks.push_back(Insert); - // A new entry block has been created for the epilogue VPlan. Hook it in. + // A new entry block has been created for the epilogue VPlan. Hook it in, as + // otherwise we would try to modify the entry to the main vector loop. VPIRBasicBlock *NewEntry = VPIRBasicBlock::fromBasicBlock(Insert); VPBasicBlock *OldEntry = Plan.getEntry(); VPBlockUtils::reassociateBlocks(OldEntry, NewEntry); Plan.setEntry(NewEntry); delete OldEntry; - connectScalarPreheaderAsBypassInVPlan(Plan); + introduceCheckBlockInVPlan(Plan, nullptr); return Insert; } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index e0fdc386cb80dd..bbaa18f451f7c0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -821,16 +821,16 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent, } #endif -VPlan::VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry, - VPIRBasicBlock *ScalarHeader) - : VPlan(Preheader, TC, ScalarHeader) { - VPBlockUtils::connectBlocks(Preheader, Entry); +VPlan::VPlan(VPBasicBlock *OriginalPreheader, VPValue *TC, + VPBasicBlock *EntryVectorPreHeader, VPIRBasicBlock *ScalarHeader) + : VPlan(OriginalPreheader, TC, ScalarHeader) { + VPBlockUtils::connectBlocks(OriginalPreheader, EntryVectorPreHeader); } -VPlan::VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry, - VPIRBasicBlock *ScalarHeader) - : VPlan(Preheader, ScalarHeader) { - VPBlockUtils::connectBlocks(Preheader, Entry); +VPlan::VPlan(VPBasicBlock *OriginalPreheader, + VPBasicBlock *EntryVectorPreHeader, VPIRBasicBlock *ScalarHeader) + : VPlan(OriginalPreheader, ScalarHeader) { + VPBlockUtils::connectBlocks(OriginalPreheader, EntryVectorPreHeader); } VPlan::~VPlan() { @@ -1187,6 +1187,7 @@ std::string VPlan::getName() const { } VPRegionBlock *VPlan::getVectorLoopRegion() { + // TODO: Cache if possible. for (VPBlockBase *B : vp_depth_first_shallow(getEntry())) if (auto *R = dyn_cast(B)) return R; @@ -1200,6 +1201,16 @@ const VPRegionBlock *VPlan::getVectorLoopRegion() const { return nullptr; } +VPBasicBlock *VPlan::getScalarPreheader() const { + auto *MiddleVPBB = + cast(getVectorLoopRegion()->getSingleSuccessor()); + auto *LastSucc = MiddleVPBB->getSuccessors().back(); + // If scalar preheader is connected to VPlan, it is the last successor of + // MiddleVPBB. If this last successor is a VPIRBasicBlock, it is the Exit + // block rather than the scalar preheader. + return isa(LastSucc) ? nullptr : cast(LastSucc); +} + LLVM_DUMP_METHOD void VPlan::printDOT(raw_ostream &O) const { VPlanPrinter Printer(O, *this); @@ -1292,15 +1303,6 @@ VPlan *VPlan::duplicate() { return NewPlan; } -VPBasicBlock *VPlan::getScalarPreheader() { - auto *MiddleVPBB = - cast(getVectorLoopRegion()->getSingleSuccessor()); - auto *LastSucc = MiddleVPBB->getSuccessors().back(); - // Order is strict: if the last successor is VPIRBasicBlock, it must be the - // single exit. - return isa(LastSucc) ? nullptr : cast(LastSucc); -} - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) Twine VPlanPrinter::getUID(const VPBlockBase *Block) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index df5fe83342d4be..d7ad21556f5440 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -621,6 +621,12 @@ class VPBlockBase { /// Remove all the successors of this block. void clearSuccessors() { Successors.clear(); } + /// Swap successors of the block. The block must have exactly 2 successors. + void swapSuccessors() { + assert(Successors.size() == 2 && "must have 2 successors to swap"); + std::swap(Successors[0], Successors[1]); + } + /// The method which generates the output IR that correspond to this /// VPBlockBase, thereby "executing" the VPlan. virtual void execute(VPTransformState *State) = 0; @@ -3814,14 +3820,15 @@ class VPlan { } /// Constructor variants that take disconnected preheader and entry blocks, - /// connecting them as part of construction. Only used to reduce the need of - /// code changes during transition. - VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry, - VPIRBasicBlock *ScalarHeader); - VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry, + /// connecting them as part of construction. + /// FIXME: Only used to reduce the need of code changes during transition. + VPlan(VPBasicBlock *OriginalPreheader, VPValue *TC, + VPBasicBlock *EntryVectorPreHeader, VPIRBasicBlock *ScalarHeader); + VPlan(VPBasicBlock *OriginalPreheader, VPBasicBlock *EntryVectorPreHeader, VPIRBasicBlock *ScalarHeader); - /// Construct a VPlan with \p Entry to the plan. + /// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader + /// wrapping the original header of the scalar loop. VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader) : Entry(Entry), ScalarHeader(ScalarHeader) { Entry->setPlan(this); @@ -3883,9 +3890,7 @@ class VPlan { } /// Return the VPBasicBlock for the preheader of the scalar loop. - VPBasicBlock *getScalarPreheader() const { - return cast(ScalarHeader->getSinglePredecessor()); - } + VPBasicBlock *getScalarPreheader() const; /// Return the VPIRBasicBlock wrapping the header of the scalar loop. VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; } @@ -4020,14 +4025,14 @@ class VPlan { } /// \return The block corresponding to the original preheader. + /// FIXME: There's no separate preheader any longer and Entry now serves the + /// same purpose as the original preheader. Remove after transition. VPBasicBlock *getPreheader() { return Entry; } const VPBasicBlock *getPreheader() const { return Entry; } /// Clone the current VPlan, update all VPValues of the new VPlan and cloned /// recipes to refer to the clones, and return it. VPlan *duplicate(); - - VPBasicBlock *getScalarPreheader(); }; #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) From 22eeebed6de4340b0cd71b070ea204d6de0aac4e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 5 Dec 2024 17:56:51 +0000 Subject: [PATCH 13/23] !fixup address latest comments, thanks! --- .../Transforms/Vectorize/LoopVectorize.cpp | 6 +- llvm/lib/Transforms/Vectorize/VPlan.cpp | 10 +- llvm/lib/Transforms/Vectorize/VPlan.h | 4 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +- .../Transforms/Vectorize/VPlanVerifier.cpp | 1 - .../extractvalue-no-scalarization-required.ll | 1 + .../RISCV/riscv-vector-reverse.ll | 169 ++++++++++++++++++ .../RISCV/vplan-vp-select-intrinsics.ll | 6 +- .../LoopVectorize/vplan-predicate-switch.ll | 2 +- .../LoopVectorize/vplan-printing.ll | 1 + 10 files changed, 185 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 3ec7c315033d52..969c585e91e5c2 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2432,7 +2432,6 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) { /// vector preheader and its predecessor, also connecting the new block to the /// scalar preheader. static void introduceCheckBlockInVPlan(VPlan &Plan, BasicBlock *CheckIRBB) { - VPBlockBase *ScalarPH = Plan.getScalarPreheader(); VPBlockBase *VectorPH = Plan.getVectorPreheader(); VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor(); @@ -2530,7 +2529,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) { LoopBypassBlocks.push_back(TCCheckBlock); // TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here. - introduceCheckBlockInVPlan(Plan, nullptr); + introduceCheckBlockInVPlan(Plan, TCCheckBlock); } BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { @@ -7913,7 +7912,6 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass, setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false); ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI); - // Connect TCCheckblock to the VPlan. introduceCheckBlockInVPlan(Plan, TCCheckBlock); return TCCheckBlock; } @@ -8068,7 +8066,7 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( Plan.setEntry(NewEntry); delete OldEntry; - introduceCheckBlockInVPlan(Plan, nullptr); + introduceCheckBlockInVPlan(Plan, Insert); return Insert; } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index bbaa18f451f7c0..9315241c1071c7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -862,9 +862,8 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy, VPIRBasicBlock *Entry = VPIRBasicBlock::fromBasicBlock(TheLoop->getLoopPreheader()); VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph"); - // Connect entry only to vector preheader initially. Edges to the scalar - // preheader will be inserted later, during skeleton creation when runtime - // guards are added as needed. + // Connect entry only to vector preheader initially. Entry will also be connected to the scalar preheader later, during skeleton creation when runtime guards are added as needed. + // Note that when executing the VPlan for an epilogue vector loop, the original entry block here will be replaced by a new VPIRBasicBlock wrapping the entry to the epilogue vector loop after generating code for the main vector loop. VPBlockUtils::connectBlocks(Entry, VecPreheader); VPIRBasicBlock *ScalarHeader = VPIRBasicBlock::fromBasicBlock(TheLoop->getHeader()); @@ -1021,12 +1020,11 @@ void VPlan::execute(VPTransformState *State) { // blocks are created during skeleton creation, so we can only create the // VPIRBasicBlocks now during VPlan execution rather than earlier during VPlan // construction. - replaceVPBBWithIRVPBB(getVectorPreheader(), VectorPreHeader); BasicBlock *MiddleBB = State->CFG.ExitBB; BasicBlock *ScalarPh = MiddleBB->getSingleSuccessor(); + replaceVPBBWithIRVPBB(getVectorPreheader(), VectorPreHeader); + replaceVPBBWithIRVPBB(getMiddleBlock(), MiddleBB); replaceVPBBWithIRVPBB(getScalarPreheader(), ScalarPh); - VPBasicBlock *MiddleVPBB = getMiddleBlock(); - replaceVPBBWithIRVPBB(MiddleVPBB, MiddleBB); LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << State->VF << ", UF=" << getUF() << '\n'); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d7ad21556f5440..324fbc08554f68 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3844,8 +3844,8 @@ class VPlan { } /// Create initial VPlan, having an "entry" VPBasicBlock (wrapping - /// original scalar pre-header ) which contains SCEV expansions that need - /// to happen before the CFG is modified; a VPBasicBlock for the vector + /// original scalar pre-header) which contains SCEV expansions that need + /// to happen before the CFG is modified (when executing a VPlan for the epilogue vector loop, the original entry needs to be replaced by the new entry for the epilogue vector loop); a VPBasicBlock for the vector /// pre-header, followed by a region for the vector loop, followed by the /// middle VPBasicBlock. If a check is needed to guard executing the scalar /// epilogue loop, it will be added to the middle block, together with diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index b49c7955369b96..22c05c0f0850aa 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3248,6 +3248,7 @@ void VPExpandSCEVRecipe::execute(VPTransformState &State) { const DataLayout &DL = State.CFG.PrevBB->getDataLayout(); SCEVExpander Exp(SE, DL, "induction"); + Value *Res = Exp.expandCodeFor(Expr, Expr->getType(), &*State.Builder.GetInsertPoint()); State.ExpandedSCEVs[Expr] = Res; @@ -3501,7 +3502,7 @@ void VPScalarPHIRecipe::execute(VPTransformState &State) { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPScalarPHIRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << Indent << "SCALAR-PHI"; + O << Indent << "SCALAR-PHI "; printAsOperand(O, SlotTracker); O << " = phi "; printOperands(O, SlotTracker); diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 5c8c3bc7b66de1..be420a873bef52 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -194,7 +194,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { errs() << "not in a VPIRBasicBlock!\n"; return false; } - for (const VPValue *V : R.definedValues()) { for (const VPUser *U : V->users()) { auto *UI = dyn_cast(U); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll index aa78113ebaa48c..f97184c21c9d9c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll @@ -64,6 +64,7 @@ exit: declare float @powf(float, float) readnone nounwind ; Ensure the extractvalue + add instructions are hoisted out +; CM-LABEL: Checking a loop in 'test_getVectorCallCost' ; CM: vector.ph: ; CM: CLONE ir<%a> = extractvalue ir<%sv> ; CM: CLONE ir<%b> = extractvalue ir<%sv> diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index d718b75fac9625..4fc50a43fe1179 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -144,6 +144,90 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop ; CHECK: Executing best plan with VF=vscale x 4, UF=1 ; CHECK-NEXT: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in ir<[[VF:%.+]]> = VF +; CHECK-NEXT: Live-in ir<[[VFxUF:%.+]]>.1 = VF * UF +; CHECK-NEXT: Live-in ir<[[VEC_TC:%.+]]> = vector-trip-count +; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %0 = zext i32 %n to i64 +; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %3 = add nsw i64 %0, -1 +; CHECK-NEXT: IR %4 = add i32 %n, -1 +; CHECK-NEXT: IR %5 = trunc i64 %3 to i32 +; CHECK-NEXT: IR %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5) +; CHECK-NEXT: IR %mul.result = extractvalue { i32, i1 } %mul, 0 +; CHECK-NEXT: IR %mul.overflow = extractvalue { i32, i1 } %mul, 1 +; CHECK-NEXT: IR %6 = sub i32 %4, %mul.result +; CHECK-NEXT: IR %7 = icmp ugt i32 %6, %4 +; CHECK-NEXT: IR %8 = or i1 %7, %mul.overflow +; CHECK-NEXT: IR %9 = icmp ugt i64 %3, 4294967295 +; CHECK-NEXT: IR %10 = or i1 %8, %9 +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %11 = call i64 @llvm.vscale.i64() +; CHECK-NEXT: IR %12 = mul i64 %11, 4 +; CHECK-NEXT: IR %13 = mul i64 %12, 4 +; CHECK-NEXT: IR %14 = sub i64 %B1, %A2 +; CHECK-NEXT: IR %diff.check = icmp ult i64 %14, %13 +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %15 = call i64 @llvm.vscale.i64() +; CHECK-NEXT: IR %16 = mul i64 %15, 4 +; CHECK-NEXT: IR %n.mod.vf = urem i64 %0, %16 +; CHECK-NEXT: IR %n.vec = sub i64 %0, %n.mod.vf +; CHECK-NEXT: IR %ind.end = sub i64 %0, %n.vec +; CHECK-NEXT: IR %.cast = trunc i64 %n.vec to i32 +; CHECK-NEXT: IR %ind.end3 = sub i32 %n, %.cast +; CHECK-NEXT: IR %17 = call i64 @llvm.vscale.i64() +; CHECK-NEXT: IR %18 = mul i64 %17, 4 +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: vector loop: { +; CHECK-NEXT: vector.body: +; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]> +; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, ir<[[VF]]> +; CHECK-NEXT: WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%add9> = add ir<[[L]]>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, ir<[[VF]]> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>.1 +; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VEC_TC]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<[[VEC_TC]]> +; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR [[RESUME_1:%.+]] = phi i64 +; CHECK-NEXT: IR [[RESUME_2:%.+]] = phi i32 +; CHECK-NEXT: Successor(s): ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %indvars.iv = phi i64 [ [[RESUME_1]], %scalar.ph ], [ %indvars.iv.next, %for.body ] +; CHECK-NEXT: IR %i.0.in8 = phi i32 [ [[RESUME_2]], %scalar.ph ], [ %i.0, %for.body ] +; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK: LV: Loop does not require scalar epilogue ; entry: %cmp7 = icmp sgt i32 %n, 0 @@ -306,6 +390,91 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Found a vectorizable loop (vscale x 4) in ; CHECK-NEXT: LEV: Epilogue vectorization is not profitable for this loop ; CHECK: Executing best plan with VF=vscale x 4, UF=1 +; CHECK-NEXT: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' { +; CHECK-NEXT: Live-in ir<[[VF:%.+]]> = VF +; CHECK-NEXT: Live-in ir<[[VFxUF:%.+]]>.1 = VF * UF +; CHECK-NEXT: Live-in ir<[[VEC_TC:%.+]]> = vector-trip-count +; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %0 = zext i32 %n to i64 +; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %3 = add nsw i64 %0, -1 +; CHECK-NEXT: IR %4 = add i32 %n, -1 +; CHECK-NEXT: IR %5 = trunc i64 %3 to i32 +; CHECK-NEXT: IR %mul = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 %5) +; CHECK-NEXT: IR %mul.result = extractvalue { i32, i1 } %mul, 0 +; CHECK-NEXT: IR %mul.overflow = extractvalue { i32, i1 } %mul, 1 +; CHECK-NEXT: IR %6 = sub i32 %4, %mul.result +; CHECK-NEXT: IR %7 = icmp ugt i32 %6, %4 +; CHECK-NEXT: IR %8 = or i1 %7, %mul.overflow +; CHECK-NEXT: IR %9 = icmp ugt i64 %3, 4294967295 +; CHECK-NEXT: IR %10 = or i1 %8, %9 +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %11 = call i64 @llvm.vscale.i64() +; CHECK-NEXT: IR %12 = mul i64 %11, 4 +; CHECK-NEXT: IR %13 = mul i64 %12, 4 +; CHECK-NEXT: IR %14 = sub i64 %B1, %A2 +; CHECK-NEXT: IR %diff.check = icmp ult i64 %14, %13 +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %15 = call i64 @llvm.vscale.i64() +; CHECK-NEXT: IR %16 = mul i64 %15, 4 +; CHECK-NEXT: IR %n.mod.vf = urem i64 %0, %16 +; CHECK-NEXT: IR %n.vec = sub i64 %0, %n.mod.vf +; CHECK-NEXT: IR %ind.end = sub i64 %0, %n.vec +; CHECK-NEXT: IR %.cast = trunc i64 %n.vec to i32 +; CHECK-NEXT: IR %ind.end3 = sub i32 %n, %.cast +; CHECK-NEXT: IR %17 = call i64 @llvm.vscale.i64() +; CHECK-NEXT: IR %18 = mul i64 %17, 4 +; CHECK-NEXT: Successor(s): vector loop +; CHECK-EMPTY: +; CHECK-NEXT: vector loop: { +; CHECK-NEXT: vector.body: +; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:.+]]> +; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx>, ir<[[VF]]> +; CHECK-NEXT: WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<[[L]]>, ir<1.000000e+00> +; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = reverse-vector-pointer inbounds ir<%arrayidx3>, ir<[[VF]]> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%conv1> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>.1 +; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VEC_TC]]> +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK-NEXT: Successor(s): ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, ir<[[VEC_TC]]> +; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> +; CHECK-NEXT: Successor(s): ir-bb, ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR [[RESUME_1:%.+]] = phi i64 +; CHECK-NEXT: IR [[RESUME_2:%.+]] = phi i32 +; CHECK-NEXT: Successor(s): ir-bb +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: IR %indvars.iv = phi i64 [ [[RESUME_1]], %scalar.ph ], [ %indvars.iv.next, %for.body ] +; CHECK-NEXT: IR %i.0.in8 = phi i32 [ [[RESUME_2]], %scalar.ph ], [ %i.0, %for.body ] +; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 +; CHECK-NEXT: No successors +; CHECK-NEXT: } +; CHECK: LV: Loop does not require scalar epilogue ; entry: %cmp7 = icmp sgt i32 %n, 0 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll index 523cf1bd798b68..49bcabc292b0db 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll @@ -27,8 +27,8 @@ ; IF-EVL: vector loop: { ; IF-EVL-NEXT: vector.body: - ; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION - ; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEX:%.+]]> + ; IF-EVL-NEXT: SCALAR-PHI vp<[[IV:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT_EXIT:%.+]]> + ; IF-EVL-NEXT: SCALAR-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEX:%.+]]> ; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> ; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> ; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1> @@ -47,7 +47,7 @@ ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, ir<%8> + ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<%8> ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, ir<%n.vec> ; IF-EVL-NEXT: No successors ; IF-EVL-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll index f31ecca50a3701..e545c2bc2daa9f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll @@ -19,7 +19,7 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> +; CHECK-NEXT: SCALAR-PHI vp<[[CAN_IV:%.+]]> = phi ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[PTR:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[WIDE_PTR:%.+]]> = vector-pointer vp<[[PTR]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 47c0671cfffe3a..99f74c8188912f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -505,6 +505,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb ; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block) +; CHECK-NEXT: No successors ; CHECK-NEXT:} entry: From 886bcc214ca052dcb2d72759f33ac2bd39d50772 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 5 Dec 2024 19:43:06 +0000 Subject: [PATCH 14/23] !fixup restore part of assert and fix formatting --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 ++++++-- llvm/lib/Transforms/Vectorize/VPlan.h | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 9315241c1071c7..982f541670fbd3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -862,8 +862,12 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy, VPIRBasicBlock *Entry = VPIRBasicBlock::fromBasicBlock(TheLoop->getLoopPreheader()); VPBasicBlock *VecPreheader = new VPBasicBlock("vector.ph"); - // Connect entry only to vector preheader initially. Entry will also be connected to the scalar preheader later, during skeleton creation when runtime guards are added as needed. - // Note that when executing the VPlan for an epilogue vector loop, the original entry block here will be replaced by a new VPIRBasicBlock wrapping the entry to the epilogue vector loop after generating code for the main vector loop. + // Connect entry only to vector preheader initially. Entry will also be + // connected to the scalar preheader later, during skeleton creation when + // runtime guards are added as needed. Note that when executing the VPlan for + // an epilogue vector loop, the original entry block here will be replaced by + // a new VPIRBasicBlock wrapping the entry to the epilogue vector loop after + // generating code for the main vector loop. VPBlockUtils::connectBlocks(Entry, VecPreheader); VPIRBasicBlock *ScalarHeader = VPIRBasicBlock::fromBasicBlock(TheLoop->getHeader()); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 324fbc08554f68..6f51407b7d05f1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3845,7 +3845,9 @@ class VPlan { /// Create initial VPlan, having an "entry" VPBasicBlock (wrapping /// original scalar pre-header) which contains SCEV expansions that need - /// to happen before the CFG is modified (when executing a VPlan for the epilogue vector loop, the original entry needs to be replaced by the new entry for the epilogue vector loop); a VPBasicBlock for the vector + /// to happen before the CFG is modified (when executing a VPlan for the + /// epilogue vector loop, the original entry needs to be replaced by the new + /// entry for the epilogue vector loop); a VPBasicBlock for the vector /// pre-header, followed by a region for the vector loop, followed by the /// middle VPBasicBlock. If a check is needed to guard executing the scalar /// epilogue loop, it will be added to the middle block, together with @@ -4176,6 +4178,8 @@ class VPBlockUtils { unsigned PredIdx = -1u, unsigned SuccIdx = -1u) { assert((From->getParent() == To->getParent()) && "Can't connect two block with different parents"); + assert((SuccIdx != -1u || From->getNumSuccessors() < 2) && + "Blocks can't have more than two successors."); if (SuccIdx == -1u) From->appendSuccessor(To); else From 65ac2d70bcf4fb8bf9176f874c500c103871c721 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 5 Dec 2024 19:53:47 +0000 Subject: [PATCH 15/23] !fixup more formatting fixes --- llvm/lib/Transforms/Vectorize/VPlan.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 6f51407b7d05f1..8c983649043bf4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -4179,7 +4179,7 @@ class VPBlockUtils { assert((From->getParent() == To->getParent()) && "Can't connect two block with different parents"); assert((SuccIdx != -1u || From->getNumSuccessors() < 2) && - "Blocks can't have more than two successors."); + "Blocks can't have more than two successors."); if (SuccIdx == -1u) From->appendSuccessor(To); else From 2f7d5302fb0caba7ca89997623fe554a51117589 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 6 Dec 2024 16:41:40 +0000 Subject: [PATCH 16/23] [VPlan] Use RPOT for VPlan codegen and printing. This split off changes for more complex CFGs in VPlan from both https://github.com/llvm/llvm-project/pull/114292 https://github.com/llvm/llvm-project/pull/112138 This reduces their respective diffs. --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 +- .../AArch64/sve2-histcnt-vplan.ll | 12 +-- .../AArch64/synthesize-mask-for-call.ll | 36 +++---- .../widen-call-with-intrinsic-or-libfunc.ll | 12 +-- .../RISCV/riscv-vector-reverse.ll | 24 ++--- .../RISCV/vplan-vp-intrinsics-reduction.ll | 32 +++--- .../first-order-recurrence-chains-vplan.ll | 24 ++--- ...-order-recurrence-sink-replicate-region.ll | 38 +++---- .../interleave-and-scalarize-only.ll | 12 +-- .../LoopVectorize/vplan-iv-transforms.ll | 6 +- .../LoopVectorize/vplan-predicate-switch.ll | 6 +- .../vplan-printing-before-execute.ll | 12 +-- .../vplan-printing-outer-loop.ll | 6 +- .../LoopVectorize/vplan-printing.ll | 98 +++++++++---------- .../vplan-sink-scalars-and-merge-vf1.ll | 6 +- .../vplan-sink-scalars-and-merge.ll | 12 +-- 16 files changed, 171 insertions(+), 169 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index b801d1863e252c..6cd0916a1c15cf 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1139,7 +1139,9 @@ void VPlan::print(raw_ostream &O) const { getPreheader()->print(O, "", SlotTracker); } - for (const VPBlockBase *Block : vp_depth_first_shallow(getEntry())) { + ReversePostOrderTraversal> + RPOT(getEntry()); + for (const VPBlockBase *Block : RPOT) { O << '\n'; Block->print(O, "", SlotTracker); } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll index 6257d3325f9796..9739611a8b6e48 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll @@ -41,9 +41,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]] ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -51,6 +48,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ;; Check that the vectorized plan contains a histogram recipe instead. @@ -83,9 +83,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond [[TC_CHECK]] ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -93,6 +90,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %N ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll index af1c63bc405a38..050bc7c346bf26 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -39,9 +39,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -49,6 +46,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -81,9 +81,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -91,6 +88,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ;; If we have a masked variant at one VF and an unmasked variant at a different @@ -128,9 +128,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -138,6 +135,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -170,9 +170,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -180,6 +177,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ;; If we have two variants at different VFs, neither of which are masked, we @@ -216,9 +216,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -226,6 +223,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -258,9 +258,6 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -268,6 +265,9 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] ; CHECK: IR %exitcond = icmp eq i64 %indvars.iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } define void @test_v4_v4m(ptr noalias %a, ptr readonly %b) #3 { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index afc2fd5a049ad1..7a67f37c726622 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -37,9 +37,6 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -47,6 +44,9 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { @@ -79,9 +79,6 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -89,6 +86,9 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %cmp = icmp ne i64 %iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; ; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index d68556fca4774f..1e4e748df6cea8 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -91,9 +91,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -102,6 +99,9 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] @@ -182,9 +182,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -193,6 +190,9 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: LV: Loop does not require scalar epilogue ; @@ -305,9 +305,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -316,6 +313,9 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %indvars.iv = phi i64 [ %0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] ; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] @@ -396,9 +396,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -407,6 +404,9 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %for.body.preheader ], [ %i.0, %for.body ] ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: LV: Loop does not require scalar epilogue ; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll index ab541f6fa94e66..a474d926a6303f 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll @@ -59,10 +59,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-cond ir ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; IF-EVL-OUTLOOP-EMPTY: -; IF-EVL-OUTLOOP-NEXT: ir-bb: -; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; IF-EVL-OUTLOOP-NEXT: No successors -; IF-EVL-OUTLOOP-EMPTY: ; IF-EVL-OUTLOOP-NEXT: scalar.ph: ; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb @@ -72,6 +68,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; IF-EVL-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; IF-EVL-OUTLOOP-NEXT: No successors +; IF-EVL-OUTLOOP-EMPTY: +; IF-EVL-OUTLOOP-NEXT: ir-bb: +; IF-EVL-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) +; IF-EVL-OUTLOOP-NEXT: No successors ; IF-EVL-OUTLOOP-NEXT: } ; @@ -109,10 +109,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: EMIT branch-on-cond ir ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; IF-EVL-INLOOP-EMPTY: -; IF-EVL-INLOOP-NEXT: ir-bb: -; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; IF-EVL-INLOOP-NEXT: No successors -; IF-EVL-INLOOP-EMPTY: ; IF-EVL-INLOOP-NEXT: scalar.ph: ; IF-EVL-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb @@ -122,6 +118,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; IF-EVL-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; IF-EVL-INLOOP-NEXT: No successors +; IF-EVL-INLOOP-EMPTY: +; IF-EVL-INLOOP-NEXT: ir-bb: +; IF-EVL-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) +; IF-EVL-INLOOP-NEXT: No successors ; IF-EVL-INLOOP-NEXT: } ; @@ -155,10 +155,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-OUTLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]> ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; NO-VP-OUTLOOP-EMPTY: -; NO-VP-OUTLOOP-NEXT: ir-bb: -; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; NO-VP-OUTLOOP-NEXT: No successors -; NO-VP-OUTLOOP-EMPTY: ; NO-VP-OUTLOOP-NEXT: scalar.ph: ; NO-VP-OUTLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb @@ -168,6 +164,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-OUTLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; NO-VP-OUTLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; NO-VP-OUTLOOP-NEXT: No successors +; NO-VP-OUTLOOP-EMPTY: +; NO-VP-OUTLOOP-NEXT: ir-bb: +; NO-VP-OUTLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) +; NO-VP-OUTLOOP-NEXT: No successors ; NO-VP-OUTLOOP-NEXT: } ; @@ -201,10 +201,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-INLOOP-NEXT: EMIT branch-on-cond vp<[[BOC]]> ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb, scalar.ph ; NO-VP-INLOOP-EMPTY: -; NO-VP-INLOOP-NEXT: ir-bb: -; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) -; NO-VP-INLOOP-NEXT: No successors -; NO-VP-INLOOP-EMPTY: ; NO-VP-INLOOP-NEXT: scalar.ph: ; NO-VP-INLOOP-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RDX]]>, ir<%start> ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb @@ -214,6 +210,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) { ; NO-VP-INLOOP-NEXT: IR %rdx = phi i32 [ %start, %entry ], [ %add, %for.body ] ; NO-VP-INLOOP: IR %exitcond.not = icmp eq i64 %iv.next, %n ; NO-VP-INLOOP-NEXT: No successors +; NO-VP-INLOOP-EMPTY: +; NO-VP-INLOOP-NEXT: ir-bb: +; NO-VP-INLOOP-NEXT: IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block) +; NO-VP-INLOOP-NEXT: No successors ; NO-VP-INLOOP-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index 517de8be5c9987..d2d2063fd90583 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -39,9 +39,6 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22> ; CHECK-NEXT: EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33> @@ -53,6 +50,9 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -115,9 +115,6 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22> ; CHECK-NEXT: EMIT vp<[[RESUME_2_P:%.*]]>.1 = resume-phi vp<[[RESUME_2]]>.1, ir<33> @@ -131,6 +128,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -195,9 +195,6 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0> @@ -208,6 +205,9 @@ define i32 @test_chained_first_order_recurrences_4(ptr %base, i64 %x) { ; CHECK-NEXT: IR %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]> from scalar.ph) ; CHECK-NEXT: IR %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1 from scalar.ph) ; CHECK: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -269,9 +269,6 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: EMIT vp<[[RESUME_X:%.+]]> = resume-phi vp<[[EXT_X]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[RESUME_Y:%.+]]>.1 = resume-phi vp<[[EXT_Y]]>.1, ir<0> @@ -282,6 +279,9 @@ define i32 @test_chained_first_order_recurrences_5_hoist_to_load(ptr %base) { ; CHECK-NEXT: IR %for.x = phi i64 [ %for.x.next, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_X]]> from scalar.ph) ; CHECK-NEXT: IR %for.y = phi i32 [ %for.x.prev, %loop ], [ 0, %entry ] (extra operand: vp<[[RESUME_Y]]>.1 from scalar.ph) ; CHECK: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index fbcd9cb95f2fc8..3d8403b1eb9c01 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -77,9 +77,6 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -89,6 +86,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -164,9 +164,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -176,6 +173,9 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -234,10 +234,6 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[RESUME_RED:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<1234> @@ -249,6 +245,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: IR %and.red = phi i32 [ 1234, %entry ], [ %and.red.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -347,9 +347,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -359,6 +356,9 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 20001 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -444,9 +444,6 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -456,6 +453,9 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %C = icmp sgt i32 %iv.next, %recur.next ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -530,9 +530,6 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_1_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; CHECK-NEXT: Successor(s): ir-bb @@ -542,6 +539,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: IR %.pn = phi i32 [ 0, %entry ], [ %l, %loop ] (extra operand: vp<[[RESUME_1_P]]> from scalar.ph) ; CHECK: IR %ec = icmp ugt i64 %iv, 3 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 0e961d1e4eefad..53a0a308c79a9c 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -112,9 +112,6 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; DBG-NEXT: Successor(s): ir-bb, scalar.ph ; DBG-EMPTY: -; DBG-NEXT: ir-bb: -; DBG-NEXT: No successors -; DBG-EMPTY: ; DBG-NEXT: scalar.ph: ; DBG-NEXT: Successor(s): ir-bb ; DBG-EMPTY: @@ -123,6 +120,9 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: IR %d = phi i1 [ false, %entry ], [ %d.next, %loop.latch ] ; DBG-NEXT: IR %d.next = xor i1 %d, true ; DBG-NEXT: No successors +; DBG-EMPTY: +; DBG-NEXT: ir-bb: +; DBG-NEXT: No successors ; DBG-NEXT: } define void @test_scalarize_with_branch_cond(ptr %src, ptr %dst) { @@ -218,9 +218,6 @@ exit: ; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; DBG-NEXT: Successor(s): ir-bb, scalar.ph ; DBG-EMPTY: -; DBG-NEXT: ir-bb: -; DBG-NEXT: No successors -; DBG-EMPTY: ; DBG-NEXT: scalar.ph: ; DBG-NEXT: EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<0> ; DBG-NEXT: Successor(s): ir-bb @@ -230,6 +227,9 @@ exit: ; DBG-NEXT: IR %for = phi i32 [ 0, %entry ], [ %iv.trunc, %loop ] (extra operand: vp<[[RESUME_P]]> from scalar.ph) ; DBG: IR %ec = icmp slt i32 %iv.next.trunc, %n ; DBG-NEXT: No successors +; DBG-EMPTY: +; DBG-NEXT: ir-bb: +; DBG-NEXT: No successors ; DBG-NEXT: } define void @first_order_recurrence_using_induction(i32 %n, ptr %dst) { diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll index 1f815899ed55c2..323a638fd63549 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -31,9 +31,6 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -41,6 +38,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next.p, %loop.latch ] ; CHECK: IR %iv.next = add i64 %iv, 1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll index 53f5a5658fb68d..a9ed4fb94a4721 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll @@ -87,9 +87,6 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -97,6 +94,9 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ] ; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll index c9612ced3eee01..5a18da42f92b84 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll @@ -42,9 +42,6 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -53,6 +50,9 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: IR %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ] ; CHECK: IR %cmp = icmp eq i64 %iv.next, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; ; CHECK: Executing best plan with VF=8, UF=2 @@ -95,9 +95,6 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -106,6 +103,9 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: IR %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ] ; CHECK: IR %cmp = icmp eq i64 %iv.next, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll index 50d406d0c04164..5e3116eae85483 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -47,9 +47,6 @@ define void @foo(i64 %n) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -59,6 +56,9 @@ define void @foo(i64 %n) { ; CHECK-NEXT: IR store i64 %outer.iv, ptr %gep.1, align 4 ; CHECK-NEXT: IR %add = add nsw i64 %outer.iv, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } entry: br label %outer.header diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 195f6a48640e54..32e55bff94b3e5 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -38,9 +38,6 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -48,6 +45,9 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -105,9 +105,6 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -115,6 +112,9 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %for.body.preheader ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -170,10 +170,6 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00> ; CHECK-NEXT: Successor(s): ir-bb @@ -182,6 +178,10 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -233,9 +233,6 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00> ; CHECK-NEXT: Successor(s): ir-bb @@ -245,6 +242,9 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: IR %red = phi float [ %red.next, %for.body ], [ 0.000000e+00, %entry ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -319,9 +319,6 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -329,6 +326,9 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: IR %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ] ; CHECK-NEXT: IR %cmp = icmp ult i64 %i, 5 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -396,9 +396,6 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -406,6 +403,9 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] ; CHECK: IR %cmp = icmp slt i64 %iv.next, 1024 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -475,10 +475,6 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RED_RESUME:%.+]]> = resume-phi vp<[[RED_RES]]>, ir<0.000000e+00> ; CHECK-NEXT: Successor(s): ir-bb @@ -488,6 +484,10 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: IR %sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %for.body ] (extra operand: vp<[[RED_EX]]> from middle.block) +; CHECK-NEXT: No successors ; CHECK-NEXT:} entry: @@ -566,9 +566,6 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -576,6 +573,9 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %if.end ] ; CHECK: IR %cmp1 = icmp slt i32 %lsd, 100 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT:} ; entry: @@ -651,9 +651,6 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -661,6 +658,9 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ] ; CHECK: IR %iv.next = add i64 %iv, %inc ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -715,10 +715,6 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]> from middle.block) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -726,6 +722,10 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %ec = icmp eq i32 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %lcssa = phi i32 [ %add, %loop ] (extra operand: vp<[[EXIT]]> from middle.block) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -779,9 +779,6 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -789,6 +786,9 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -844,9 +844,6 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -854,6 +851,9 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -929,9 +929,6 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -939,6 +936,9 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.loop ] ; CHECK: IR %ifcond = fcmp oeq float %ld.value, 5.0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -1003,9 +1003,6 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -1013,6 +1010,9 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond = icmp eq i64 %iv.next, %n ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -1110,10 +1110,6 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]> from middle.block) -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: EMIT vp<[[RESUME_P:%.*]]> = resume-phi vp<[[RESUME_1]]>, ir<22> ; CHECK-NEXT: Successor(s): ir-bb @@ -1123,6 +1119,10 @@ define i16 @print_first_order_recurrence_and_result(ptr %ptr) { ; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK: IR %exitcond.not = icmp eq i64 %iv.next, 1000 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: IR %for.1.lcssa = phi i16 [ %for.1, %loop ] (extra operand: vp<[[FOR_RESULT]]> from middle.block) +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll index cdeffeff84d03a..823b1cef93ce74 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -49,9 +49,6 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -60,6 +57,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: IR %tmp1 = phi i64 [ %tmp7, %for.inc ], [ 0, %entry ] ; CHECK: IR %tmp5 = trunc i32 %tmp4 to i8 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index 8872ad4e3897de..330a6b1715c78e 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -1073,9 +1073,6 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -1089,6 +1086,9 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: IR store i32 %l, ptr %gep.dst, align 16 ; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: @@ -1164,9 +1164,6 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, scalar.ph ; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors -; CHECK-EMPTY: ; CHECK-NEXT: scalar.ph: ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: @@ -1176,6 +1173,9 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv.next, align 1 ; CHECK-NEXT: IR %c.1 = icmp eq i8 %l, 0 ; CHECK-NEXT: No successors +; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: From 4c76bec679e53bddaad1545e69d9936d83805f6b Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 6 Dec 2024 18:07:10 +0000 Subject: [PATCH 17/23] !fixup update test --- .../RISCV/riscv-vector-reverse.ll | 28 +++++++++---------- .../LoopVectorize/vplan-predicate-switch.ll | 10 +++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index f8c2cf70f2abac..2128c63f85a3c1 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -213,19 +213,19 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR [[RESUME_1:%.+]] = phi i64 -; CHECK-NEXT: IR [[RESUME_2:%.+]] = phi i32 +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = resume-phi ir<%ind.end>, ir<%0> +; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]>.1 = resume-phi ir<%ind.end3>, ir<%n> ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %indvars.iv = phi i64 [ [[RESUME_1]], %scalar.ph ], [ %indvars.iv.next, %for.body ] -; CHECK-NEXT: IR %i.0.in8 = phi i32 [ [[RESUME_2]], %scalar.ph ], [ %i.0, %for.body ] +; CHECK-NEXT: IR %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME_1]]> from ir-bb) +; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME_2]]>.1 from ir-bb) ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: LV: Loop does not require scalar epilogue ; @@ -460,19 +460,19 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR [[RESUME_1:%.+]] = phi i64 -; CHECK-NEXT: IR [[RESUME_2:%.+]] = phi i32 +; CHECK-NEXT: EMIT vp<[[RESUME_1:%.+]]> = resume-phi ir<%ind.end>, ir<%0> +; CHECK-NEXT: EMIT vp<[[RESUME_2:%.+]]>.1 = resume-phi ir<%ind.end3>, ir<%n> ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %indvars.iv = phi i64 [ [[RESUME_1]], %scalar.ph ], [ %indvars.iv.next, %for.body ] -; CHECK-NEXT: IR %i.0.in8 = phi i32 [ [[RESUME_2]], %scalar.ph ], [ %i.0, %for.body ] +; CHECK-NEXT: IR %indvars.iv = phi i64 [ %0, %scalar.ph ], [ %indvars.iv.next, %for.body ] (extra operand: vp<[[RESUME_1]]> from ir-bb) +; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME_2]]>.1 from ir-bb) ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: LV: Loop does not require scalar epilogue ; diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll index b819e3e8e94eb7..895b2314cfe89a 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll @@ -90,17 +90,17 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %bc.resume.val = phi ptr [ %ind.end, %middle.block ], [ %start, %entry ] +; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi ir<%ind.end>, ir<%start> ; CHECK-NEXT: Successor(s): ir-bb ; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: -; CHECK-NEXT: IR %ptr.iv = phi ptr [ %bc.resume.val, %scalar.ph ], [ %ptr.iv.next, %loop.latch ] +; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %scalar.ph ], [ %ptr.iv.next, %loop.latch ] (extra operand: vp<[[RESUME]]> from ir-bb) ; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: From 7f0875860064da3c93655ac1b1eece7831782047 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 6 Dec 2024 18:07:10 +0000 Subject: [PATCH 18/23] !fixup update test --- .../Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll | 6 ------ .../test/Transforms/LoopVectorize/vplan-predicate-switch.ll | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index a1dec2f7103ee0..2128c63f85a3c1 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -226,9 +226,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME_2]]>.1 from ir-bb) ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: LV: Loop does not require scalar epilogue ; @@ -476,9 +473,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %scalar.ph ], [ %i.0, %for.body ] (extra operand: vp<[[RESUME_2]]>.1 from ir-bb) ; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: LV: Loop does not require scalar epilogue ; diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll index cddd59985f2b6e..895b2314cfe89a 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll @@ -90,6 +90,9 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[MIDDLE_CMP]]> ; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[RESUME:%.+]]> = resume-phi ir<%ind.end>, ir<%start> ; CHECK-NEXT: Successor(s): ir-bb @@ -98,9 +101,6 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) { ; CHECK-NEXT: IR %ptr.iv = phi ptr [ %start, %scalar.ph ], [ %ptr.iv.next, %loop.latch ] (extra operand: vp<[[RESUME]]> from ir-bb) ; CHECK-NEXT: IR %l = load i8, ptr %ptr.iv, align 1 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: From 16a6246416f86c2ea7ff0284af1e8d0c7a06c2d8 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 7 Dec 2024 11:51:11 +0000 Subject: [PATCH 19/23] !Fixup update after merge --- llvm/lib/Transforms/Vectorize/VPlan.h | 12 +++----- .../RISCV/vplan-vp-select-intrinsics.ll | 9 ------ .../vplan-printing-before-execute.ll | 6 ++-- .../AArch64/hoist-runtime-checks.ll | 28 +++++++++---------- .../X86/hoist-load-of-baseptr.ll | 8 +++--- .../X86/preserve-access-group.ll | 10 +++---- .../Transforms/Vectorize/VPlanHCFGTest.cpp | 3 +- .../Transforms/Vectorize/VPlanTest.cpp | 9 +++--- .../Vectorize/VPlanVerifierTest.cpp | 2 +- 9 files changed, 38 insertions(+), 49 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 8c983649043bf4..78a89374d899a0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -622,6 +622,7 @@ class VPBlockBase { void clearSuccessors() { Successors.clear(); } /// Swap successors of the block. The block must have exactly 2 successors. + // TODO: This should be part of introducing conditional branch recipes rather than being independent. void swapSuccessors() { assert(Successors.size() == 2 && "must have 2 successors to swap"); std::swap(Successors[0], Successors[1]); @@ -3766,6 +3767,7 @@ class VPlan { /// VPBasicBlock corresponding to the original preheader. Used to place /// VPExpandSCEV recipes for expressions used during skeleton creation and the /// rest of VPlan execution. + /// When this VPlan is used for the epilogue vector loop, the entry will be replaced by a new entry block created during skeleton creation. VPBasicBlock *Entry; /// VPIRBasicBlock wrapping the header of the original scalar loop. @@ -3846,14 +3848,8 @@ class VPlan { /// Create initial VPlan, having an "entry" VPBasicBlock (wrapping /// original scalar pre-header) which contains SCEV expansions that need /// to happen before the CFG is modified (when executing a VPlan for the - /// epilogue vector loop, the original entry needs to be replaced by the new - /// entry for the epilogue vector loop); a VPBasicBlock for the vector - /// pre-header, followed by a region for the vector loop, followed by the - /// middle VPBasicBlock. If a check is needed to guard executing the scalar - /// epilogue loop, it will be added to the middle block, together with - /// VPBasicBlocks for the scalar preheader and exit blocks. - /// \p InductionTy is the type of the canonical induction and used for related - /// values, like the trip count expression. + /// epilogue vector loop, the original entry needs to be replaced by a new + /// one); a VPBasicBlock for the vector pre-header, followed by a region for the vector loop, followed by the middle VPBasicBlock. If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle block, together with VPBasicBlocks for the scalar preheader and exit blocks. \p InductionTy is the type of the canonical induction and used for related values, like the trip count expression. static VPlanPtr createInitialVPlan(Type *InductionTy, PredicatedScalarEvolution &PSE, bool RequiresScalarEpilogueCheck, diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll index 296618db69c316..53c9fb0c604daa 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll @@ -40,24 +40,15 @@ ; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]> ; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]> ; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]> -<<<<<<< HEAD - ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%10>, ir<%11>, vp<[[EVL]]>) -======= ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<[[LD2]]>, ir<[[SUB]]>, vp<[[EVL]]>) ->>>>>>> origin/main ; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]> ; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]> ; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]> ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]> ; IF-EVL-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 ; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> -<<<<<<< HEAD - ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<%8> - ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, ir<%n.vec> -======= ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<[[VFUF]]> ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, ir<[[VTC]]> ->>>>>>> origin/main ; IF-EVL-NEXT: No successors ; IF-EVL-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll index 94ddea0071c347..f07d1af47af02e 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll @@ -99,6 +99,9 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: EMIT branch-on-cond vp<[[C]]> ; CHECK-NEXT: Successor(s): ir-bb, ir-bb ; CHECK-EMPTY: +; CHECK-NEXT: ir-bb: +; CHECK-NEXT: No successors +; CHECK-EMPTY: ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: EMIT vp<[[RESUME1:%.+]]> = resume-phi ir<%ind.end>, ir<%and> ; CHECK-NEXT: EMIT vp<[[RESUME2:%.+]]>.1 = resume-phi ir<%ind.end1>, ir<%A> @@ -109,9 +112,6 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: IR %p.src = phi ptr [ %A, %scalar.ph ], [ %p.src.next, %loop ] (extra operand: vp<[[RESUME2]]>.1 from ir-bb) ; CHECK: IR %cmp = icmp eq i64 %iv.next, 0 ; CHECK-NEXT: No successors -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: No successors ; CHECK-NEXT: } ; entry: diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll index 4423b89d815656..7097171ab78c5f 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll @@ -18,12 +18,8 @@ define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %c ; CHECK: for.body.preheader10: ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER13:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: for.body.preheader13: -; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[N_VEC:%.*]], [[MIDDLE_BLOCK:%.*]] ] -; CHECK-NEXT: [[SUM_07_PH:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[TMP7:%.*]], [[MIDDLE_BLOCK]] ] -; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_VEC]] = and i64 [[TMP0]], 4294967288 +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967288 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -37,12 +33,16 @@ define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %c ; CHECK-NEXT: [[TMP5]] = add <4 x i32> [[WIDE_LOAD12]], [[VEC_PHI11]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP5]], [[TMP4]] -; CHECK-NEXT: [[TMP7]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) +; CHECK-NEXT: [[TMP7:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER13]] +; CHECK: for.body.preheader13: +; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[SUM_07_PH:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.cond.cleanup: ; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[ADD:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]] @@ -133,12 +133,8 @@ define dso_local noundef i32 @sum_prefix_with_sum(ptr %s.coerce0, i64 %s.coerce1 ; CHECK: for.body.preheader8: ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER11:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: for.body.preheader11: -; CHECK-NEXT: [[I_07_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER8]] ], [ [[N_VEC:%.*]], [[SPAN_CHECKED_ACCESS_EXIT:%.*]] ] -; CHECK-NEXT: [[RET_0_LCSSA:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER8]] ], [ [[ADD:%.*]], [[SPAN_CHECKED_ACCESS_EXIT]] ] -; CHECK-NEXT: br label [[FOR_BODY1:%.*]] ; CHECK: vector.ph: -; CHECK-NEXT: [[N_VEC]] = and i64 [[N]], -8 +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -8 ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ] @@ -152,12 +148,16 @@ define dso_local noundef i32 @sum_prefix_with_sum(ptr %s.coerce0, i64 %s.coerce1 ; CHECK-NEXT: [[TMP4]] = add <4 x i32> [[WIDE_LOAD10]], [[VEC_PHI9]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP5]], label [[SPAN_CHECKED_ACCESS_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP5]], label [[SPAN_CHECKED_ACCESS_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP4]], [[TMP3]] -; CHECK-NEXT: [[ADD]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) +; CHECK-NEXT: [[ADD:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER11]] +; CHECK: for.body.preheader11: +; CHECK-NEXT: [[I_07_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER8]] ], [ [[N_VEC]], [[SPAN_CHECKED_ACCESS_EXIT]] ] +; CHECK-NEXT: [[RET_0_LCSSA:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER8]] ], [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ] +; CHECK-NEXT: br label [[FOR_BODY1:%.*]] ; CHECK: for.cond.cleanup: ; CHECK-NEXT: [[RET_0_LCSSA1:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ], [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ], [ [[ADD1:%.*]], [[FOR_BODY1]] ] ; CHECK-NEXT: ret i32 [[RET_0_LCSSA1]] diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll index 63e85731e8c700..8bc40cfc5cc8b8 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll @@ -53,9 +53,6 @@ define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8 ; O2-NEXT: br i1 [[CMP24_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4_PREHEADER:%.*]] ; O2: for.body4.preheader: ; O2-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY4_PREHEADER9:%.*]], label [[VECTOR_BODY:%.*]] -; O2: for.body4.preheader9: -; O2-NEXT: [[J_05_PH:%.*]] = phi i64 [ 0, [[FOR_BODY4_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK:%.*]] ] -; O2-NEXT: br label [[FOR_BODY4:%.*]] ; O2: vector.body: ; O2-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[FOR_BODY4_PREHEADER]] ] ; O2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[INDEX]] @@ -68,9 +65,12 @@ define dso_local void @_Z7computeRSt6vectorIiSaIiEEy(ptr noundef nonnull align 8 ; O2-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP2]], align 4, !tbaa [[TBAA0]] ; O2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; O2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; O2-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; O2-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; O2: middle.block: ; O2-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP3]], label [[FOR_BODY4_PREHEADER9]] +; O2: for.body4.preheader9: +; O2-NEXT: [[J_05_PH:%.*]] = phi i64 [ 0, [[FOR_BODY4_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] +; O2-NEXT: br label [[FOR_BODY4:%.*]] ; O2: for.cond.cleanup: ; O2-NEXT: ret void ; O2: for.cond.cleanup3: diff --git a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll index d79ffb0149ff87..1e062041b12869 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll @@ -17,11 +17,8 @@ define void @test(i32 noundef %nface, i32 noundef %ncell, ptr noalias noundef %f ; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[FACE_CELL]], i64 [[TMP0]] ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[NFACE]], 4 ; CHECK-NEXT: br i1 [[TMP1]], label %[[FOR_BODY_PREHEADER14:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[FOR_BODY_PREHEADER14]]: -; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[UNROLL_ITER:%.*]], %[[MIDDLE_BLOCK:.*]] ] -; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[UNROLL_ITER]] = and i64 [[TMP0]], 2147483644 +; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[TMP0]], 2147483644 ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] @@ -40,10 +37,13 @@ define void @test(i32 noundef %nface, i32 noundef %ncell, ptr noalias noundef %f ; CHECK-NEXT: tail call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP8]], <4 x ptr> [[TMP4]], i32 8, <4 x i1> splat (i1 true)), !tbaa [[TBAA5]], !llvm.access.group [[ACC_GRP4]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDVARS_IV_EPIL]], 4 ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[UNROLL_ITER]] -; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UNROLL_ITER]], [[TMP0]] ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY_PREHEADER14]] +; CHECK: [[FOR_BODY_PREHEADER14]]: +; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[UNROLL_ITER]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_COND_CLEANUP]]: ; CHECK-NEXT: ret void ; CHECK: [[FOR_BODY]]: diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp index 93277eed8be129..1b362d1d26bdd3 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -107,8 +107,9 @@ edge [fontname=Courier, fontsize=30] compound=true N0 [label = "ir-bb\:\l" + - "No successors\l" + "Successor(s): vector.ph\l" ] + N0 -> N1 [ label=""] N1 [label = "vector.ph:\l" + "Successor(s): vector loop\l" diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 3179cfc676ab67..bc8bcc3447ea0a 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -747,8 +747,9 @@ compound=true N0 [label = "preheader:\l" + " EMIT vp\<%1\> = add\l" + - "No successors\l" + "Successor(s): bb1\l" ] + N0 -> N1 [ label=""] N1 [label = "bb1:\l" + " EMIT vp\<%2\> = add\l" + @@ -840,7 +841,7 @@ vp<%1> = original trip-count preheader: EMIT vp<%1> = sub -No successors +Successor(s): bb1 bb1: EMIT vp<%2> = add @@ -864,7 +865,7 @@ vp<%1> = original trip-count preheader: EMIT vp<%1> = sub -No successors +Successor(s): bb1 bb1: EMIT vp<%2> = add @@ -888,7 +889,7 @@ vp<%1> = original trip-count preheader: EMIT vp<%1> = sub -No successors +Successor(s): bb1 bb1: EMIT vp<%2> = add diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index edb3f8a2952942..bc4f3943447fde 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -227,7 +227,6 @@ TEST(VPVerifierTest, BlockOutsideRegionWithParent) { VPRegionBlock *R1 = new VPRegionBlock(VPBB2, VPBB2, "R1"); VPBlockUtils::connectBlocks(VPBB1, R1); - VPBB1->setParent(R1); auto TC = std::make_unique(); LLVMContext C; @@ -235,6 +234,7 @@ TEST(VPVerifierTest, BlockOutsideRegionWithParent) { VPIRBasicBlock *ScalarHeaderVPBB = new VPIRBasicBlock(ScalarHeader); VPBlockUtils::connectBlocks(R1, ScalarHeaderVPBB); VPlan Plan(VPPH, &*TC, VPBB1, ScalarHeaderVPBB); + VPBB1->setParent(R1); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); From b7b43a88ebcc56066546637a8e36a6d593aac5eb Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 7 Dec 2024 13:42:12 +0000 Subject: [PATCH 20/23] !fixup fix formatting --- llvm/lib/Transforms/Vectorize/VPlan.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 78a89374d899a0..23b00becc83738 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -622,7 +622,8 @@ class VPBlockBase { void clearSuccessors() { Successors.clear(); } /// Swap successors of the block. The block must have exactly 2 successors. - // TODO: This should be part of introducing conditional branch recipes rather than being independent. + // TODO: This should be part of introducing conditional branch recipes rather + // than being independent. void swapSuccessors() { assert(Successors.size() == 2 && "must have 2 successors to swap"); std::swap(Successors[0], Successors[1]); @@ -3767,7 +3768,8 @@ class VPlan { /// VPBasicBlock corresponding to the original preheader. Used to place /// VPExpandSCEV recipes for expressions used during skeleton creation and the /// rest of VPlan execution. - /// When this VPlan is used for the epilogue vector loop, the entry will be replaced by a new entry block created during skeleton creation. + /// When this VPlan is used for the epilogue vector loop, the entry will be + /// replaced by a new entry block created during skeleton creation. VPBasicBlock *Entry; /// VPIRBasicBlock wrapping the header of the original scalar loop. @@ -3849,7 +3851,12 @@ class VPlan { /// original scalar pre-header) which contains SCEV expansions that need /// to happen before the CFG is modified (when executing a VPlan for the /// epilogue vector loop, the original entry needs to be replaced by a new - /// one); a VPBasicBlock for the vector pre-header, followed by a region for the vector loop, followed by the middle VPBasicBlock. If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle block, together with VPBasicBlocks for the scalar preheader and exit blocks. \p InductionTy is the type of the canonical induction and used for related values, like the trip count expression. + /// one); a VPBasicBlock for the vector pre-header, followed by a region for + /// the vector loop, followed by the middle VPBasicBlock. If a check is needed + /// to guard executing the scalar epilogue loop, it will be added to the + /// middle block, together with VPBasicBlocks for the scalar preheader and + /// exit blocks. \p InductionTy is the type of the canonical induction and + /// used for related values, like the trip count expression. static VPlanPtr createInitialVPlan(Type *InductionTy, PredicatedScalarEvolution &PSE, bool RequiresScalarEpilogueCheck, From 4a51d2900da06c08c7b85d3c49a19e7a539ea986 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 12 Dec 2024 09:35:07 +0000 Subject: [PATCH 21/23] !fixup use first successor for middle block. --- llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index ec88ab32f0443a..81ea23f6f11e07 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3891,10 +3891,10 @@ class VPlan { /// whether to execute the scalar tail loop or the exit block from the loop /// latch. const VPBasicBlock *getMiddleBlock() const { - return cast(getScalarPreheader()->getSinglePredecessor()); + return cast(getScalarPreheader()->getPredecessors()[0]); } VPBasicBlock *getMiddleBlock() { - return cast(getScalarPreheader()->getSinglePredecessor()); + return cast(getScalarPreheader()->getPredecessors()[0]); } /// Return the VPBasicBlock for the preheader of the scalar loop. From a0af5832fcaa5bd2c3bc9332f4f52a5a9f152d35 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 12 Dec 2024 09:53:22 +0000 Subject: [PATCH 22/23] !fixup update after merging #112138 --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 10 ---------- llvm/lib/Transforms/Vectorize/VPlan.h | 4 +++- .../LoopVectorize/uncountable-early-exit-vplan.ll | 6 +++--- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index cdd26329cff348..0ede3012e16593 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1182,16 +1182,6 @@ const VPRegionBlock *VPlan::getVectorLoopRegion() const { return nullptr; } -VPBasicBlock *VPlan::getScalarPreheader() const { - auto *MiddleVPBB = - cast(getVectorLoopRegion()->getSingleSuccessor()); - auto *LastSucc = MiddleVPBB->getSuccessors().back(); - // If scalar preheader is connected to VPlan, it is the last successor of - // MiddleVPBB. If this last successor is a VPIRBasicBlock, it is the Exit - // block rather than the scalar preheader. - return isa(LastSucc) ? nullptr : cast(LastSucc); -} - LLVM_DUMP_METHOD void VPlan::printDOT(raw_ostream &O) const { VPlanPrinter Printer(O, *this); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 81ea23f6f11e07..ab552cde9adc5d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3898,7 +3898,9 @@ class VPlan { } /// Return the VPBasicBlock for the preheader of the scalar loop. - VPBasicBlock *getScalarPreheader() const; + VPBasicBlock *getScalarPreheader() const { + return cast(getScalarHeader()->getSinglePredecessor()); + } /// Return the VPIRBasicBlock wrapping the header of the scalar loop. VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; } diff --git a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll index 368c766e6b3c70..730dbfe84070ab 100644 --- a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll @@ -15,7 +15,7 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() { ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %src = alloca [128 x i32], align 4 ; CHECK-NEXT: IR call void @init(ptr %src) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -99,7 +99,7 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values() { ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %src = alloca [128 x i32], align 4 ; CHECK-NEXT: IR call void @init(ptr %src) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -176,7 +176,7 @@ define i64 @multi_exiting_to_same_exit_live_in_exit_values_2() { ; CHECK-NEXT: ir-bb: ; CHECK-NEXT: IR %src = alloca [128 x i32], align 4 ; CHECK-NEXT: IR call void @init(ptr %src) -; CHECK-NEXT: No successors +; CHECK-NEXT: Successor(s): vector.ph ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop From 968598b7792bda791710b3c62e80fad209523773 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 12 Dec 2024 09:58:59 +0000 Subject: [PATCH 23/23] !fixup use front instead of [0] --- llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index ab552cde9adc5d..ae68e1fc63a139 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3891,10 +3891,10 @@ class VPlan { /// whether to execute the scalar tail loop or the exit block from the loop /// latch. const VPBasicBlock *getMiddleBlock() const { - return cast(getScalarPreheader()->getPredecessors()[0]); + return cast(getScalarPreheader()->getPredecessors().front()); } VPBasicBlock *getMiddleBlock() { - return cast(getScalarPreheader()->getPredecessors()[0]); + return cast(getScalarPreheader()->getPredecessors().front()); } /// Return the VPBasicBlock for the preheader of the scalar loop.