diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index e8653498d32a12..9647267c217172 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7692,8 +7692,7 @@ DenseMap LoopVectorizationPlanner::executePlan( BestVPlan.execute(&State); // 2.5 Collect reduction resume values. - auto *ExitVPBB = - cast(BestVPlan.getVectorLoopRegion()->getSingleSuccessor()); + auto *ExitVPBB = BestVPlan.getMiddleBlock(); for (VPRecipeBase &R : *ExitVPBB) { createAndCollectMergePhiForReduction( dyn_cast(&R), State, OrigLoop, @@ -8778,8 +8777,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, static SetVector collectUsersInExitBlock( Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan, const MapVector &Inductions) { - auto *MiddleVPBB = - cast(Plan.getVectorLoopRegion()->getSingleSuccessor()); + auto *MiddleVPBB = Plan.getMiddleBlock(); // No edge from the middle block to the unique exit block has been inserted // and there is nothing to fix from vector loop; phis should have incoming // from scalar loop only. @@ -8825,8 +8823,7 @@ addUsersInExitBlock(VPlan &Plan, if (ExitUsersToFix.empty()) return; - auto *MiddleVPBB = - cast(Plan.getVectorLoopRegion()->getSingleSuccessor()); + auto *MiddleVPBB = Plan.getMiddleBlock(); VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi()); // Introduce extract for exiting values and update the VPIRInstructions @@ -8863,7 +8860,7 @@ static void addLiveOutsForFirstOrderRecurrences( // TODO: Should be replaced by // Plan->getScalarLoopRegion()->getSinglePredecessor() in the future once the // scalar region is modeled as well. - auto *MiddleVPBB = cast(VectorRegion->getSingleSuccessor()); + auto *MiddleVPBB = Plan.getMiddleBlock(); VPBasicBlock *ScalarPHVPBB = nullptr; if (MiddleVPBB->getNumSuccessors() == 2) { // Order is strict: first is the exit block, second is the scalar preheader. @@ -9065,8 +9062,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { bool NeedsBlends = BB != HeaderBB && !BB->phis().empty(); return Legal->blockNeedsPredication(BB) || NeedsBlends; }); - auto *MiddleVPBB = - cast(Plan->getVectorLoopRegion()->getSingleSuccessor()); + auto *MiddleVPBB = Plan->getMiddleBlock(); VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi(); for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) { // Relevant instructions from basic block BB will be grouped into VPRecipe @@ -9277,8 +9273,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( using namespace VPlanPatternMatch; VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion(); VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock(); - VPBasicBlock *MiddleVPBB = - cast(VectorLoopRegion->getSingleSuccessor()); + VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock(); for (VPRecipeBase &R : Header->phis()) { auto *PhiR = dyn_cast(&R); if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered())) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index c1b97791331bcf..deeeca57357e26 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1030,8 +1030,8 @@ void VPlan::execute(VPTransformState *State) { // skeleton creation, so we can only create the VPIRBasicBlocks now during // VPlan execution rather than earlier during VPlan construction. BasicBlock *MiddleBB = State->CFG.ExitBB; - VPBasicBlock *MiddleVPBB = - cast(getVectorLoopRegion()->getSingleSuccessor()); + VPBasicBlock *MiddleVPBB = getMiddleBlock(); + // Find the VPBB for the scalar preheader, relying on the current structure // when creating the middle block and its successrs: if there's a single // predecessor, it must be the scalar preheader. Otherwise, the second diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 59a084401cc9bf..8ea50767aa783f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3689,6 +3689,14 @@ class VPlan { VPBasicBlock *getEntry() { return Entry; } const VPBasicBlock *getEntry() const { return Entry; } + /// Methods to support access to the middle block. + const VPBasicBlock *getMiddleBlock() const { + return cast(getVectorLoopRegion()->getSingleSuccessor()); + } + VPBasicBlock *getMiddleBlock() { + return cast(getVectorLoopRegion()->getSingleSuccessor()); + } + /// The trip count of the original loop. VPValue *getTripCount() const { assert(TripCount && "trip count needs to be set before accessing it"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 41f13cc2d9a978..19c1a910b49c66 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -212,8 +212,7 @@ bool VPRecipeBase::mayHaveSideEffects() const { void VPLiveOut::fixPhi(VPlan &Plan, VPTransformState &State) { VPValue *ExitValue = getOperand(0); - VPBasicBlock *MiddleVPBB = - cast(Plan.getVectorLoopRegion()->getSingleSuccessor()); + VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock(); VPRecipeBase *ExitingRecipe = ExitValue->getDefiningRecipe(); auto *ExitingVPBB = ExitingRecipe ? ExitingRecipe->getParent() : nullptr; // Values leaving the vector loop reach live out phi's in the exiting block diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 7ea5ee341cc547..1cccd8a5bab7fe 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -248,8 +248,7 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { return false; } - VPBlockBase *MiddleBB = - IRBB->getPlan()->getVectorLoopRegion()->getSingleSuccessor(); + const VPBasicBlock *MiddleBB = IRBB->getPlan()->getMiddleBlock(); if (IRBB != IRBB->getPlan()->getPreheader() && IRBB->getSinglePredecessor() != MiddleBB) { errs() << "VPIRBasicBlock can only be used as pre-header or a successor of "