[LoopUnroll] Add UnrollMultiExit to loop unrolling options (NFC) #124462
+20
−10
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-llvm-analysis Author: Florian Hahn (fhahn) ChangesAdd an extra knob to UnrollingPreferences to let backends control whether to allow multi-exit unrolling on a per-loop basis. This gives backends more fine-grained control on deciding if multi-exit unrolling is profitable for a given loop and uarch. Similar to 4226e0a. Full diff: https://github.com/llvm/llvm-project/pull/124462.diff 6 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 71b204f9c3fec7..c8a34bdc801085 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -622,6 +622,9 @@ class TargetTransformInfo {
/// Don't allow runtime unrolling if expanding the trip count takes more
/// than SCEVExpansionBudget.
unsigned SCEVExpansionBudget;
+ /// Allow runtime unrolling multi-exit loops. Should only be set if the
+ /// target determined that multi-exit unrolling is profitable for the loop.
+ bool UnrollMultiExit;
};
/// Get target-customized preferences for the generic loop unrolling
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 8cf17ced458c82..3f9c14880ad471 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -76,6 +76,7 @@ struct UnrollLoopOptions {
bool ForgetAllSCEV;
const Instruction *Heart = nullptr;
unsigned SCEVExpansionBudget;
+ bool UnrollMultiExit = false;
};
LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -91,7 +92,8 @@ bool UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, Loop **ResultLoop = nullptr);
+ unsigned SCEVExpansionBudget, bool UnrollMultiExit,
+ Loop **ResultLoop = nullptr);
LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 090348809e5719..84e4630640134e 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -220,6 +220,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.UnrollAndJamInnerLoopThreshold = 60;
UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
+ UP.UnrollMultiExit = false;
// Override with any target specific settings
TTI.getUnrollingPreferences(L, SE, UP, &ORE);
@@ -1352,6 +1353,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ULO.ForgetAllSCEV = ForgetAllSCEV;
ULO.Heart = getLoopConvergenceHeart(L);
ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
+ ULO.UnrollMultiExit = UP.UnrollMultiExit;
LoopUnrollResult UnrollResult = UnrollLoop(
L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
if (UnrollResult == LoopUnrollResult::Unmodified)
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index b11d92836a998f..623031f532ab32 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -590,10 +590,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
: isEpilogProfitable(L);
if (ULO.Runtime &&
- !UnrollRuntimeLoopRemainder(
- L, ULO.Count, ULO.AllowExpensiveTripCount, EpilogProfitability,
- ULO.UnrollRemainder, ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
- PreserveLCSSA, ULO.SCEVExpansionBudget, RemainderLoop)) {
+ !UnrollRuntimeLoopRemainder(L, ULO.Count, ULO.AllowExpensiveTripCount,
+ EpilogProfitability, ULO.UnrollRemainder,
+ ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
+ PreserveLCSSA, ULO.SCEVExpansionBudget,
+ ULO.UnrollMultiExit, RemainderLoop)) {
if (ULO.Force)
ULO.Runtime = false;
else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index b0bc55cd64c377..63210c75d697a7 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -583,7 +583,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, Loop **ResultLoop) {
+ unsigned SCEVExpansionBudget, bool UnrollMultiExit, Loop **ResultLoop) {
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
LLVM_DEBUG(L->dump());
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -632,8 +632,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
if (!PreserveLCSSA)
return false;
- if (!canProfitablyUnrollMultiExitLoop(L, OtherExits, LatchExit,
- UseEpilogRemainder)) {
+ if (!UnrollMultiExit && !canProfitablyUnrollMultiExitLoop(
+ L, OtherExits, LatchExit, UseEpilogRemainder)) {
LLVM_DEBUG(
dbgs()
<< "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
diff --git a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
index b49e37d9eee984..fb02c89c77a10e 100644
--- a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -73,6 +73,6 @@ while.end: ; preds = %while.cond
bool ret =
UnrollRuntimeLoopRemainder(L, 4, true, false, false, false, &LI, &SE, &DT,
- &AC, /*TTI=*/nullptr, PreserveLCSSA, 4);
+ &AC, /*TTI=*/nullptr, PreserveLCSSA, 4, false);
EXPECT_FALSE(ret);
}
|
|
@llvm/pr-subscribers-llvm-transforms Author: Florian Hahn (fhahn) ChangesAdd an extra knob to UnrollingPreferences to let backends control whether to allow multi-exit unrolling on a per-loop basis. This gives backends more fine-grained control on deciding if multi-exit unrolling is profitable for a given loop and uarch. Similar to 4226e0a. Full diff: https://github.com/llvm/llvm-project/pull/124462.diff 6 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 71b204f9c3fec7..c8a34bdc801085 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -622,6 +622,9 @@ class TargetTransformInfo {
/// Don't allow runtime unrolling if expanding the trip count takes more
/// than SCEVExpansionBudget.
unsigned SCEVExpansionBudget;
+ /// Allow runtime unrolling multi-exit loops. Should only be set if the
+ /// target determined that multi-exit unrolling is profitable for the loop.
+ bool UnrollMultiExit;
};
/// Get target-customized preferences for the generic loop unrolling
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 8cf17ced458c82..3f9c14880ad471 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -76,6 +76,7 @@ struct UnrollLoopOptions {
bool ForgetAllSCEV;
const Instruction *Heart = nullptr;
unsigned SCEVExpansionBudget;
+ bool UnrollMultiExit = false;
};
LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
@@ -91,7 +92,8 @@ bool UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, Loop **ResultLoop = nullptr);
+ unsigned SCEVExpansionBudget, bool UnrollMultiExit,
+ Loop **ResultLoop = nullptr);
LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 090348809e5719..84e4630640134e 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -220,6 +220,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.UnrollAndJamInnerLoopThreshold = 60;
UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
+ UP.UnrollMultiExit = false;
// Override with any target specific settings
TTI.getUnrollingPreferences(L, SE, UP, &ORE);
@@ -1352,6 +1353,7 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ULO.ForgetAllSCEV = ForgetAllSCEV;
ULO.Heart = getLoopConvergenceHeart(L);
ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
+ ULO.UnrollMultiExit = UP.UnrollMultiExit;
LoopUnrollResult UnrollResult = UnrollLoop(
L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
if (UnrollResult == LoopUnrollResult::Unmodified)
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index b11d92836a998f..623031f532ab32 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -590,10 +590,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
: isEpilogProfitable(L);
if (ULO.Runtime &&
- !UnrollRuntimeLoopRemainder(
- L, ULO.Count, ULO.AllowExpensiveTripCount, EpilogProfitability,
- ULO.UnrollRemainder, ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
- PreserveLCSSA, ULO.SCEVExpansionBudget, RemainderLoop)) {
+ !UnrollRuntimeLoopRemainder(L, ULO.Count, ULO.AllowExpensiveTripCount,
+ EpilogProfitability, ULO.UnrollRemainder,
+ ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
+ PreserveLCSSA, ULO.SCEVExpansionBudget,
+ ULO.UnrollMultiExit, RemainderLoop)) {
if (ULO.Force)
ULO.Runtime = false;
else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index b0bc55cd64c377..63210c75d697a7 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -583,7 +583,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
bool UseEpilogRemainder, bool UnrollRemainder, bool ForgetAllSCEV,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
const TargetTransformInfo *TTI, bool PreserveLCSSA,
- unsigned SCEVExpansionBudget, Loop **ResultLoop) {
+ unsigned SCEVExpansionBudget, bool UnrollMultiExit, Loop **ResultLoop) {
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
LLVM_DEBUG(L->dump());
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -632,8 +632,8 @@ bool llvm::UnrollRuntimeLoopRemainder(
if (!PreserveLCSSA)
return false;
- if (!canProfitablyUnrollMultiExitLoop(L, OtherExits, LatchExit,
- UseEpilogRemainder)) {
+ if (!UnrollMultiExit && !canProfitablyUnrollMultiExitLoop(
+ L, OtherExits, LatchExit, UseEpilogRemainder)) {
LLVM_DEBUG(
dbgs()
<< "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
diff --git a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
index b49e37d9eee984..fb02c89c77a10e 100644
--- a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -73,6 +73,6 @@ while.end: ; preds = %while.cond
bool ret =
UnrollRuntimeLoopRemainder(L, 4, true, false, false, false, &LI, &SE, &DT,
- &AC, /*TTI=*/nullptr, PreserveLCSSA, 4);
+ &AC, /*TTI=*/nullptr, PreserveLCSSA, 4, false);
EXPECT_FALSE(ret);
}
|
nikic
reviewed
Jan 26, 2025
| @@ -622,6 +622,9 @@ class TargetTransformInfo { | |||
| /// Don't allow runtime unrolling if expanding the trip count takes more | |||
| /// than SCEVExpansionBudget. | |||
| unsigned SCEVExpansionBudget; | |||
| /// Allow runtime unrolling multi-exit loops. Should only be set if the | |||
| /// target determined that multi-exit unrolling is profitable for the loop. | |||
| bool UnrollMultiExit; | |||
There was a problem hiding this comment.
Suggested change
| bool UnrollMultiExit; | |
| bool RuntimeUnrollMultiExit; |
To clarify that this affects only runtime unrolling?
Add an extra knob to UnrollingPreferences to let backends control whether to allow multi-exit unrolling on a per-loop basis. This gives backends more fine-grained control on deciding if multi-exit unrolling is profitable for a given loop and uarch. Similar to 4226e0a.
fhahn
force-pushed
the
tti-unroll-multi-exit
branch
from
5c7b461 to
e2ae7f8
Compare
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Add an extra knob to UnrollingPreferences to let backends control whether to allow multi-exit unrolling on a per-loop basis.
This gives backends more fine-grained control on deciding if multi-exit unrolling is profitable for a given loop and uarch. Similar to 4226e0a.