Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner: refactor tryToGetIndexJoin #45580

Closed
wants to merge 5 commits into from
Closed

planner: refactor tryToGetIndexJoin #45580

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f48e0cd
fixup
qw4990 Jul 26, 2023
e6273b9
fixup
qw4990 Jul 26, 2023
677a010
fixup
qw4990 Jul 26, 2023
4f02ffb
fixup
qw4990 Jul 26, 2023
d9d4a20
fixup
qw4990 Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
257 changes: 112 additions & 145 deletions planner/core/exhaust_physical_plans.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1957,151 +1957,6 @@ func (ijHelper *indexJoinBuildHelper) buildTemplateRange(matchedKeyCnt int, eqAn
return
}

func filterIndexJoinBySessionVars(sc sessionctx.Context, indexJoins []PhysicalPlan) []PhysicalPlan {
if sc.GetSessionVars().EnableIndexMergeJoin {
return indexJoins
}
for i := len(indexJoins) - 1; i >= 0; i-- {
if _, ok := indexJoins[i].(*PhysicalIndexMergeJoin); ok {
indexJoins = append(indexJoins[:i], indexJoins[i+1:]...)
}
}
return indexJoins
}

// tryToGetIndexJoin will get index join by hints. If we can generate a valid index join by hint, the second return value
// will be true, which means we force to choose this index join. Otherwise we will select a join algorithm with min-cost.
func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJoins []PhysicalPlan, canForced bool) {
inljRightOuter := (p.preferJoinType & preferLeftAsINLJInner) > 0
inljLeftOuter := (p.preferJoinType & preferRightAsINLJInner) > 0
hasINLJHint := inljLeftOuter || inljRightOuter

inlhjRightOuter := (p.preferJoinType & preferLeftAsINLHJInner) > 0
inlhjLeftOuter := (p.preferJoinType & preferRightAsINLHJInner) > 0
hasINLHJHint := inlhjLeftOuter || inlhjRightOuter

inlmjRightOuter := (p.preferJoinType & preferLeftAsINLMJInner) > 0
inlmjLeftOuter := (p.preferJoinType & preferRightAsINLMJInner) > 0
hasINLMJHint := inlmjLeftOuter || inlmjRightOuter

forceLeftOuter := inljLeftOuter || inlhjLeftOuter || inlmjLeftOuter
forceRightOuter := inljRightOuter || inlhjRightOuter || inlmjRightOuter
needForced := forceLeftOuter || forceRightOuter

defer func() {
// refine error message
// If the required property is not empty, we will enforce it and try the hint again.
// So we only need to generate warning message when the property is empty.
if !canForced && needForced && prop.IsSortItemEmpty() {
// Construct warning message prefix.
var errMsg string
switch {
case hasINLJHint:
errMsg = "Optimizer Hint INL_JOIN or TIDB_INLJ is inapplicable"
case hasINLHJHint:
errMsg = "Optimizer Hint INL_HASH_JOIN is inapplicable"
case hasINLMJHint:
errMsg = "Optimizer Hint INL_MERGE_JOIN is inapplicable"
}
if p.hintInfo != nil && p.preferJoinType > 0 {
t := p.hintInfo.indexNestedLoopJoinTables
switch {
case len(t.inljTables) != 0 && ((p.preferJoinType&preferLeftAsINLJInner > 0) || (p.preferJoinType&preferRightAsINLJInner > 0)):
errMsg = fmt.Sprintf("Optimizer Hint %s or %s is inapplicable",
restore2JoinHint(HintINLJ, t.inljTables), restore2JoinHint(TiDBIndexNestedLoopJoin, t.inljTables))
case len(t.inlhjTables) != 0 && ((p.preferJoinType&preferLeftAsINLHJInner > 0) || (p.preferJoinType&preferRightAsINLHJInner > 0)):
errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(HintINLHJ, t.inlhjTables))
case len(t.inlmjTables) != 0 && ((p.preferJoinType&preferLeftAsINLMJInner > 0) || (p.preferJoinType&preferRightAsINLMJInner > 0)):
errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(HintINLMJ, t.inlmjTables))
}
}

// Append inapplicable reason.
if len(p.EqualConditions) == 0 {
errMsg += " without column equal ON condition"
}

// Generate warning message to client.
warning := ErrInternal.GenWithStack(errMsg)
p.SCtx().GetSessionVars().StmtCtx.AppendWarning(warning)
}
}()

// supportLeftOuter and supportRightOuter indicates whether this type of join
// supports the left side or right side to be the outer side.
var supportLeftOuter, supportRightOuter bool
switch p.JoinType {
case SemiJoin, AntiSemiJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin, LeftOuterJoin:
supportLeftOuter = true
case RightOuterJoin:
supportRightOuter = true
case InnerJoin:
supportLeftOuter, supportRightOuter = true, true
}

var allLeftOuterJoins, allRightOuterJoins, forcedLeftOuterJoins, forcedRightOuterJoins []PhysicalPlan
if supportLeftOuter {
allLeftOuterJoins = p.getIndexJoinByOuterIdx(prop, 0)
forcedLeftOuterJoins = make([]PhysicalPlan, 0, len(allLeftOuterJoins))
for _, j := range allLeftOuterJoins {
switch j.(type) {
case *PhysicalIndexJoin:
if inljLeftOuter {
forcedLeftOuterJoins = append(forcedLeftOuterJoins, j)
}
case *PhysicalIndexHashJoin:
if inlhjLeftOuter {
forcedLeftOuterJoins = append(forcedLeftOuterJoins, j)
}
case *PhysicalIndexMergeJoin:
if inlmjLeftOuter {
forcedLeftOuterJoins = append(forcedLeftOuterJoins, j)
}
}
}
switch {
case len(forcedLeftOuterJoins) == 0 && !supportRightOuter:
return filterIndexJoinBySessionVars(p.SCtx(), allLeftOuterJoins), false
case len(forcedLeftOuterJoins) != 0 && (!supportRightOuter || (forceLeftOuter && !forceRightOuter)):
return forcedLeftOuterJoins, true
}
}
if supportRightOuter {
allRightOuterJoins = p.getIndexJoinByOuterIdx(prop, 1)
forcedRightOuterJoins = make([]PhysicalPlan, 0, len(allRightOuterJoins))
for _, j := range allRightOuterJoins {
switch j.(type) {
case *PhysicalIndexJoin:
if inljRightOuter {
forcedRightOuterJoins = append(forcedRightOuterJoins, j)
}
case *PhysicalIndexHashJoin:
if inlhjRightOuter {
forcedRightOuterJoins = append(forcedRightOuterJoins, j)
}
case *PhysicalIndexMergeJoin:
if inlmjRightOuter {
forcedRightOuterJoins = append(forcedRightOuterJoins, j)
}
}
}
switch {
case len(forcedRightOuterJoins) == 0 && !supportLeftOuter:
return filterIndexJoinBySessionVars(p.SCtx(), allRightOuterJoins), false
case len(forcedRightOuterJoins) != 0 && (!supportLeftOuter || (forceRightOuter && !forceLeftOuter)):
return forcedRightOuterJoins, true
}
}

canForceLeft := len(forcedLeftOuterJoins) != 0 && forceLeftOuter
canForceRight := len(forcedRightOuterJoins) != 0 && forceRightOuter
canForced = canForceLeft || canForceRight
if canForced {
return append(forcedLeftOuterJoins, forcedRightOuterJoins...), true
}
return filterIndexJoinBySessionVars(p.SCtx(), append(allLeftOuterJoins, allRightOuterJoins...)), false
}

func checkChildFitBC(p Plan) bool {
if p.StatsInfo().HistColl == nil {
return p.SCtx().GetSessionVars().BroadcastJoinThresholdCount == -1 || p.StatsInfo().Count() < p.SCtx().GetSessionVars().BroadcastJoinThresholdCount
Expand Down Expand Up @@ -2226,6 +2081,118 @@ func (p *LogicalJoin) preferMppBCJ() bool {
return checkChildFitBC(p.children[0]) || checkChildFitBC(p.children[1])
}

func (p *LogicalJoin) filterIndexJoinByVars(candidates []PhysicalPlan) []PhysicalPlan {
if !p.SCtx().GetSessionVars().EnableIndexMergeJoin {
result := make([]PhysicalPlan, 0, len(candidates))
for _, candidate := range candidates {
switch candidate.(type) {
case *PhysicalIndexMergeJoin:
default:
result = append(result, candidate)
}
}
candidates = result
}
return candidates
}

func (p *LogicalJoin) filterIndexJoinByHints(prop *property.PhysicalProperty, leftAsInner, rightAsInner []PhysicalPlan) ([]PhysicalPlan, bool) {
candidates := make([]PhysicalPlan, 0, len(leftAsInner)+len(rightAsInner))
candidates = append(candidates, leftAsInner...)
candidates = append(candidates, rightAsInner...)

const left, right = 0, 1
const indexJoin, indexMergeJoin, indexHashJoin = 0, 1, 2
preferJoins := make([]PhysicalPlan, 0, len(candidates))
for i, candidate := range candidates {
innerSide := left
if i >= len(leftAsInner) {
innerSide = right
}
joinMethod := indexJoin
switch candidate.(type) {
case *PhysicalIndexMergeJoin:
joinMethod = indexMergeJoin
case *PhysicalIndexHashJoin:
joinMethod = indexHashJoin
}

if (p.preferJoin(preferLeftAsINLJInner) && !(innerSide == left && joinMethod == indexJoin)) ||
(p.preferJoin(preferRightAsINLJInner) && !(innerSide == right && joinMethod == indexJoin)) ||
(p.preferJoin(preferLeftAsINLHJInner) && !(innerSide == left && joinMethod == indexHashJoin)) ||
(p.preferJoin(preferRightAsINLHJInner) && !(innerSide == right && joinMethod == indexHashJoin)) ||
(p.preferJoin(preferLeftAsINLMJInner) && !(innerSide == left && joinMethod == indexMergeJoin)) ||
(p.preferJoin(preferRightAsINLMJInner) && !(innerSide == right && joinMethod == indexMergeJoin)) {
continue
}

preferJoins = append(preferJoins, candidate)
}

if p.preferJoin(preferLeftAsINLJInner, preferRightAsINLJInner, preferLeftAsINLHJInner,
preferRightAsINLHJInner, preferLeftAsINLMJInner, preferRightAsINLMJInner) {
if len(preferJoins) > 0 {
return preferJoins, true
}
// Cannot find any index join under the current hints, generate some warnings.
// If the required property is not empty, we will enforce it and try the hint again.
// So we only need to generate warning messages when the property is empty.
if prop.IsSortItemEmpty() {
t := p.hintInfo.indexNestedLoopJoinTables
var errMsg string
if p.preferJoin(preferLeftAsINLJInner, preferRightAsINLJInner) {
errMsg = fmt.Sprintf("Optimizer Hint %s or %s is inapplicable", restore2JoinHint(HintINLJ, t.inljTables), restore2JoinHint(TiDBIndexNestedLoopJoin, t.inljTables))
} else if p.preferJoin(preferLeftAsINLHJInner, preferRightAsINLHJInner) {
errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(HintINLHJ, t.inlhjTables))
} else if p.preferJoin(preferLeftAsINLMJInner, preferRightAsINLMJInner) {
errMsg = fmt.Sprintf("Optimizer Hint %s is inapplicable", restore2JoinHint(HintINLMJ, t.inlmjTables))
}
if len(p.EqualConditions) == 0 {
errMsg += " without column equal ON condition"
}
warning := ErrInternal.GenWithStack(errMsg)
p.SCtx().GetSessionVars().StmtCtx.AppendWarning(warning)
}
}
return candidates, false
}

func (p *LogicalJoin) preferJoin(flags ...uint) bool {
for _, flag := range flags {
if (p.preferJoinType & flag) > 0 {
return true
}
}
return false
}

// tryToGetIndexJoin will get index join by hints. If we can generate a valid index join by hint, the second return value
// will be true, which means we force to choose this index join. Otherwise we will select a join algorithm with min-cost.
func (p *LogicalJoin) tryToGetIndexJoin(prop *property.PhysicalProperty) (indexJoins []PhysicalPlan, canForced bool) {
// supportLeftOuter and supportRightOuter indicates whether this type of join
// supports the left side or right side to be the outer side.
var supportLeftOuter, supportRightOuter bool
switch p.JoinType {
case SemiJoin, AntiSemiJoin, LeftOuterSemiJoin, AntiLeftOuterSemiJoin, LeftOuterJoin:
supportLeftOuter = true
case RightOuterJoin:
supportRightOuter = true
case InnerJoin:
supportLeftOuter, supportRightOuter = true, true
}
var leftAsInner, rightAsInner []PhysicalPlan
if supportLeftOuter {
rightAsInner = p.getIndexJoinByOuterIdx(prop, 0)
}
if supportRightOuter {
leftAsInner = p.getIndexJoinByOuterIdx(prop, 1)
}

leftAsInner = p.filterIndexJoinByVars(leftAsInner)
rightAsInner = p.filterIndexJoinByVars(rightAsInner)
return p.filterIndexJoinByHints(prop, leftAsInner, rightAsInner)
}

// LogicalJoin can generates hash join, index join and sort merge join.
// Firstly we check the hint, if hint is figured by user, we force to choose the corresponding physical plan.
// If the hint is not matched, it will get other candidates.
Expand Down
3 changes: 3 additions & 0 deletions planner/core/planbuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,9 @@ func restore2TableHint(hintTables ...hintTableInfo) string {
}

func restore2JoinHint(hintType string, hintTables []hintTableInfo) string {
if len(hintTables) == 0 {
return strings.ToUpper(hintType)
}
buffer := bytes.NewBufferString("/*+ ")
buffer.WriteString(strings.ToUpper(hintType))
buffer.WriteString("(")
Expand Down