Skip to content

Commit

Permalink
planner: update cost formulas of Index*Join of model2 (#39296)
Browse files Browse the repository at this point in the history 
ref #35240
  • Loading branch information
@qw4990
qw4990 authored Nov 23, 2022
1 parent 65585a5 commit 7c46100
Show file tree
Hide file tree
Showing 6 changed files with 90 additions and 74 deletions.
24 changes: 12 additions & 12 deletions executor/explainfor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,12 @@ func TestPointGetUserVarPlanCache(t *testing.T) {
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( // can use idx_a
`Projection_9 1.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b`,
`└─MergeJoin_10 1.00 root inner join, left key:test.t2.a, right key:test.t1.a`,
` ├─Selection_42(Build) 10.00 root eq(test.t1.a, 1)`,
` │ └─TableReader_41 10.00 root data:TableRangeScan_40`,
` │ └─TableRangeScan_40 10.00 cop[tikv] table:t1 range:[1,1], keep order:true, stats:pseudo`,
` └─Selection_39(Probe) 0.80 root not(isnull(test.t2.a))`,
` └─Point_Get_38 1.00 root table:t2, index:idx_a(a) `))
`└─IndexJoin_17 1.00 root inner join, inner:TableReader_13, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a)`,
` ├─Selection_44(Build) 0.80 root not(isnull(test.t2.a))`,
` │ └─Point_Get_43 1.00 root table:t2, index:idx_a(a) `,
` └─TableReader_13(Probe) 0.00 root data:Selection_12`,
` └─Selection_12 0.00 cop[tikv] eq(test.t1.a, 1)`,
` └─TableRangeScan_11 0.80 cop[tikv] table:t1 range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo`))

tk.MustExec("set @a=2")
tk.MustQuery("execute stmt using @a").Check(testkit.Rows(
Expand All @@ -478,12 +478,12 @@ func TestPointGetUserVarPlanCache(t *testing.T) {
tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps})
tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Check(testkit.Rows( // can use idx_a
`Projection_9 1.00 root test.t1.a, test.t1.b, test.t2.a, test.t2.b`,
`└─MergeJoin_10 1.00 root inner join, left key:test.t2.a, right key:test.t1.a`,
` ├─Selection_42(Build) 10.00 root eq(test.t1.a, 2)`,
` │ └─TableReader_41 10.00 root data:TableRangeScan_40`,
` │ └─TableRangeScan_40 10.00 cop[tikv] table:t1 range:[2,2], keep order:true, stats:pseudo`,
` └─Selection_39(Probe) 0.80 root not(isnull(test.t2.a))`,
` └─Point_Get_38 1.00 root table:t2, index:idx_a(a) `))
`└─IndexJoin_17 1.00 root inner join, inner:TableReader_13, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a)`,
` ├─Selection_44(Build) 0.80 root not(isnull(test.t2.a))`,
` │ └─Point_Get_43 1.00 root table:t2, index:idx_a(a) `,
` └─TableReader_13(Probe) 0.00 root data:Selection_12`,
` └─Selection_12 0.00 cop[tikv] eq(test.t1.a, 2)`,
` └─TableRangeScan_11 0.80 cop[tikv] table:t1 range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo`))
tk.MustQuery("execute stmt using @a").Check(testkit.Rows(
"2 4 2 2",
))
Expand Down
44 changes: 30 additions & 14 deletions planner/core/plan_cost_ver2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -534,57 +534,73 @@ func (p *PhysicalHashJoin) getPlanCostVer2(taskType property.TaskType, option *P
return p.planCostVer2.label(p), nil
}

// getPlanCostVer2 returns the plan-cost of this sub-plan, which is:
// plan-cost = build-child-cost + build-filter-cost +
// (probe-cost + probe-filter-cost) / concurrency
// probe-cost = probe-child-cost * build-rows / batchRatio
func (p *PhysicalIndexJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
func (p *PhysicalIndexJoin) getIndexJoinCostVer2(taskType property.TaskType, option *PlanCostOption, indexJoinType int) (costVer2, error) {
if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) {
return p.planCostVer2, nil
}

build, probe := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx]
buildRows := getCardinality(build, option.CostFlag)
buildRowSize := getAvgRowSize(build.Stats(), build.Schema().Columns)
probeRowsOne := getCardinality(probe, option.CostFlag)
probeRowsTot := probeRowsOne * buildRows
probeRowSize := getAvgRowSize(probe.Stats(), probe.Schema().Columns)
buildFilters, probeFilters := p.LeftConditions, p.RightConditions
probeConcurrency := float64(p.ctx.GetSessionVars().IndexLookupJoinConcurrency())
cpuFactor := getTaskCPUFactorVer2(p, taskType)
requestFactor := getTaskRequestFactorVer2(p, taskType)
memFactor := getTaskMemFactorVer2(p, taskType)

buildFilterCost := filterCostVer2(option, buildRows, buildFilters, cpuFactor)
buildChildCost, err := build.getPlanCostVer2(taskType, option)
if err != nil {
return zeroCostVer2, err
}
buildTaskCost := newCostVer2(option, cpuFactor,
buildRows*10*cpuFactor.Value,
"cpu(%v*10*%v)", buildRows, cpuFactor)

probeFilterCost := filterCostVer2(option, probeRowsTot, probeFilters, cpuFactor)
probeChildCost, err := probe.getPlanCostVer2(taskType, option)
if err != nil {
return zeroCostVer2, err
}

var hashTableCost costVer2
switch indexJoinType {
case 1: // IndexHashJoin
hashTableCost = hashBuildCostVer2(option, buildRows, buildRowSize, cols2Exprs(p.RightJoinKeys), cpuFactor, memFactor)
case 2: // IndexMergeJoin
hashTableCost = newZeroCostVer2(traceCost(option))
default: // IndexJoin
hashTableCost = hashBuildCostVer2(option, probeRowsTot, probeRowSize, cols2Exprs(p.LeftJoinKeys), cpuFactor, memFactor)
}

// IndexJoin executes a batch of rows at a time, so the actual cost of this part should be
// `innerCostPerBatch * numberOfBatches` instead of `innerCostPerRow * numberOfOuterRow`.
// Use an empirical value batchRatio to handle this now.
// TODO: remove this empirical value.
batchRatio := 1800.0
batchRatio := 6.0
probeCost := divCostVer2(mulCostVer2(probeChildCost, buildRows), batchRatio)
numTasks := math.Max(buildRows/batchRatio, 0.001)
doubleReadCost := doubleReadCostVer2(option, numTasks, requestFactor)

p.planCostVer2 = sumCostVer2(buildChildCost, buildFilterCost, divCostVer2(sumCostVer2(probeCost, probeFilterCost, doubleReadCost), probeConcurrency))
p.planCostVer2 = sumCostVer2(buildChildCost, buildFilterCost, buildTaskCost, divCostVer2(sumCostVer2(probeCost, probeFilterCost, hashTableCost), probeConcurrency))
p.planCostInit = true
return p.planCostVer2.label(p), nil
}

// getPlanCostVer2 returns the plan-cost of this sub-plan, which is:
// plan-cost = build-child-cost + build-filter-cost +
// (probe-cost + probe-filter-cost) / concurrency
// probe-cost = probe-child-cost * build-rows / batchRatio
func (p *PhysicalIndexJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
return p.getIndexJoinCostVer2(taskType, option, 0)
}

func (p *PhysicalIndexHashJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
// TODO: distinguish IndexHashJoin with IndexJoin
return p.PhysicalIndexJoin.getPlanCostVer2(taskType, option)
return p.getIndexJoinCostVer2(taskType, option, 1)
}

func (p *PhysicalIndexMergeJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (costVer2, error) {
// TODO: distinguish IndexMergeJoin with IndexJoin
return p.PhysicalIndexJoin.getPlanCostVer2(taskType, option)
return p.getIndexJoinCostVer2(taskType, option, 2)
}

// getPlanCostVer2 returns the plan-cost of this sub-plan, which is:
Expand Down
8 changes: 4 additions & 4 deletions planner/core/testdata/integration_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1502,8 +1502,8 @@
{
"SQL": "explain format = 'brief' SELECT t1.pk FROM t1 INNER JOIN t2 ON t1.col1 = t2.pk INNER JOIN t3 ON t1.col3 = t3.pk WHERE t2.col1 IN ('a' , 'b') AND t3.keycol = 'c' AND t1.col2 = 'a' AND t1.col1 != 'abcdef' AND t1.col1 != 'aaaaaa'",
"Plan": [
"IndexJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"IndexHashJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexHashJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"│ ├─IndexLookUp(Build) 10.00 root ",
"│ │ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t3, index:keycol(keycol, pad1, pad2) range:[\"c\",\"c\"], keep order:false, stats:pseudo",
"│ │ └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
Expand All @@ -1522,8 +1522,8 @@
{
"SQL": "explain format = 'brief' SELECT t1.pk FROM t1 LEFT JOIN t2 ON t1.col1 = t2.pk LEFT JOIN t3 ON t1.col3 = t3.pk WHERE t2.col1 IN ('a' , 'b') AND t3.keycol = 'c' AND t1.col2 = 'a' AND t1.col1 != 'abcdef' AND t1.col1 != 'aaaaaa'",
"Plan": [
"IndexJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"IndexHashJoin 13.81 root inner join, inner:IndexLookUp, outer key:test.t1.col1, inner key:test.t2.pk, equal cond:eq(test.t1.col1, test.t2.pk)",
"├─IndexHashJoin(Build) 12.50 root inner join, inner:IndexLookUp, outer key:test.t3.pk, inner key:test.t1.col3, equal cond:eq(test.t3.pk, test.t1.col3)",
"│ ├─IndexLookUp(Build) 10.00 root ",
"│ │ ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t3, index:keycol(keycol, pad1, pad2) range:[\"c\",\"c\"], keep order:false, stats:pseudo",
"│ │ └─TableRowIDScan(Probe) 10.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
Expand Down
Loading

0 comments on commit 7c46100

Please sign in to comment.