Skip to content

Commit

Permalink
planner: support push part of limit prop to IndexMerge (#41054)
Browse files Browse the repository at this point in the history
ref #41028
  • Loading branch information
winoros authored Feb 19, 2023
1 parent 79558c0 commit 60dda69
Show file tree
Hide file tree
Showing 13 changed files with 494 additions and 154 deletions.
60 changes: 30 additions & 30 deletions cmd/explaintest/r/index_merge.result
Original file line number Diff line number Diff line change
Expand Up @@ -391,14 +391,14 @@ Delete_11 N/A root N/A
└─SelectLock_17 4056.68 root for update 0
└─HashJoin_33 4056.68 root inner join, equal:[eq(test.t1.c1, test.t1.c1)]
├─HashAgg_36(Build) 3245.34 root group by:test.t1.c1, funcs:firstrow(test.t1.c1)->test.t1.c1
│ └─IndexMerge_41 2248.30 root type: union
│ ├─IndexRangeScan_37(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_38(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_40(Probe) 2248.30 cop[tikv] not(isnull(test.t1.c1)), or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
│ └─TableRowIDScan_39 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─TableReader_44(Probe) 9990.00 root data:Selection_43
└─Selection_43 9990.00 cop[tikv] not(isnull(test.t1.c1))
└─TableFullScan_42 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
│ └─IndexMerge_45 2248.30 root type: union
│ ├─IndexRangeScan_41(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_42(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_44(Probe) 2248.30 cop[tikv] not(isnull(test.t1.c1)), or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
│ └─TableRowIDScan_43 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─TableReader_48(Probe) 9990.00 root data:Selection_47
└─Selection_47 9990.00 cop[tikv] not(isnull(test.t1.c1))
└─TableFullScan_46 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
delete from t1 where c1 in (select /*+ use_index_merge(t1) */ c1 from t1 where c1 < 10 or c2 < 10 and c3 < 10) order by 1;
select * from t1;
c1 c2 c3
Expand All @@ -409,14 +409,14 @@ Update_10 N/A root N/A
└─SelectLock_14 4056.68 root for update 0
└─HashJoin_30 4056.68 root inner join, equal:[eq(test.t1.c1, test.t1.c1)]
├─HashAgg_33(Build) 3245.34 root group by:test.t1.c1, funcs:firstrow(test.t1.c1)->test.t1.c1
│ └─IndexMerge_38 2248.30 root type: union
│ ├─IndexRangeScan_34(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_35(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_37(Probe) 2248.30 cop[tikv] not(isnull(test.t1.c1)), or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
│ └─TableRowIDScan_36 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─TableReader_41(Probe) 9990.00 root data:Selection_40
└─Selection_40 9990.00 cop[tikv] not(isnull(test.t1.c1))
└─TableFullScan_39 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
│ └─IndexMerge_42 2248.30 root type: union
│ ├─IndexRangeScan_38(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
│ ├─IndexRangeScan_39(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
│ └─Selection_41(Probe) 2248.30 cop[tikv] not(isnull(test.t1.c1)), or(lt(test.t1.c1, 10), and(lt(test.t1.c2, 10), lt(test.t1.c3, 10)))
│ └─TableRowIDScan_40 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─TableReader_45(Probe) 9990.00 root data:Selection_44
└─Selection_44 9990.00 cop[tikv] not(isnull(test.t1.c1))
└─TableFullScan_43 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
update t1 set c1 = 100, c2 = 100, c3 = 100 where c1 in (select /*+ use_index_merge(t1) */ c1 from t1 where c1 < 10 or c2 < 10 and c3 < 10);
select * from t1;
c1 c2 c3
Expand Down Expand Up @@ -471,25 +471,25 @@ insert into t1 values(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, 5);
explain select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and c3 < 10 order by 1 limit 1 offset 2;
id estRows task access object operator info
TopN_10 1.00 root test.t1.c1, offset:2, count:1
└─IndexMerge_19 1841.86 root type: union
├─IndexRangeScan_15(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_16(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_18(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
└─TableRowIDScan_17 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─IndexMerge_23 1841.86 root type: union
├─IndexRangeScan_19(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_20(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_22(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
└─TableRowIDScan_21 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ * from t1 where (c1 < 10 or c2 < 10) and c3 < 10 order by 1 limit 1 offset 2;
c1 c2 c3
3 3 3
///// GROUP BY
explain select /*+ use_index_merge(t1) */ sum(c1) from t1 where (c1 < 10 or c2 < 10) and c3 < 10 group by c1 order by 1;
id estRows task access object operator info
Sort_6 1473.49 root Column#5
└─HashAgg_11 1473.49 root group by:Column#10, funcs:sum(Column#9)->Column#5
└─Projection_18 1841.86 root cast(test.t1.c1, decimal(10,0) BINARY)->Column#9, test.t1.c1
└─IndexMerge_16 1841.86 root type: union
├─IndexRangeScan_12(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_13(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_15(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
└─TableRowIDScan_14 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─HashAgg_11 1473.49 root group by:Column#13, funcs:sum(Column#12)->Column#5
└─Projection_22 1841.86 root cast(test.t1.c1, decimal(10,0) BINARY)->Column#12, test.t1.c1
└─IndexMerge_20 1841.86 root type: union
├─IndexRangeScan_16(Build) 3323.33 cop[tikv] table:t1, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
├─IndexRangeScan_17(Build) 3323.33 cop[tikv] table:t1, index:c2(c2) range:[-inf,10), keep order:false, stats:pseudo
└─Selection_19(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
└─TableRowIDScan_18 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
select /*+ use_index_merge(t1) */ sum(c1) from t1 where (c1 < 10 or c2 < 10) and c3 < 10 group by c1 order by 1;
sum(c1)
1
Expand Down Expand Up @@ -536,8 +536,8 @@ Sort_16 1841.86 root test.t1.c1
│ └─Selection_25(Probe) 1841.86 cop[tikv] lt(test.t1.c3, 10)
│ └─TableRowIDScan_24 5542.21 cop[tikv] table:t1 keep order:false, stats:pseudo
└─TopN_29(Probe) 1841.86 root test.t2.c1, offset:2, count:1
└─HashAgg_36 4900166.23 root group by:Column#21, funcs:avg(Column#19)->Column#9, funcs:firstrow(Column#20)->test.t2.c1
└─Projection_48 6125207.79 root cast(test.t2.c1, decimal(10,0) BINARY)->Column#19, test.t2.c1, test.t2.c1
└─HashAgg_35 4900166.23 root group by:Column#24, funcs:avg(Column#22)->Column#9, funcs:firstrow(Column#23)->test.t2.c1
└─Projection_53 6125207.79 root cast(test.t2.c1, decimal(10,0) BINARY)->Column#22, test.t2.c1, test.t2.c1
└─IndexMerge_41 6125207.79 root type: union
├─Selection_38(Build) 6121.12 cop[tikv] eq(test.t1.c1, test.t2.c1)
│ └─IndexRangeScan_37 6121120.92 cop[tikv] table:t2, index:c1(c1) range:[-inf,10), keep order:false, stats:pseudo
Expand Down
27 changes: 27 additions & 0 deletions planner/core/casetest/physical_plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2191,3 +2191,30 @@ func TestHashAggPushdownToTiFlashCompute(t *testing.T) {
tk.MustQuery("explain format = 'brief' " + ts).Check(testkit.Rows(output[i].Plan...))
}
}

func TestIndexMergeOrderPushDown(t *testing.T) {
var (
input []string
output []struct {
SQL string
Plan []string
Warning []string
}
)
planSuiteData := GetPlanSuiteData()
planSuiteData.LoadTestCases(t, &input, &output)
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)

tk.MustExec("use test")
tk.MustExec("set tidb_cost_model_version=1")
tk.MustExec("create table t (a int, b int, c int, index idx(a, c), index idx2(b, c))")

for i, ts := range input {
testdata.OnRecord(func() {
output[i].SQL = ts
output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery("explain format = 'brief' " + ts).Rows())
})
tk.MustQuery("explain format = 'brief' " + ts).Check(testkit.Rows(output[i].Plan...))
}
}
4 changes: 2 additions & 2 deletions planner/core/casetest/testdata/integration_suite_out.json
Original file line number Diff line number Diff line change
Expand Up @@ -2099,7 +2099,7 @@
" └─TableRowIDScan_14(Probe) 0.00 186.61 cop[tikv] table:t keep order:false"
],
"Warnings": [
"Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copDoubleReadTask}"
"Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copMultiReadTask}"
]
},
{
Expand All @@ -2111,7 +2111,7 @@
"└─TableRowIDScan_11(Probe) 0.00 186.61 cop[tikv] table:t keep order:false"
],
"Warnings": [
"Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copDoubleReadTask}"
"Note 1105 [idx_b] remain after pruning paths for t given Prop{SortItems: [], TaskTp: copMultiReadTask}"
]
}
]
Expand Down
14 changes: 14 additions & 0 deletions planner/core/casetest/testdata/plan_suite_in.json
Original file line number Diff line number Diff line change
Expand Up @@ -1210,5 +1210,19 @@
"select f from t use index() where f not in (1,2,3) and f not in (3,4,5) -- intersection of two not in. Not done yet",
"select f from t use index() where f not in (1,2,3) and f in (1,2,3) -- intersection of in and not in. Not done yet"
]
},
{
"name": "TestIndexMergeOrderPushDown",
"cases": [
"select * from t where a = 1 or b = 1 order by c limit 2",
"select * from t where a = 1 or b in (1, 2, 3) order by c limit 2",
"select * from t where a in (1, 2, 3) or b = 1 order by c limit 2",
"select * from t where a in (1, 2, 3) or b in (1, 2, 3) order by c limit 2",
"select * from t where (a = 1 and c = 2) or (b = 1) order by c limit 2",
"select * from t where (a = 1 and c = 2) or b in (1, 2, 3) order by c limit 2",
"select * from t where (a = 1 and c = 2) or (b in (1, 2, 3) and c = 3) order by c limit 2",
"select * from t where (a = 1 or b = 2) and c = 3 order by c limit 2",
"select * from t where (a = 1 or b = 2) and c in (1, 2, 3) order by c limit 2"
]
}
]
120 changes: 117 additions & 3 deletions planner/core/casetest/testdata/plan_suite_out.json
Original file line number Diff line number Diff line change
Expand Up @@ -382,11 +382,11 @@
"└─ExchangeSender 4439.11 mpp[tiflash] ExchangeType: PassThrough",
" └─Projection 4439.11 mpp[tiflash] test.t.a, Column#5",
" └─Projection 4439.11 mpp[tiflash] Column#5, test.t.a",
" └─HashAgg 4439.11 mpp[tiflash] group by:test.t.a, test.t.c, funcs:sum(Column#13)->Column#5, funcs:firstrow(test.t.a)->test.t.a",
" └─HashAgg 4439.11 mpp[tiflash] group by:test.t.a, test.t.c, funcs:sum(Column#16)->Column#5, funcs:firstrow(test.t.a)->test.t.a",
" └─ExchangeReceiver 4439.11 mpp[tiflash] ",
" └─ExchangeSender 4439.11 mpp[tiflash] ExchangeType: HashPartition, Compression: FAST, Hash Cols: [name: test.t.a, collate: binary], [name: test.t.c, collate: binary]",
" └─HashAgg 4439.11 mpp[tiflash] group by:Column#16, Column#17, funcs:sum(Column#15)->Column#13",
" └─Projection 5548.89 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#15, test.t.a, test.t.c",
" └─HashAgg 4439.11 mpp[tiflash] group by:Column#19, Column#20, funcs:sum(Column#18)->Column#16",
" └─Projection 5548.89 mpp[tiflash] cast(test.t.b, decimal(10,0) BINARY)->Column#18, test.t.a, test.t.c",
" └─Selection 5548.89 mpp[tiflash] or(lt(test.t.b, 2), gt(test.t.a, 2))",
" └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo"
],
Expand Down Expand Up @@ -7649,5 +7649,119 @@
"Best": "TableReader(Table(t)->Sel([not(in(test.t.f, 1, 2, 3)) in(test.t.f, 1, 2, 3)]))"
}
]
},
{
"Name": "TestIndexMergeOrderPushDown",
"Cases": [
{
"SQL": "select * from t where a = 1 or b = 1 order by c limit 2",
"Plan": [
"TopN 2.00 root test.t.c, offset:0, count:2",
"└─IndexMerge 19.99 root type: union",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:true, stats:pseudo",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 19.99 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where a = 1 or b in (1, 2, 3) order by c limit 2",
"Plan": [
"TopN 2.00 root test.t.c, offset:0, count:2",
"└─IndexMerge 2.00 root type: union",
" ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 30.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], [2,2], [3,3], keep order:false, stats:pseudo",
" └─TopN(Probe) 2.00 cop[tikv] test.t.c, offset:0, count:2",
" └─TableRowIDScan 39.97 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where a in (1, 2, 3) or b = 1 order by c limit 2",
"Plan": [
"TopN 2.00 root test.t.c, offset:0, count:2",
"└─IndexMerge 2.00 root type: union",
" ├─IndexRangeScan(Build) 30.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], [2,2], [3,3], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:false, stats:pseudo",
" └─TopN(Probe) 2.00 cop[tikv] test.t.c, offset:0, count:2",
" └─TableRowIDScan 39.97 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where a in (1, 2, 3) or b in (1, 2, 3) order by c limit 2",
"Plan": [
"TopN 2.00 root test.t.c, offset:0, count:2",
"└─IndexMerge 2.00 root type: union",
" ├─IndexRangeScan(Build) 30.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], [2,2], [3,3], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 30.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], [2,2], [3,3], keep order:false, stats:pseudo",
" └─TopN(Probe) 2.00 cop[tikv] test.t.c, offset:0, count:2",
" └─TableRowIDScan 59.91 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where (a = 1 and c = 2) or (b = 1) order by c limit 2",
"Plan": [
"TopN 2.00 root test.t.c, offset:0, count:2",
"└─IndexMerge 10.10 root type: union",
" ├─Limit(Build) 0.10 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 0.10 cop[tikv] table:t, index:idx(a, c) range:[1 2,1 2], keep order:true, stats:pseudo",
" ├─Limit(Build) 2.00 cop[tikv] offset:0, count:2",
" │ └─IndexRangeScan 2.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], keep order:true, stats:pseudo",
" └─TableRowIDScan(Probe) 10.10 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where (a = 1 and c = 2) or b in (1, 2, 3) order by c limit 2",
"Plan": [
"TopN 2.00 root test.t.c, offset:0, count:2",
"└─IndexMerge 2.00 root type: union",
" ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:idx(a, c) range:[1 2,1 2], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 30.00 cop[tikv] table:t, index:idx2(b, c) range:[1,1], [2,2], [3,3], keep order:false, stats:pseudo",
" └─TopN(Probe) 2.00 cop[tikv] test.t.c, offset:0, count:2",
" └─TableRowIDScan 30.10 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where (a = 1 and c = 2) or (b in (1, 2, 3) and c = 3) order by c limit 2",
"Plan": [
"TopN 0.40 root test.t.c, offset:0, count:2",
"└─IndexMerge 0.40 root type: union",
" ├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:idx(a, c) range:[1 2,1 2], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 0.30 cop[tikv] table:t, index:idx2(b, c) range:[1 3,1 3], [2 3,2 3], [3 3,3 3], keep order:false, stats:pseudo",
" └─TableRowIDScan(Probe) 0.40 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where (a = 1 or b = 2) and c = 3 order by c limit 2",
"Plan": [
"TopN 0.02 root test.t.c, offset:0, count:2",
"└─IndexMerge 0.02 root type: union",
" ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx2(b, c) range:[2,2], keep order:false, stats:pseudo",
" └─Selection(Probe) 0.02 cop[tikv] eq(test.t.c, 3)",
" └─TableRowIDScan 19.99 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "select * from t where (a = 1 or b = 2) and c in (1, 2, 3) order by c limit 2",
"Plan": [
"TopN 0.06 root test.t.c, offset:0, count:2",
"└─IndexMerge 0.06 root type: union",
" ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx(a, c) range:[1,1], keep order:false, stats:pseudo",
" ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:idx2(b, c) range:[2,2], keep order:false, stats:pseudo",
" └─Selection(Probe) 0.06 cop[tikv] in(test.t.c, 1, 2, 3)",
" └─TableRowIDScan 19.99 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
}
]
}
]
Loading

0 comments on commit 60dda69

Please sign in to comment.