Skip to content

Commit

Permalink
planner, table: list partition support prune non point range (#56725)
Browse files Browse the repository at this point in the history 
close #56673
  • Loading branch information
@Defined2014
Defined2014 authored Oct 25, 2024
1 parent 5faeca4 commit 3f0514e
Show file tree
Hide file tree
Showing 8 changed files with 234 additions and 88 deletions.
20 changes: 13 additions & 7 deletions pkg/planner/core/casetest/partition/testdata/integration_partition_suite_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,21 @@
"explain format = 'brief' select * from tcollist where a in (2)",
"explain format = 'brief' select * from tcollist where a in (7)",
"explain format = 'brief' select * from tcollist where a in (0,5)",
// not in: not support
// not in
"explain format = 'brief' select * from tlist where a not in (0, 1, 2, 3, 4, 5, 6, 7, 8)",
"explain format = 'brief' select * from tcollist where a not in (0, 1, 2, 3, 4, 5, 6, 7, 8)",
// GE: not support
"explain format = 'brief' select * from tlist where a >= 7",
"explain format = 'brief' select * from tcollist where a >= 7",
// LE: not support
"explain format = 'brief' select * from tlist where a <= 7",
"explain format = 'brief' select * from tcollist where a <= 7",
// GE
"explain format = 'brief' select * from tlist where a >= 8",
"explain format = 'brief' select * from tcollist where a >= 8",
// LE
"explain format = 'brief' select * from tlist where a <= 6",
"explain format = 'brief' select * from tcollist where a <= 6",
// GT
"explain format = 'brief' select * from tlist where a > 8",
"explain format = 'brief' select * from tcollist where a > 8",
// LT
"explain format = 'brief' select * from tlist where a < 6",
"explain format = 'brief' select * from tcollist where a < 6",
// or
"explain format = 'brief' select * from tlist where a=0 or a=7",
"explain format = 'brief' select * from tlist where a=0 or a=1 or a=6 or a=11",
Expand Down
133 changes: 87 additions & 46 deletions pkg/planner/core/casetest/partition/testdata/integration_partition_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,24 +91,14 @@
{
"SQL": "explain format = 'brief' select * from tlist where a not in (0, 1, 2, 3, 4, 5, 6, 7, 8)",
"DynamicPlan": [
"TableReader 3583.33 root partition:all data:Selection",
"TableReader 3583.33 root partition:p3 data:Selection",
"└─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))",
" └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 14333.33 root ",
"├─TableReader 3583.33 root data:Selection",
"│ └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo",
"├─TableReader 3583.33 root data:Selection",
"│ └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo",
"├─TableReader 3583.33 root data:Selection",
"│ └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo",
"└─TableReader 3583.33 root data:Selection",
" └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo"
"TableReader 3583.33 root data:Selection",
"└─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo"
]
},
{
Expand All @@ -125,88 +115,139 @@
]
},
{
"SQL": "explain format = 'brief' select * from tlist where a >= 7",
"SQL": "explain format = 'brief' select * from tlist where a >= 8",
"DynamicPlan": [
"TableReader 3333.33 root partition:all data:Selection",
"└─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 7)",
"TableReader 3333.33 root partition:p2,p3 data:Selection",
"└─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 13333.33 root ",
"├─TableReader 3333.33 root data:Selection",
"│ └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 7)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo",
"├─TableReader 3333.33 root data:Selection",
"│ └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 7)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo",
"PartitionUnion 6666.67 root ",
"├─TableReader 3333.33 root data:Selection",
"│ └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 7)",
"│ └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 8)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo",
"└─TableReader 3333.33 root data:Selection",
" └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 7)",
" └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tlist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tcollist where a >= 7",
"SQL": "explain format = 'brief' select * from tcollist where a >= 8",
"DynamicPlan": [
"TableReader 3333.33 root partition:p2,p3 data:Selection",
"└─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tcollist.a, 7)",
"└─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tcollist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 6666.67 root ",
"├─TableReader 3333.33 root data:Selection",
"│ └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tcollist.a, 7)",
"│ └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tcollist.a, 8)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p2 keep order:false, stats:pseudo",
"└─TableReader 3333.33 root data:Selection",
" └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tcollist.a, 7)",
" └─Selection 3333.33 cop[tikv] ge(list_partition_pruning.tcollist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p3 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tlist where a <= 7",
"SQL": "explain format = 'brief' select * from tlist where a <= 6",
"DynamicPlan": [
"TableReader 3323.33 root partition:all data:Selection",
"└─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 7)",
"TableReader 3323.33 root partition:p0,p1,p2 data:Selection",
"└─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 13293.33 root ",
"PartitionUnion 9970.00 root ",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 7)",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 7)",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 7)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo",
"└─TableReader 3323.33 root data:Selection",
" └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 7)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo"
" └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tcollist where a <= 7",
"SQL": "explain format = 'brief' select * from tcollist where a <= 6",
"DynamicPlan": [
"TableReader 3323.33 root partition:p0,p1,p2 data:Selection",
"└─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 7)",
"└─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 9970.00 root ",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 7)",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p0 keep order:false, stats:pseudo",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 7)",
"│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo",
"└─TableReader 3323.33 root data:Selection",
" └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 7)",
" └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p2 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tlist where a > 8",
"DynamicPlan": [
"TableReader 3333.33 root partition:p3 data:Selection",
"└─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo"
],
"StaticPlan": [
"TableReader 3333.33 root data:Selection",
"└─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tcollist where a > 8",
"DynamicPlan": [
"TableReader 3333.33 root partition:p3 data:Selection",
"└─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo"
],
"StaticPlan": [
"TableReader 3333.33 root data:Selection",
"└─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.a, 8)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p3 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tlist where a < 6",
"DynamicPlan": [
"TableReader 3323.33 root partition:p0,p1 data:Selection",
"└─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 6646.67 root ",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo",
"└─TableReader 3323.33 root data:Selection",
" └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tcollist where a < 6",
"DynamicPlan": [
"TableReader 3323.33 root partition:p0,p1 data:Selection",
"└─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo"
],
"StaticPlan": [
"PartitionUnion 6646.67 root ",
"├─TableReader 3323.33 root data:Selection",
"│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)",
"│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p0 keep order:false, stats:pseudo",
"└─TableReader 3323.33 root data:Selection",
" └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)",
" └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo"
]
},
{
"SQL": "explain format = 'brief' select * from tlist where a=0 or a=7",
"DynamicPlan": [
Expand Down
4 changes: 2 additions & 2 deletions pkg/planner/core/logical_plan_trace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ func TestSingleRuleTraceStep(t *testing.T) {
assertRuleName: "partition_processor",
assertRuleSteps: []assertTraceStep{
{
assertReason: "DataSource_1 has multiple needed partitions[p1,p2] after pruning",
assertAction: "DataSource_1 becomes PartitionUnion_6 with children[TableScan_7,TableScan_8]",
assertReason: "DataSource_1 doesn't have needed partition table after pruning",
assertAction: "DataSource_1 becomes TableDual_4",
},
},
},
Expand Down
40 changes: 26 additions & 14 deletions pkg/planner/core/rule_partition_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -781,25 +781,37 @@ func (l *listPartitionPruner) findUsedListPartitions(conds []expression.Expressi
used := make(map[int]struct{}, len(ranges))
tc := l.ctx.GetSessionVars().StmtCtx.TypeCtx()
for _, r := range ranges {
if !r.IsPointNullable(tc) {
return l.fullRange, nil
}
if len(r.HighVal) != len(exprCols) {
return l.fullRange, nil
}
value, isNull, err := pruneExpr.EvalInt(l.ctx.GetExprCtx().GetEvalCtx(), chunk.MutRowFromDatums(r.HighVal).ToRow())
if err != nil {
return nil, err
}
partitionIdx := l.listPrune.LocatePartition(value, isNull)
partitionIdx = l.pi.GetOverlappingDroppingPartitionIdx(partitionIdx)
if partitionIdx == -1 {
continue
var idxs map[int]struct{}
if !r.IsPointNullable(tc) {
// Only support `pruneExpr` is a Column
if _, ok := pruneExpr.(*expression.Column); !ok {
return l.fullRange, nil
}
idxs, err = l.listPrune.LocatePartitionByRange(l.ctx.GetExprCtx().GetEvalCtx(), r)
if err != nil {
return nil, err
}
} else {
value, isNull, err := pruneExpr.EvalInt(l.ctx.GetExprCtx().GetEvalCtx(), chunk.MutRowFromDatums(r.HighVal).ToRow())
if err != nil {
return nil, err
}
idxs = make(map[int]struct{})
idxs[l.listPrune.LocatePartition(l.ctx.GetExprCtx().GetEvalCtx(), value, isNull)] = struct{}{}
}
if len(l.partitionNames) > 0 && !l.findByName(l.partitionNames, l.pi.Definitions[partitionIdx].Name.L) {
continue
for idx := range idxs {
idx = l.pi.GetOverlappingDroppingPartitionIdx(idx)
if idx == -1 {
continue
}
if len(l.partitionNames) > 0 && !l.findByName(l.partitionNames, l.pi.Definitions[idx].Name.L) {
continue
}
used[idx] = struct{}{}
}
used[partitionIdx] = struct{}{}
}
return used, nil
}
Expand Down
Loading

0 comments on commit 3f0514e

Please sign in to comment.