From d7fccb31232d564984c7d125243b33f9d3734a1f Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Mon, 11 Nov 2024 17:09:21 +0800 Subject: [PATCH] planner: fix list partition with non point range locates to a wrong partition (#57072) close pingcap/tidb#57062 --- .../partition/integration_partition_test.go | 10 +- .../integration_partition_suite_in.json | 7 +- .../integration_partition_suite_out.json | 164 +++++++++++++++--- pkg/planner/core/rule_partition_processor.go | 2 +- pkg/table/tables/partition.go | 18 +- 5 files changed, 172 insertions(+), 29 deletions(-) diff --git a/pkg/planner/core/casetest/partition/integration_partition_test.go b/pkg/planner/core/casetest/partition/integration_partition_test.go index 706baf88f4188..8436d1788301b 100644 --- a/pkg/planner/core/casetest/partition/integration_partition_test.go +++ b/pkg/planner/core/casetest/partition/integration_partition_test.go @@ -34,16 +34,18 @@ func TestListPartitionPruning(t *testing.T) { tk.MustExec("create database list_partition_pruning") tk.MustExec("use list_partition_pruning") tk.MustExec("drop table if exists tlist") - tk.MustExec(`create table tlist (a int) partition by list (a) ( + tk.MustExec(`create table tlist (a int, b int) partition by list (a) ( partition p0 values in (0, 1, 2), partition p1 values in (3, 4, 5), partition p2 values in (6, 7, 8), - partition p3 values in (9, 10, 11))`) - tk.MustExec(`create table tcollist (a int) partition by list columns(a) ( + partition p3 values in (9, 10, 11), + partition p4 values in (-1))`) + tk.MustExec(`create table tcollist (a int, b int) partition by list columns(a) ( partition p0 values in (0, 1, 2), partition p1 values in (3, 4, 5), partition p2 values in (6, 7, 8), - partition p3 values in (9, 10, 11))`) + partition p3 values in (9, 10, 11), + partition p4 values in (-1))`) tk.MustExec(`analyze table tlist`) tk.MustExec(`analyze table tcollist`) diff --git a/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_in.json b/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_in.json index c45821aa50e67..7fe00b8fc0a50 100644 --- a/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_in.json +++ b/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_in.json @@ -53,7 +53,12 @@ "explain format = 'brief' replace tlist select * from tlist where a in (2)", "explain format = 'brief' replace tlist select * from tlist where a in (0, 5)", "explain format = 'brief' replace tcollist select * from tcollist where a in (2)", - "explain format = 'brief' replace tcollist select * from tcollist where a in (0, 5)" + "explain format = 'brief' replace tcollist select * from tcollist where a in (0, 5)", + // for non partition column + "explain format = 'brief' select * from tlist where b > 0", + "explain format = 'brief' select * from tcollist where b > 0", + "explain format = 'brief' select * from tlist where b < 0", + "explain format = 'brief' select * from tcollist where b < 0" ] }, { diff --git a/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_out.json b/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_out.json index fbdb4ea477f96..5677735c6cd88 100644 --- a/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_out.json +++ b/pkg/planner/core/casetest/partition/testdata/integration_partition_suite_out.json @@ -91,27 +91,35 @@ { "SQL": "explain format = 'brief' select * from tlist where a not in (0, 1, 2, 3, 4, 5, 6, 7, 8)", "DynamicPlan": [ - "TableReader 3583.33 root partition:p3 data:Selection", + "TableReader 3583.33 root partition:p3,p4 data:Selection", "└─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", " └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo" ], "StaticPlan": [ - "TableReader 3583.33 root data:Selection", - "└─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", - " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo" + "PartitionUnion 7166.67 root ", + "├─TableReader 3583.33 root data:Selection", + "│ └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo", + "└─TableReader 3583.33 root data:Selection", + " └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tlist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", + " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p4 keep order:false, stats:pseudo" ] }, { "SQL": "explain format = 'brief' select * from tcollist where a not in (0, 1, 2, 3, 4, 5, 6, 7, 8)", "DynamicPlan": [ - "TableReader 3583.33 root partition:p3 data:Selection", + "TableReader 3583.33 root partition:p3,p4 data:Selection", "└─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tcollist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", " └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo" ], "StaticPlan": [ - "TableReader 3583.33 root data:Selection", - "└─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tcollist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", - " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p3 keep order:false, stats:pseudo" + "PartitionUnion 7166.67 root ", + "├─TableReader 3583.33 root data:Selection", + "│ └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tcollist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p3 keep order:false, stats:pseudo", + "└─TableReader 3583.33 root data:Selection", + " └─Selection 3583.33 cop[tikv] not(in(list_partition_pruning.tcollist.a, 0, 1, 2, 3, 4, 5, 6, 7, 8))", + " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p4 keep order:false, stats:pseudo" ] }, { @@ -151,41 +159,47 @@ { "SQL": "explain format = 'brief' select * from tlist where a <= 6", "DynamicPlan": [ - "TableReader 3323.33 root partition:p0,p1,p2 data:Selection", + "TableReader 3323.33 root partition:p0,p1,p2,p4 data:Selection", "└─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)", " └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo" ], "StaticPlan": [ - "PartitionUnion 9970.00 root ", + "PartitionUnion 13293.33 root ", "├─TableReader 3323.33 root data:Selection", "│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)", "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo", "├─TableReader 3323.33 root data:Selection", "│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)", "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo", "└─TableReader 3323.33 root data:Selection", " └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tlist.a, 6)", - " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo" + " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p4 keep order:false, stats:pseudo" ] }, { "SQL": "explain format = 'brief' select * from tcollist where a <= 6", "DynamicPlan": [ - "TableReader 3323.33 root partition:p0,p1,p2 data:Selection", + "TableReader 3323.33 root partition:p0,p1,p2,p4 data:Selection", "└─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)", " └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo" ], "StaticPlan": [ - "PartitionUnion 9970.00 root ", + "PartitionUnion 13293.33 root ", "├─TableReader 3323.33 root data:Selection", "│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)", "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p0 keep order:false, stats:pseudo", "├─TableReader 3323.33 root data:Selection", "│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)", "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p2 keep order:false, stats:pseudo", "└─TableReader 3323.33 root data:Selection", " └─Selection 3323.33 cop[tikv] le(list_partition_pruning.tcollist.a, 6)", - " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p2 keep order:false, stats:pseudo" + " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p4 keep order:false, stats:pseudo" ] }, { @@ -217,35 +231,41 @@ { "SQL": "explain format = 'brief' select * from tlist where a < 6", "DynamicPlan": [ - "TableReader 3323.33 root partition:p0,p1 data:Selection", + "TableReader 3323.33 root partition:p0,p1,p4 data:Selection", "└─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)", " └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo" ], "StaticPlan": [ - "PartitionUnion 6646.67 root ", + "PartitionUnion 9970.00 root ", "├─TableReader 3323.33 root data:Selection", "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)", "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo", "└─TableReader 3323.33 root data:Selection", " └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.a, 6)", - " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo" + " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p4 keep order:false, stats:pseudo" ] }, { "SQL": "explain format = 'brief' select * from tcollist where a < 6", "DynamicPlan": [ - "TableReader 3323.33 root partition:p0,p1 data:Selection", + "TableReader 3323.33 root partition:p0,p1,p4 data:Selection", "└─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)", " └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo" ], "StaticPlan": [ - "PartitionUnion 6646.67 root ", + "PartitionUnion 9970.00 root ", "├─TableReader 3323.33 root data:Selection", "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)", "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo", "└─TableReader 3323.33 root data:Selection", " └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.a, 6)", - " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo" + " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p4 keep order:false, stats:pseudo" ] }, { @@ -659,6 +679,110 @@ " └─Selection 20.00 cop[tikv] in(list_partition_pruning.tcollist.a, 0, 5)", " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo" ] + }, + { + "SQL": "explain format = 'brief' select * from tlist where b > 0", + "DynamicPlan": [ + "TableReader 3333.33 root partition:all data:Selection", + "└─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo" + ], + "StaticPlan": [ + "PartitionUnion 16666.67 root ", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo", + "└─TableReader 3333.33 root data:Selection", + " └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tlist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p4 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select * from tcollist where b > 0", + "DynamicPlan": [ + "TableReader 3333.33 root partition:all data:Selection", + "└─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo" + ], + "StaticPlan": [ + "PartitionUnion 16666.67 root ", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p2 keep order:false, stats:pseudo", + "├─TableReader 3333.33 root data:Selection", + "│ └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p3 keep order:false, stats:pseudo", + "└─TableReader 3333.33 root data:Selection", + " └─Selection 3333.33 cop[tikv] gt(list_partition_pruning.tcollist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p4 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select * from tlist where b < 0", + "DynamicPlan": [ + "TableReader 3323.33 root partition:all data:Selection", + "└─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tlist keep order:false, stats:pseudo" + ], + "StaticPlan": [ + "PartitionUnion 16616.67 root ", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p1 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p2 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p3 keep order:false, stats:pseudo", + "└─TableReader 3323.33 root data:Selection", + " └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tlist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tlist, partition:p4 keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain format = 'brief' select * from tcollist where b < 0", + "DynamicPlan": [ + "TableReader 3323.33 root partition:all data:Selection", + "└─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tcollist keep order:false, stats:pseudo" + ], + "StaticPlan": [ + "PartitionUnion 16616.67 root ", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p0 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p1 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p2 keep order:false, stats:pseudo", + "├─TableReader 3323.33 root data:Selection", + "│ └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.b, 0)", + "│ └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p3 keep order:false, stats:pseudo", + "└─TableReader 3323.33 root data:Selection", + " └─Selection 3323.33 cop[tikv] lt(list_partition_pruning.tcollist.b, 0)", + " └─TableFullScan 10000.00 cop[tikv] table:tcollist, partition:p4 keep order:false, stats:pseudo" + ] } ] }, diff --git a/pkg/planner/core/rule_partition_processor.go b/pkg/planner/core/rule_partition_processor.go index adf30ffbbcaf7..cd873d114633e 100644 --- a/pkg/planner/core/rule_partition_processor.go +++ b/pkg/planner/core/rule_partition_processor.go @@ -781,7 +781,7 @@ func (l *listPartitionPruner) findUsedListPartitions(conds []expression.Expressi used := make(map[int]struct{}, len(ranges)) tc := l.ctx.GetSessionVars().StmtCtx.TypeCtx() for _, r := range ranges { - if len(r.HighVal) != len(exprCols) { + if len(r.HighVal) != len(exprCols) || r.IsFullRange(false) { return l.fullRange, nil } var idxs map[int]struct{} diff --git a/pkg/table/tables/partition.go b/pkg/table/tables/partition.go index a6b05f8f5c36c..53d40983a234e 100644 --- a/pkg/table/tables/partition.go +++ b/pkg/table/tables/partition.go @@ -1024,6 +1024,7 @@ func (lp *ForListPruning) LocatePartition(ctx exprctx.EvalContext, value int64, // LocatePartitionByRange locates partition by the range // Only could process `column op value` right now. func (lp *ForListPruning) LocatePartitionByRange(ctx exprctx.EvalContext, r *ranger.Range) (idxs map[int]struct{}, err error) { + idxs = make(map[int]struct{}) lowVal, highVal := r.LowVal[0], r.HighVal[0] if r.LowVal[0].Kind() == types.KindMinNotNull { lowVal = types.GetMinValue(lp.PruneExpr.GetType(ctx)) @@ -1033,15 +1034,27 @@ func (lp *ForListPruning) LocatePartitionByRange(ctx exprctx.EvalContext, r *ran highVal = types.GetMaxValue(lp.PruneExpr.GetType(ctx)) } - highInt64, _, err := lp.PruneExpr.EvalInt(ctx, chunk.MutRowFromDatums([]types.Datum{highVal}).ToRow()) + highInt64, isNull, err := lp.PruneExpr.EvalInt(ctx, chunk.MutRowFromDatums([]types.Datum{highVal}).ToRow()) if err != nil { return nil, err } + if isNull { + return nil, errors.Errorf("Internal error, `r.HighVal` cannot be null") + } - lowInt64, _, err := lp.PruneExpr.EvalInt(ctx, chunk.MutRowFromDatums([]types.Datum{lowVal}).ToRow()) + lowInt64, isNull, err := lp.PruneExpr.EvalInt(ctx, chunk.MutRowFromDatums([]types.Datum{lowVal}).ToRow()) if err != nil { return nil, err } + if isNull { + // If low value is null, add `lp.nullPartitionIdx` into idxs map. + if !r.LowExclude && lp.nullPartitionIdx != -1 { + idxs[lp.nullPartitionIdx] = struct{}{} + } else { + dt := types.GetMinValue(lp.PruneExpr.GetType(ctx)) + lowInt64 = dt.GetInt64() + } + } var lowKey, highKey uint64 @@ -1051,7 +1064,6 @@ func (lp *ForListPruning) LocatePartitionByRange(ctx exprctx.EvalContext, r *ran lowKey, highKey = codec.EncodeIntToCmpUint(lowInt64), codec.EncodeIntToCmpUint(highInt64) } - idxs = make(map[int]struct{}) lp.valueToPartitionIdxBTree.AscendRange(&btreeListItem{key: lowKey}, &btreeListItem{key: highKey}, func(item *btreeListItem) bool { if item.key == lowKey && r.LowExclude { return true