From 8208e73eafe107e3f11706a44fbd1cf671081781 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Fri, 15 Nov 2024 12:58:30 +0800 Subject: [PATCH 1/6] This is an automated cherry-pick of #57344 Signed-off-by: ti-chi-bot --- pkg/planner/core/partition_pruning_test.go | 54 ++++++++++++++++++++ pkg/planner/core/rule_partition_processor.go | 15 +++--- pkg/util/ranger/detacher.go | 14 +++-- 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/pkg/planner/core/partition_pruning_test.go b/pkg/planner/core/partition_pruning_test.go index 4268f890348bf..1f7f04092e182 100644 --- a/pkg/planner/core/partition_pruning_test.go +++ b/pkg/planner/core/partition_pruning_test.go @@ -578,6 +578,60 @@ func TestPartitionRangeColumnsForExpr(t *testing.T) { } } +func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { + tc := prepareTestCtx(t, "create table t (a varchar(255) COLLATE utf8mb4_0900_ai_ci, b varchar(255) COLLATE utf8mb4_unicode_ci)", "a,b") + lessThan := make([][]*expression.Expression, 0, 6) + partDefs := [][]string{ + {"'i'", "'i'"}, + {"MAXVALUE", "MAXVALUE"}, + } + for i := range partDefs { + l := make([]*expression.Expression, 0, 2) + for j := range []int{0, 1} { + v := partDefs[i][j] + var e *expression.Expression + if v == "MAXVALUE" { + e = nil // MAXVALUE + } else { + expr, err := expression.ParseSimpleExpr(tc.sctx, v, expression.WithInputSchemaAndNames(tc.schema, tc.names, nil)) + require.NoError(t, err) + e = &expr + } + l = append(l, e) + } + lessThan = append(lessThan, l) + } + pruner := &rangeColumnsPruner{lessThan, tc.columns[:2]} + cases := []struct { + input string + result partitionRangeOR + }{ + {"a = 'q'", partitionRangeOR{{1, 2}}}, + {"a = 'Q'", partitionRangeOR{{1, 2}}}, + {"a = 'a'", partitionRangeOR{{0, 1}}}, + {"a = 'A'", partitionRangeOR{{0, 1}}}, + {"a > 'a'", partitionRangeOR{{0, 2}}}, + {"a > 'q'", partitionRangeOR{{1, 2}}}, + {"a = 'i' and b = 'q'", partitionRangeOR{{1, 2}}}, + {"a = 'i' and b = 'Q'", partitionRangeOR{{1, 2}}}, + {"a = 'i' and b = 'a'", partitionRangeOR{{0, 1}}}, + {"a = 'i' and b = 'A'", partitionRangeOR{{0, 1}}}, + {"a = 'i' and b > 'a'", partitionRangeOR{{0, 2}}}, + {"a = 'i' and b > 'q'", partitionRangeOR{{1, 2}}}, + {"a = 'i' or a = 'h'", partitionRangeOR{{0, 2}}}, + {"a = 'h' and a = 'j'", partitionRangeOR{}}, + } + + for _, ca := range cases { + expr, err := expression.ParseSimpleExpr(tc.sctx, ca.input, expression.WithInputSchemaAndNames(tc.schema, tc.names, nil)) + require.NoError(t, err) + result := fullRange(len(lessThan)) + e := expression.SplitCNFItems(expr) + result = partitionRangeForCNFExpr(tc.sctx, e, pruner, result) + require.Truef(t, equalPartitionRangeOR(ca.result, result), "unexpected: %v %v != %v", ca.input, ca.result, result) + } +} + func benchmarkRangeColumnsPruner(b *testing.B, parts int) { tc := prepareBenchCtx("create table t (a bigint unsigned, b int, c int)", "a") if tc == nil { diff --git a/pkg/planner/core/rule_partition_processor.go b/pkg/planner/core/rule_partition_processor.go index 3da234a6486cb..9460469cf845e 100644 --- a/pkg/planner/core/rule_partition_processor.go +++ b/pkg/planner/core/rule_partition_processor.go @@ -1175,7 +1175,11 @@ func multiColumnRangeColumnsPruner(sctx sessionctx.Context, exprs []expression.E lens = append(lens, columnsPruner.partCols[i].RetType.GetFlen()) } +<<<<<<< HEAD res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, exprs, columnsPruner.partCols, lens, sctx.GetSessionVars().RangeMaxSize) +======= + res, err := ranger.DetachCondAndBuildRangeForPartition(sctx.GetRangerCtx(), exprs, columnsPruner.partCols, lens, sctx.GetSessionVars().RangeMaxSize) +>>>>>>> 3c70a289b27 (planner: fix RANGE COLUMNS partition prune gives wrong result with special collation (#57344)) if err != nil { return fullRange(len(columnsPruner.lessThan)) } @@ -1190,16 +1194,12 @@ func multiColumnRangeColumnsPruner(sctx sessionctx.Context, exprs []expression.E rangeOr := make([]partitionRange, 0, len(res.Ranges)) - comparer := make([]collate.Collator, 0, len(columnsPruner.partCols)) - for i := range columnsPruner.partCols { - comparer = append(comparer, collate.GetCollator(columnsPruner.partCols[i].RetType.GetCollate())) - } gotError := false // Create a sort.Search where the compare loops over ColumnValues // Loop over the different ranges and extend/include all the partitions found for idx := range res.Ranges { - minComparer := minCmp(sctx, res.Ranges[idx].LowVal, columnsPruner, comparer, res.Ranges[idx].LowExclude, &gotError) - maxComparer := maxCmp(sctx, res.Ranges[idx].HighVal, columnsPruner, comparer, res.Ranges[idx].HighExclude, &gotError) + minComparer := minCmp(sctx, res.Ranges[idx].LowVal, columnsPruner, res.Ranges[idx].Collators, res.Ranges[idx].LowExclude, &gotError) + maxComparer := maxCmp(sctx, res.Ranges[idx].HighVal, columnsPruner, res.Ranges[idx].Collators, res.Ranges[idx].HighExclude, &gotError) if gotError { // the compare function returned error, use all partitions. return fullRange(len(columnsPruner.lessThan)) @@ -1797,10 +1797,9 @@ func makeRangeColumnPruner(columns []*expression.Column, pi *model.PartitionInfo if len(pi.Definitions) != len(from.LessThan) { return nil, errors.Trace(fmt.Errorf("internal error len(pi.Definitions) != len(from.LessThan) %d != %d", len(pi.Definitions), len(from.LessThan))) } - schema := expression.NewSchema(columns...) partCols := make([]*expression.Column, len(offsets)) for i, offset := range offsets { - partCols[i] = schema.Columns[offset] + partCols[i] = columns[offset] } lessThan := make([][]*expression.Expression, 0, len(from.LessThan)) for i := range from.LessThan { diff --git a/pkg/util/ranger/detacher.go b/pkg/util/ranger/detacher.go index 5586aaea06fae..c5e309610f001 100644 --- a/pkg/util/ranger/detacher.go +++ b/pkg/util/ranger/detacher.go @@ -261,7 +261,7 @@ func extractBestCNFItemRanges(sctx sessionctx.Context, conds []expression.Expres // We build ranges for `(a,b) in ((1,1),(1,2))` and get `[1 1, 1 1] [1 2, 1 2]`, which are point ranges and we can // append `c = 1` to the point ranges. However, if we choose to merge consecutive ranges here, we get `[1 1, 1 2]`, // which are not point ranges, and we cannot append `c = 1` anymore. - res, err := detachCondAndBuildRangeWithoutMerging(sctx, tmpConds, cols, lengths, rangeMaxSize, convertToSortKey) + res, err := detachCondAndBuildRange(sctx, tmpConds, cols, lengths, rangeMaxSize, convertToSortKey, false) if err != nil { return nil, nil, err } @@ -418,7 +418,7 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi if eqOrInCount > 0 { newCols := d.cols[eqOrInCount:] newLengths := d.lengths[eqOrInCount:] - tailRes, err := DetachCondAndBuildRangeForIndex(d.sctx, newConditions, newCols, newLengths, d.rangeMaxSize) + tailRes, err := detachCondAndBuildRange(d.sctx, newConditions, newCols, newLengths, d.rangeMaxSize, d.convertToSortKey, d.mergeConsecutive) if err != nil { return nil, err } @@ -892,16 +892,22 @@ func DetachCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []expre return d.detachCondAndBuildRangeForCols() } +<<<<<<< HEAD // detachCondAndBuildRangeWithoutMerging detaches the index filters from table filters and uses them to build ranges. // When building ranges, it doesn't merge consecutive ranges. func detachCondAndBuildRangeWithoutMerging(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, lengths []int, rangeMaxSize int64, convertToSortKey bool) (*DetachRangeResult, error) { +======= +// detachCondAndBuildRange detaches the index filters from table filters and uses them to build ranges. +func detachCondAndBuildRange(sctx *rangerctx.RangerContext, conditions []expression.Expression, cols []*expression.Column, + lengths []int, rangeMaxSize int64, convertToSortKey bool, mergeConsecutive bool) (*DetachRangeResult, error) { +>>>>>>> 3c70a289b27 (planner: fix RANGE COLUMNS partition prune gives wrong result with special collation (#57344)) d := &rangeDetacher{ sctx: sctx, allConds: conditions, cols: cols, lengths: lengths, - mergeConsecutive: false, + mergeConsecutive: mergeConsecutive, convertToSortKey: convertToSortKey, rangeMaxSize: rangeMaxSize, } @@ -914,7 +920,7 @@ func detachCondAndBuildRangeWithoutMerging(sctx sessionctx.Context, conditions [ // The returned values are encapsulated into a struct DetachRangeResult, see its comments for explanation. func DetachCondAndBuildRangeForPartition(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, lengths []int, rangeMaxSize int64) (*DetachRangeResult, error) { - return detachCondAndBuildRangeWithoutMerging(sctx, conditions, cols, lengths, rangeMaxSize, false) + return detachCondAndBuildRange(sctx, conditions, cols, lengths, rangeMaxSize, false, false) } type rangeDetacher struct { From e532cf44fd33950b021b18e984aadf45e16c7d1e Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Tue, 3 Dec 2024 18:07:48 +0800 Subject: [PATCH 2/6] Update rule_partition_processor.go --- pkg/planner/core/rule_partition_processor.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/planner/core/rule_partition_processor.go b/pkg/planner/core/rule_partition_processor.go index 9460469cf845e..34811b32ffa3b 100644 --- a/pkg/planner/core/rule_partition_processor.go +++ b/pkg/planner/core/rule_partition_processor.go @@ -1175,11 +1175,7 @@ func multiColumnRangeColumnsPruner(sctx sessionctx.Context, exprs []expression.E lens = append(lens, columnsPruner.partCols[i].RetType.GetFlen()) } -<<<<<<< HEAD - res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, exprs, columnsPruner.partCols, lens, sctx.GetSessionVars().RangeMaxSize) -======= - res, err := ranger.DetachCondAndBuildRangeForPartition(sctx.GetRangerCtx(), exprs, columnsPruner.partCols, lens, sctx.GetSessionVars().RangeMaxSize) ->>>>>>> 3c70a289b27 (planner: fix RANGE COLUMNS partition prune gives wrong result with special collation (#57344)) + res, err := ranger.DetachCondAndBuildRangeForPartition(sctx, exprs, columnsPruner.partCols, lens, sctx.GetSessionVars().RangeMaxSize) if err != nil { return fullRange(len(columnsPruner.lessThan)) } From df8528b0b8ef9171501eff5ac3337bcab1aeaad4 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Tue, 3 Dec 2024 18:09:56 +0800 Subject: [PATCH 3/6] Update detacher.go --- pkg/util/ranger/detacher.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/util/ranger/detacher.go b/pkg/util/ranger/detacher.go index c5e309610f001..bea2d7fe43b24 100644 --- a/pkg/util/ranger/detacher.go +++ b/pkg/util/ranger/detacher.go @@ -892,16 +892,9 @@ func DetachCondAndBuildRangeForIndex(sctx sessionctx.Context, conditions []expre return d.detachCondAndBuildRangeForCols() } -<<<<<<< HEAD -// detachCondAndBuildRangeWithoutMerging detaches the index filters from table filters and uses them to build ranges. -// When building ranges, it doesn't merge consecutive ranges. -func detachCondAndBuildRangeWithoutMerging(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, - lengths []int, rangeMaxSize int64, convertToSortKey bool) (*DetachRangeResult, error) { -======= // detachCondAndBuildRange detaches the index filters from table filters and uses them to build ranges. -func detachCondAndBuildRange(sctx *rangerctx.RangerContext, conditions []expression.Expression, cols []*expression.Column, +func detachCondAndBuildRange(sctx sessionctx.Context, conditions []expression.Expression, cols []*expression.Column, lengths []int, rangeMaxSize int64, convertToSortKey bool, mergeConsecutive bool) (*DetachRangeResult, error) { ->>>>>>> 3c70a289b27 (planner: fix RANGE COLUMNS partition prune gives wrong result with special collation (#57344)) d := &rangeDetacher{ sctx: sctx, allConds: conditions, From 12ca22c5aef6266569401553711a8215011df3c2 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Tue, 3 Dec 2024 19:19:52 +0800 Subject: [PATCH 4/6] Update partition_pruning_test.go --- pkg/planner/core/partition_pruning_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/planner/core/partition_pruning_test.go b/pkg/planner/core/partition_pruning_test.go index 1f7f04092e182..b154e9a24444e 100644 --- a/pkg/planner/core/partition_pruning_test.go +++ b/pkg/planner/core/partition_pruning_test.go @@ -593,7 +593,7 @@ func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { if v == "MAXVALUE" { e = nil // MAXVALUE } else { - expr, err := expression.ParseSimpleExpr(tc.sctx, v, expression.WithInputSchemaAndNames(tc.schema, tc.names, nil)) + expr, err := expression.ParseSimpleExpr(tc.sctx, v, tc.schema, tc.names) require.NoError(t, err) e = &expr } @@ -623,7 +623,7 @@ func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { } for _, ca := range cases { - expr, err := expression.ParseSimpleExpr(tc.sctx, ca.input, expression.WithInputSchemaAndNames(tc.schema, tc.names, nil)) + expr, err := expression.ParseSimpleExpr(tc.sctx, ca.input, tc.schema, tc.names) require.NoError(t, err) result := fullRange(len(lessThan)) e := expression.SplitCNFItems(expr) From 76e35dc185ed6b6624e77017f1c93d31b87268e7 Mon Sep 17 00:00:00 2001 From: Hangjie Mo Date: Tue, 3 Dec 2024 19:20:22 +0800 Subject: [PATCH 5/6] Update partition_pruning_test.go --- pkg/planner/core/partition_pruning_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/planner/core/partition_pruning_test.go b/pkg/planner/core/partition_pruning_test.go index b154e9a24444e..8930bc6c8458a 100644 --- a/pkg/planner/core/partition_pruning_test.go +++ b/pkg/planner/core/partition_pruning_test.go @@ -593,7 +593,7 @@ func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { if v == "MAXVALUE" { e = nil // MAXVALUE } else { - expr, err := expression.ParseSimpleExpr(tc.sctx, v, tc.schema, tc.names) + expr, err := expression.ParseSimpleExprsWithNames(tc.sctx, v, tc.schema, tc.names) require.NoError(t, err) e = &expr } @@ -623,7 +623,7 @@ func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { } for _, ca := range cases { - expr, err := expression.ParseSimpleExpr(tc.sctx, ca.input, tc.schema, tc.names) + expr, err := expression.ParseSimpleExprsWithNames(tc.sctx, ca.input, tc.schema, tc.names) require.NoError(t, err) result := fullRange(len(lessThan)) e := expression.SplitCNFItems(expr) From e3e3644da4c59ac00fd256ceb06f10de29413cc8 Mon Sep 17 00:00:00 2001 From: Jason Mo Date: Tue, 3 Dec 2024 19:36:15 +0800 Subject: [PATCH 6/6] update --- pkg/planner/core/partition_pruning_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/planner/core/partition_pruning_test.go b/pkg/planner/core/partition_pruning_test.go index 8930bc6c8458a..92b2b251f0a57 100644 --- a/pkg/planner/core/partition_pruning_test.go +++ b/pkg/planner/core/partition_pruning_test.go @@ -595,7 +595,7 @@ func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { } else { expr, err := expression.ParseSimpleExprsWithNames(tc.sctx, v, tc.schema, tc.names) require.NoError(t, err) - e = &expr + e = &expr[0] } l = append(l, e) } @@ -626,7 +626,7 @@ func TestPartitionRangeColumnsForExprWithSpecialCollation(t *testing.T) { expr, err := expression.ParseSimpleExprsWithNames(tc.sctx, ca.input, tc.schema, tc.names) require.NoError(t, err) result := fullRange(len(lessThan)) - e := expression.SplitCNFItems(expr) + e := expression.SplitCNFItems(expr[0]) result = partitionRangeForCNFExpr(tc.sctx, e, pruner, result) require.Truef(t, equalPartitionRangeOR(ca.result, result), "unexpected: %v %v != %v", ca.input, ca.result, result) }