Skip to content

Commit

Permalink
planner: Columns in string pruning (#32626) (#32721)
Browse files Browse the repository at this point in the history 
close #32626
  • Loading branch information
@mjonss
mjonss authored Mar 8, 2022
1 parent c40d9c4 commit 7bf5e4e
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 12 deletions.
128 changes: 117 additions & 11 deletions planner/core/partition_pruner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,21 +125,127 @@ func TestRangeColumnPartitionPruningForIn(t *testing.T) {
" └─Selection 20.00 cop[tikv] in(test_range_col_in.t2.a, -1, 1)",
" └─TableFullScan 10000.00 cop[tikv] table:t2, partition:p1 keep order:false, stats:pseudo"))

// for other types, the in-pruning shouldn't be working for safety
tk.MustExec(`create table t3 (a varchar(10)) partition by range columns(a) (
partition p0 values less than ("aaa"),
partition p1 values less than ("bbb"),
partition p2 values less than ("ccc"))`)
tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'aab')`).Check(testkit.Rows("PartitionUnion 60.00 root ",
"├─TableReader 20.00 root data:Selection",
"│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")",
"│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p0 keep order:false, stats:pseudo",
"├─TableReader 20.00 root data:Selection",
"│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")",
"│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo",
"└─TableReader 20.00 root data:Selection",
" └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, \"aaa\", \"aab\")",
" └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p2 keep order:false, stats:pseudo"))
tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'aab')`).Check(testkit.Rows(
`TableReader 20.00 root data:Selection`,
`└─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, "aaa", "aab")`,
` └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo`))
tk.MustQuery(`explain format='brief' select a from t3 where a in ('aaa', 'bu')`).Check(testkit.Rows(
`PartitionUnion 40.00 root `,
`├─TableReader 20.00 root data:Selection`,
`│ └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, "aaa", "bu")`,
`│ └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p1 keep order:false, stats:pseudo`,
`└─TableReader 20.00 root data:Selection`,
` └─Selection 20.00 cop[tikv] in(test_range_col_in.t3.a, "aaa", "bu")`,
` └─TableFullScan 10000.00 cop[tikv] table:t3, partition:p2 keep order:false, stats:pseudo`))
}

func TestRangeColumnPartitionPruningForInString(t *testing.T) {
store, clean := testkit.CreateMockStore(t)
defer clean()
tk := testkit.NewTestKit(t, store)
tk.MustExec("drop database if exists test_range_col_in_string")
tk.MustExec("create database test_range_col_in_string")

tk.MustExec("use test_range_col_in_string")
tk.MustExec("set names utf8mb4 collate utf8mb4_bin")
tk.MustExec("set @@session.tidb_partition_prune_mode='static'")

type testStruct struct {
sql string
partitions string
rows []string
}

extractPartitions := func(res *testkit.Result) string {
planStrings := testdata.ConvertRowsToStrings(res.Rows())
partitions := []string{}
for _, s := range planStrings {
parts := getFieldValue("partition:", s)
if parts != "" {
partitions = append(partitions, strings.Split(parts, ",")...)
}
}
return strings.Join(partitions, ",")
}
checkColumnStringPruningTests := func(tests []testStruct) {
modes := []string{"dynamic", "static"}
for _, mode := range modes {
tk.MustExec(`set @@tidb_partition_prune_mode = '` + mode + `'`)
for _, test := range tests {
explainResult := tk.MustQuery("explain format = 'brief' " + test.sql)
partitions := strings.ToLower(extractPartitions(explainResult))
require.Equal(t, test.partitions, partitions, "Mode: %s sql: %s", mode, test.sql)
tk.MustQuery(test.sql).Sort().Check(testkit.Rows(test.rows...))
}
}
}
tk.MustExec("create table t (a varchar(255) charset utf8mb4 collate utf8mb4_bin) partition by range columns(a)" +
`( partition pNull values less than (""),` +
`partition pAAAA values less than ("AAAA"),` +
`partition pCCC values less than ("CCC"),` +
`partition pShrimpsandwich values less than ("Räksmörgås"),` +
`partition paaa values less than ("aaa"),` +
`partition pSushi values less than ("🍣🍣🍣"),` +
`partition pMax values less than (MAXVALUE))`)
tk.MustExec(`insert into t values (NULL), ("a"), ("Räkmacka"), ("🍣 is life"), ("🍺 after work?"), ("🍺🍺🍺🍺🍺 for oktoberfest"),("AA"),("aa"),("AAA"),("aaa")`)
tests := []testStruct{
// Lower case partition names due to issue#32719
{sql: `select * from t where a IS NULL`, partitions: "pnull", rows: []string{"<nil>"}},
{sql: `select * from t where a = 'AA'`, partitions: "paaaa", rows: []string{"AA"}},
{sql: `select * from t where a = 'AA' collate utf8mb4_general_ci`, partitions: "paaaa", rows: []string{"AA"}}, // Notice that the it not uses _bin collation for partition => 'aa' not found! #32749
{sql: `select * from t where a = 'aa'`, partitions: "paaa", rows: []string{"aa"}},
{sql: `select * from t where a = 'aa' collate utf8mb4_general_ci`, partitions: "paaaa", rows: []string{"AA"}}, // Notice that the it not uses _bin collation for partition => 'aa' not found! #32749
{sql: `select * from t where a = 'AAA'`, partitions: "paaaa", rows: []string{"AAA"}},
{sql: `select * from t where a = 'AB'`, partitions: "pccc", rows: []string{}},
{sql: `select * from t where a = 'aB'`, partitions: "paaa", rows: []string{}},
{sql: `select * from t where a = '🍣'`, partitions: "psushi", rows: []string{}},
{sql: `select * from t where a in ('🍣 is life', "Räkmacka", "🍺🍺🍺🍺 after work?")`, partitions: "pshrimpsandwich,psushi,pmax", rows: []string{"Räkmacka", "🍣 is life"}},
{sql: `select * from t where a in ('AAA', 'aa')`, partitions: "paaaa,paaa", rows: []string{"AAA", "aa"}},
{sql: `select * from t where a in ('AAA' collate utf8mb4_general_ci, 'aa')`, partitions: "paaaa,paaa", rows: []string{"AA", "AAA", "aa"}}, // aaa missing due to #32749
{sql: `select * from t where a in ('AAA', 'aa' collate utf8mb4_general_ci)`, partitions: "paaaa", rows: []string{"AA", "AAA"}}, // aa, aaa missing due to #32749
}
checkColumnStringPruningTests(tests)
tk.MustExec(`set names utf8mb4 collate utf8mb4_general_ci`)
checkColumnStringPruningTests(tests)
tk.MustExec(`set names utf8mb4 collate utf8mb4_unicode_ci`)
checkColumnStringPruningTests(tests)
tk.MustExec("drop table t")
tk.MustExec("create table t (a varchar(255) charset utf8mb4 collate utf8mb4_general_ci) partition by range columns(a)" +
`( partition pNull values less than (""),` +
`partition paaa values less than ("aaa"),` +
`partition pAAAA values less than ("AAAA"),` +
`partition pCCC values less than ("CCC"),` +
`partition pShrimpsandwich values less than ("Räksmörgås"),` +
`partition pSushi values less than ("🍣🍣🍣"),` +
`partition pMax values less than (MAXVALUE))`)
tk.MustExec(`insert into t values (NULL), ("a"), ("Räkmacka"), ("🍣 is life"), ("🍺 after work?"), ("🍺🍺🍺🍺🍺 for oktoberfest"),("AA"),("aa"),("AAA"),("aaa")`)

tests = []testStruct{
// Lower case partition names due to issue#32719
{sql: `select * from t where a IS NULL`, partitions: "pnull", rows: []string{"<nil>"}},
{sql: `select * from t where a = 'AA'`, partitions: "paaa", rows: []string{"AA", "aa"}},
{sql: `select * from t where a = 'AA' collate utf8mb4_bin`, partitions: "paaa", rows: []string{"AA"}},
{sql: `select * from t where a = 'AAA'`, partitions: "paaaa", rows: []string{"AAA", "aaa"}},
{sql: `select * from t where a = 'AAA' collate utf8mb4_bin`, partitions: "paaa", rows: []string{}}, // Notice that the it uses _bin collation for partition => not found! #32749
{sql: `select * from t where a = 'AB'`, partitions: "pccc", rows: []string{}},
{sql: `select * from t where a = 'aB'`, partitions: "pccc", rows: []string{}},
{sql: `select * from t where a = '🍣'`, partitions: "psushi", rows: []string{}},
{sql: `select * from t where a in ('🍣 is life', "Räkmacka", "🍺🍺🍺🍺 after work?")`, partitions: "pshrimpsandwich,psushi,pmax", rows: []string{"Räkmacka", "🍣 is life"}},
{sql: `select * from t where a in ('AA', 'aaa')`, partitions: "paaa,paaaa", rows: []string{"AA", "AAA", "aa", "aaa"}},
{sql: `select * from t where a in ('AAA' collate utf8mb4_bin, 'aa')`, partitions: "paaa", rows: []string{"aa"}}, // AAA missing due to #32749, why is AA missing?
{sql: `select * from t where a in ('AAA', 'aa' collate utf8mb4_bin)`, partitions: "paaaa,psushi", rows: []string{"AAA"}}, // aa, aaa missing due to #32749 also all missing paaa
}

tk.MustExec(`set names utf8mb4 collate utf8mb4_bin`)
checkColumnStringPruningTests(tests)
tk.MustExec(`set names utf8mb4 collate utf8mb4_general_ci`)
checkColumnStringPruningTests(tests)
tk.MustExec(`set names utf8mb4 collate utf8mb4_unicode_ci`)
checkColumnStringPruningTests(tests)
}

func TestListPartitionPruner(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion planner/core/rule_partition_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1051,7 +1051,7 @@ func partitionRangeColumnForInExpr(sctx sessionctx.Context, args []expression.Ex
return pruner.fullRange()
}
switch constExpr.Value.Kind() {
case types.KindInt64, types.KindUint64, types.KindMysqlTime: // for safety, only support int and datetime now
case types.KindInt64, types.KindUint64, types.KindMysqlTime, types.KindString: // for safety, only support string,int and datetime now
case types.KindNull:
result = append(result, partitionRange{0, 1})
continue
Expand Down

0 comments on commit 7bf5e4e

Please sign in to comment.