From ff61407df4631a287951b8637c36a76216861b74 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 2 Jun 2021 16:14:32 +0800 Subject: [PATCH 1/8] fixup --- executor/partition_table_test.go | 36 ++++++++++++++++++++++++++++++++ planner/core/optimizer.go | 30 ++++++++++++++++++++++++++ planner/core/task.go | 5 +++++ sessionctx/variable/session.go | 2 ++ 4 files changed, 73 insertions(+) diff --git a/executor/partition_table_test.go b/executor/partition_table_test.go index d8ae8abcd9476..6f354f399cff8 100644 --- a/executor/partition_table_test.go +++ b/executor/partition_table_test.go @@ -2254,6 +2254,42 @@ func (s *partitionTableSuite) TestDirectReadingWithAgg(c *C) { } } +func (s *partitionTableSuite) TestIssue24636(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("create database test_issue_24636") + tk.MustExec("use test_issue_24636") + + tk.MustExec(`CREATE TABLE t (a int, b date, c int, PRIMARY KEY (a,b)) + PARTITION BY RANGE ( TO_DAYS(b) ) ( + PARTITION p0 VALUES LESS THAN (737821), + PARTITION p1 VALUES LESS THAN (738289) + )`) + tk.MustExec(`INSERT INTO t (a, b, c) VALUES(0, '2021-05-05', 0)`) + tk.MustQuery(`select c from t use index(primary) where a=0 limit 1`).Check(testkit.Rows("0")) + + tk.MustExec(` + CREATE TABLE test_partition ( + a varchar(100) NOT NULL, + b date NOT NULL, + c varchar(100) NOT NULL, + d datetime DEFAULT NULL, + e datetime DEFAULT NULL, + f bigint(20) DEFAULT NULL, + g bigint(20) DEFAULT NULL, + h bigint(20) DEFAULT NULL, + i bigint(20) DEFAULT NULL, + j bigint(20) DEFAULT NULL, + k bigint(20) DEFAULT NULL, + l bigint(20) DEFAULT NULL, + PRIMARY KEY (a,b,c) /*T![clustered_index] NONCLUSTERED */ + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin + PARTITION BY RANGE ( TO_DAYS(b) ) ( + PARTITION pmin VALUES LESS THAN (737821), + PARTITION p20200601 VALUES LESS THAN (738289))`) + tk.MustExec(`INSERT INTO test_partition (a, b, c, d, e, f, g, h, i, j, k, l) VALUES('aaa', '2021-05-05', '428ff6a1-bb37-42ac-9883-33d7a29961e6', '2021-05-06 08:13:38', '2021-05-06 13:28:08', 0, 8, 3, 0, 9, 1, 0)`) + tk.MustQuery(`select c,j,l from test_partition where c='428ff6a1-bb37-42ac-9883-33d7a29961e6' and a='aaa' limit 0, 200`).Check(testkit.Rows("428ff6a1-bb37-42ac-9883-33d7a29961e6 9 0")) +} + func (s *partitionTableSuite) TestIdexMerge(c *C) { if israce.RaceEnabled { c.Skip("exhaustive types test, skip race test") diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 59c228767171a..16f14d11a9e8b 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -15,7 +15,9 @@ package core import ( "context" + "fmt" "math" + "strings" "github.com/pingcap/errors" "github.com/pingcap/parser/ast" @@ -132,12 +134,38 @@ func CheckTableLock(ctx sessionctx.Context, is infoschema.InfoSchema, vs []visit return nil } +func DEBUGLOGCIAL(sctx sessionctx.Context, prefix string, p LogicalPlan) { + if !sctx.GetSessionVars().DEBUG { + return + } + fmt.Println(prefix, p.ExplainID().String(), len(p.Schema().Columns), p.Schema()) + for _, child := range p.Children() { + DEBUGLOGCIAL(sctx, prefix+" ", child) + } +} + +func DEBUGPHYSICAL(sctx sessionctx.Context, prefix string, p PhysicalPlan) { + if !sctx.GetSessionVars().DEBUG { + return + } + fmt.Println(prefix, p.ExplainID().String(), len(p.Schema().Columns), p.Schema()) + for _, child := range p.Children() { + DEBUGPHYSICAL(sctx, prefix+" ", child) + } +} + // DoOptimize optimizes a logical plan to a physical plan. func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic LogicalPlan) (PhysicalPlan, float64, error) { // if there is something after flagPrunColumns, do flagPrunColumnsAgain if flag&flagPrunColumns > 0 && flag-flagPrunColumns > flagPrunColumns { flag |= flagPrunColumnsAgain } + + if strings.Contains(ToString(logic), "test_partition") { + sctx.GetSessionVars().DEBUG = true + } + + DEBUGLOGCIAL(sctx, ">> begin: ", logic) logic, err := logicalOptimize(ctx, flag, logic) if err != nil { return nil, 0, err @@ -149,7 +177,9 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic if planCounter == 0 { planCounter = -1 } + DEBUGLOGCIAL(sctx, ">> after logical: ", logic) physical, cost, err := physicalOptimize(logic, &planCounter) + DEBUGPHYSICAL(sctx, ">> afer phy: ", physical) if err != nil { return nil, 0, err } diff --git a/planner/core/task.go b/planner/core/task.go index f4c1d1ca72c65..b301915b24e10 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1121,6 +1121,11 @@ func (p *PhysicalLimit) sinkIntoIndexLookUp(t task) bool { return false } } + // If this happens, some Projection Operator must be inlined into this Limit. + // For safety, do not sink Limit into this LookUp in this case. + if p.Schema().Len() != reader.Schema().Len() { + return false + } // We can sink Limit into IndexLookUpReader only if tablePlan contains no Selection. ts, isTableScan := reader.tablePlan.(*PhysicalTableScan) if !isTableScan { diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 6e6d766bd8bf5..736e9e2899af2 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -555,6 +555,8 @@ type SessionVars struct { SQLMode mysql.SQLMode + DEBUG bool + // AutoIncrementIncrement and AutoIncrementOffset indicates the autoID's start value and increment. AutoIncrementIncrement int From 71894069da2aa75087565376781ce695817dd314 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 2 Jun 2021 19:13:15 +0800 Subject: [PATCH 2/8] remove debug code --- planner/core/optimizer.go | 32 +------------------------------- sessionctx/variable/session.go | 2 -- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 16f14d11a9e8b..3e3fe8dbec877 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -15,10 +15,6 @@ package core import ( "context" - "fmt" - "math" - "strings" - "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" @@ -36,6 +32,7 @@ import ( utilhint "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/set" "go.uber.org/atomic" + "math" ) // OptimizeAstNode optimizes the query to a physical plan directly. @@ -134,26 +131,6 @@ func CheckTableLock(ctx sessionctx.Context, is infoschema.InfoSchema, vs []visit return nil } -func DEBUGLOGCIAL(sctx sessionctx.Context, prefix string, p LogicalPlan) { - if !sctx.GetSessionVars().DEBUG { - return - } - fmt.Println(prefix, p.ExplainID().String(), len(p.Schema().Columns), p.Schema()) - for _, child := range p.Children() { - DEBUGLOGCIAL(sctx, prefix+" ", child) - } -} - -func DEBUGPHYSICAL(sctx sessionctx.Context, prefix string, p PhysicalPlan) { - if !sctx.GetSessionVars().DEBUG { - return - } - fmt.Println(prefix, p.ExplainID().String(), len(p.Schema().Columns), p.Schema()) - for _, child := range p.Children() { - DEBUGPHYSICAL(sctx, prefix+" ", child) - } -} - // DoOptimize optimizes a logical plan to a physical plan. func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic LogicalPlan) (PhysicalPlan, float64, error) { // if there is something after flagPrunColumns, do flagPrunColumnsAgain @@ -161,11 +138,6 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic flag |= flagPrunColumnsAgain } - if strings.Contains(ToString(logic), "test_partition") { - sctx.GetSessionVars().DEBUG = true - } - - DEBUGLOGCIAL(sctx, ">> begin: ", logic) logic, err := logicalOptimize(ctx, flag, logic) if err != nil { return nil, 0, err @@ -177,9 +149,7 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic if planCounter == 0 { planCounter = -1 } - DEBUGLOGCIAL(sctx, ">> after logical: ", logic) physical, cost, err := physicalOptimize(logic, &planCounter) - DEBUGPHYSICAL(sctx, ">> afer phy: ", physical) if err != nil { return nil, 0, err } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 736e9e2899af2..6e6d766bd8bf5 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -555,8 +555,6 @@ type SessionVars struct { SQLMode mysql.SQLMode - DEBUG bool - // AutoIncrementIncrement and AutoIncrementOffset indicates the autoID's start value and increment. AutoIncrementIncrement int From b8e5fcb16cbb35e0f1b3bc8cd5d3c15a2b6b257a Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 2 Jun 2021 19:14:15 +0800 Subject: [PATCH 3/8] make linter happy --- planner/core/optimizer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 3e3fe8dbec877..1fc9a2ed15406 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -15,6 +15,8 @@ package core import ( "context" + "math" + "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/auth" @@ -32,7 +34,6 @@ import ( utilhint "github.com/pingcap/tidb/util/hint" "github.com/pingcap/tidb/util/set" "go.uber.org/atomic" - "math" ) // OptimizeAstNode optimizes the query to a physical plan directly. From d49d4cdfc2b09104f3f1e132401ee4ec0b7670a5 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Wed, 2 Jun 2021 19:15:08 +0800 Subject: [PATCH 4/8] fixup --- planner/core/optimizer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 1fc9a2ed15406..59c228767171a 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -138,7 +138,6 @@ func DoOptimize(ctx context.Context, sctx sessionctx.Context, flag uint64, logic if flag&flagPrunColumns > 0 && flag-flagPrunColumns > flagPrunColumns { flag |= flagPrunColumnsAgain } - logic, err := logicalOptimize(ctx, flag, logic) if err != nil { return nil, 0, err From 1e7d79a71b21855b5115e6cbcbb93008fa83d91d Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 3 Jun 2021 16:02:31 +0800 Subject: [PATCH 5/8] fixup --- planner/core/exhaust_physical_plans.go | 1 + 1 file changed, 1 insertion(+) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index b2ba91ff99dcd..af56f07bb3c83 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2008,6 +2008,7 @@ func (lt *LogicalTopN) getPhysLimits(prop *property.PhysicalProperty) []Physical Count: lt.Count, Offset: lt.Offset, }.Init(lt.ctx, lt.stats, lt.blockOffset, resultProp) + limit.SetSchema(lt.Schema()) ret = append(ret, limit) } return ret From 1ab3db8c72d49d39963c2307034dbec9e5a00143 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 3 Jun 2021 16:34:29 +0800 Subject: [PATCH 6/8] update comments --- planner/core/task.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/planner/core/task.go b/planner/core/task.go index b301915b24e10..29581820becca 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1121,11 +1121,16 @@ func (p *PhysicalLimit) sinkIntoIndexLookUp(t task) bool { return false } } - // If this happens, some Projection Operator must be inlined into this Limit. - // For safety, do not sink Limit into this LookUp in this case. + + // If this happens, some Projection Operator must be inlined into this Limit. (issues/14428) + // For example, if the original plan is `IndexLookUp(col1, col2) -> Limit(col1, col2) -> Project(col1)`, + // then after inlining the Project, it will be `IndexLookUp(col1, col2) -> Limit(col1)` here. + // If the Limit is sunk into the IndexLookUp, the IndexLookUp's schema needs to be updated as well, + // but updating it here is not safe, so do not sink Limit into this IndexLookUp in this case now. if p.Schema().Len() != reader.Schema().Len() { return false } + // We can sink Limit into IndexLookUpReader only if tablePlan contains no Selection. ts, isTableScan := reader.tablePlan.(*PhysicalTableScan) if !isTableScan { From 648d483e2be6a73f1c361e107e974db5bee34628 Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Thu, 3 Jun 2021 16:37:11 +0800 Subject: [PATCH 7/8] update explaintest --- cmd/explaintest/r/explain_easy.result | 11 ++++++----- cmd/explaintest/r/explain_easy_stats.result | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/explaintest/r/explain_easy.result b/cmd/explaintest/r/explain_easy.result index 214e51b366de1..c2c61bf79b58e 100644 --- a/cmd/explaintest/r/explain_easy.result +++ b/cmd/explaintest/r/explain_easy.result @@ -124,11 +124,12 @@ Projection 10000.00 root eq(test.t1.c2, test.t2.c2)->Column#11 └─Apply 10000.00 root CARTESIAN left outer join ├─TableReader(Build) 10000.00 root data:TableFullScan │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo - └─Projection(Probe) 1.00 root test.t2.c1, test.t2.c2 - └─IndexLookUp 1.00 root limit embedded(offset:0, count:1) - ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 1.00 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true, stats:pseudo - └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─Limit(Probe) 1.00 root offset:0, count:1 + └─Projection 1.00 root test.t2.c1, test.t2.c2 + └─IndexLookUp 1.00 root + ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 + │ └─IndexRangeScan 1.00 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain format = 'brief' select * from t1 order by c1 desc limit 1; id estRows task access object operator info Limit 1.00 root offset:0, count:1 diff --git a/cmd/explaintest/r/explain_easy_stats.result b/cmd/explaintest/r/explain_easy_stats.result index e76b0d4d233a6..8305dfa929e2b 100644 --- a/cmd/explaintest/r/explain_easy_stats.result +++ b/cmd/explaintest/r/explain_easy_stats.result @@ -106,11 +106,12 @@ Projection 1999.00 root eq(test.t1.c2, test.t2.c2)->Column#11 └─Apply 1999.00 root CARTESIAN left outer join ├─TableReader(Build) 1999.00 root data:TableFullScan │ └─TableFullScan 1999.00 cop[tikv] table:t1 keep order:false - └─Projection(Probe) 1.00 root test.t2.c1, test.t2.c2 - └─IndexLookUp 1.00 root limit embedded(offset:0, count:1) - ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 - │ └─IndexRangeScan 1.25 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true - └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─Limit(Probe) 1.00 root offset:0, count:1 + └─Projection 1.00 root test.t2.c1, test.t2.c2 + └─IndexLookUp 1.00 root + ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1 + │ └─IndexRangeScan 1.25 cop[tikv] table:t2, index:c1(c1) range: decided by [eq(test.t1.c1, test.t2.c1)], keep order:true + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t2 keep order:false, stats:pseudo explain format = 'brief' select * from t1 order by c1 desc limit 1; id estRows task access object operator info Limit 1.00 root offset:0, count:1 From 56c2bad094bff487ce6c1e0a47b92c22ad7ed03f Mon Sep 17 00:00:00 2001 From: Yuanjia Zhang Date: Fri, 4 Jun 2021 10:49:40 +0800 Subject: [PATCH 8/8] fix ci --- planner/core/testdata/integration_suite_out.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 77aa5b1494da7..72d5c968d7468 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -14,11 +14,12 @@ { "SQL": "explain format = 'brief' select * from tbl use index(idx_b_c) where b > 1 order by b desc limit 2,1", "Plan": [ - "Projection 1.00 root test.tbl.a, test.tbl.b, test.tbl.c", - "└─IndexLookUp 1.00 root limit embedded(offset:2, count:1)", - " ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3", - " │ └─IndexRangeScan 3.00 cop[tikv] table:tbl, index:idx_b_c(b, c) range:(1,+inf], keep order:true, desc", - " └─TableRowIDScan(Probe) 1.00 cop[tikv] table:tbl keep order:false, stats:pseudo" + "Limit 1.00 root offset:2, count:1", + "└─Projection 3.00 root test.tbl.a, test.tbl.b, test.tbl.c", + " └─IndexLookUp 3.00 root ", + " ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3", + " │ └─IndexRangeScan 3.00 cop[tikv] table:tbl, index:idx_b_c(b, c) range:(1,+inf], keep order:true, desc", + " └─TableRowIDScan(Probe) 3.00 cop[tikv] table:tbl keep order:false, stats:pseudo" ] }, {