diff --git a/executor/set_test.go b/executor/set_test.go index 935848e966c0c..e07876274cce2 100644 --- a/executor/set_test.go +++ b/executor/set_test.go @@ -801,6 +801,14 @@ func TestSetVar(t *testing.T) { tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("0")) // default value tk.MustExec("set global tidb_enable_foreign_key = 1") tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("1")) + + // test variable 'tidb_opt_inline_cte' + tk.MustQuery("select @@session.tidb_opt_inline_cte").Check(testkit.Rows("0")) // default value is 0 + tk.MustExec("set session tidb_opt_inline_cte=1") + tk.MustQuery("select @@session.tidb_opt_inline_cte").Check(testkit.Rows("1")) + tk.MustQuery("select @@global.tidb_opt_inline_cte").Check(testkit.Rows("0")) // default value is 0 + tk.MustExec("set global tidb_opt_inline_cte=1") + tk.MustQuery("select @@global.tidb_opt_inline_cte").Check(testkit.Rows("1")) } func TestGetSetNoopVars(t *testing.T) { diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index e57d951959054..55ddc3bdcb7f2 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -3841,11 +3841,17 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L b.isForUpdateRead = true } - // Verify Merge hints in the current query, we will update parameters for those that meet the rules, and warn those that do not. - //If the current query uses Merge Hint and the query is a CTE, we update the HINT information for the current query. - //If the current query is not a CTE query (it may be a subquery within a CTE query or an external non-CTE query), we will give a warning. - //In particular, recursive CTE have separate warnings, so they are no longer called. - if hints := b.TableHints(); hints != nil && hints.MergeHints.preferMerge { + // If the session variable "tidb_opt_inline_cte" is true, all of CTEs will be inlined. + // Otherwise, whether CTEs are inlined depends on whether the merge() hint is declared. + if b.ctx.GetSessionVars().EnableInlineCTE() { + if b.buildingCTE && b.isCTE { + b.outerCTEs[len(b.outerCTEs)-1].isInline = true + } + } else if hints := b.TableHints(); hints != nil && hints.MergeHints.preferMerge { + // Verify Merge hints in the current query, we will update parameters for those that meet the rules, and warn those that do not. + // If the current query uses Merge Hint and the query is a CTE, we update the HINT information for the current query. + // If the current query is not a CTE query (it may be a subquery within a CTE query or an external non-CTE query), we will give a warning. + // In particular, recursive CTE have separate warnings, so they are no longer called. if b.buildingCTE { if b.isCTE { b.outerCTEs[len(b.outerCTEs)-1].isInline = hints.MergeHints.preferMerge @@ -4243,10 +4249,9 @@ func (b *PlanBuilder) tryBuildCTE(ctx context.Context, tn *ast.TableName, asName lp.SetSchema(getResultCTESchema(cte.seedLP.Schema(), b.ctx.GetSessionVars())) if cte.recurLP != nil && cte.isInline { - b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack("Hint merge() is inapplicable. Please check whether the CTE use recursive.")) + b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack("Recursive cte cannot be inlined.")) } if cte.recurLP == nil && cte.isInline { - lp.MergeHints.preferMerge = cte.isInline saveCte := make([]*cteInfo, len(b.outerCTEs[i:])) copy(saveCte, b.outerCTEs[i:]) b.outerCTEs = b.outerCTEs[:i] diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index ed7df91314e47..464bb682a48aa 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -1916,7 +1916,6 @@ type LogicalCTE struct { cteAsName model.CIStr seedStat *property.StatsInfo isOuterMostCTE bool - MergeHints MergeHintInfo } // LogicalCTETable is for CTE table diff --git a/planner/core/physical_plan_test.go b/planner/core/physical_plan_test.go index 61a2f5dd87e11..597c17e94ffb1 100644 --- a/planner/core/physical_plan_test.go +++ b/planner/core/physical_plan_test.go @@ -1153,6 +1153,61 @@ func TestCTEMergeHint(t *testing.T) { } } +func TestInlineCTE(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t;") + tk.MustExec("CREATE TABLE `t` (`a` int(11));") + tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);") + + var ( + input []string + output []struct { + SQL string + Plan []string + Warning []string + } + ) + planSuiteData := core.GetPlanSuiteData() + planSuiteData.LoadTestCases(t, &input, &output) + + for i, ts := range input { + testdata.OnRecord(func() { + output[i].SQL = ts + }) + if strings.HasPrefix(ts, "set") { + tk.MustExec(ts) + continue + } + testdata.OnRecord(func() { + output[i].SQL = ts + output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery("explain format='brief' " + ts).Rows()) + }) + tk.MustQuery("explain format='brief' " + ts).Check(testkit.Rows(output[i].Plan...)) + + comment := fmt.Sprintf("case:%v sql:%s", i, ts) + warnings := tk.Session().GetSessionVars().StmtCtx.GetWarnings() + testdata.OnRecord(func() { + if len(warnings) > 0 { + output[i].Warning = make([]string, len(warnings)) + for j, warning := range warnings { + output[i].Warning[j] = warning.Err.Error() + } + } + }) + if len(output[i].Warning) == 0 { + require.Len(t, warnings, 0) + } else { + require.Len(t, warnings, len(output[i].Warning), comment) + for j, warning := range warnings { + require.Equal(t, stmtctx.WarnLevelWarning, warning.Level, comment) + require.Equal(t, output[i].Warning[j], warning.Err.Error(), comment) + } + } + } +} + func TestPushdownDistinctEnable(t *testing.T) { var ( input []string diff --git a/planner/core/testdata/plan_suite_in.json b/planner/core/testdata/plan_suite_in.json index 8c86842c4be4e..908ea504dadaf 100644 --- a/planner/core/testdata/plan_suite_in.json +++ b/planner/core/testdata/plan_suite_in.json @@ -596,6 +596,19 @@ "with cte1 as (with cte2 as (with cte3 as (select /*+ MERGE() */ * from t2) select /*+ MERGE() */ * from cte3) select * from cte2,(select * from t1) ttt) select * from cte1,(select * from t3) ttw;" ] }, + { + "name": "TestInlineCTE", + "cases": [ + "set tidb_opt_inline_cte=1;", // enable inline CTE + "with cte as (select * from t) select * from cte;", // inline + "with cte as (select /*+ MERGE() */ * from t) select * from cte;", // inline + "with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;", // Recursive cte cannot be inlined. + "set tidb_opt_inline_cte=0;", // disable inline CTE + "with cte as (select * from t) select * from cte;", // cannot be inlined + "with cte as (select /*+ MERGE() */ * from t) select * from cte;", // inline, merge hint override session variable + "with recursive cte1(c1) as (select 1 union select /*+ MERGE() */ c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;" // Recursive cte cannot be inlined. + ] + }, { "name": "TestPushdownDistinctEnable", "cases": [ diff --git a/planner/core/testdata/plan_suite_out.json b/planner/core/testdata/plan_suite_out.json index 613bc7890e486..6a403116e5d66 100644 --- a/planner/core/testdata/plan_suite_out.json +++ b/planner/core/testdata/plan_suite_out.json @@ -1598,7 +1598,7 @@ "Name": "TestCTEMergeHint", "Cases": [ { - "SQL": "with cte as (select /*+ MERGE()*/ * from tc where tc.a < 60) select * from cte where cte.a <18", + "SQL": "with cte as (select /*+ MERGE() */ * from tc where tc.a < 60) select * from cte where cte.a <18", "Plan": [ "TableReader 4.00 root data:Selection", "└─Selection 4.00 cop[tikv] lt(test.tc.a, 18), lt(test.tc.a, 60)", @@ -1627,7 +1627,7 @@ "Warning": null }, { - "SQL": "WITH cte1 AS (SELECT /*+ MERGE()*/ a FROM tc), cte2 AS (SELECT /*+ MERGE()*/ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;", + "SQL": "WITH cte1 AS (SELECT /*+ MERGE() */ a FROM tc), cte2 AS (SELECT /*+ MERGE()*/ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;", "Plan": [ "HashJoin 7.00 root inner join, equal:[eq(test.tc.a, test.te.c)]", "├─TableReader(Build) 7.00 root data:Selection", @@ -1640,7 +1640,7 @@ "Warning": null }, { - "SQL": "WITH cte1 AS (SELECT a FROM tc), cte2 AS (SELECT /*+ MERGE()*/ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;", + "SQL": "WITH cte1 AS (SELECT a FROM tc), cte2 AS (SELECT /*+ MERGE() */ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;", "Plan": [ "Projection 4.48 root test.tc.a, test.te.c", "└─HashJoin 4.48 root inner join, equal:[eq(test.te.c, test.tc.a)]", @@ -1668,7 +1668,7 @@ " └─CTETable 1.00 root Scan on CTE_0" ], "Warning": [ - "[planner:1815]Hint merge() is inapplicable. Please check whether the CTE use recursive." + "[planner:1815]Recursive cte cannot be inlined." ] }, { @@ -1863,7 +1863,7 @@ "Warning": null }, { - "SQL": "with cte2 as (with cte4 as (select /*+ merge() */ * from tc) select * from te, cte4) select * from cte2;", + "SQL": "with cte2 as (with cte4 as (select /*+ merge() */ * from tc) select * from te, cte4) select * from cte2;", "Plan": [ "CTEFullScan 49.00 root CTE:cte2 data:CTE_0", "CTE_0 49.00 root Non-Recursive CTE", @@ -1928,6 +1928,85 @@ } ] }, + { + "Name": "TestInlineCTE", + "Cases": [ + { + "SQL": "set tidb_opt_inline_cte=1;", + "Plan": null, + "Warning": null + }, + { + "SQL": "with cte as (select * from t) select * from cte;", + "Plan": [ + "TableReader 10000.00 root data:TableFullScan", + "└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warning": null + }, + { + "SQL": "with cte as (select /*+ MERGE() */ * from t) select * from cte;", + "Plan": [ + "TableReader 10000.00 root data:TableFullScan", + "└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warning": null + }, + { + "SQL": "with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;", + "Plan": [ + "CTEFullScan 2.00 root CTE:cte1 data:CTE_0", + "CTE_0 2.00 root Recursive CTE", + "├─Projection(Seed Part) 1.00 root 1->Column#2", + "│ └─TableDual 1.00 root rows:1", + "└─Projection(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5", + " └─Selection 0.80 root lt(Column#3, 100)", + " └─CTETable 1.00 root Scan on CTE_0" + ], + "Warning": [ + "[planner:1815]Recursive cte cannot be inlined." + ] + }, + { + "SQL": "set tidb_opt_inline_cte=0;", + "Plan": null, + "Warning": null + }, + { + "SQL": "with cte as (select * from t) select * from cte;", + "Plan": [ + "CTEFullScan 10000.00 root CTE:cte data:CTE_0", + "CTE_0 10000.00 root Non-Recursive CTE", + "└─TableReader(Seed Part) 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warning": null + }, + { + "SQL": "with cte as (select /*+ MERGE() */ * from t) select * from cte;", + "Plan": [ + "TableReader 10000.00 root data:TableFullScan", + "└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warning": null + }, + { + "SQL": "with recursive cte1(c1) as (select 1 union select /*+ MERGE() */ c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;", + "Plan": [ + "CTEFullScan 2.00 root CTE:cte1 data:CTE_0", + "CTE_0 2.00 root Recursive CTE", + "├─Projection(Seed Part) 1.00 root 1->Column#2", + "│ └─TableDual 1.00 root rows:1", + "└─Projection(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5", + " └─Selection 0.80 root lt(Column#3, 100)", + " └─CTETable 1.00 root Scan on CTE_0" + ], + "Warning": [ + "[planner:1815]Recursive cte cannot be inlined." + ] + } + ] + }, { "Name": "TestPushdownDistinctEnable", "Cases": [ diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 192ed495a873c..c476b81410925 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -834,6 +834,8 @@ type SessionVars struct { diskFactorV2 float64 // concurrencyFactorV2 is the concurrency factor for the Cost Model Ver2. concurrencyFactorV2 float64 + // enableInlineCTE is used to enable/disable inline CTE. + enableInlineCTE bool // CopTiFlashConcurrencyFactor is the concurrency number of computation in tiflash coprocessor. CopTiFlashConcurrencyFactor float64 @@ -1498,6 +1500,7 @@ func NewSessionVars() *SessionVars { memoryFactor: DefOptMemoryFactor, diskFactor: DefOptDiskFactor, concurrencyFactor: DefOptConcurrencyFactor, + enableInlineCTE: DefOptInlineCTE, EnableVectorizedExpression: DefEnableVectorizedExpression, CommandValue: uint32(mysql.ComSleep), TiDBOptJoinReorderThreshold: DefTiDBOptJoinReorderThreshold, @@ -3021,3 +3024,7 @@ func (s *SessionVars) GetNegateStrMatchDefaultSelectivity() float64 { } return 1 - s.GetStrMatchDefaultSelectivity() } + +func (s *SessionVars) EnableInlineCTE() bool { + return s.enableInlineCTE +} diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index c542793cf7c19..e1ae43ef3c82e 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1242,6 +1242,10 @@ var defaultSysVars = []*SysVar{ s.concurrencyFactorV2 = tidbOptFloat64(val, DefOptConcurrencyFactorV2) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptInlineCTE, Value: BoolToOnOff(DefOptInlineCTE), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.enableInlineCTE = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexJoinBatchSize, Value: strconv.Itoa(DefIndexJoinBatchSize), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error { s.IndexJoinBatchSize = tidbOptPositiveInt32(val, DefIndexJoinBatchSize) return nil diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 1402106dab1a2..a4b57da46c567 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -289,6 +289,8 @@ const ( TiDBOptDiskFactor = "tidb_opt_disk_factor" // TiDBOptConcurrencyFactor is the CPU cost of additional one goroutine. TiDBOptConcurrencyFactor = "tidb_opt_concurrency_factor" + // TiDBOptInlineCTE is used to enable/disable inline CTE + TiDBOptInlineCTE = "tidb_opt_inline_cte" // Variables for the Cost Model Ver2 // TiDBOptCPUFactorV2 is the CPU factor for the Cost Model Ver2 @@ -868,6 +870,7 @@ const ( DefOptMemoryFactorV2 = 0.001 DefOptDiskFactorV2 = 1.5 DefOptConcurrencyFactorV2 = 3.0 + DefOptInlineCTE = false DefOptInSubqToJoinAndAgg = true DefOptPreferRangeScan = false DefBatchInsert = false