Skip to content

Commit

Permalink
planner: add a session variable "tidb_opt_inline_cte"
Browse files Browse the repository at this point in the history 
This pr adds a new session variable,
which is mainly used to control whether all CTEs in the entire session are inlined or not.
The default value is false, which means that inline cte is not enabled by default.
(But if the user directly specifies the merge hint, it can still be turned on)
If the variable is set to true, it means that all CTEs for this session try to enable inlining.

Fixed pingcap#36514
  • Loading branch information
@elsa0520
elsa0520 committed Sep 7, 2022
1 parent 9036de3 commit 9bfa8f2
Show file tree
Hide file tree
Showing 9 changed files with 186 additions and 13 deletions.
8 changes: 8 additions & 0 deletions executor/set_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,14 @@ func TestSetVar(t *testing.T) {
tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("0")) // default value
tk.MustExec("set global tidb_enable_foreign_key = 1")
tk.MustQuery("select @@global.tidb_enable_foreign_key").Check(testkit.Rows("1"))

// test variable 'tidb_opt_inline_cte'
tk.MustQuery("select @@session.tidb_opt_inline_cte").Check(testkit.Rows("0")) // default value is 0
tk.MustExec("set session tidb_opt_inline_cte=1")
tk.MustQuery("select @@session.tidb_opt_inline_cte").Check(testkit.Rows("1"))
tk.MustQuery("select @@global.tidb_opt_inline_cte").Check(testkit.Rows("0")) // default value is 0
tk.MustExec("set global tidb_opt_inline_cte=1")
tk.MustQuery("select @@global.tidb_opt_inline_cte").Check(testkit.Rows("1"))
}

func TestGetSetNoopVars(t *testing.T) {
Expand Down
19 changes: 12 additions & 7 deletions planner/core/logical_plan_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3841,11 +3841,17 @@ func (b *PlanBuilder) buildSelect(ctx context.Context, sel *ast.SelectStmt) (p L
b.isForUpdateRead = true
}

// Verify Merge hints in the current query, we will update parameters for those that meet the rules, and warn those that do not.
//If the current query uses Merge Hint and the query is a CTE, we update the HINT information for the current query.
//If the current query is not a CTE query (it may be a subquery within a CTE query or an external non-CTE query), we will give a warning.
//In particular, recursive CTE have separate warnings, so they are no longer called.
if hints := b.TableHints(); hints != nil && hints.MergeHints.preferMerge {
// If the session variable "tidb_opt_inline_cte" is true, all of CTEs will be inlined.
// Otherwise, whether CTEs are inlined depends on whether the merge() hint is declared.
if b.ctx.GetSessionVars().EnableInlineCTE() {
if b.buildingCTE && b.isCTE {
b.outerCTEs[len(b.outerCTEs)-1].isInline = true
}
} else if hints := b.TableHints(); hints != nil && hints.MergeHints.preferMerge {
// Verify Merge hints in the current query, we will update parameters for those that meet the rules, and warn those that do not.
// If the current query uses Merge Hint and the query is a CTE, we update the HINT information for the current query.
// If the current query is not a CTE query (it may be a subquery within a CTE query or an external non-CTE query), we will give a warning.
// In particular, recursive CTE have separate warnings, so they are no longer called.
if b.buildingCTE {
if b.isCTE {
b.outerCTEs[len(b.outerCTEs)-1].isInline = hints.MergeHints.preferMerge
Expand Down Expand Up @@ -4243,10 +4249,9 @@ func (b *PlanBuilder) tryBuildCTE(ctx context.Context, tn *ast.TableName, asName
lp.SetSchema(getResultCTESchema(cte.seedLP.Schema(), b.ctx.GetSessionVars()))

if cte.recurLP != nil && cte.isInline {
b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack("Hint merge() is inapplicable. Please check whether the CTE use recursive."))
b.ctx.GetSessionVars().StmtCtx.AppendWarning(ErrInternal.GenWithStack("Recursive cte cannot be inlined."))
}
if cte.recurLP == nil && cte.isInline {
lp.MergeHints.preferMerge = cte.isInline
saveCte := make([]*cteInfo, len(b.outerCTEs[i:]))
copy(saveCte, b.outerCTEs[i:])
b.outerCTEs = b.outerCTEs[:i]
Expand Down
1 change: 0 additions & 1 deletion planner/core/logical_plans.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1916,7 +1916,6 @@ type LogicalCTE struct {
cteAsName model.CIStr
seedStat *property.StatsInfo
isOuterMostCTE bool
MergeHints MergeHintInfo
}

// LogicalCTETable is for CTE table
Expand Down
55 changes: 55 additions & 0 deletions planner/core/physical_plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1153,6 +1153,61 @@ func TestCTEMergeHint(t *testing.T) {
}
}

func TestInlineCTE(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("drop table if exists t;")
tk.MustExec("CREATE TABLE `t` (`a` int(11));")
tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);")

var (
input []string
output []struct {
SQL string
Plan []string
Warning []string
}
)
planSuiteData := core.GetPlanSuiteData()
planSuiteData.LoadTestCases(t, &input, &output)

for i, ts := range input {
testdata.OnRecord(func() {
output[i].SQL = ts
})
if strings.HasPrefix(ts, "set") {
tk.MustExec(ts)
continue
}
testdata.OnRecord(func() {
output[i].SQL = ts
output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery("explain format='brief' " + ts).Rows())
})
tk.MustQuery("explain format='brief' " + ts).Check(testkit.Rows(output[i].Plan...))

comment := fmt.Sprintf("case:%v sql:%s", i, ts)
warnings := tk.Session().GetSessionVars().StmtCtx.GetWarnings()
testdata.OnRecord(func() {
if len(warnings) > 0 {
output[i].Warning = make([]string, len(warnings))
for j, warning := range warnings {
output[i].Warning[j] = warning.Err.Error()
}
}
})
if len(output[i].Warning) == 0 {
require.Len(t, warnings, 0)
} else {
require.Len(t, warnings, len(output[i].Warning), comment)
for j, warning := range warnings {
require.Equal(t, stmtctx.WarnLevelWarning, warning.Level, comment)
require.Equal(t, output[i].Warning[j], warning.Err.Error(), comment)
}
}
}
}

func TestPushdownDistinctEnable(t *testing.T) {
var (
input []string
Expand Down
13 changes: 13 additions & 0 deletions planner/core/testdata/plan_suite_in.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,19 @@
"with cte1 as (with cte2 as (with cte3 as (select /*+ MERGE() */ * from t2) select /*+ MERGE() */ * from cte3) select * from cte2,(select * from t1) ttt) select * from cte1,(select * from t3) ttw;"
]
},
{
"name": "TestInlineCTE",
"cases": [
"set tidb_opt_inline_cte=1;", // enable inline CTE
"with cte as (select * from t) select * from cte;", // inline
"with cte as (select /*+ MERGE() */ * from t) select * from cte;", // inline
"with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;", // Recursive cte cannot be inlined.
"set tidb_opt_inline_cte=0;", // disable inline CTE
"with cte as (select * from t) select * from cte;", // cannot be inlined
"with cte as (select /*+ MERGE() */ * from t) select * from cte;", // inline, merge hint override session variable
"with recursive cte1(c1) as (select 1 union select /*+ MERGE() */ c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;" // Recursive cte cannot be inlined.
]
},
{
"name": "TestPushdownDistinctEnable",
"cases": [
Expand Down
89 changes: 84 additions & 5 deletions planner/core/testdata/plan_suite_out.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1598,7 +1598,7 @@
"Name": "TestCTEMergeHint",
"Cases": [
{
"SQL": "with cte as (select /*+ MERGE()*/ * from tc where tc.a < 60) select * from cte where cte.a <18",
"SQL": "with cte as (select /*+ MERGE() */ * from tc where tc.a < 60) select * from cte where cte.a <18",
"Plan": [
"TableReader 4.00 root data:Selection",
"└─Selection 4.00 cop[tikv] lt(test.tc.a, 18), lt(test.tc.a, 60)",
Expand Down Expand Up @@ -1627,7 +1627,7 @@
"Warning": null
},
{
"SQL": "WITH cte1 AS (SELECT /*+ MERGE()*/ a FROM tc), cte2 AS (SELECT /*+ MERGE()*/ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;",
"SQL": "WITH cte1 AS (SELECT /*+ MERGE() */ a FROM tc), cte2 AS (SELECT /*+ MERGE()*/ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;",
"Plan": [
"HashJoin 7.00 root inner join, equal:[eq(test.tc.a, test.te.c)]",
"├─TableReader(Build) 7.00 root data:Selection",
Expand All @@ -1640,7 +1640,7 @@
"Warning": null
},
{
"SQL": "WITH cte1 AS (SELECT a FROM tc), cte2 AS (SELECT /*+ MERGE()*/ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;",
"SQL": "WITH cte1 AS (SELECT a FROM tc), cte2 AS (SELECT /*+ MERGE() */ c FROM te) SELECT * FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;",
"Plan": [
"Projection 4.48 root test.tc.a, test.te.c",
"└─HashJoin 4.48 root inner join, equal:[eq(test.te.c, test.tc.a)]",
Expand Down Expand Up @@ -1668,7 +1668,7 @@
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Hint merge() is inapplicable. Please check whether the CTE use recursive."
"[planner:1815]Recursive cte cannot be inlined."
]
},
{
Expand Down Expand Up @@ -1863,7 +1863,7 @@
"Warning": null
},
{
"SQL": "with cte2 as (with cte4 as (select /*+ merge() */ * from tc) select * from te, cte4) select * from cte2;",
"SQL": "with cte2 as (with cte4 as (select /*+ merge() */ * from tc) select * from te, cte4) select * from cte2;",
"Plan": [
"CTEFullScan 49.00 root CTE:cte2 data:CTE_0",
"CTE_0 49.00 root Non-Recursive CTE",
Expand Down Expand Up @@ -1928,6 +1928,85 @@
}
]
},
{
"Name": "TestInlineCTE",
"Cases": [
{
"SQL": "set tidb_opt_inline_cte=1;",
"Plan": null,
"Warning": null
},
{
"SQL": "with cte as (select * from t) select * from cte;",
"Plan": [
"TableReader 10000.00 root data:TableFullScan",
"└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with cte as (select /*+ MERGE() */ * from t) select * from cte;",
"Plan": [
"TableReader 10000.00 root data:TableFullScan",
"└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select 1 union select c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;",
"Plan": [
"CTEFullScan 2.00 root CTE:cte1 data:CTE_0",
"CTE_0 2.00 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 1->Column#2",
"│ └─TableDual 1.00 root rows:1",
"└─Projection(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5",
" └─Selection 0.80 root lt(Column#3, 100)",
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive cte cannot be inlined."
]
},
{
"SQL": "set tidb_opt_inline_cte=0;",
"Plan": null,
"Warning": null
},
{
"SQL": "with cte as (select * from t) select * from cte;",
"Plan": [
"CTEFullScan 10000.00 root CTE:cte data:CTE_0",
"CTE_0 10000.00 root Non-Recursive CTE",
"└─TableReader(Seed Part) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with cte as (select /*+ MERGE() */ * from t) select * from cte;",
"Plan": [
"TableReader 10000.00 root data:TableFullScan",
"└─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warning": null
},
{
"SQL": "with recursive cte1(c1) as (select 1 union select /*+ MERGE() */ c1 + 1 c1 from cte1 where c1 < 100) select * from cte1;",
"Plan": [
"CTEFullScan 2.00 root CTE:cte1 data:CTE_0",
"CTE_0 2.00 root Recursive CTE",
"├─Projection(Seed Part) 1.00 root 1->Column#2",
"│ └─TableDual 1.00 root rows:1",
"└─Projection(Recursive Part) 0.80 root cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5",
" └─Selection 0.80 root lt(Column#3, 100)",
" └─CTETable 1.00 root Scan on CTE_0"
],
"Warning": [
"[planner:1815]Recursive cte cannot be inlined."
]
}
]
},
{
"Name": "TestPushdownDistinctEnable",
"Cases": [
Expand Down
7 changes: 7 additions & 0 deletions sessionctx/variable/session.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,8 @@ type SessionVars struct {
diskFactorV2 float64
// concurrencyFactorV2 is the concurrency factor for the Cost Model Ver2.
concurrencyFactorV2 float64
// enableInlineCTE is used to enable/disable inline CTE.
enableInlineCTE bool

// CopTiFlashConcurrencyFactor is the concurrency number of computation in tiflash coprocessor.
CopTiFlashConcurrencyFactor float64
Expand Down Expand Up @@ -1498,6 +1500,7 @@ func NewSessionVars() *SessionVars {
memoryFactor: DefOptMemoryFactor,
diskFactor: DefOptDiskFactor,
concurrencyFactor: DefOptConcurrencyFactor,
enableInlineCTE: DefOptInlineCTE,
EnableVectorizedExpression: DefEnableVectorizedExpression,
CommandValue: uint32(mysql.ComSleep),
TiDBOptJoinReorderThreshold: DefTiDBOptJoinReorderThreshold,
Expand Down Expand Up @@ -3021,3 +3024,7 @@ func (s *SessionVars) GetNegateStrMatchDefaultSelectivity() float64 {
}
return 1 - s.GetStrMatchDefaultSelectivity()
}

func (s *SessionVars) EnableInlineCTE() bool {
return s.enableInlineCTE
}
4 changes: 4 additions & 0 deletions sessionctx/variable/sysvar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1242,6 +1242,10 @@ var defaultSysVars = []*SysVar{
s.concurrencyFactorV2 = tidbOptFloat64(val, DefOptConcurrencyFactorV2)
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBOptInlineCTE, Value: BoolToOnOff(DefOptInlineCTE), Type: TypeBool, SetSession: func(s *SessionVars, val string) error {
s.enableInlineCTE = TiDBOptOn(val)
return nil
}},
{Scope: ScopeGlobal | ScopeSession, Name: TiDBIndexJoinBatchSize, Value: strconv.Itoa(DefIndexJoinBatchSize), Type: TypeUnsigned, MinValue: 1, MaxValue: math.MaxInt32, SetSession: func(s *SessionVars, val string) error {
s.IndexJoinBatchSize = tidbOptPositiveInt32(val, DefIndexJoinBatchSize)
return nil
Expand Down
3 changes: 3 additions & 0 deletions sessionctx/variable/tidb_vars.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,8 @@ const (
TiDBOptDiskFactor = "tidb_opt_disk_factor"
// TiDBOptConcurrencyFactor is the CPU cost of additional one goroutine.
TiDBOptConcurrencyFactor = "tidb_opt_concurrency_factor"
// TiDBOptInlineCTE is used to enable/disable inline CTE
TiDBOptInlineCTE = "tidb_opt_inline_cte"

// Variables for the Cost Model Ver2
// TiDBOptCPUFactorV2 is the CPU factor for the Cost Model Ver2
Expand Down Expand Up @@ -868,6 +870,7 @@ const (
DefOptMemoryFactorV2 = 0.001
DefOptDiskFactorV2 = 1.5
DefOptConcurrencyFactorV2 = 3.0
DefOptInlineCTE = false
DefOptInSubqToJoinAndAgg = true
DefOptPreferRangeScan = false
DefBatchInsert = false
Expand Down

0 comments on commit 9bfa8f2

Please sign in to comment.