Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

planner: Log warnings when agg function can not be pushdown in explain statement (#25553) #25737

Merged
merged 5 commits into from
Jul 22, 2021
Merged

planner: Log warnings when agg function can not be pushdown in explain statement (#25553) #25737

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d196c2e
cherry pick #25553 to release-5.1
windtalker Jun 24, 2021
9105630
Merge branch 'release-5.1' into release-5.1-27489d43b4c7
windtalker Jul 19, 2021
4499af9
fix conflict
windtalker Jul 19, 2021
13bb972
Merge branch 'release-5.1' into release-5.1-27489d43b4c7
ti-chi-bot Jul 22, 2021
c0da3ea
Merge branch 'release-5.1' into release-5.1-27489d43b4c7
ti-chi-bot Jul 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions executor/show_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,16 @@ func (s *testSuite5) TestShowWarningsForExprPushdown(c *C) {
tk.MustExec("explain select * from show_warnings_expr_pushdown where date_add(value, interval 1 day) = '2020-01-01'")
c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1))
tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv"))
tk.MustExec("explain select max(date_add(value, interval 1 day)) from show_warnings_expr_pushdown group by a")
c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2))
tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv", "Warning|1105|Aggregation can not be pushed to tikv because arguments of AggFunc `max` contains unsupported exprs"))
tk.MustExec("explain select max(a) from show_warnings_expr_pushdown group by date_add(value, interval 1 day)")
c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2))
tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv", "Warning|1105|Aggregation can not be pushed to tikv because groupByItems contain unsupported exprs"))
tk.MustExec("set tidb_opt_distinct_agg_push_down=0")
tk.MustExec("explain select max(distinct a) from show_warnings_expr_pushdown group by value")
c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1))
tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct"))
}

func (s *testSuite5) TestShowGrantsPrivilege(c *C) {
Expand Down
15 changes: 13 additions & 2 deletions planner/core/exhaust_physical_plans.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"math"
"sort"

"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/parser/ast"
"github.com/pingcap/parser/mysql"
Expand Down Expand Up @@ -2355,17 +2356,24 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P

// TODO: support more operators and distinct later
func (la *LogicalAggregation) checkCanPushDownToMPP() bool {
hasUnsupportedDistinct := false
for _, agg := range la.AggFuncs {
// MPP does not support distinct except count distinct now
if agg.HasDistinct {
if agg.Name != ast.AggFuncCount {
return false
hasUnsupportedDistinct = true
}
}
// MPP does not support AggFuncApproxCountDistinct now
if agg.Name == ast.AggFuncApproxCountDistinct {
return false
hasUnsupportedDistinct = true
}
}
if hasUnsupportedDistinct {
if la.ctx.GetSessionVars().StmtCtx.InExplainStmt {
la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in mpp mode because it contains agg function with distinct"))
}
return false
}
return CheckAggCanPushCop(la.ctx, la.AggFuncs, la.GroupByItems, kv.TiFlash)
}
Expand Down Expand Up @@ -2451,6 +2459,9 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy
if la.HasDistinct() {
// TODO: remove after the cost estimation of distinct pushdown is implemented.
if !la.ctx.GetSessionVars().AllowDistinctAggPushDown {
if la.ctx.GetSessionVars().StmtCtx.InExplainStmt {
la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct"))
}
taskTypes = []property.TaskType{property.RootTaskType}
}
} else if !la.aggHints.preferAggToCop {
Expand Down
55 changes: 34 additions & 21 deletions planner/core/task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1401,37 +1401,50 @@ func (sel *PhysicalSelection) attach2Task(tasks ...task) task {
func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, groupByItems []expression.Expression, storeType kv.StoreType) bool {
sc := sctx.GetSessionVars().StmtCtx
client := sctx.GetClient()
ret := true
reason := ""
for _, aggFunc := range aggFuncs {
// if the aggFunc contain VirtualColumn or CorrelatedColumn, it can not be pushed down.
if expression.ContainVirtualColumn(aggFunc.Args) || expression.ContainCorrelatedColumn(aggFunc.Args) {
sctx.GetSessionVars().RaiseWarningWhenMPPEnforced(
"MPP mode may be blocked because expressions of AggFunc `" + aggFunc.Name + "` contain virtual column or correlated column, which is not supported now.")
return false
}
pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc)
if pb == nil {
sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because AggFunc `" + aggFunc.Name + "` is not supported now.")
return false
reason = "expressions of AggFunc `" + aggFunc.Name + "` contain virtual column or correlated column, which is not supported now"
ret = false
break
}
if !aggregation.CheckAggPushDown(aggFunc, storeType) {
if sc.InExplainStmt {
storageName := storeType.Name()
if storeType == kv.UnSpecified {
storageName = "storage layer"
}
sc.AppendWarning(errors.New("Agg function '" + aggFunc.Name + "' can not be pushed to " + storageName))
}
return false
reason = "AggFunc `" + aggFunc.Name + "` is not supported now"
ret = false
break
}
if !expression.CanExprsPushDown(sc, aggFunc.Args, client, storeType) {
return false
reason = "arguments of AggFunc `" + aggFunc.Name + "` contains unsupported exprs"
ret = false
break
}
pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc)
if pb == nil {
reason = "AggFunc `" + aggFunc.Name + "` can not be converted to pb expr"
ret = false
break
}
}
if expression.ContainVirtualColumn(groupByItems) {
sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because groupByItems contain virtual column, which is not supported now.")
return false
if ret && expression.ContainVirtualColumn(groupByItems) {
reason = "groupByItems contain virtual columns, which is not supported now"
ret = false
}
if ret && !expression.CanExprsPushDown(sc, groupByItems, client, storeType) {
reason = "groupByItems contain unsupported exprs"
ret = false
}

if !ret && sc.InExplainStmt {
sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because " + reason)
storageName := storeType.Name()
if storeType == kv.UnSpecified {
storageName = "storage layer"
}
sc.AppendWarning(errors.New("Aggregation can not be pushed to " + storageName + " because " + reason))
}
return expression.CanExprsPushDown(sc, groupByItems, client, storeType)
return ret
}

// AggInfo stores the information of an Aggregation.
Expand Down
33 changes: 23 additions & 10 deletions planner/core/testdata/enforce_mpp_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,10 +294,14 @@
" └─TableRowIDScan_40(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
"Warn": [
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.",
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.",
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.",
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now."
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"Aggregation can not be pushed to tikv because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now",
"Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now"
]
},
{
Expand All @@ -309,8 +313,10 @@
" └─TableFullScan_9 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo"
],
"Warn": [
"MPP mode may be blocked because groupByItems contain virtual column, which is not supported now.",
"MPP mode may be blocked because groupByItems contain virtual column, which is not supported now."
"MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now",
"Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now",
"MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now",
"Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now"
]
},
{
Expand All @@ -322,9 +328,12 @@
" └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo"
],
"Warn": [
"MPP mode may be blocked because AggFunc `group_concat` is not supported now.",
"MPP mode may be blocked because AggFunc `group_concat` is not supported now.",
"MPP mode may be blocked because AggFunc `group_concat` is not supported now."
"MPP mode may be blocked because AggFunc `group_concat` is not supported now",
"Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now",
"MPP mode may be blocked because AggFunc `group_concat` is not supported now",
"Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now",
"MPP mode may be blocked because AggFunc `group_concat` is not supported now",
"Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now"
]
},
{
Expand All @@ -337,7 +346,11 @@
],
"Warn": [
"Scalar function 'md5'(signature: MD5) can not be pushed to tiflash",
"Scalar function 'md5'(signature: MD5) can not be pushed to tiflash"
"MPP mode may be blocked because groupByItems contain unsupported exprs",
"Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs",
"Scalar function 'md5'(signature: MD5) can not be pushed to tiflash",
"MPP mode may be blocked because groupByItems contain unsupported exprs",
"Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs"
]
},
{
Expand Down