From d196c2e39cc5f3259cd09e653f846c2782e9f768 Mon Sep 17 00:00:00 2001 From: xufei Date: Thu, 24 Jun 2021 15:21:23 +0800 Subject: [PATCH 1/2] cherry pick #25553 to release-5.1 Signed-off-by: ti-srebot --- executor/show_test.go | 10 ++++ planner/core/exhaust_physical_plans.go | 15 ++++- planner/core/task.go | 55 ++++++++++++------- .../core/testdata/enforce_mpp_suite_out.json | 44 +++++++++++++++ 4 files changed, 101 insertions(+), 23 deletions(-) diff --git a/executor/show_test.go b/executor/show_test.go index 4109559b30928..2d8a8a0a03e44 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -149,6 +149,16 @@ func (s *testSuite5) TestShowWarningsForExprPushdown(c *C) { tk.MustExec("explain select * from show_warnings_expr_pushdown where date_add(value, interval 1 day) = '2020-01-01'") c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1)) tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv")) + tk.MustExec("explain select max(date_add(value, interval 1 day)) from show_warnings_expr_pushdown group by a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2)) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv", "Warning|1105|Aggregation can not be pushed to tikv because arguments of AggFunc `max` contains unsupported exprs")) + tk.MustExec("explain select max(a) from show_warnings_expr_pushdown group by date_add(value, interval 1 day)") + c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2)) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv", "Warning|1105|Aggregation can not be pushed to tikv because groupByItems contain unsupported exprs")) + tk.MustExec("set tidb_opt_distinct_agg_push_down=0") + tk.MustExec("explain select max(distinct a) from show_warnings_expr_pushdown group by value") + c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1)) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct")) } func (s *testSuite5) TestShowGrantsPrivilege(c *C) { diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index 37dbd04c10792..0f1b568f5281c 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -19,6 +19,7 @@ import ( "math" "sort" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" @@ -2355,17 +2356,24 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P // TODO: support more operators and distinct later func (la *LogicalAggregation) checkCanPushDownToMPP() bool { + hasUnsupportedDistinct := false for _, agg := range la.AggFuncs { // MPP does not support distinct except count distinct now if agg.HasDistinct { if agg.Name != ast.AggFuncCount { - return false + hasUnsupportedDistinct = true } } // MPP does not support AggFuncApproxCountDistinct now if agg.Name == ast.AggFuncApproxCountDistinct { - return false + hasUnsupportedDistinct = true + } + } + if hasUnsupportedDistinct { + if la.ctx.GetSessionVars().StmtCtx.InExplainStmt { + la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in mpp mode because it contains agg function with distinct")) } + return false } return CheckAggCanPushCop(la.ctx, la.AggFuncs, la.GroupByItems, kv.TiFlash) } @@ -2447,6 +2455,9 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy if la.HasDistinct() { // TODO: remove after the cost estimation of distinct pushdown is implemented. if !la.ctx.GetSessionVars().AllowDistinctAggPushDown { + if la.ctx.GetSessionVars().StmtCtx.InExplainStmt { + la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct")) + } taskTypes = []property.TaskType{property.RootTaskType} } } else if !la.aggHints.preferAggToCop { diff --git a/planner/core/task.go b/planner/core/task.go index 26261836f5e07..d346f3d8a46aa 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1401,37 +1401,50 @@ func (sel *PhysicalSelection) attach2Task(tasks ...task) task { func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, groupByItems []expression.Expression, storeType kv.StoreType) bool { sc := sctx.GetSessionVars().StmtCtx client := sctx.GetClient() + ret := true + reason := "" for _, aggFunc := range aggFuncs { // if the aggFunc contain VirtualColumn or CorrelatedColumn, it can not be pushed down. if expression.ContainVirtualColumn(aggFunc.Args) || expression.ContainCorrelatedColumn(aggFunc.Args) { - sctx.GetSessionVars().RaiseWarningWhenMPPEnforced( - "MPP mode may be blocked because expressions of AggFunc `" + aggFunc.Name + "` contain virtual column or correlated column, which is not supported now.") - return false - } - pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc) - if pb == nil { - sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because AggFunc `" + aggFunc.Name + "` is not supported now.") - return false + reason = "expressions of AggFunc `" + aggFunc.Name + "` contain virtual column or correlated column, which is not supported now" + ret = false + break } if !aggregation.CheckAggPushDown(aggFunc, storeType) { - if sc.InExplainStmt { - storageName := storeType.Name() - if storeType == kv.UnSpecified { - storageName = "storage layer" - } - sc.AppendWarning(errors.New("Agg function '" + aggFunc.Name + "' can not be pushed to " + storageName)) - } - return false + reason = "AggFunc `" + aggFunc.Name + "` is not supported now" + ret = false + break } if !expression.CanExprsPushDown(sc, aggFunc.Args, client, storeType) { - return false + reason = "arguments of AggFunc `" + aggFunc.Name + "` contains unsupported exprs" + ret = false + break + } + pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc) + if pb == nil { + reason = "AggFunc `" + aggFunc.Name + "` can not be converted to pb expr" + ret = false + break } } - if expression.ContainVirtualColumn(groupByItems) { - sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because groupByItems contain virtual column, which is not supported now.") - return false + if ret && expression.ContainVirtualColumn(groupByItems) { + reason = "groupByItems contain virtual columns, which is not supported now" + ret = false + } + if ret && !expression.CanExprsPushDown(sc, groupByItems, client, storeType) { + reason = "groupByItems contain unsupported exprs" + ret = false + } + + if !ret && sc.InExplainStmt { + sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because " + reason) + storageName := storeType.Name() + if storeType == kv.UnSpecified { + storageName = "storage layer" + } + sc.AppendWarning(errors.New("Aggregation can not be pushed to " + storageName + " because " + reason)) } - return expression.CanExprsPushDown(sc, groupByItems, client, storeType) + return ret } // AggInfo stores the information of an Aggregation. diff --git a/planner/core/testdata/enforce_mpp_suite_out.json b/planner/core/testdata/enforce_mpp_suite_out.json index 372a69d73513f..8e59dc532487c 100644 --- a/planner/core/testdata/enforce_mpp_suite_out.json +++ b/planner/core/testdata/enforce_mpp_suite_out.json @@ -294,10 +294,23 @@ " └─TableRowIDScan_40(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warn": [ +<<<<<<< HEAD "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.", "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.", "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.", "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now." +======= + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tikv because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now" +>>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { @@ -309,8 +322,17 @@ " └─TableFullScan_9 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": [ +<<<<<<< HEAD "MPP mode may be blocked because groupByItems contain virtual column, which is not supported now.", "MPP mode may be blocked because groupByItems contain virtual column, which is not supported now." +======= + "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", + "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", + "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", + "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", + "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", + "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now" +>>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { @@ -322,9 +344,20 @@ " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": [ +<<<<<<< HEAD "MPP mode may be blocked because AggFunc `group_concat` is not supported now.", "MPP mode may be blocked because AggFunc `group_concat` is not supported now.", "MPP mode may be blocked because AggFunc `group_concat` is not supported now." +======= + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now" +>>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { @@ -337,7 +370,18 @@ ], "Warn": [ "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", +<<<<<<< HEAD "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash" +======= + "MPP mode may be blocked because groupByItems contain unsupported exprs", + "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", + "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", + "MPP mode may be blocked because groupByItems contain unsupported exprs", + "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", + "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", + "MPP mode may be blocked because groupByItems contain unsupported exprs", + "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs" +>>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { From 4499af9141c0b7a1b435bc19c21c522c26e54b41 Mon Sep 17 00:00:00 2001 From: xufei Date: Mon, 19 Jul 2021 09:23:30 +0800 Subject: [PATCH 2/2] fix conflict --- .../core/testdata/enforce_mpp_suite_out.json | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/planner/core/testdata/enforce_mpp_suite_out.json b/planner/core/testdata/enforce_mpp_suite_out.json index 8e59dc532487c..39facfb69cff0 100644 --- a/planner/core/testdata/enforce_mpp_suite_out.json +++ b/planner/core/testdata/enforce_mpp_suite_out.json @@ -294,12 +294,6 @@ " └─TableRowIDScan_40(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" ], "Warn": [ -<<<<<<< HEAD - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now.", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now." -======= "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", @@ -307,10 +301,7 @@ "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", "Aggregation can not be pushed to tikv because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now" ->>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { @@ -322,17 +313,10 @@ " └─TableFullScan_9 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": [ -<<<<<<< HEAD - "MPP mode may be blocked because groupByItems contain virtual column, which is not supported now.", - "MPP mode may be blocked because groupByItems contain virtual column, which is not supported now." -======= - "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", - "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now" ->>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { @@ -344,20 +328,12 @@ " └─TableFullScan_11 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" ], "Warn": [ -<<<<<<< HEAD - "MPP mode may be blocked because AggFunc `group_concat` is not supported now.", - "MPP mode may be blocked because AggFunc `group_concat` is not supported now.", - "MPP mode may be blocked because AggFunc `group_concat` is not supported now." -======= - "MPP mode may be blocked because AggFunc `group_concat` is not supported now", - "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", "MPP mode may be blocked because AggFunc `group_concat` is not supported now", "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", "MPP mode may be blocked because AggFunc `group_concat` is not supported now", "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", "MPP mode may be blocked because AggFunc `group_concat` is not supported now", "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now" ->>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, { @@ -370,18 +346,11 @@ ], "Warn": [ "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", -<<<<<<< HEAD - "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash" -======= - "MPP mode may be blocked because groupByItems contain unsupported exprs", - "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", - "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", "MPP mode may be blocked because groupByItems contain unsupported exprs", "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", "MPP mode may be blocked because groupByItems contain unsupported exprs", "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs" ->>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) ] }, {