From a162e06890cf2de32e80723620ff1e005f257456 Mon Sep 17 00:00:00 2001 From: xufei Date: Thu, 24 Jun 2021 15:21:23 +0800 Subject: [PATCH 1/3] cherry pick #25553 to release-5.0 Signed-off-by: ti-srebot --- executor/show_test.go | 10 + planner/core/exhaust_physical_plans.go | 15 +- planner/core/task.go | 50 +- .../core/testdata/enforce_mpp_suite_out.json | 462 ++++++++++++++++++ 4 files changed, 525 insertions(+), 12 deletions(-) create mode 100644 planner/core/testdata/enforce_mpp_suite_out.json diff --git a/executor/show_test.go b/executor/show_test.go index 7bebc1843db76..48f166b64da74 100644 --- a/executor/show_test.go +++ b/executor/show_test.go @@ -149,6 +149,16 @@ func (s *testSuite5) TestShowWarningsForExprPushdown(c *C) { tk.MustExec("explain select * from show_warnings_expr_pushdown where date_add(value, interval 1 day) = '2020-01-01'") c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1)) tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv")) + tk.MustExec("explain select max(date_add(value, interval 1 day)) from show_warnings_expr_pushdown group by a") + c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2)) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv", "Warning|1105|Aggregation can not be pushed to tikv because arguments of AggFunc `max` contains unsupported exprs")) + tk.MustExec("explain select max(a) from show_warnings_expr_pushdown group by date_add(value, interval 1 day)") + c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(2)) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Scalar function 'date_add'(signature: AddDateDatetimeInt) can not be pushed to tikv", "Warning|1105|Aggregation can not be pushed to tikv because groupByItems contain unsupported exprs")) + tk.MustExec("set tidb_opt_distinct_agg_push_down=0") + tk.MustExec("explain select max(distinct a) from show_warnings_expr_pushdown group by value") + c.Assert(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(1)) + tk.MustQuery("show warnings").Check(testutil.RowsWithSep("|", "Warning|1105|Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct")) } func (s *testSuite5) TestShowGrantsPrivilege(c *C) { diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index c81a26aeae9d9..a03f576ce2cc5 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -19,6 +19,7 @@ import ( "math" "sort" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/model" @@ -2342,17 +2343,24 @@ func (la *LogicalAggregation) getStreamAggs(prop *property.PhysicalProperty) []P // TODO: support more operators and distinct later func (la *LogicalAggregation) checkCanPushDownToMPP() bool { + hasUnsupportedDistinct := false for _, agg := range la.AggFuncs { // MPP does not support distinct except count distinct now if agg.HasDistinct { if agg.Name != ast.AggFuncCount { - return false + hasUnsupportedDistinct = true } } // MPP does not support AggFuncApproxCountDistinct now if agg.Name == ast.AggFuncApproxCountDistinct { - return false + hasUnsupportedDistinct = true + } + } + if hasUnsupportedDistinct { + if la.ctx.GetSessionVars().StmtCtx.InExplainStmt { + la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in mpp mode because it contains agg function with distinct")) } + return false } return CheckAggCanPushCop(la.ctx, la.AggFuncs, la.GroupByItems, kv.TiFlash) } @@ -2434,6 +2442,9 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy if la.HasDistinct() { // TODO: remove after the cost estimation of distinct pushdown is implemented. if !la.ctx.GetSessionVars().AllowDistinctAggPushDown { + if la.ctx.GetSessionVars().StmtCtx.InExplainStmt { + la.ctx.GetSessionVars().StmtCtx.AppendWarning(errors.New("Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct")) + } taskTypes = []property.TaskType{property.RootTaskType} } } else if !la.aggHints.preferAggToCop { diff --git a/planner/core/task.go b/planner/core/task.go index a4720e860a7b6..864e17456e0be 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1358,33 +1358,63 @@ func (sel *PhysicalSelection) attach2Task(tasks ...task) task { func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFuncDesc, groupByItems []expression.Expression, storeType kv.StoreType) bool { sc := sctx.GetSessionVars().StmtCtx client := sctx.GetClient() + ret := true + reason := "" for _, aggFunc := range aggFuncs { // if the aggFunc contain VirtualColumn or CorrelatedColumn, it can not be pushed down. if expression.ContainVirtualColumn(aggFunc.Args) || expression.ContainCorrelatedColumn(aggFunc.Args) { +<<<<<<< HEAD return false } pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc) if pb == nil { return false +======= + reason = "expressions of AggFunc `" + aggFunc.Name + "` contain virtual column or correlated column, which is not supported now" + ret = false + break +>>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) } if !aggregation.CheckAggPushDown(aggFunc, storeType) { - if sc.InExplainStmt { - storageName := storeType.Name() - if storeType == kv.UnSpecified { - storageName = "storage layer" - } - sc.AppendWarning(errors.New("Agg function '" + aggFunc.Name + "' can not be pushed to " + storageName)) - } - return false + reason = "AggFunc `" + aggFunc.Name + "` is not supported now" + ret = false + break } if !expression.CanExprsPushDown(sc, aggFunc.Args, client, storeType) { - return false + reason = "arguments of AggFunc `" + aggFunc.Name + "` contains unsupported exprs" + ret = false + break + } + pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc) + if pb == nil { + reason = "AggFunc `" + aggFunc.Name + "` can not be converted to pb expr" + ret = false + break } } +<<<<<<< HEAD if expression.ContainVirtualColumn(groupByItems) { return false +======= + if ret && expression.ContainVirtualColumn(groupByItems) { + reason = "groupByItems contain virtual columns, which is not supported now" + ret = false + } + if ret && !expression.CanExprsPushDown(sc, groupByItems, client, storeType) { + reason = "groupByItems contain unsupported exprs" + ret = false + } + + if !ret && sc.InExplainStmt { + sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because " + reason) + storageName := storeType.Name() + if storeType == kv.UnSpecified { + storageName = "storage layer" + } + sc.AppendWarning(errors.New("Aggregation can not be pushed to " + storageName + " because " + reason)) +>>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) } - return expression.CanExprsPushDown(sc, groupByItems, client, storeType) + return ret } // AggInfo stores the information of an Aggregation. diff --git a/planner/core/testdata/enforce_mpp_suite_out.json b/planner/core/testdata/enforce_mpp_suite_out.json new file mode 100644 index 0000000000000..63787fde4ff7c --- /dev/null +++ b/planner/core/testdata/enforce_mpp_suite_out.json @@ -0,0 +1,462 @@ +[ + { + "Name": "TestEnforceMPP", + "Cases": [ + { + "SQL": "select @@tidb_allow_mpp", + "Plan": [ + "1" + ], + "Warn": null + }, + { + "SQL": "select @@tidb_enforce_mpp", + "Plan": [ + "0" + ], + "Warn": null + }, + { + "SQL": "select @@tidb_opt_tiflash_concurrency_factor", + "Plan": [ + "24" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_allow_mpp=0", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='verbose' select count(*) from t where a=1", + "Plan": [ + "StreamAgg_24 1.00 485.00 root funcs:count(Column#6)->Column#4", + "└─IndexReader_25 1.00 32.88 root index:StreamAgg_9", + " └─StreamAgg_9 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", + " └─IndexRangeScan_23 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", + "Plan": [ + "StreamAgg_17 1.00 485.00 root funcs:count(Column#6)->Column#4", + "└─IndexReader_18 1.00 32.88 root index:StreamAgg_9", + " └─StreamAgg_9 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", + " └─IndexRangeScan_16 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", + "Plan": [ + "StreamAgg_20 1.00 285050.00 root funcs:count(Column#6)->Column#4", + "└─TableReader_21 1.00 19003.88 root data:StreamAgg_9", + " └─StreamAgg_9 1.00 19006.88 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_19 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_18 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_allow_mpp=1;", + "Plan": null, + "Warn": null + }, + { + "SQL": "set @@tidb_enforce_mpp=0;", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='verbose' select count(*) from t where a=1", + "Plan": [ + "StreamAgg_30 1.00 485.00 root funcs:count(Column#7)->Column#4", + "└─IndexReader_31 1.00 32.88 root index:StreamAgg_10", + " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#7", + " └─IndexRangeScan_29 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", + "Plan": [ + "StreamAgg_18 1.00 485.00 root funcs:count(Column#6)->Column#4", + "└─IndexReader_19 1.00 32.88 root index:StreamAgg_10", + " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", + " └─IndexRangeScan_17 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", + "Plan": [ + "HashAgg_21 1.00 11910.73 root funcs:count(Column#6)->Column#4", + "└─TableReader_23 1.00 11877.13 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_20 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_19 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_opt_tiflash_concurrency_factor = 1000000", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='verbose' select count(*) from t where a=1", + "Plan": [ + "HashAgg_24 1.00 33.89 root funcs:count(Column#6)->Column#4", + "└─TableReader_26 1.00 0.29 root data:ExchangeSender_25", + " └─ExchangeSender_25 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_23 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_22 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", + "Plan": [ + "StreamAgg_18 1.00 485.00 root funcs:count(Column#6)->Column#4", + "└─IndexReader_19 1.00 32.88 root index:StreamAgg_10", + " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", + " └─IndexRangeScan_17 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", + "Plan": [ + "HashAgg_21 1.00 33.89 root funcs:count(Column#6)->Column#4", + "└─TableReader_23 1.00 0.29 root data:ExchangeSender_22", + " └─ExchangeSender_22 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_20 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_19 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "set @@tidb_enforce_mpp=1;", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain format='verbose' select count(*) from t where a=1", + "Plan": [ + "HashAgg_25 1.00 33.60 root funcs:count(Column#6)->Column#4", + "└─TableReader_27 1.00 0.00 root data:ExchangeSender_26", + " └─ExchangeSender_26 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_10 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_24 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_23 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", + "Plan": [ + "StreamAgg_19 1.00 485.00 root funcs:count(Column#6)->Column#4", + "└─IndexReader_20 1.00 32.88 root index:StreamAgg_11", + " └─StreamAgg_11 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", + " └─IndexRangeScan_18 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because you have set a hint to read table `t` from TiKV." + ] + }, + { + "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", + "Plan": [ + "HashAgg_22 1.00 33.60 root funcs:count(Column#6)->Column#4", + "└─TableReader_24 1.00 0.00 root data:ExchangeSender_23", + " └─ExchangeSender_23 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_10 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_21 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_20 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": null + } + ] + }, + { + "Name": "TestEnforceMPPWarning1", + "Cases": [ + { + "SQL": "set @@tidb_allow_mpp=1;set @@tidb_enforce_mpp=1;", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain select count(*) from t where a=1 -- 1. no replica", + "Plan": [ + "StreamAgg_17 1.00 root funcs:count(Column#7)->Column#5", + "└─IndexReader_18 1.00 root index:StreamAgg_9", + " └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#7", + " └─IndexRangeScan_16 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because there aren't tiflash replicas of table `t`." + ] + }, + { + "SQL": "cmd: create-replica", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain select count(*) from t where a=1 -- 2. replica not ready", + "Plan": [ + "StreamAgg_17 1.00 root funcs:count(Column#7)->Column#5", + "└─IndexReader_18 1.00 root index:StreamAgg_9", + " └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#7", + " └─IndexRangeScan_16 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because tiflash replicas of table `t` not ready." + ] + }, + { + "SQL": "cmd: enable-replica", + "Plan": null, + "Warn": null + }, + { + "SQL": "set @@session.tidb_isolation_read_engines = 'tikv';", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain select count(*) from t where a=1 -- 3. isolation_engine not match", + "Plan": [ + "StreamAgg_17 1.00 root funcs:count(Column#7)->Column#5", + "└─IndexReader_18 1.00 root index:StreamAgg_9", + " └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#7", + " └─IndexRangeScan_16 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because 'tidb_isolation_read_engines'(value: 'tikv') not match, need 'tiflash'." + ] + }, + { + "SQL": "set @@session.tidb_isolation_read_engines = 'tikv, tiflash';", + "Plan": null, + "Warn": null + }, + { + "SQL": "explain select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1 -- 4. hint use tikv", + "Plan": [ + "StreamAgg_19 1.00 root funcs:count(Column#7)->Column#5", + "└─IndexReader_20 1.00 root index:StreamAgg_11", + " └─StreamAgg_11 1.00 cop[tikv] funcs:count(1)->Column#7", + " └─IndexRangeScan_18 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because you have set a hint to read table `t` from TiKV." + ] + }, + { + "SQL": "explain SELECT a, ROW_NUMBER() OVER (ORDER BY a) FROM t; -- 5. window unsupported", + "Plan": [ + "Window_7 10000.00 root row_number()->Column#6 over(order by test.t.a rows between current row and current row)", + "└─IndexReader_9 10000.00 root index:IndexFullScan_8", + " └─IndexFullScan_8 10000.00 cop[tikv] table:t, index:idx(a) keep order:true, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because operator `Window` is not supported now.", + "MPP mode may be blocked because operator `Window` is not supported now." + ] + }, + { + "SQL": "EXPLAIN SELECT t1.b FROM t t1 join t t2 where t1.a=t2.a; -- 6. virtual column", + "Plan": [ + "HashJoin_36 12487.50 root inner join, equal:[eq(test.t.a, test.t.a)]", + "├─TableReader_56(Build) 9990.00 root data:Selection_55", + "│ └─Selection_55 9990.00 cop[tiflash] not(isnull(test.t.a))", + "│ └─TableFullScan_54 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo", + "└─TableReader_50(Probe) 9990.00 root data:Selection_49", + " └─Selection_49 9990.00 cop[tiflash] not(isnull(test.t.a))", + " └─TableFullScan_48 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because column `test.t.b` is a virtual column which is not supported now." + ] + }, + { + "SQL": "EXPLAIN SELECT count(b) from t where a=1; -- 7. agg func has virtual column", + "Plan": [ + "StreamAgg_11 1.00 root funcs:count(test.t.b)->Column#5", + "└─IndexLookUp_42 10.00 root ", + " ├─IndexRangeScan_40(Build) 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo", + " └─TableRowIDScan_41(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tikv because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", + "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now" + ] + }, + { + "SQL": "EXPLAIN SELECT count(*) from t group by b; -- 8. group by virtual column", + "Plan": [ + "HashAgg_6 8000.00 root group by:test.t.b, funcs:count(1)->Column#5", + "└─Projection_12 10000.00 root test.t.b", + " └─TableReader_11 10000.00 root data:TableFullScan_10", + " └─TableFullScan_10 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", + "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", + "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", + "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", + "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", + "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now" + ] + }, + { + "SQL": "EXPLAIN SELECT group_concat(a) from t; -- 9. agg func not supported", + "Plan": [ + "HashAgg_6 1.00 root funcs:group_concat(Column#6 separator \",\")->Column#5", + "└─Projection_31 10000.00 root cast(test.t.a, var_string(20))->Column#6", + " └─TableReader_14 10000.00 root data:TableFullScan_12", + " └─TableFullScan_12 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", + "MPP mode may be blocked because AggFunc `group_concat` is not supported now", + "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now" + ] + }, + { + "SQL": "EXPLAIN SELECT count(a) from t group by md5(a); -- 10. scalar func not supported", + "Plan": [ + "HashAgg_6 8000.00 root group by:Column#7, funcs:count(Column#6)->Column#5", + "└─Projection_19 10000.00 root test.t.a, md5(cast(test.t.a, var_string(20)))->Column#7", + " └─TableReader_12 10000.00 root data:TableFullScan_10", + " └─TableFullScan_10 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", + "MPP mode may be blocked because groupByItems contain unsupported exprs", + "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", + "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", + "MPP mode may be blocked because groupByItems contain unsupported exprs", + "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", + "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", + "MPP mode may be blocked because groupByItems contain unsupported exprs", + "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs" + ] + }, + { + "SQL": "EXPLAIN SELECT count(a) from t where c=1; -- 11. type not supported", + "Plan": [ + "HashAgg_7 1.00 root funcs:count(test.t.a)->Column#5", + "└─Selection_17 10000.00 root eq(test.t.c, 00:00:01.000000)", + " └─TableReader_16 10000.00 root data:TableFullScan_15", + " └─TableFullScan_15 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", + "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", + "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", + "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", + "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type" + ] + } + ] + }, + { + "Name": "TestEnforceMPPWarning2", + "Cases": [ + { + "SQL": "set @@tidb_allow_mpp=1;set @@tidb_enforce_mpp=1;", + "Plan": null, + "Warn": null + }, + { + "SQL": "set @@tidb_partition_prune_mode=static;", + "Plan": null, + "Warn": null + }, + { + "SQL": "EXPLAIN SELECT count(*) from t where a=1; -- 12. static partition prune", + "Plan": [ + "StreamAgg_32 1.00 root funcs:count(Column#6)->Column#4", + "└─TableReader_33 1.00 root data:StreamAgg_13", + " └─StreamAgg_13 1.00 batchCop[tiflash] funcs:count(1)->Column#6", + " └─Selection_31 10.00 batchCop[tiflash] eq(test.t.a, 1)", + " └─TableFullScan_30 10000.00 batchCop[tiflash] table:t, partition:p0 keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because table `t`is a partition table which is not supported when `@@tidb_partition_prune_mode=static`." + ] + }, + { + "SQL": "set @@tidb_partition_prune_mode=dynamic;", + "Plan": null, + "Warn": null + } + ] + }, + { + "Name": "TestEnforceMPPWarning3", + "Cases": [ + { + "SQL": "set @@tidb_allow_mpp=1;set @@tidb_enforce_mpp=1;", + "Plan": null, + "Warn": null + }, + { + "SQL": "cmd: enable-new-collation", + "Plan": null, + "Warn": null + }, + { + "SQL": "EXPLAIN SELECT count(*) from t group by b; -- 13. new collation FIXME", + "Plan": [ + "HashAgg_24 8000.00 root group by:test.t.b, funcs:count(Column#7)->Column#4", + "└─TableReader_26 8000.00 root data:ExchangeSender_25", + " └─ExchangeSender_25 8000.00 batchCop[tiflash] ExchangeType: PassThrough", + " └─HashAgg_11 8000.00 batchCop[tiflash] group by:test.t.b, funcs:count(1)->Column#7", + " └─TableFullScan_21 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ], + "Warn": [ + "MPP mode may be blocked because when `new_collation_enabled` is true, HashJoin or HashAgg with string key is not supported now.", + "MPP mode may be blocked because when `new_collation_enabled` is true, HashJoin or HashAgg with string key is not supported now.", + "MPP mode may be blocked because when `new_collation_enabled` is true, HashJoin or HashAgg with string key is not supported now." + ] + }, + { + "SQL": "EXPLAIN SELECT * from t t1 join t t2 on t1.b=t2.b; -- 13. new collation FIXME", + "Plan": [ + "TableReader_34 12487.50 root data:ExchangeSender_33", + "└─ExchangeSender_33 12487.50 cop[tiflash] ExchangeType: PassThrough", + " └─HashJoin_32 12487.50 cop[tiflash] inner join, equal:[eq(test.t.b, test.t.b)]", + " ├─ExchangeReceiver_15(Build) 9990.00 cop[tiflash] ", + " │ └─ExchangeSender_14 9990.00 cop[tiflash] ExchangeType: Broadcast", + " │ └─Selection_13 9990.00 cop[tiflash] not(isnull(test.t.b))", + " │ └─TableFullScan_12 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo", + " └─Selection_17(Probe) 9990.00 cop[tiflash] not(isnull(test.t.b))", + " └─TableFullScan_16 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo" + ], + "Warn": null + } + ] + } +] From 02f7793cfd48a54137cb92850d133ae08b581575 Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 20 Jul 2021 14:09:42 +0800 Subject: [PATCH 2/3] resolve conflict --- planner/core/task.go | 14 - .../core/testdata/enforce_mpp_suite_out.json | 462 ------------------ 2 files changed, 476 deletions(-) delete mode 100644 planner/core/testdata/enforce_mpp_suite_out.json diff --git a/planner/core/task.go b/planner/core/task.go index 864e17456e0be..26ac8bc46ccc0 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1363,17 +1363,9 @@ func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFunc for _, aggFunc := range aggFuncs { // if the aggFunc contain VirtualColumn or CorrelatedColumn, it can not be pushed down. if expression.ContainVirtualColumn(aggFunc.Args) || expression.ContainCorrelatedColumn(aggFunc.Args) { -<<<<<<< HEAD - return false - } - pb := aggregation.AggFuncToPBExpr(sc, client, aggFunc) - if pb == nil { - return false -======= reason = "expressions of AggFunc `" + aggFunc.Name + "` contain virtual column or correlated column, which is not supported now" ret = false break ->>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) } if !aggregation.CheckAggPushDown(aggFunc, storeType) { reason = "AggFunc `" + aggFunc.Name + "` is not supported now" @@ -1392,10 +1384,6 @@ func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFunc break } } -<<<<<<< HEAD - if expression.ContainVirtualColumn(groupByItems) { - return false -======= if ret && expression.ContainVirtualColumn(groupByItems) { reason = "groupByItems contain virtual columns, which is not supported now" ret = false @@ -1406,13 +1394,11 @@ func CheckAggCanPushCop(sctx sessionctx.Context, aggFuncs []*aggregation.AggFunc } if !ret && sc.InExplainStmt { - sctx.GetSessionVars().RaiseWarningWhenMPPEnforced("MPP mode may be blocked because " + reason) storageName := storeType.Name() if storeType == kv.UnSpecified { storageName = "storage layer" } sc.AppendWarning(errors.New("Aggregation can not be pushed to " + storageName + " because " + reason)) ->>>>>>> 27489d43b... planner: Log warnings when agg function can not be pushdown in explain statement (#25553) } return ret } diff --git a/planner/core/testdata/enforce_mpp_suite_out.json b/planner/core/testdata/enforce_mpp_suite_out.json deleted file mode 100644 index 63787fde4ff7c..0000000000000 --- a/planner/core/testdata/enforce_mpp_suite_out.json +++ /dev/null @@ -1,462 +0,0 @@ -[ - { - "Name": "TestEnforceMPP", - "Cases": [ - { - "SQL": "select @@tidb_allow_mpp", - "Plan": [ - "1" - ], - "Warn": null - }, - { - "SQL": "select @@tidb_enforce_mpp", - "Plan": [ - "0" - ], - "Warn": null - }, - { - "SQL": "select @@tidb_opt_tiflash_concurrency_factor", - "Plan": [ - "24" - ], - "Warn": null - }, - { - "SQL": "set @@tidb_allow_mpp=0", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain format='verbose' select count(*) from t where a=1", - "Plan": [ - "StreamAgg_24 1.00 485.00 root funcs:count(Column#6)->Column#4", - "└─IndexReader_25 1.00 32.88 root index:StreamAgg_9", - " └─StreamAgg_9 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", - " └─IndexRangeScan_23 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", - "Plan": [ - "StreamAgg_17 1.00 485.00 root funcs:count(Column#6)->Column#4", - "└─IndexReader_18 1.00 32.88 root index:StreamAgg_9", - " └─StreamAgg_9 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", - " └─IndexRangeScan_16 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", - "Plan": [ - "StreamAgg_20 1.00 285050.00 root funcs:count(Column#6)->Column#4", - "└─TableReader_21 1.00 19003.88 root data:StreamAgg_9", - " └─StreamAgg_9 1.00 19006.88 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_19 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_18 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "set @@tidb_allow_mpp=1;", - "Plan": null, - "Warn": null - }, - { - "SQL": "set @@tidb_enforce_mpp=0;", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain format='verbose' select count(*) from t where a=1", - "Plan": [ - "StreamAgg_30 1.00 485.00 root funcs:count(Column#7)->Column#4", - "└─IndexReader_31 1.00 32.88 root index:StreamAgg_10", - " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#7", - " └─IndexRangeScan_29 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", - "Plan": [ - "StreamAgg_18 1.00 485.00 root funcs:count(Column#6)->Column#4", - "└─IndexReader_19 1.00 32.88 root index:StreamAgg_10", - " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", - " └─IndexRangeScan_17 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", - "Plan": [ - "HashAgg_21 1.00 11910.73 root funcs:count(Column#6)->Column#4", - "└─TableReader_23 1.00 11877.13 root data:ExchangeSender_22", - " └─ExchangeSender_22 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_20 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_19 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "set @@tidb_opt_tiflash_concurrency_factor = 1000000", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain format='verbose' select count(*) from t where a=1", - "Plan": [ - "HashAgg_24 1.00 33.89 root funcs:count(Column#6)->Column#4", - "└─TableReader_26 1.00 0.29 root data:ExchangeSender_25", - " └─ExchangeSender_25 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_23 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_22 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", - "Plan": [ - "StreamAgg_18 1.00 485.00 root funcs:count(Column#6)->Column#4", - "└─IndexReader_19 1.00 32.88 root index:StreamAgg_10", - " └─StreamAgg_10 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", - " └─IndexRangeScan_17 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", - "Plan": [ - "HashAgg_21 1.00 33.89 root funcs:count(Column#6)->Column#4", - "└─TableReader_23 1.00 0.29 root data:ExchangeSender_22", - " └─ExchangeSender_22 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg_9 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_20 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_19 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "set @@tidb_enforce_mpp=1;", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain format='verbose' select count(*) from t where a=1", - "Plan": [ - "HashAgg_25 1.00 33.60 root funcs:count(Column#6)->Column#4", - "└─TableReader_27 1.00 0.00 root data:ExchangeSender_26", - " └─ExchangeSender_26 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg_10 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_24 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_23 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": null - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1", - "Plan": [ - "StreamAgg_19 1.00 485.00 root funcs:count(Column#6)->Column#4", - "└─IndexReader_20 1.00 32.88 root index:StreamAgg_11", - " └─StreamAgg_11 1.00 35.88 cop[tikv] funcs:count(1)->Column#6", - " └─IndexRangeScan_18 10.00 455.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because you have set a hint to read table `t` from TiKV." - ] - }, - { - "SQL": "explain format='verbose' select /*+ read_from_storage(tiflash[t]) */ count(*) from t where a=1", - "Plan": [ - "HashAgg_22 1.00 33.60 root funcs:count(Column#6)->Column#4", - "└─TableReader_24 1.00 0.00 root data:ExchangeSender_23", - " └─ExchangeSender_23 1.00 285050.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg_10 1.00 285050.00 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_21 10.00 285020.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_20 10000.00 255020.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": null - } - ] - }, - { - "Name": "TestEnforceMPPWarning1", - "Cases": [ - { - "SQL": "set @@tidb_allow_mpp=1;set @@tidb_enforce_mpp=1;", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain select count(*) from t where a=1 -- 1. no replica", - "Plan": [ - "StreamAgg_17 1.00 root funcs:count(Column#7)->Column#5", - "└─IndexReader_18 1.00 root index:StreamAgg_9", - " └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#7", - " └─IndexRangeScan_16 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because there aren't tiflash replicas of table `t`." - ] - }, - { - "SQL": "cmd: create-replica", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain select count(*) from t where a=1 -- 2. replica not ready", - "Plan": [ - "StreamAgg_17 1.00 root funcs:count(Column#7)->Column#5", - "└─IndexReader_18 1.00 root index:StreamAgg_9", - " └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#7", - " └─IndexRangeScan_16 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because tiflash replicas of table `t` not ready." - ] - }, - { - "SQL": "cmd: enable-replica", - "Plan": null, - "Warn": null - }, - { - "SQL": "set @@session.tidb_isolation_read_engines = 'tikv';", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain select count(*) from t where a=1 -- 3. isolation_engine not match", - "Plan": [ - "StreamAgg_17 1.00 root funcs:count(Column#7)->Column#5", - "└─IndexReader_18 1.00 root index:StreamAgg_9", - " └─StreamAgg_9 1.00 cop[tikv] funcs:count(1)->Column#7", - " └─IndexRangeScan_16 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because 'tidb_isolation_read_engines'(value: 'tikv') not match, need 'tiflash'." - ] - }, - { - "SQL": "set @@session.tidb_isolation_read_engines = 'tikv, tiflash';", - "Plan": null, - "Warn": null - }, - { - "SQL": "explain select /*+ read_from_storage(tikv[t]) */ count(*) from t where a=1 -- 4. hint use tikv", - "Plan": [ - "StreamAgg_19 1.00 root funcs:count(Column#7)->Column#5", - "└─IndexReader_20 1.00 root index:StreamAgg_11", - " └─StreamAgg_11 1.00 cop[tikv] funcs:count(1)->Column#7", - " └─IndexRangeScan_18 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because you have set a hint to read table `t` from TiKV." - ] - }, - { - "SQL": "explain SELECT a, ROW_NUMBER() OVER (ORDER BY a) FROM t; -- 5. window unsupported", - "Plan": [ - "Window_7 10000.00 root row_number()->Column#6 over(order by test.t.a rows between current row and current row)", - "└─IndexReader_9 10000.00 root index:IndexFullScan_8", - " └─IndexFullScan_8 10000.00 cop[tikv] table:t, index:idx(a) keep order:true, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because operator `Window` is not supported now.", - "MPP mode may be blocked because operator `Window` is not supported now." - ] - }, - { - "SQL": "EXPLAIN SELECT t1.b FROM t t1 join t t2 where t1.a=t2.a; -- 6. virtual column", - "Plan": [ - "HashJoin_36 12487.50 root inner join, equal:[eq(test.t.a, test.t.a)]", - "├─TableReader_56(Build) 9990.00 root data:Selection_55", - "│ └─Selection_55 9990.00 cop[tiflash] not(isnull(test.t.a))", - "│ └─TableFullScan_54 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo", - "└─TableReader_50(Probe) 9990.00 root data:Selection_49", - " └─Selection_49 9990.00 cop[tiflash] not(isnull(test.t.a))", - " └─TableFullScan_48 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because column `test.t.b` is a virtual column which is not supported now." - ] - }, - { - "SQL": "EXPLAIN SELECT count(b) from t where a=1; -- 7. agg func has virtual column", - "Plan": [ - "StreamAgg_11 1.00 root funcs:count(test.t.b)->Column#5", - "└─IndexLookUp_42 10.00 root ", - " ├─IndexRangeScan_40(Build) 10.00 cop[tikv] table:t, index:idx(a) range:[1,1], keep order:false, stats:pseudo", - " └─TableRowIDScan_41(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "Aggregation can not be pushed to tikv because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "MPP mode may be blocked because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now", - "Aggregation can not be pushed to tiflash because expressions of AggFunc `count` contain virtual column or correlated column, which is not supported now" - ] - }, - { - "SQL": "EXPLAIN SELECT count(*) from t group by b; -- 8. group by virtual column", - "Plan": [ - "HashAgg_6 8000.00 root group by:test.t.b, funcs:count(1)->Column#5", - "└─Projection_12 10000.00 root test.t.b", - " └─TableReader_11 10000.00 root data:TableFullScan_10", - " └─TableFullScan_10 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", - "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", - "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", - "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now", - "MPP mode may be blocked because groupByItems contain virtual columns, which is not supported now", - "Aggregation can not be pushed to tiflash because groupByItems contain virtual columns, which is not supported now" - ] - }, - { - "SQL": "EXPLAIN SELECT group_concat(a) from t; -- 9. agg func not supported", - "Plan": [ - "HashAgg_6 1.00 root funcs:group_concat(Column#6 separator \",\")->Column#5", - "└─Projection_31 10000.00 root cast(test.t.a, var_string(20))->Column#6", - " └─TableReader_14 10000.00 root data:TableFullScan_12", - " └─TableFullScan_12 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because AggFunc `group_concat` is not supported now", - "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", - "MPP mode may be blocked because AggFunc `group_concat` is not supported now", - "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", - "MPP mode may be blocked because AggFunc `group_concat` is not supported now", - "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now", - "MPP mode may be blocked because AggFunc `group_concat` is not supported now", - "Aggregation can not be pushed to tiflash because AggFunc `group_concat` is not supported now" - ] - }, - { - "SQL": "EXPLAIN SELECT count(a) from t group by md5(a); -- 10. scalar func not supported", - "Plan": [ - "HashAgg_6 8000.00 root group by:Column#7, funcs:count(Column#6)->Column#5", - "└─Projection_19 10000.00 root test.t.a, md5(cast(test.t.a, var_string(20)))->Column#7", - " └─TableReader_12 10000.00 root data:TableFullScan_10", - " └─TableFullScan_10 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": [ - "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", - "MPP mode may be blocked because groupByItems contain unsupported exprs", - "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", - "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", - "MPP mode may be blocked because groupByItems contain unsupported exprs", - "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs", - "Scalar function 'md5'(signature: MD5) can not be pushed to tiflash", - "MPP mode may be blocked because groupByItems contain unsupported exprs", - "Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs" - ] - }, - { - "SQL": "EXPLAIN SELECT count(a) from t where c=1; -- 11. type not supported", - "Plan": [ - "HashAgg_7 1.00 root funcs:count(test.t.a)->Column#5", - "└─Selection_17 10000.00 root eq(test.t.c, 00:00:01.000000)", - " └─TableReader_16 10000.00 root data:TableFullScan_15", - " └─TableFullScan_15 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": [ - "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", - "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", - "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", - "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type", - "Expr 'test.t.c' can not be pushed to TiFlash because it contains Duration type" - ] - } - ] - }, - { - "Name": "TestEnforceMPPWarning2", - "Cases": [ - { - "SQL": "set @@tidb_allow_mpp=1;set @@tidb_enforce_mpp=1;", - "Plan": null, - "Warn": null - }, - { - "SQL": "set @@tidb_partition_prune_mode=static;", - "Plan": null, - "Warn": null - }, - { - "SQL": "EXPLAIN SELECT count(*) from t where a=1; -- 12. static partition prune", - "Plan": [ - "StreamAgg_32 1.00 root funcs:count(Column#6)->Column#4", - "└─TableReader_33 1.00 root data:StreamAgg_13", - " └─StreamAgg_13 1.00 batchCop[tiflash] funcs:count(1)->Column#6", - " └─Selection_31 10.00 batchCop[tiflash] eq(test.t.a, 1)", - " └─TableFullScan_30 10000.00 batchCop[tiflash] table:t, partition:p0 keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because table `t`is a partition table which is not supported when `@@tidb_partition_prune_mode=static`." - ] - }, - { - "SQL": "set @@tidb_partition_prune_mode=dynamic;", - "Plan": null, - "Warn": null - } - ] - }, - { - "Name": "TestEnforceMPPWarning3", - "Cases": [ - { - "SQL": "set @@tidb_allow_mpp=1;set @@tidb_enforce_mpp=1;", - "Plan": null, - "Warn": null - }, - { - "SQL": "cmd: enable-new-collation", - "Plan": null, - "Warn": null - }, - { - "SQL": "EXPLAIN SELECT count(*) from t group by b; -- 13. new collation FIXME", - "Plan": [ - "HashAgg_24 8000.00 root group by:test.t.b, funcs:count(Column#7)->Column#4", - "└─TableReader_26 8000.00 root data:ExchangeSender_25", - " └─ExchangeSender_25 8000.00 batchCop[tiflash] ExchangeType: PassThrough", - " └─HashAgg_11 8000.00 batchCop[tiflash] group by:test.t.b, funcs:count(1)->Column#7", - " └─TableFullScan_21 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" - ], - "Warn": [ - "MPP mode may be blocked because when `new_collation_enabled` is true, HashJoin or HashAgg with string key is not supported now.", - "MPP mode may be blocked because when `new_collation_enabled` is true, HashJoin or HashAgg with string key is not supported now.", - "MPP mode may be blocked because when `new_collation_enabled` is true, HashJoin or HashAgg with string key is not supported now." - ] - }, - { - "SQL": "EXPLAIN SELECT * from t t1 join t t2 on t1.b=t2.b; -- 13. new collation FIXME", - "Plan": [ - "TableReader_34 12487.50 root data:ExchangeSender_33", - "└─ExchangeSender_33 12487.50 cop[tiflash] ExchangeType: PassThrough", - " └─HashJoin_32 12487.50 cop[tiflash] inner join, equal:[eq(test.t.b, test.t.b)]", - " ├─ExchangeReceiver_15(Build) 9990.00 cop[tiflash] ", - " │ └─ExchangeSender_14 9990.00 cop[tiflash] ExchangeType: Broadcast", - " │ └─Selection_13 9990.00 cop[tiflash] not(isnull(test.t.b))", - " │ └─TableFullScan_12 10000.00 cop[tiflash] table:t1 keep order:false, stats:pseudo", - " └─Selection_17(Probe) 9990.00 cop[tiflash] not(isnull(test.t.b))", - " └─TableFullScan_16 10000.00 cop[tiflash] table:t2 keep order:false, stats:pseudo" - ], - "Warn": null - } - ] - } -] From 09cc539749e69b961cf907e64d870d0173927f54 Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 20 Jul 2021 15:54:31 +0800 Subject: [PATCH 3/3] fix ci --- planner/core/exhaust_physical_plans.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index eb04683603318..0c2369bdae7bc 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2445,7 +2445,7 @@ func (la *LogicalAggregation) getHashAggs(prop *property.PhysicalProperty) []Phy taskTypes = append(taskTypes, property.CopTiFlashLocalReadTaskType) } canPushDownToTiFlash := la.canPushToCop(kv.TiFlash) - canPushDownToMPP := la.ctx.GetSessionVars().AllowMPPExecution && la.checkCanPushDownToMPP() && canPushDownToTiFlash + canPushDownToMPP := canPushDownToTiFlash && la.ctx.GetSessionVars().AllowMPPExecution && la.checkCanPushDownToMPP() if la.HasDistinct() { // TODO: remove after the cost estimation of distinct pushdown is implemented. if !la.ctx.GetSessionVars().AllowDistinctAggPushDown {