From 02ceded78bac34285d3ce1aceda86b848bad673d Mon Sep 17 00:00:00 2001 From: Elsa <111482174+elsa0520@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:51:10 +0800 Subject: [PATCH] planner: add new logical rule for constant propagation (#46544) close pingcap/tidb#15082 --- cmd/explaintest/r/explain_cte.result | 46 +-- executor/index_advise_test.go | 21 +- expression/constant_propagation.go | 14 +- planner/cardinality/selectivity_test.go | 48 ++- .../testdata/cardinality_suite_in.json | 8 + .../testdata/cardinality_suite_out.json | 39 +++ planner/core/BUILD.bazel | 2 + .../hint/testdata/integration_suite_out.json | 50 +-- planner/core/logical_plan_builder.go | 1 + planner/core/main_test.go | 5 + planner/core/optimizer.go | 2 + planner/core/physical_plan_trace_test.go | 3 +- planner/core/plan.go | 6 + planner/core/rule_constant_propagation.go | 284 ++++++++++++++++++ .../core/rule_constant_propagation_test.go | 153 ++++++++++ planner/core/rule_eliminate_projection.go | 11 + .../rule_constant_propagation_suite_in.json | 43 +++ .../rule_constant_propagation_suite_out.json | 243 +++++++++++++++ 18 files changed, 887 insertions(+), 92 deletions(-) create mode 100644 planner/core/rule_constant_propagation.go create mode 100644 planner/core/rule_constant_propagation_test.go create mode 100644 planner/core/testdata/rule_constant_propagation_suite_in.json create mode 100644 planner/core/testdata/rule_constant_propagation_suite_out.json diff --git a/cmd/explaintest/r/explain_cte.result b/cmd/explaintest/r/explain_cte.result index 6de5e74e259a3..f5647afff215f 100644 --- a/cmd/explaintest/r/explain_cte.result +++ b/cmd/explaintest/r/explain_cte.result @@ -511,17 +511,17 @@ a b explain insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; id estRows task access object operator info Insert_1 N/A root N/A -└─HashJoin_25 15.61 root inner join, equal:[eq(explain_cte.t2.c, explain_cte.t1.a)] - ├─HashJoin_27(Build) 12.49 root inner join, equal:[eq(explain_cte.t3.e, explain_cte.t2.d)] - │ ├─TableReader_30(Build) 9.99 root data:Selection_29 - │ │ └─Selection_29 9.99 cop[tikv] eq(explain_cte.t3.f, 1234), not(isnull(explain_cte.t3.e)) - │ │ └─TableFullScan_28 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo - │ └─TableReader_33(Probe) 9980.01 root data:Selection_32 - │ └─Selection_32 9980.01 cop[tikv] not(isnull(explain_cte.t2.c)), not(isnull(explain_cte.t2.d)) - │ └─TableFullScan_31 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo - └─TableReader_36(Probe) 9990.00 root data:Selection_35 - └─Selection_35 9990.00 cop[tikv] not(isnull(explain_cte.t1.a)) - └─TableFullScan_34 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─HashJoin_26 15.61 root inner join, equal:[eq(explain_cte.t2.c, explain_cte.t1.a)] + ├─HashJoin_28(Build) 12.49 root inner join, equal:[eq(explain_cte.t3.e, explain_cte.t2.d)] + │ ├─TableReader_31(Build) 9.99 root data:Selection_30 + │ │ └─Selection_30 9.99 cop[tikv] eq(explain_cte.t3.f, 1234), not(isnull(explain_cte.t3.e)) + │ │ └─TableFullScan_29 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo + │ └─TableReader_34(Probe) 9980.01 root data:Selection_33 + │ └─Selection_33 9980.01 cop[tikv] not(isnull(explain_cte.t2.c)), not(isnull(explain_cte.t2.d)) + │ └─TableFullScan_32 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableReader_37(Probe) 9990.00 root data:Selection_36 + └─Selection_36 9990.00 cop[tikv] not(isnull(explain_cte.t1.a)) + └─TableFullScan_35 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo insert into t1 select t1.a, t1.b from t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; select * from t1; a b @@ -530,18 +530,18 @@ a b explain delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; id estRows task access object operator info Delete_17 N/A root N/A -└─Projection_22 15.61 root explain_cte.t1.a, explain_cte.t1.b, explain_cte.t1._tidb_rowid, explain_cte.t2.c - └─HashJoin_24 15.61 root inner join, equal:[eq(explain_cte.t2.c, explain_cte.t1.a)] - ├─HashJoin_26(Build) 12.49 root inner join, equal:[eq(explain_cte.t3.e, explain_cte.t2.d)] - │ ├─TableReader_29(Build) 9.99 root data:Selection_28 - │ │ └─Selection_28 9.99 cop[tikv] eq(explain_cte.t3.f, 1234), not(isnull(explain_cte.t3.e)) - │ │ └─TableFullScan_27 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo - │ └─TableReader_32(Probe) 9980.01 root data:Selection_31 - │ └─Selection_31 9980.01 cop[tikv] not(isnull(explain_cte.t2.c)), not(isnull(explain_cte.t2.d)) - │ └─TableFullScan_30 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo - └─TableReader_35(Probe) 9990.00 root data:Selection_34 - └─Selection_34 9990.00 cop[tikv] not(isnull(explain_cte.t1.a)) - └─TableFullScan_33 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_23 15.61 root explain_cte.t1.a, explain_cte.t1.b, explain_cte.t1._tidb_rowid, explain_cte.t2.c + └─HashJoin_25 15.61 root inner join, equal:[eq(explain_cte.t2.c, explain_cte.t1.a)] + ├─HashJoin_27(Build) 12.49 root inner join, equal:[eq(explain_cte.t3.e, explain_cte.t2.d)] + │ ├─TableReader_30(Build) 9.99 root data:Selection_29 + │ │ └─Selection_29 9.99 cop[tikv] eq(explain_cte.t3.f, 1234), not(isnull(explain_cte.t3.e)) + │ │ └─TableFullScan_28 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo + │ └─TableReader_33(Probe) 9980.01 root data:Selection_32 + │ └─Selection_32 9980.01 cop[tikv] not(isnull(explain_cte.t2.c)), not(isnull(explain_cte.t2.d)) + │ └─TableFullScan_31 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo + └─TableReader_36(Probe) 9990.00 root data:Selection_35 + └─Selection_35 9990.00 cop[tikv] not(isnull(explain_cte.t1.a)) + └─TableFullScan_34 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo delete from t1 using t1 inner join (select t2.c from t2 inner join (with temp as (select e from t3 where t3.f = 1234) select e from temp) tt on t2.d = tt.e) t on t1.a = t.c; select * from t1; a b diff --git a/executor/index_advise_test.go b/executor/index_advise_test.go index af7a593838584..92dac62b12345 100644 --- a/executor/index_advise_test.go +++ b/executor/index_advise_test.go @@ -101,23 +101,22 @@ and b.txn_accno = a.new_accno;` rows := [][]interface{}{ {"Update_8"}, {"└─IndexJoin_14"}, - {" ├─TableReader_25(Build)"}, - {" │ └─Selection_24"}, - {" │ └─TableFullScan_23"}, + {" ├─TableReader_23(Build)"}, + {" │ └─Selection_22"}, + {" │ └─TableFullScan_21"}, {" └─IndexReader_12(Probe)"}, - {" └─Selection_11"}, - {" └─IndexRangeScan_10"}, + {" └─IndexRangeScan_11"}, } tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='ON'") tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) rows = [][]interface{}{ {"Update_8"}, - {"└─HashJoin_10"}, - {" ├─IndexReader_17(Build)"}, - {" │ └─IndexRangeScan_16"}, - {" └─TableReader_14(Probe)"}, - {" └─Selection_13"}, - {" └─TableFullScan_12"}, + {"└─HashJoin_12"}, + {" ├─TableReader_15(Build)"}, + {" │ └─Selection_14"}, + {" │ └─TableFullScan_13"}, + {" └─IndexReader_18(Probe)"}, + {" └─IndexRangeScan_17"}, } tk.MustExec("set @@session.tidb_enable_inl_join_inner_multi_pattern='OFF'") tk.MustQuery("explain "+sql).CheckAt([]int{0}, rows) diff --git a/expression/constant_propagation.go b/expression/constant_propagation.go index 75147dcac3723..0fa8104f9f0e8 100644 --- a/expression/constant_propagation.go +++ b/expression/constant_propagation.go @@ -69,7 +69,8 @@ func (s *basePropConstSolver) tryToUpdateEQList(col *Column, con *Constant) (boo return true, false } -func validEqualCondHelper(ctx sessionctx.Context, eq *ScalarFunction, colIsLeft bool) (*Column, *Constant) { +// ValidCompareConstantPredicateHelper checks if the predicate is a compare constant predicate, like "Column xxx Constant" +func ValidCompareConstantPredicateHelper(eq *ScalarFunction, colIsLeft bool) (*Column, *Constant) { var col *Column var con *Constant colOk := false @@ -97,14 +98,14 @@ func validEqualCondHelper(ctx sessionctx.Context, eq *ScalarFunction, colIsLeft } // validEqualCond checks if the cond is an expression like [column eq constant]. -func validEqualCond(ctx sessionctx.Context, cond Expression) (*Column, *Constant) { +func validEqualCond(cond Expression) (*Column, *Constant) { if eq, ok := cond.(*ScalarFunction); ok { if eq.FuncName.L != ast.EQ { return nil, nil } - col, con := validEqualCondHelper(ctx, eq, true) + col, con := ValidCompareConstantPredicateHelper(eq, true) if col == nil { - return validEqualCondHelper(ctx, eq, false) + return ValidCompareConstantPredicateHelper(eq, false) } return col, con } @@ -296,7 +297,7 @@ func (s *propConstSolver) pickNewEQConds(visited []bool) (retMapper map[int]*Con if visited[i] { continue } - col, con := validEqualCond(s.ctx, cond) + col, con := validEqualCond(cond) // Then we check if this CNF item is a false constant. If so, we will set the whole condition to false. var ok bool if col == nil { @@ -357,6 +358,7 @@ func (s *propConstSolver) solve(conditions []Expression) []Expression { } // PropagateConstant propagate constant values of deterministic predicates in a condition. +// This is a constant propagation logic for expression list such as ['a=1', 'a=b'] func PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression { return newPropConstSolver().PropagateConstant(ctx, conditions) } @@ -400,7 +402,7 @@ func (s *propOuterJoinConstSolver) pickEQCondsOnOuterCol(retMapper map[int]*Cons if visited[i+condsOffset] { continue } - col, con := validEqualCond(s.ctx, cond) + col, con := validEqualCond(cond) // Then we check if this CNF item is a false constant. If so, we will set the whole condition to false. var ok bool if col == nil { diff --git a/planner/cardinality/selectivity_test.go b/planner/cardinality/selectivity_test.go index 0f897386553f4..37e53b68f760d 100644 --- a/planner/cardinality/selectivity_test.go +++ b/planner/cardinality/selectivity_test.go @@ -1202,33 +1202,29 @@ func TestIndexJoinInnerRowCountUpperBound(t *testing.T) { stat := h.GetTableStats(tblInfo) stat.HistColl = mockStatsTbl.HistColl - query := "explain format = 'brief' " + - "select /*+ inl_join(t2) */ * from (select * from t where t.a < 1) as t1 join t t2 where t2.a = 0 and t1.a = t2.b" - - testKit.MustQuery(query).Check(testkit.Rows( - "IndexJoin 1000000.00 root inner join, inner:IndexLookUp, outer key:test.t.a, inner key:test.t.b, equal cond:eq(test.t.a, test.t.b)", - "├─TableReader(Build) 1000.00 root data:Selection", - "│ └─Selection 1000.00 cop[tikv] lt(test.t.a, 1), not(isnull(test.t.a))", - "│ └─TableFullScan 500000.00 cop[tikv] table:t keep order:false, stats:pseudo", - "└─IndexLookUp(Probe) 1000000.00 root ", - " ├─Selection(Build) 500000000.00 cop[tikv] not(isnull(test.t.b))", - " │ └─IndexRangeScan 500000000.00 cop[tikv] table:t2, index:idx(b) range: decided by [eq(test.t.b, test.t.a)], keep order:false, stats:pseudo", - " └─Selection(Probe) 1000000.00 cop[tikv] eq(test.t.a, 0)", - " └─TableRowIDScan 500000000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", - )) + var ( + input []string + output []struct { + Query string + Result []string + } + ) - testKit.MustExec("set @@tidb_opt_fix_control = '44855:ON'") - testKit.MustQuery(query).Check(testkit.Rows( - "IndexJoin 1000000.00 root inner join, inner:IndexLookUp, outer key:test.t.a, inner key:test.t.b, equal cond:eq(test.t.a, test.t.b)", - "├─TableReader(Build) 1000.00 root data:Selection", - "│ └─Selection 1000.00 cop[tikv] lt(test.t.a, 1), not(isnull(test.t.a))", - "│ └─TableFullScan 500000.00 cop[tikv] table:t keep order:false, stats:pseudo", - "└─IndexLookUp(Probe) 1000000.00 root ", - " ├─Selection(Build) 1000000.00 cop[tikv] not(isnull(test.t.b))", - " │ └─IndexRangeScan 1000000.00 cop[tikv] table:t2, index:idx(b) range: decided by [eq(test.t.b, test.t.a)], keep order:false, stats:pseudo", - " └─Selection(Probe) 1000000.00 cop[tikv] eq(test.t.a, 0)", - " └─TableRowIDScan 1000000.00 cop[tikv] table:t2 keep order:false, stats:pseudo", - )) + suiteData := cardinality.GetCardinalitySuiteData() + suiteData.LoadTestCases(t, &input, &output) + for i := 0; i < len(input); i++ { + testdata.OnRecord(func() { + output[i].Query = input[i] + }) + if !strings.HasPrefix(input[i], "explain") { + testKit.MustExec(input[i]) + continue + } + testdata.OnRecord(func() { + output[i].Result = testdata.ConvertRowsToStrings(testKit.MustQuery(input[i]).Rows()) + }) + testKit.MustQuery(input[i]).Check(testkit.Rows(output[i].Result...)) + } } func TestOrderingIdxSelectivityThreshold(t *testing.T) { diff --git a/planner/cardinality/testdata/cardinality_suite_in.json b/planner/cardinality/testdata/cardinality_suite_in.json index 9560538250e76..9e9348905e348 100644 --- a/planner/cardinality/testdata/cardinality_suite_in.json +++ b/planner/cardinality/testdata/cardinality_suite_in.json @@ -454,5 +454,13 @@ "select * from t where a = 100 and b = 350", "select * from t where a < -1500 and b > 400 and b < 403" ] + }, + { + "name": "TestIndexJoinInnerRowCountUpperBound", + "cases": [ + "explain format = 'brief' select /*+ inl_join(t2) */ * from (select * from t where t.a < 1) as t1 join t t2 where t2.a = 0 and t1.a = t2.b", + "set @@tidb_opt_fix_control = '44855:ON'", + "explain format = 'brief' select /*+ inl_join(t2) */ * from (select * from t where t.a < 1) as t1 join t t2 where t2.a = 0 and t1.a = t2.b" + ] } ] diff --git a/planner/cardinality/testdata/cardinality_suite_out.json b/planner/cardinality/testdata/cardinality_suite_out.json index 7568b66ea8ca4..d28055fa54b12 100644 --- a/planner/cardinality/testdata/cardinality_suite_out.json +++ b/planner/cardinality/testdata/cardinality_suite_out.json @@ -4630,5 +4630,44 @@ ] } ] + }, + { + "Name": "TestIndexJoinInnerRowCountUpperBound", + "Cases": [ + { + "Query": "explain format = 'brief' select /*+ inl_join(t2) */ * from (select * from t where t.a < 1) as t1 join t t2 where t2.a = 0 and t1.a = t2.b", + "Result": [ + "Projection 2000.00 root test.t.a, test.t.b, test.t.a, test.t.b", + "└─IndexJoin 2000.00 root inner join, inner:IndexLookUp, outer key:test.t.a, inner key:test.t.b, equal cond:eq(test.t.a, test.t.b)", + " ├─TableReader(Build) 1000.00 root data:Selection", + " │ └─Selection 1000.00 cop[tikv] lt(test.t.a, 1), not(isnull(test.t.a))", + " │ └─TableFullScan 500000.00 cop[tikv] table:t keep order:false, stats:pseudo", + " └─IndexLookUp(Probe) 2000.00 root ", + " ├─Selection(Build) 1000000.00 cop[tikv] lt(test.t.b, 1), not(isnull(test.t.b))", + " │ └─IndexRangeScan 500000000.00 cop[tikv] table:t2, index:idx(b) range: decided by [eq(test.t.b, test.t.a)], keep order:false, stats:pseudo", + " └─Selection(Probe) 2000.00 cop[tikv] eq(test.t.a, 0)", + " └─TableRowIDScan 1000000.00 cop[tikv] table:t2 keep order:false, stats:pseudo" + ] + }, + { + "Query": "set @@tidb_opt_fix_control = '44855:ON'", + "Result": null + }, + { + "Query": "explain format = 'brief' select /*+ inl_join(t2) */ * from (select * from t where t.a < 1) as t1 join t t2 where t2.a = 0 and t1.a = t2.b", + "Result": [ + "Projection 2000.00 root test.t.a, test.t.b, test.t.a, test.t.b", + "└─IndexJoin 2000.00 root inner join, inner:IndexLookUp, outer key:test.t.a, inner key:test.t.b, equal cond:eq(test.t.a, test.t.b)", + " ├─TableReader(Build) 1000.00 root data:Selection", + " │ └─Selection 1000.00 cop[tikv] lt(test.t.a, 1), not(isnull(test.t.a))", + " │ └─TableFullScan 500000.00 cop[tikv] table:t keep order:false, stats:pseudo", + " └─IndexLookUp(Probe) 2000.00 root ", + " ├─Selection(Build) 1000000.00 cop[tikv] lt(test.t.b, 1), not(isnull(test.t.b))", + " │ └─IndexRangeScan 1000000.00 cop[tikv] table:t2, index:idx(b) range: decided by [eq(test.t.b, test.t.a)], keep order:false, stats:pseudo", + " └─Selection(Probe) 2000.00 cop[tikv] eq(test.t.a, 0)", + " └─TableRowIDScan 1000000.00 cop[tikv] table:t2 keep order:false, stats:pseudo" + ] + } + ] } ] diff --git a/planner/core/BUILD.bazel b/planner/core/BUILD.bazel index 643509993f70f..1dfd3bd328de8 100644 --- a/planner/core/BUILD.bazel +++ b/planner/core/BUILD.bazel @@ -50,6 +50,7 @@ go_library( "rule_aggregation_skew_rewrite.go", "rule_build_key_info.go", "rule_column_pruning.go", + "rule_constant_propagation.go", "rule_decorrelate.go", "rule_derive_topn_from_window.go", "rule_eliminate_projection.go", @@ -221,6 +222,7 @@ go_test( "planbuilder_test.go", "point_get_plan_test.go", "preprocess_test.go", + "rule_constant_propagation_test.go", "rule_generate_column_substitute_test.go", "rule_join_reorder_dp_test.go", "rule_join_reorder_test.go", diff --git a/planner/core/casetest/hint/testdata/integration_suite_out.json b/planner/core/casetest/hint/testdata/integration_suite_out.json index 5bab4452ff26b..917ecf7cbae45 100644 --- a/planner/core/casetest/hint/testdata/integration_suite_out.json +++ b/planner/core/casetest/hint/testdata/integration_suite_out.json @@ -2381,43 +2381,43 @@ { "SQL": "explain with d1 as (\n select a from (\n select a from (\n select /*+ qb_name(qb, v4) use_index(t4@qb, idx_a) */ a from v4 where a < 10\n ) as t0 where a < 9\n ) as t1 where a < 8\n), d2 as (select /*+ qb_name(qb2, v4) use_index(t4@qb2, idx_b) */ a from v4 where b < 10)\n\nselect * from (select * from d1) as t0 join (select * from d2) as t1;", "Plan": [ - "HashJoin_41 6944.44 root CARTESIAN inner join", - "├─IndexLookUp_50(Build) 83.33 root ", - "│ ├─IndexRangeScan_47(Build) 250.00 cop[tikv] table:t4, index:idx_b(b) range:(3,10), keep order:false, stats:pseudo", - "│ └─Selection_49(Probe) 83.33 cop[tikv] gt(test.t4.a, 2)", - "│ └─TableRowIDScan_48 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo", - "└─IndexLookUp_46(Probe) 83.33 root ", - " ├─IndexRangeScan_43(Build) 250.00 cop[tikv] table:t4, index:idx_a(a) range:(2,8), keep order:false, stats:pseudo", - " └─Selection_45(Probe) 83.33 cop[tikv] gt(test.t4.b, 3)", - " └─TableRowIDScan_44 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo" + "HashJoin_42 6944.44 root CARTESIAN inner join", + "├─IndexLookUp_51(Build) 83.33 root ", + "│ ├─IndexRangeScan_48(Build) 250.00 cop[tikv] table:t4, index:idx_b(b) range:(3,10), keep order:false, stats:pseudo", + "│ └─Selection_50(Probe) 83.33 cop[tikv] gt(test.t4.a, 2)", + "│ └─TableRowIDScan_49 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo", + "└─IndexLookUp_47(Probe) 83.33 root ", + " ├─IndexRangeScan_44(Build) 250.00 cop[tikv] table:t4, index:idx_a(a) range:(2,8), keep order:false, stats:pseudo", + " └─Selection_46(Probe) 83.33 cop[tikv] gt(test.t4.b, 3)", + " └─TableRowIDScan_45 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo" ], "Warn": null }, { "SQL": "explain with d1 as (\n select a from (\n select a from (\n select a from v4 where a < 10\n ) as t0 where a < 9\n ) as t1 where a < 8\n), d2 as (select a from v4 where b < 10)\n\nselect /*+ qb_name(qb, v4@sel_4) use_index(t4@qb, idx_a) qb_name(qb2, v4@sel_5) use_index(t4@qb, idx_b) */ * from (select * from d1) as t0 join (select * from d2) as t1;", "Plan": [ - "HashJoin_41 6944.44 root CARTESIAN inner join", - "├─TableReader_53(Build) 83.33 root data:Selection_52", - "│ └─Selection_52 83.33 cop[tikv] gt(test.t4.a, 2), gt(test.t4.b, 3), lt(test.t4.b, 10)", - "│ └─TableFullScan_51 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo", - "└─IndexLookUp_46(Probe) 83.33 root ", - " ├─IndexRangeScan_43(Build) 250.00 cop[tikv] table:t4, index:idx_a(a) range:(2,8), keep order:false, stats:pseudo", - " └─Selection_45(Probe) 83.33 cop[tikv] gt(test.t4.b, 3)", - " └─TableRowIDScan_44 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo" + "HashJoin_42 6944.44 root CARTESIAN inner join", + "├─TableReader_54(Build) 83.33 root data:Selection_53", + "│ └─Selection_53 83.33 cop[tikv] gt(test.t4.a, 2), gt(test.t4.b, 3), lt(test.t4.b, 10)", + "│ └─TableFullScan_52 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo", + "└─IndexLookUp_47(Probe) 83.33 root ", + " ├─IndexRangeScan_44(Build) 250.00 cop[tikv] table:t4, index:idx_a(a) range:(2,8), keep order:false, stats:pseudo", + " └─Selection_46(Probe) 83.33 cop[tikv] gt(test.t4.b, 3)", + " └─TableRowIDScan_45 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo" ], "Warn": null }, { "SQL": "explain with d1 as (\n select a from (\n select a from (\n select /*+ qb_name(qb, v5) use_index(t4@qb, idx_a) */ a from v4 where a < 10\n ) as t0 where a < 9\n ) as t1 where a < 8\n), d2 as (select /*+ qb_name(qb2, v4) use_index(t4@qb2, idx_b) */ a from v4 where b < 10)\n\nselect * from (select * from d1) as t0 join (select * from d2) as t1;", "Plan": [ - "HashJoin_41 6944.44 root CARTESIAN inner join", - "├─IndexLookUp_57(Build) 83.33 root ", - "│ ├─IndexRangeScan_54(Build) 250.00 cop[tikv] table:t4, index:idx_b(b) range:(3,10), keep order:false, stats:pseudo", - "│ └─Selection_56(Probe) 83.33 cop[tikv] gt(test.t4.a, 2)", - "│ └─TableRowIDScan_55 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo", - "└─TableReader_45(Probe) 83.33 root data:Selection_44", - " └─Selection_44 83.33 cop[tikv] gt(test.t4.a, 2), gt(test.t4.b, 3), lt(test.t4.a, 10), lt(test.t4.a, 8), lt(test.t4.a, 9)", - " └─TableFullScan_43 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo" + "HashJoin_42 6944.44 root CARTESIAN inner join", + "├─IndexLookUp_58(Build) 83.33 root ", + "│ ├─IndexRangeScan_55(Build) 250.00 cop[tikv] table:t4, index:idx_b(b) range:(3,10), keep order:false, stats:pseudo", + "│ └─Selection_57(Probe) 83.33 cop[tikv] gt(test.t4.a, 2)", + "│ └─TableRowIDScan_56 250.00 cop[tikv] table:t4 keep order:false, stats:pseudo", + "└─TableReader_46(Probe) 83.33 root data:Selection_45", + " └─Selection_45 83.33 cop[tikv] gt(test.t4.a, 2), gt(test.t4.b, 3), lt(test.t4.a, 10), lt(test.t4.a, 8), lt(test.t4.a, 9)", + " └─TableFullScan_44 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo" ], "Warn": [ "The qb_name hint qb is unused, please check whether the table list in the qb_name hint qb is correct" diff --git a/planner/core/logical_plan_builder.go b/planner/core/logical_plan_builder.go index 67a8595a49d0d..102c3e84a31dc 100644 --- a/planner/core/logical_plan_builder.go +++ b/planner/core/logical_plan_builder.go @@ -523,6 +523,7 @@ func (b *PlanBuilder) buildResultSetNode(ctx context.Context, node ast.ResultSet case *ast.SelectStmt: ci := b.prepareCTECheckForSubQuery() defer resetCTECheckForSubQuery(ci) + b.optFlag = b.optFlag | flagConstantPropagation p, err = b.buildSelect(ctx, v) case *ast.SetOprStmt: ci := b.prepareCTECheckForSubQuery() diff --git a/planner/core/main_test.go b/planner/core/main_test.go index 81a1b98d06c96..e270e6ae1a48e 100644 --- a/planner/core/main_test.go +++ b/planner/core/main_test.go @@ -36,6 +36,7 @@ func TestMain(m *testing.M) { testDataMap.LoadTestSuiteData("testdata", "index_merge_suite") testDataMap.LoadTestSuiteData("testdata", "runtime_filter_generator_suite") testDataMap.LoadTestSuiteData("testdata", "join_reorder_suite") + testDataMap.LoadTestSuiteData("testdata", "rule_constant_propagation_suite") indexMergeSuiteData = testDataMap["index_merge_suite"] planSuiteUnexportedData = testDataMap["plan_suite_unexported"] @@ -67,3 +68,7 @@ func GetRuntimeFilterGeneratorData() testdata.TestData { func GetJoinReorderData() testdata.TestData { return testDataMap["join_reorder_suite"] } + +func GetRuleConstantPropagationData() testdata.TestData { + return testDataMap["rule_constant_propagation_suite"] +} diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index 7519935dfe8eb..df38bb17ca41c 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -74,6 +74,7 @@ const ( flagSkewDistinctAgg flagEliminateProjection flagMaxMinEliminate + flagConstantPropagation flagPredicatePushDown flagEliminateOuterJoin flagPartitionProcessor @@ -100,6 +101,7 @@ var optRuleList = []logicalOptRule{ &skewDistinctAggRewriter{}, &projectionEliminator{}, &maxMinEliminator{}, + &constantPropagationSolver{}, &ppdSolver{}, &outerJoinEliminator{}, &partitionProcessor{}, diff --git a/planner/core/physical_plan_trace_test.go b/planner/core/physical_plan_trace_test.go index 2344ffe204b90..583720af678f5 100644 --- a/planner/core/physical_plan_trace_test.go +++ b/planner/core/physical_plan_trace_test.go @@ -159,7 +159,8 @@ func TestPhysicalOptimizerTrace(t *testing.T) { plan, err := builder.Build(context.TODO(), stmt) require.NoError(t, err) flag := uint64(0) - flag = flag | 1<<1 | 1<<3 | 1<<8 | 1<<12 | 1<<15 + // flagGcSubstitute | flagStabilizeResults | flagSkewDistinctAgg | flagEliminateOuterJoin | flagPushDownAgg + flag |= flag | 1<<1 | 1<<3 | 1<<8 | 1<<13 | 1<<16 _, _, err = core.DoOptimize(context.TODO(), sctx, flag, plan.(core.LogicalPlan)) require.NoError(t, err) otrace := sctx.GetSessionVars().StmtCtx.OptimizeTracer.Physical diff --git a/planner/core/plan.go b/planner/core/plan.go index 80db51e37ea19..da472ded3f259 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -282,6 +282,12 @@ type LogicalPlan interface { // predicateSimplification consolidates different predcicates on a column and its equivalence classes. predicateSimplification(opt *logicalOptimizeOp) LogicalPlan + // constantPropagation generate new constant predicate according to column equivalence relation + constantPropagation(parentPlan LogicalPlan, currentChildIdx int, opt *logicalOptimizeOp) (newRoot LogicalPlan) + + // pullUpConstantPredicates recursive find constant predicate, used for the constant propagation rule + pullUpConstantPredicates() []expression.Expression + // recursiveDeriveStats derives statistic info between plans. recursiveDeriveStats(colGroups [][]*expression.Column) (*property.StatsInfo, error) diff --git a/planner/core/rule_constant_propagation.go b/planner/core/rule_constant_propagation.go new file mode 100644 index 0000000000000..b4ccd6ca7ca86 --- /dev/null +++ b/planner/core/rule_constant_propagation.go @@ -0,0 +1,284 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "context" + + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/parser/ast" +) + +// constantPropagationSolver can support constant propagated cross-query block. +// This is a logical optimize rule. +// It mainly used for the sub query in FromList and propagated the constant predicate +// from sub query to outer query. +// In the future, it will support propagate constant in WhereClause and SelectList. +// +// Example 1: +// Query: select * from t, (select * from s where s.id>1) tmp where t.id=tmp.id +// Optimized: select * from t, (select * from s where s.id>1) tmp where t.id=tmp.id and tmp.id>1 +// +// Process: +// 1. Match the Join + selection pattern and find the candidate constant predicate, such as 's.id>1' +// 2. Pull up the candidate constant predicate, above of Join node. +// The new selection will be created with the new constant predicate. 'tmp.id>1' +// +// Steps 1 and 2 will be called recursively +// 3. (ppdSolver in rule_predicate_push_down.go) Push down constant predicate +// and propagate constant predicate into other side. 't.id>1' +type constantPropagationSolver struct { +} + +// **Preorder traversal** of logic tree +// Step1: constant propagation current plan node +// Step2: optimize all of child +// +// For step1, different logical plan have their own logic for constant propagation, +// which is mainly implemented in the interface "constantPropagation" of LogicalPlan. +// Currently only the Logical Join implements this function. (Used for the subquery in FROM List) +// In the future, the Logical Apply will implements this function. (Used for the subquery in WHERE or SELECT list) +func (cp *constantPropagationSolver) optimize(_ context.Context, p LogicalPlan, opt *logicalOptimizeOp) (LogicalPlan, error) { + // constant propagation root plan + newRoot := p.constantPropagation(nil, 0, opt) + + // recursive optimize + for i, children := range p.Children() { + cp.execOptimize(children, p, i, opt) + } + + if newRoot == nil { + return p, nil + } + return newRoot, nil +} + +// execOptimize optimize constant propagation exclude root plan node +func (cp *constantPropagationSolver) execOptimize(currentPlan LogicalPlan, parentPlan LogicalPlan, currentChildIdx int, opt *logicalOptimizeOp) { + if parentPlan == nil { + // Attention: The function 'execOptimize' could not handle the root plan, so the parent plan could not be nil. + return + } + // constant propagation + currentPlan.constantPropagation(parentPlan, currentChildIdx, opt) + // recursive optimize + for i, children := range currentPlan.Children() { + cp.execOptimize(children, currentPlan, i, opt) + } +} + +func (*constantPropagationSolver) name() string { + return "constant_propagation" +} + +func (*baseLogicalPlan) constantPropagation(_ LogicalPlan, _ int, _ *logicalOptimizeOp) (newRoot LogicalPlan) { + // Only LogicalJoin can apply constant propagation + // Other Logical plan do nothing + return nil +} + +// Implemented the logic of constant propagation in From List +// Query: select * from t, (select a, b from s where s.a>1) tmp where tmp.a=t.a +// Origin logical plan: +/* + +----------------+ + | LogicalJoin | + +-------^--------+ + | + +-------------+--------------+ + | | ++-----+------+ +------+------+ +| Projection | | TableScan | ++-----^------+ +-------------+ + | + | ++-----+------+ +| Selection | +| s.a>1 | ++------------+ +*/ +// 1. 'pullUpConstantPredicates': Call this function until find selection and pull up the constant predicate layer by layer +// LogicalSelection: find the s.a>1 +// LogicalProjection: get the s.a>1 and pull up it, changed to tmp.a>1 +// 2. 'addCandidateSelection': Add selection above of LogicalJoin, +// put all predicates pulled up from the lower layer into the current new selection. +// LogicalSelection: tmp.a >1 +// +// Optimized plan: +/* + +----------------+ + | Selection | + | tmp.a>1 | + +-------^--------+ + | + +-------+--------+ + | LogicalJoin | + +-------^--------+ + | + +-------------+--------------+ + | | ++-----+------+ +------+------+ +| Projection | | TableScan | ++-----^------+ +-------------+ + | + | ++-----+------+ +| Selection | +| s.a>1 | ++------------+ +*/ +// Return nil if the root of plan has not been changed +// Return new root if the root of plan is changed to selection +func (logicalJoin *LogicalJoin) constantPropagation(parentPlan LogicalPlan, currentChildIdx int, opt *logicalOptimizeOp) (newRoot LogicalPlan) { + // step1: get constant predicate from left or right according to the JoinType + var getConstantPredicateFromLeft bool + var getConstantPredicateFromRight bool + switch logicalJoin.JoinType { + case LeftOuterJoin: + getConstantPredicateFromLeft = true + case RightOuterJoin: + getConstantPredicateFromRight = true + case InnerJoin: + getConstantPredicateFromLeft = true + getConstantPredicateFromRight = true + default: + return + } + var candidateConstantPredicates []expression.Expression + if getConstantPredicateFromLeft { + candidateConstantPredicates = logicalJoin.children[0].pullUpConstantPredicates() + } + if getConstantPredicateFromRight { + candidateConstantPredicates = append(candidateConstantPredicates, logicalJoin.children[1].pullUpConstantPredicates()...) + } + if len(candidateConstantPredicates) == 0 { + return + } + + // step2: add selection above of LogicalJoin + return addCandidateSelection(logicalJoin, currentChildIdx, parentPlan, candidateConstantPredicates, opt) +} + +func (*baseLogicalPlan) pullUpConstantPredicates() []expression.Expression { + // Only LogicalProjection and LogicalSelection can get constant predicates + // Other Logical plan return nil + return nil +} + +func (selection *LogicalSelection) pullUpConstantPredicates() []expression.Expression { + var result []expression.Expression + for _, candidatePredicate := range selection.Conditions { + // the candidate predicate should be a constant and compare predicate + match := validCompareConstantPredicate(candidatePredicate) + if match { + result = append(result, candidatePredicate) + } + } + return result +} + +func (projection *LogicalProjection) pullUpConstantPredicates() []expression.Expression { + // projection has no column expr + if !canProjectionBeEliminatedLoose(projection) { + return nil + } + candidateConstantPredicates := projection.children[0].pullUpConstantPredicates() + // replace predicate by projection expr + // candidate predicate : a=1 + // projection: a as a' + // result predicate : a'=1 + replace := make(map[string]*expression.Column) + for i, expr := range projection.Exprs { + replace[string(expr.HashCode(nil))] = projection.Schema().Columns[i] + } + result := make([]expression.Expression, 0, len(candidateConstantPredicates)) + for _, predicate := range candidateConstantPredicates { + // The column of predicate must exist in projection exprs + columns := expression.ExtractColumns(predicate) + // The number of columns in candidate predicate must be 1. + if len(columns) != 1 { + continue + } + if replace[string(columns[0].HashCode(nil))] == nil { + // The column of predicate will not appear on the upper level + // This means that this predicate does not apply to the constant propagation optimization rule + // For example: select * from t, (select b from s where s.a=1) tmp where t.b=s.b + continue + } + clonePredicate := predicate.Clone() + ResolveExprAndReplace(clonePredicate, replace) + result = append(result, clonePredicate) + } + return result +} + +// validComparePredicate checks if the predicate is an expression like [column '>'|'>='|'<'|'<='|'=' constant]. +// return param1: return true, if the predicate is a compare constant predicate. +// return param2: return the column side of predicate. +func validCompareConstantPredicate(candidatePredicate expression.Expression) bool { + scalarFunction, ok := candidatePredicate.(*expression.ScalarFunction) + if !ok { + return false + } + if scalarFunction.FuncName.L != ast.GT && scalarFunction.FuncName.L != ast.GE && + scalarFunction.FuncName.L != ast.LT && scalarFunction.FuncName.L != ast.LE && + scalarFunction.FuncName.L != ast.EQ { + return false + } + column, _ := expression.ValidCompareConstantPredicateHelper(scalarFunction, true) + if column == nil { + column, _ = expression.ValidCompareConstantPredicateHelper(scalarFunction, false) + } + if column == nil { + return false + } + return true +} + +// Add a new selection between parent plan and current plan with candidate predicates +/* ++-------------+ +-------------+ +| parentPlan | | parentPlan | ++-----^-------+ +-----^-------+ + | --addCandidateSelection---> | ++-----+-------+ +-----------+--------------+ +| currentPlan | | selection | ++-------------+ | candidate predicate | + +-----------^--------------+ + | + | + +----+--------+ + | currentPlan | + +-------------+ +*/ +// If the currentPlan at the top of query plan, return new root plan (selection) +// Else return nil +func addCandidateSelection(currentPlan LogicalPlan, currentChildIdx int, parentPlan LogicalPlan, + candidatePredicates []expression.Expression, opt *logicalOptimizeOp) (newRoot LogicalPlan) { + // generate a new selection for candidatePredicates + selection := LogicalSelection{Conditions: candidatePredicates}.Init(currentPlan.SCtx(), currentPlan.SelectBlockOffset()) + // add selection above of p + if parentPlan == nil { + newRoot = selection + } else { + parentPlan.SetChild(currentChildIdx, selection) + } + selection.SetChildren(currentPlan) + appendAddSelectionTraceStep(parentPlan, currentPlan, selection, opt) + if parentPlan == nil { + return newRoot + } + return nil +} diff --git a/planner/core/rule_constant_propagation_test.go b/planner/core/rule_constant_propagation_test.go new file mode 100644 index 0000000000000..2da017a7eb2e8 --- /dev/null +++ b/planner/core/rule_constant_propagation_test.go @@ -0,0 +1,153 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core_test + +import ( + "testing" + + "github.com/pingcap/tidb/planner/core" + "github.com/pingcap/tidb/testkit" + "github.com/pingcap/tidb/testkit/testdata" +) + +func TestRuleConstantPropagation(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + // create table + tk.MustExec("use test") + tk.MustExec("create table t (id int, name varchar(10));") + tk.MustExec("create table s (id int, name varchar(10));") + testData := core.GetRuleConstantPropagationData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + sql = "explain " + sql + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } +} + +func TestDifferentJoinTypeConstantPropagation(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + // create table + tk.MustExec("use test") + tk.MustExec("create table t (id int, name varchar(10));") + tk.MustExec("create table s (id int, name varchar(10));") + testData := core.GetRuleConstantPropagationData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + sql = "explain " + sql + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } +} + +func TestSelectionThroughPlanNode(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + // create table + tk.MustExec("use test") + tk.MustExec("create table t (id int, name varchar(10));") + tk.MustExec("create table s (id int, name varchar(10));") + testData := core.GetRuleConstantPropagationData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + sql = "explain " + sql + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } +} + +func TestUpdate(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + // create table + tk.MustExec("use test") + tk.MustExec("create table t (id int, name varchar(10));") + tk.MustExec("create table s (id int, name varchar(10));") + testData := core.GetRuleConstantPropagationData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + sql = "explain " + sql + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } +} + +func TestMultiSubtreeMatch(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + // create table + tk.MustExec("use test") + tk.MustExec("create table t (id int, name varchar(10));") + tk.MustExec("create table s (id int, name varchar(10));") + testData := core.GetRuleConstantPropagationData() + var ( + input []string + output []struct { + SQL string + Output []string + } + ) + testData.LoadTestCases(t, &input, &output) + for i, sql := range input { + sql = "explain " + sql + testdata.OnRecord(func() { + output[i].SQL = sql + output[i].Output = testdata.ConvertRowsToStrings(tk.MustQuery(sql).Rows()) + }) + tk.MustQuery(sql).Check(testkit.Rows(output[i].Output...)) + } +} diff --git a/planner/core/rule_eliminate_projection.go b/planner/core/rule_eliminate_projection.go index 5943f74fd70b5..00b274da76db2 100644 --- a/planner/core/rule_eliminate_projection.go +++ b/planner/core/rule_eliminate_projection.go @@ -156,6 +156,13 @@ func eliminatePhysicalProjection(p PhysicalPlan) PhysicalPlan { return newRoot } +// For select, insert, delete list +// The projection eliminate in logical optimize will optimize the projection under the projection, window, agg +// The projection eliminate in post optimize will optimize other projection + +// For update stmt +// The projection eliminate in logical optimize has been forbidden. +// The projection eliminate in post optimize will optimize the projection under the projection, window, agg (the condition is same as logical optimize) type projectionEliminator struct { } @@ -184,6 +191,7 @@ func (pe *projectionEliminator) eliminate(p LogicalPlan, replace map[string]*exp p.Children()[i] = pe.eliminate(child, replace, childFlag, opt) } + // replace logical plan schema switch x := p.(type) { case *LogicalJoin: x.schema = buildLogicalJoinSchema(x.JoinType, x) @@ -194,7 +202,10 @@ func (pe *projectionEliminator) eliminate(p LogicalPlan, replace map[string]*exp resolveColumnAndReplace(dst, replace) } } + // replace all of exprs in logical plan p.ReplaceExprColumns(replace) + + // eliminate duplicate projection: projection with child projection if isProj { if child, ok := p.Children()[0].(*LogicalProjection); ok && !ExprsHasSideEffects(child.Exprs) { for i := range proj.Exprs { diff --git a/planner/core/testdata/rule_constant_propagation_suite_in.json b/planner/core/testdata/rule_constant_propagation_suite_in.json new file mode 100644 index 0000000000000..1b5bf9e2ed396 --- /dev/null +++ b/planner/core/testdata/rule_constant_propagation_suite_in.json @@ -0,0 +1,43 @@ +[ + { + "Name": "TestRuleConstantPropagation", + "Cases": [ + "select * from t, (select * from s where s.id>1) tmp where t.id=tmp.id; -- inner join", + "select * from t, (select * from s where s.id>1) tmp where t.name=tmp.name; -- can't, without id equal predicate", + "select * from t, (select name from s where s.id>1) tmp where t.name=tmp.name; -- can't, projection without id column", + "select * from t, (select id as id1, name as name1 from s where s.id>1) tmp where t.id=tmp.id1; -- projection above of s.id>1", + "select * from t, (select id +1 as id1 from s where s.id>1) tmp where t.id=tmp.id1; -- can't optimize, projection has column function" + ] + }, + { + "Name": "TestDifferentJoinTypeConstantPropagation", + "Cases": [ + "select * from (select * from t where t.id >1) tmp1, (select * from s where s.id <4) tmp2 where tmp1.id=tmp2.id; -- inner join, both children can be optimized", + "select * from (select * from t where t.id>1) tmp, s where tmp.id=s.id; -- inner join, child 0", + "select * from (select * from t where t.id>1) tmp left join s on tmp.id=s.id; -- left join, only left child can be optimized", + "select * from t left join (select * from s where s.id>1) tmp on t.id=tmp.id; -- can't, left join", + "select * from t right join (select * from s where s.id>1) tmp on t.id=tmp.id; -- right join, only right child can be optimized", + "select * from (select * from t where t.id>1) tmp right join s on tmp.id=s.id; -- can't, right join" + ] + }, + { + "Name": "TestSelectionThroughPlanNode", + "Cases": [ + "select * from t, (select id as id1 from s where s.id>1) tmp where t.id=tmp.id1; -- constant propagation can through the projection node", + "select * from t, (select id, count(name) from s where s.id>1 group by id) tmp where t.id=tmp.id; -- can't, constant propagation can't through the aggregation node", + "select * from t, (select id from s where s.id>1 order by id limit 2) tmp where t.id=tmp.id; -- can't, constant propagation can't through the sort node" + ] + }, + { + "Name": "TestUpdate", + "Cases": [ + "Update t, (select * from s where s.id>1) tmp set t.name=tmp.name where t.id=tmp.id;" + ] + }, + { + "Name": "TestMultiSubtreeMatch", + "Cases": [ + "select * from (select * from (select t.id+1 as id1, t.name from t, (select * from s where s.id>1) s1 where t.id=s1.id ) tmp order by id1) a union (select tmp.* from (select * from t where t.id <3) tmp left join s on tmp.id=s.id); -- match twice" + ] + } +] diff --git a/planner/core/testdata/rule_constant_propagation_suite_out.json b/planner/core/testdata/rule_constant_propagation_suite_out.json new file mode 100644 index 0000000000000..b3f12ab963df1 --- /dev/null +++ b/planner/core/testdata/rule_constant_propagation_suite_out.json @@ -0,0 +1,243 @@ +[ + { + "Name": "TestRuleConstantPropagation", + "Cases": [ + { + "SQL": "explain select * from t, (select * from s where s.id>1) tmp where t.id=tmp.id; -- inner join", + "Output": [ + "HashJoin_11 4166.67 root inner join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_18(Build) 3333.33 root data:Selection_17", + "│ └─Selection_17 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + "│ └─TableFullScan_16 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_15(Probe) 3333.33 root data:Selection_14", + " └─Selection_14 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " └─TableFullScan_13 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t, (select * from s where s.id>1) tmp where t.name=tmp.name; -- can't, without id equal predicate", + "Output": [ + "Projection_12 4162.50 root test.t.id, test.t.name, test.s.id, test.s.name", + "└─HashJoin_14 4162.50 root inner join, equal:[eq(test.s.name, test.t.name)]", + " ├─TableReader_17(Build) 3330.00 root data:Selection_16", + " │ └─Selection_16 3330.00 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.name))", + " │ └─TableFullScan_15 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_20(Probe) 9990.00 root data:Selection_19", + " └─Selection_19 9990.00 cop[tikv] not(isnull(test.t.name))", + " └─TableFullScan_18 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t, (select name from s where s.id>1) tmp where t.name=tmp.name; -- can't, projection without id column", + "Output": [ + "Projection_12 4162.50 root test.t.id, test.t.name, test.s.name", + "└─HashJoin_14 4162.50 root inner join, equal:[eq(test.s.name, test.t.name)]", + " ├─TableReader_17(Build) 3330.00 root data:Selection_16", + " │ └─Selection_16 3330.00 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.name))", + " │ └─TableFullScan_15 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_20(Probe) 9990.00 root data:Selection_19", + " └─Selection_19 9990.00 cop[tikv] not(isnull(test.t.name))", + " └─TableFullScan_18 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t, (select id as id1, name as name1 from s where s.id>1) tmp where t.id=tmp.id1; -- projection above of s.id>1", + "Output": [ + "HashJoin_11 4166.67 root inner join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_18(Build) 3333.33 root data:Selection_17", + "│ └─Selection_17 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + "│ └─TableFullScan_16 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_15(Probe) 3333.33 root data:Selection_14", + " └─Selection_14 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " └─TableFullScan_13 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t, (select id +1 as id1 from s where s.id>1) tmp where t.id=tmp.id1; -- can't optimize, projection has column function", + "Output": [ + "Projection_11 3333.33 root test.t.id, test.t.name, Column#7", + "└─HashJoin_13 3333.33 root inner join, equal:[eq(Column#7, test.t.id)]", + " ├─Projection_14(Build) 2666.67 root plus(test.s.id, 1)->Column#7", + " │ └─TableReader_17 2666.67 root data:Selection_16", + " │ └─Selection_16 2666.67 cop[tikv] gt(test.s.id, 1), not(isnull(plus(test.s.id, 1)))", + " │ └─TableFullScan_15 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_20(Probe) 9990.00 root data:Selection_19", + " └─Selection_19 9990.00 cop[tikv] not(isnull(test.t.id))", + " └─TableFullScan_18 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + } + ] + }, + { + "Name": "TestDifferentJoinTypeConstantPropagation", + "Cases": [ + { + "SQL": "explain select * from (select * from t where t.id >1) tmp1, (select * from s where s.id <4) tmp2 where tmp1.id=tmp2.id; -- inner join, both children can be optimized", + "Output": [ + "HashJoin_13 312.50 root inner join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_20(Build) 250.00 root data:Selection_19", + "│ └─Selection_19 250.00 cop[tikv] gt(test.s.id, 1), lt(test.s.id, 4), not(isnull(test.s.id))", + "│ └─TableFullScan_18 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_17(Probe) 250.00 root data:Selection_16", + " └─Selection_16 250.00 cop[tikv] gt(test.t.id, 1), lt(test.t.id, 4), not(isnull(test.t.id))", + " └─TableFullScan_15 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from (select * from t where t.id>1) tmp, s where tmp.id=s.id; -- inner join, child 0", + "Output": [ + "HashJoin_11 4166.67 root inner join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_18(Build) 3333.33 root data:Selection_17", + "│ └─Selection_17 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + "│ └─TableFullScan_16 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_15(Probe) 3333.33 root data:Selection_14", + " └─Selection_14 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " └─TableFullScan_13 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from (select * from t where t.id>1) tmp left join s on tmp.id=s.id; -- left join, only left child can be optimized", + "Output": [ + "HashJoin_10 4166.67 root left outer join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_17(Build) 3333.33 root data:Selection_16", + "│ └─Selection_16 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + "│ └─TableFullScan_15 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_14(Probe) 3333.33 root data:Selection_13", + " └─Selection_13 3333.33 cop[tikv] gt(test.t.id, 1)", + " └─TableFullScan_12 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t left join (select * from s where s.id>1) tmp on t.id=tmp.id; -- can't, left join", + "Output": [ + "HashJoin_9 10000.00 root left outer join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_15(Build) 3333.33 root data:Selection_14", + "│ └─Selection_14 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + "│ └─TableFullScan_13 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_12(Probe) 10000.00 root data:TableFullScan_11", + " └─TableFullScan_11 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t right join (select * from s where s.id>1) tmp on t.id=tmp.id; -- right join, only right child can be optimized", + "Output": [ + "HashJoin_10 4166.67 root right outer join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_17(Build) 3333.33 root data:Selection_16", + "│ └─Selection_16 3333.33 cop[tikv] gt(test.s.id, 1)", + "│ └─TableFullScan_15 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_14(Probe) 3333.33 root data:Selection_13", + " └─Selection_13 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " └─TableFullScan_12 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from (select * from t where t.id>1) tmp right join s on tmp.id=s.id; -- can't, right join", + "Output": [ + "HashJoin_10 10000.00 root right outer join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_13(Build) 3333.33 root data:Selection_12", + "│ └─Selection_12 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + "│ └─TableFullScan_11 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader_15(Probe) 10000.00 root data:TableFullScan_14", + " └─TableFullScan_14 10000.00 cop[tikv] table:s keep order:false, stats:pseudo" + ] + } + ] + }, + { + "Name": "TestSelectionThroughPlanNode", + "Cases": [ + { + "SQL": "explain select * from t, (select id as id1 from s where s.id>1) tmp where t.id=tmp.id1; -- constant propagation can through the projection node", + "Output": [ + "HashJoin_11 4166.67 root inner join, equal:[eq(test.t.id, test.s.id)]", + "├─TableReader_18(Build) 3333.33 root data:Selection_17", + "│ └─Selection_17 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + "│ └─TableFullScan_16 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + "└─TableReader_15(Probe) 3333.33 root data:Selection_14", + " └─Selection_14 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " └─TableFullScan_13 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t, (select id, count(name) from s where s.id>1 group by id) tmp where t.id=tmp.id; -- can't, constant propagation can't through the aggregation node", + "Output": [ + "Projection_11 3333.33 root test.t.id, test.t.name, test.s.id, Column#7", + "└─Projection_12 3333.33 root test.t.id, test.t.name, Column#7, test.s.id", + " └─HashJoin_14 3333.33 root inner join, equal:[eq(test.s.id, test.t.id)]", + " ├─HashAgg_20(Build) 2666.67 root group by:test.s.id, funcs:count(Column#8)->Column#7, funcs:firstrow(test.s.id)->test.s.id", + " │ └─TableReader_21 2666.67 root data:HashAgg_15", + " │ └─HashAgg_15 2666.67 cop[tikv] group by:test.s.id, funcs:count(test.s.name)->Column#8", + " │ └─Selection_19 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + " │ └─TableFullScan_18 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_27(Probe) 9990.00 root data:Selection_26", + " └─Selection_26 9990.00 cop[tikv] not(isnull(test.t.id))", + " └─TableFullScan_25 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "explain select * from t, (select id from s where s.id>1 order by id limit 2) tmp where t.id=tmp.id; -- can't, constant propagation can't through the sort node", + "Output": [ + "Projection_15 2.00 root test.t.id, test.t.name, test.s.id", + "└─HashJoin_17 2.00 root inner join, equal:[eq(test.s.id, test.t.id)]", + " ├─Selection_18(Build) 1.60 root not(isnull(test.s.id))", + " │ └─TopN_19 2.00 root test.s.id, offset:0, count:2", + " │ └─TableReader_27 2.00 root data:TopN_26", + " │ └─TopN_26 2.00 cop[tikv] test.s.id, offset:0, count:2", + " │ └─Selection_25 3333.33 cop[tikv] gt(test.s.id, 1)", + " │ └─TableFullScan_24 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_30(Probe) 9990.00 root data:Selection_29", + " └─Selection_29 9990.00 cop[tikv] not(isnull(test.t.id))", + " └─TableFullScan_28 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + } + ] + }, + { + "Name": "TestUpdate", + "Cases": [ + { + "SQL": "explain Update t, (select * from s where s.id>1) tmp set t.name=tmp.name where t.id=tmp.id;", + "Output": [ + "Update_8 N/A root N/A", + "└─Projection_11 4166.67 root test.t.id, test.t.name, test.t._tidb_rowid, test.s.id, test.s.name", + " └─HashJoin_12 4166.67 root inner join, equal:[eq(test.t.id, test.s.id)]", + " ├─Projection_17(Build) 3333.33 root test.s.id, test.s.name", + " │ └─TableReader_20 3333.33 root data:Selection_19", + " │ └─Selection_19 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + " │ └─TableFullScan_18 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_16(Probe) 3333.33 root data:Selection_15", + " └─Selection_15 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " └─TableFullScan_14 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + } + ] + }, + { + "Name": "TestMultiSubtreeMatch", + "Cases": [ + { + "SQL": "explain select * from (select * from (select t.id+1 as id1, t.name from t, (select * from s where s.id>1) s1 where t.id=s1.id ) tmp order by id1) a union (select tmp.* from (select * from t where t.id <3) tmp left join s on tmp.id=s.id); -- match twice", + "Output": [ + "HashAgg_24 5325.33 root group by:Column#14, Column#15, funcs:firstrow(Column#14)->Column#14, funcs:firstrow(Column#15)->Column#15", + "└─Union_25 8320.83 root ", + " ├─Projection_26 4166.67 root plus(test.t.id, 1)->Column#14, test.t.name->Column#15", + " │ └─HashJoin_27 4166.67 root inner join, equal:[eq(test.t.id, test.s.id)]", + " │ ├─TableReader_34(Build) 3333.33 root data:Selection_33", + " │ │ └─Selection_33 3333.33 cop[tikv] gt(test.s.id, 1), not(isnull(test.s.id))", + " │ │ └─TableFullScan_32 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " │ └─TableReader_31(Probe) 3333.33 root data:Selection_30", + " │ └─Selection_30 3333.33 cop[tikv] gt(test.t.id, 1), not(isnull(test.t.id))", + " │ └─TableFullScan_29 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + " └─Projection_35 4154.17 root cast(test.t.id, bigint(20) BINARY)->Column#14, test.t.name->Column#15", + " └─HashJoin_36 4154.17 root left outer join, equal:[eq(test.t.id, test.s.id)]", + " ├─TableReader_43(Build) 3323.33 root data:Selection_42", + " │ └─Selection_42 3323.33 cop[tikv] lt(test.s.id, 3), not(isnull(test.s.id))", + " │ └─TableFullScan_41 10000.00 cop[tikv] table:s keep order:false, stats:pseudo", + " └─TableReader_40(Probe) 3323.33 root data:Selection_39", + " └─Selection_39 3323.33 cop[tikv] lt(test.t.id, 3)", + " └─TableFullScan_38 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + } + ] + } +]