From 868a44d59d756f1f31f4458543b3811e4ec5954d Mon Sep 17 00:00:00 2001 From: Yiding Cui Date: Wed, 26 Dec 2018 14:00:13 +0800 Subject: [PATCH] fix unit-test and integration test --- cmd/explaintest/r/explain_complex.result | 4 ++-- cmd/explaintest/r/explain_complex_stats.result | 4 ++-- cmd/explaintest/r/tpch.result | 6 +++--- planner/core/rule_join_reorder_dp.go | 16 +++++++++++----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/explaintest/r/explain_complex.result b/cmd/explaintest/r/explain_complex.result index 99366a5f23718..77074e6f8195d 100644 --- a/cmd/explaintest/r/explain_complex.result +++ b/cmd/explaintest/r/explain_complex.result @@ -118,7 +118,7 @@ id count task operator info Projection_13 1.00 root gad.id, test.dd.id, gad.aid, gad.cm, test.dd.dic, test.dd.ip, test.dd.t, gad.p1, gad.p2, gad.p3, gad.p4, gad.p5, gad.p6_md5, gad.p7_md5, gad.ext, gad.t └─Limit_16 1.00 root offset:0, count:2500 └─HashAgg_19 1.00 root group by:gad.aid, test.dd.dic, funcs:firstrow(gad.id), firstrow(gad.aid), firstrow(gad.cm), firstrow(gad.p1), firstrow(gad.p2), firstrow(gad.p3), firstrow(gad.p4), firstrow(gad.p5), firstrow(gad.p6_md5), firstrow(gad.p7_md5), firstrow(gad.ext), firstrow(gad.t), firstrow(test.dd.id), firstrow(test.dd.dic), firstrow(test.dd.ip), firstrow(test.dd.t) - └─IndexJoin_24 0.00 root inner join, inner:IndexLookUp_23, outer key:gad.aid, inner key:test.dd.aid, other cond:eq(gad.ip, test.dd.ip) + └─IndexJoin_24 0.00 root inner join, inner:IndexLookUp_23, outer key:gad.aid, inner key:test.dd.aid, other cond:eq(gad.ip, test.dd.ip), gt(test.dd.t, gad.t) ├─IndexLookUp_33 3.33 root │ ├─IndexScan_30 3333.33 cop table:gad, index:t, range:(1478143908,+inf], keep order:false, stats:pseudo │ └─Selection_32 3.33 cop eq(gad.pt, "android") @@ -131,7 +131,7 @@ explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic a id count task operator info Projection_10 0.00 root gad.id, sdk.id, gad.aid, gad.cm, sdk.dic, sdk.ip, sdk.t, gad.p1, gad.p2, gad.p3, gad.p4, gad.p5, gad.p6_md5, gad.p7_md5, gad.ext └─Limit_13 0.00 root offset:0, count:3000 - └─IndexJoin_18 0.00 root inner join, inner:IndexLookUp_17, outer key:gad.aid, inner key:sdk.aid, other cond:eq(gad.dic, sdk.mac) + └─IndexJoin_18 0.00 root inner join, inner:IndexLookUp_17, outer key:gad.aid, inner key:sdk.aid, other cond:eq(gad.dic, sdk.mac), lt(gad.t, sdk.t) ├─IndexLookUp_27 0.00 root │ ├─IndexScan_24 3333.33 cop table:gad, index:t, range:(1477971479,+inf], keep order:false, stats:pseudo │ └─Selection_26 0.00 cop eq(gad.bm, 0), eq(gad.dit, "mac"), eq(gad.pt, "ios") diff --git a/cmd/explaintest/r/explain_complex_stats.result b/cmd/explaintest/r/explain_complex_stats.result index a2f3ff20c1af5..1569a2d3a850c 100644 --- a/cmd/explaintest/r/explain_complex_stats.result +++ b/cmd/explaintest/r/explain_complex_stats.result @@ -128,7 +128,7 @@ id count task operator info Projection_13 424.00 root gad.id, test.dd.id, gad.aid, gad.cm, test.dd.dic, test.dd.ip, test.dd.t, gad.p1, gad.p2, gad.p3, gad.p4, gad.p5, gad.p6_md5, gad.p7_md5, gad.ext, gad.t └─Limit_16 424.00 root offset:0, count:2500 └─HashAgg_19 424.00 root group by:gad.aid, test.dd.dic, funcs:firstrow(gad.id), firstrow(gad.aid), firstrow(gad.cm), firstrow(gad.p1), firstrow(gad.p2), firstrow(gad.p3), firstrow(gad.p4), firstrow(gad.p5), firstrow(gad.p6_md5), firstrow(gad.p7_md5), firstrow(gad.ext), firstrow(gad.t), firstrow(test.dd.id), firstrow(test.dd.dic), firstrow(test.dd.ip), firstrow(test.dd.t) - └─IndexJoin_24 424.00 root inner join, inner:IndexLookUp_23, outer key:gad.aid, inner key:test.dd.aid, other cond:eq(gad.ip, test.dd.ip) + └─IndexJoin_24 424.00 root inner join, inner:IndexLookUp_23, outer key:gad.aid, inner key:test.dd.aid, other cond:eq(gad.ip, test.dd.ip), gt(test.dd.t, gad.t) ├─TableReader_29 424.00 root data:Selection_28 │ └─Selection_28 424.00 cop eq(gad.bm, 0), eq(gad.pt, "android"), gt(gad.t, 1478143908) │ └─TableScan_27 1999.00 cop table:gad, range:[0,+inf], keep order:false @@ -140,7 +140,7 @@ explain select gad.id as gid,sdk.id as sid,gad.aid as aid,gad.cm as cm,sdk.dic a id count task operator info Projection_10 170.34 root gad.id, sdk.id, gad.aid, gad.cm, sdk.dic, sdk.ip, sdk.t, gad.p1, gad.p2, gad.p3, gad.p4, gad.p5, gad.p6_md5, gad.p7_md5, gad.ext └─Limit_13 170.34 root offset:0, count:3000 - └─IndexJoin_18 170.34 root inner join, inner:IndexLookUp_17, outer key:gad.aid, inner key:sdk.aid, other cond:eq(gad.dic, sdk.mac) + └─IndexJoin_18 170.34 root inner join, inner:IndexLookUp_17, outer key:gad.aid, inner key:sdk.aid, other cond:eq(gad.dic, sdk.mac), lt(gad.t, sdk.t) ├─TableReader_23 170.34 root data:Selection_22 │ └─Selection_22 170.34 cop eq(gad.bm, 0), eq(gad.dit, "mac"), eq(gad.pt, "ios"), gt(gad.t, 1477971479) │ └─TableScan_21 1999.00 cop table:gad, range:[0,+inf], keep order:false diff --git a/cmd/explaintest/r/tpch.result b/cmd/explaintest/r/tpch.result index fd41424ee84be..e06783071377e 100644 --- a/cmd/explaintest/r/tpch.result +++ b/cmd/explaintest/r/tpch.result @@ -447,7 +447,7 @@ Sort_52 768.91 root shipping.supp_nation:asc, shipping.cust_nation:asc, shipping └─Projection_54 768.91 root shipping.supp_nation, shipping.cust_nation, shipping.l_year, 14_col_0 └─HashAgg_57 768.91 root group by:shipping.cust_nation, shipping.l_year, shipping.supp_nation, funcs:sum(shipping.volume), firstrow(shipping.supp_nation), firstrow(shipping.cust_nation), firstrow(shipping.l_year) └─Projection_58 9818555.15 root n1.n_name, n2.n_name, extract("YEAR", tpch.lineitem.l_shipdate), mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount)) - └─HashLeftJoin_60 9818555.15 root inner join, inner:HashLeftJoin_78, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)] + └─HashLeftJoin_60 9818555.15 root inner join, inner:HashLeftJoin_78, equal:[eq(tpch.lineitem.l_orderkey, tpch.orders.o_orderkey)], other cond:or(and(eq(n1.n_name, "JAPAN"), eq(n2.n_name, "INDIA")), and(eq(n1.n_name, "INDIA"), eq(n2.n_name, "JAPAN"))) ├─HashRightJoin_63 24465505.20 root inner join, inner:HashLeftJoin_68, equal:[eq(tpch.supplier.s_suppkey, tpch.lineitem.l_suppkey)] │ ├─HashLeftJoin_68 40000.00 root inner join, inner:TableReader_74, equal:[eq(tpch.supplier.s_nationkey, n1.n_nationkey)] │ │ ├─TableReader_71 500000.00 root data:TableScan_70 @@ -967,7 +967,7 @@ id count task operator info Projection_17 1.00 root div(11_col_0, 7.0) └─StreamAgg_22 1.00 root funcs:sum(tpch.lineitem.l_extendedprice) └─Projection_45 293773.83 root tpch.lineitem.l_partkey, tpch.lineitem.l_quantity, tpch.lineitem.l_extendedprice, tpch.part.p_partkey, tpch.part.p_brand, tpch.part.p_container, mul(0.2, 7_col_0) - └─HashLeftJoin_46 293773.83 root inner join, inner:HashRightJoin_33, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)] + └─HashLeftJoin_46 293773.83 root inner join, inner:HashRightJoin_33, equal:[eq(tpch.lineitem.l_partkey, tpch.part.p_partkey)], other cond:lt(tpch.lineitem.l_quantity, mul(0.2, 7_col_0)) ├─TableReader_49 300005811.00 root data:TableScan_48 │ └─TableScan_48 300005811.00 cop table:lineitem, range:[-inf,+inf], keep order:false └─HashRightJoin_33 9736.49 root inner join, inner:TableReader_44, equal:[eq(tpch.part.p_partkey, tpch.lineitem.l_partkey)] @@ -1086,7 +1086,7 @@ and l_shipinstruct = 'DELIVER IN PERSON' ); id count task operator info StreamAgg_13 1.00 root funcs:sum(mul(tpch.lineitem.l_extendedprice, minus(1, tpch.lineitem.l_discount))) -└─IndexJoin_29 6286493.79 root inner join, inner:TableReader_28, outer key:tpch.lineitem.l_partkey, inner key:tpch.part.p_partkey +└─IndexJoin_29 6286493.79 root inner join, inner:TableReader_28, outer key:tpch.lineitem.l_partkey, inner key:tpch.part.p_partkey, other cond:or(and(and(eq(tpch.part.p_brand, "Brand#52"), in(tpch.part.p_container, "SM CASE", "SM BOX", "SM PACK", "SM PKG")), and(ge(tpch.lineitem.l_quantity, 4), and(le(tpch.lineitem.l_quantity, 14), le(tpch.part.p_size, 5)))), or(and(and(eq(tpch.part.p_brand, "Brand#11"), in(tpch.part.p_container, "MED BAG", "MED BOX", "MED PKG", "MED PACK")), and(ge(tpch.lineitem.l_quantity, 18), and(le(tpch.lineitem.l_quantity, 28), le(tpch.part.p_size, 10)))), and(and(eq(tpch.part.p_brand, "Brand#51"), in(tpch.part.p_container, "LG CASE", "LG BOX", "LG PACK", "LG PKG")), and(ge(tpch.lineitem.l_quantity, 29), and(le(tpch.lineitem.l_quantity, 39), le(tpch.part.p_size, 15)))))) ├─TableReader_34 6286493.79 root data:Selection_33 │ └─Selection_33 6286493.79 cop eq(tpch.lineitem.l_shipinstruct, "DELIVER IN PERSON"), in(tpch.lineitem.l_shipmode, "AIR", "AIR REG"), or(and(ge(tpch.lineitem.l_quantity, 4), le(tpch.lineitem.l_quantity, 14)), or(and(ge(tpch.lineitem.l_quantity, 18), le(tpch.lineitem.l_quantity, 28)), and(ge(tpch.lineitem.l_quantity, 29), le(tpch.lineitem.l_quantity, 39)))) │ └─TableScan_32 300005811.00 cop table:lineitem, range:[-inf,+inf], keep order:false diff --git a/planner/core/rule_join_reorder_dp.go b/planner/core/rule_join_reorder_dp.go index a47e38877a112..7403aa9b093a1 100644 --- a/planner/core/rule_join_reorder_dp.go +++ b/planner/core/rule_join_reorder_dp.go @@ -67,16 +67,19 @@ func (s *joinReorderDPSolver) solve(joinGroup []LogicalPlan, eqConds, otherConds for _, cond := range otherConds { cols := expression.ExtractColumns(cond) mask := uint(0) + ids := make([]int, 0, len(cols)) for _, col := range cols { idx, err := findNodeIndexInGroup(joinGroup, col) if err != nil { return nil, err } + ids = append(ids, idx) mask |= 1 << uint(idx) } totalNonEqEdges = append(totalNonEqEdges, joinGroupNonEqEdge{ - idMask: mask, - expr: cond, + nodeIDs: ids, + idMask: mask, + expr: cond, }) } visited := make([]bool, len(joinGroup)) @@ -90,17 +93,17 @@ func (s *joinReorderDPSolver) solve(joinGroup []LogicalPlan, eqConds, otherConds visitID2NodeID := s.bfsGraph(i, visited, adjacents, nodeID2VisitID) nodeIDMask := uint(0) for _, nodeID := range visitID2NodeID { - nodeIDMask |= uint(nodeID) + nodeIDMask |= 1 << uint(nodeID) } var subNonEqEdges []joinGroupNonEqEdge for i := len(totalNonEqEdges) - 1; i >= 0; i-- { // If this edge is not the subset of the current sub graph. - if totalNonEqEdges[i].idMask&nodeIDMask != nodeIDMask { + if totalNonEqEdges[i].idMask&nodeIDMask != totalNonEqEdges[i].idMask { continue } newMask := uint(0) for _, nodeID := range totalNonEqEdges[i].nodeIDs { - newMask |= uint(nodeID) + newMask |= 1 << uint(nodeID2VisitID[nodeID]) } totalNonEqEdges[i].idMask = newMask subNonEqEdges = append(subNonEqEdges, totalNonEqEdges[i]) @@ -240,6 +243,9 @@ func (s *joinReorderDPSolver) makeBushyJoin(cartesianJoinGroup []LogicalPlan, ot resultJoinGroup = append(resultJoinGroup, cartesianJoinGroup[i]) break } + // TODO:Since the other condition may involve no less than two tables, e.g. t1.a = t2.b+t3.c. + // So We'll need a extra stage to deal with it. + // Currently, we just add it when building cartesianJoinGroup. mergedSchema := expression.MergeSchema(cartesianJoinGroup[i].Schema(), cartesianJoinGroup[i+1].Schema()) var usedOtherConds []expression.Expression for i := len(otherConds) - 1; i >= 0; i-- {