diff --git a/fe/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/src/main/java/org/apache/doris/analysis/Analyzer.java index d59e4e036a8787..23ee0f0f8eae10 100644 --- a/fe/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -767,7 +767,6 @@ private void registerConjunct(Expr e) { ArrayList tupleIds = Lists.newArrayList(); ArrayList slotIds = Lists.newArrayList(); e.getIds(tupleIds, slotIds); - // register full join conjuncts registerFullOuterJoinedConjunct(e); @@ -803,7 +802,7 @@ private void registerConjunct(Expr e) { if (binaryPred.getOp() != BinaryPredicate.Operator.EQ) { return; } - if (tupleIds.size() != 2) { + if (tupleIds.size() < 2) { return; } @@ -1188,6 +1187,20 @@ public boolean isFullOuterJoined(Expr e) { return globalState.fullOuterJoinedConjuncts.containsKey(e.getId()); } + public TableRef getOjRef(Expr e) { + return globalState.ojClauseByConjunct.get(e.getId()); + } + + /** + * Returns false if 'e' originates from an outer-join On-clause and it is incorrect to + * evaluate 'e' at a node materializing 'tids'. Returns true otherwise. + */ + public boolean canEvalOuterJoinedConjunct(Expr e, List tids) { + TableRef outerJoin = getOjRef(e); + if (outerJoin == null) return true; + return tids.containsAll(outerJoin.getAllTableRefIds()); + } + /** * Returns list of candidate equi-join conjuncts to be evaluated by the join node * that is specified by the table ref ids of its left and right children. @@ -1195,27 +1208,59 @@ public boolean isFullOuterJoined(Expr e) { * from its On-clause are returned. If an equi-join conjunct is full outer joined, * then it is only added to the result if this join is the one to full-outer join it. */ - public List getEqJoinConjuncts(TupleId id, TableRef rhsRef) { - List conjunctIds = globalState.eqJoinConjuncts.get(id); - if (conjunctIds == null) { - return null; + public List getEqJoinConjuncts(List lhsTblRefIds, + List rhsTblRefIds) { + // Contains all equi-join conjuncts that have one child fully bound by one of the + // rhs table ref ids (the other child is not bound by that rhs table ref id). + List conjunctIds = Lists.newArrayList(); + for (TupleId rhsId: rhsTblRefIds) { + List cids = globalState.eqJoinConjuncts.get(rhsId); + if (cids == null) continue; + for (ExprId eid: cids) { + if (!conjunctIds.contains(eid)) conjunctIds.add(eid); + } } - List result = Lists.newArrayList(); + + // Since we currently prevent join re-reordering across outer joins, we can never + // have a bushy outer join with multiple rhs table ref ids. A busy outer join can + // only be constructed with an inline view (which has a single table ref id). List ojClauseConjuncts = null; - if (rhsRef != null) { - Preconditions.checkState(rhsRef.getJoinOp().isOuterJoin()); - ojClauseConjuncts = globalState.conjunctsByOjClause.get(rhsRef.getId()); + if (rhsTblRefIds.size() == 1) { + ojClauseConjuncts = globalState.conjunctsByOjClause.get(rhsTblRefIds.get(0)); } - for (ExprId conjunctId : conjunctIds) { + + // List of table ref ids that the join node will 'materialize'. + List nodeTblRefIds = Lists.newArrayList(lhsTblRefIds); + nodeTblRefIds.addAll(rhsTblRefIds); + List result = Lists.newArrayList(); + for (ExprId conjunctId: conjunctIds) { Expr e = globalState.conjuncts.get(conjunctId); Preconditions.checkState(e != null); - if (ojClauseConjuncts != null) { - if (ojClauseConjuncts.contains(conjunctId)) { - result.add(e); - } - } else { - result.add(e); + if (!canEvalFullOuterJoinedConjunct(e, nodeTblRefIds) || + !canEvalAntiJoinedConjunct(e, nodeTblRefIds) || + !canEvalOuterJoinedConjunct(e, nodeTblRefIds)) { + continue; } + + if (ojClauseConjuncts != null && !ojClauseConjuncts.contains(conjunctId)) continue; + result.add(e); + } + return result; + } + + /** + * return equal conjuncts, used by OlapScanNode.normalizePredicate and SelectStmt.reorderTable + */ + public List getEqJoinConjuncts(TupleId id) { + final List conjunctIds = globalState.eqJoinConjuncts.get(id); + if (conjunctIds == null) { + return Lists.newArrayList(); + } + final List result = Lists.newArrayList(); + for (ExprId conjunctId : conjunctIds) { + final Expr e = globalState.conjuncts.get(conjunctId); + Preconditions.checkState(e != null); + result.add(e); } return result; } @@ -1224,7 +1269,7 @@ public List getEqJoinConjuncts(TupleId id, TableRef rhsRef) { * Returns list of candidate equi-join conjuncts excluding auxiliary predicates */ public List getEqJoinConjunctsExcludeAuxPredicates(TupleId id) { - final List candidateEqJoinPredicates = getEqJoinConjuncts(id, null); + final List candidateEqJoinPredicates = getEqJoinConjuncts(id); final Iterator iterator = candidateEqJoinPredicates.iterator(); while (iterator.hasNext()) { final Expr expr = iterator.next(); diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java index af4995fffaa14d..eb10ed6150b121 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -567,7 +567,7 @@ protected boolean reorderTable(Analyzer analyzer, TableRef firstRef) while (i < fromClause_.size()) { TableRef tblRef = fromClause_.get(i); // get all equal - List eqJoinPredicates = analyzer.getEqJoinConjuncts(tblRef.getId(), null); + List eqJoinPredicates = analyzer.getEqJoinConjuncts(tblRef.getId()); List tuple_list = Lists.newArrayList(); Expr.getIds(eqJoinPredicates, tuple_list, null); for (TupleId tid : tuple_list) { diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index 42fd43ff97785a..578c3fc693ab16 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -294,16 +294,14 @@ public int compare(MaterializedIndex index1, MaterializedIndex index2) private void normalizePredicate(Analyzer analyzer) throws UserException { // 1. Get Columns which has eqJoin on it - List eqJoinPredicate = analyzer.getEqJoinConjuncts(desc.getId(), null); - if (null != eqJoinPredicate) { + List eqJoinPredicate = analyzer.getEqJoinConjuncts(desc.getId()); + for (Expr expr : eqJoinPredicate) { for (SlotDescriptor slot : desc.getSlots()) { - for (Expr expr : eqJoinPredicate) { - for (int i = 0; i < 2; ++i) { - if (expr.getChild(i).isBound(slot.getId())) { - eqJoinColumns.add(slot.getColumn().getName()); - LOG.debug("Add eqJoinColumn: ColName=" + slot.getColumn().getName()); - break; - } + for (int i = 0; i < 2; i++) { + if (expr.getChild(i).isBound(slot.getId())) { + eqJoinColumns.add(slot.getColumn().getName()); + LOG.debug("Add eqJoinColumn: ColName=" + slot.getColumn().getName()); + break; } } } diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 245a8357417cf0..21c87db5fbefe9 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1170,25 +1170,18 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef) * inner joins, but only from the JOIN clause Returns the conjuncts in 'joinConjuncts' (in which " = " is * returned as Pair(, )) and also in their original form in 'joinPredicates'. */ - private void getHashLookupJoinConjuncts(Analyzer analyzer, List lhsIds, TableRef rhs, + private void getHashLookupJoinConjuncts(Analyzer analyzer, PlanNode left, PlanNode right, List> joinConjuncts, List joinPredicates, - Reference errMsg) { + Reference errMsg, JoinOperator op) { joinConjuncts.clear(); joinPredicates.clear(); - TupleId rhsId = rhs.getId(); - // List rhsIds = rhs.getMaterializedTupleIds(); - List rhsIds = rhsId.asList(); + final List lhsIds = left.getTblRefIds(); + final List rhsIds = right.getTblRefIds(); List candidates; - if (rhs.getJoinOp().isOuterJoin()) { - // TODO: create test for this - Preconditions.checkState(rhs.getOnClause() != null); - candidates = analyzer.getEqJoinConjuncts(rhsId, rhs); - } else { - candidates = analyzer.getEqJoinConjuncts(rhsId, null); - } + candidates = analyzer.getEqJoinConjuncts(lhsIds, rhsIds); if (candidates == null) { - if (rhs.getJoinOp().isOuterJoin() || rhs.getJoinOp().isSemiAntiJoin()) { - errMsg.setRef("non-equal " + rhs.getJoinOp().toString() + " is not supported"); + if (op.isOuterJoin() || op.isSemiAntiJoin()) { + errMsg.setRef("non-equal " + op.toString() + " is not supported"); LOG.warn(errMsg); } LOG.info("no candidates for join."); @@ -1244,8 +1237,8 @@ private PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, TableRef oute Reference errMsg = new Reference(); // get eq join predicates for the TableRefs' ids (not the PlanNodes' ids, which // are materialized) - getHashLookupJoinConjuncts(analyzer, outer.getTblRefIds(), innerRef, eqJoinConjuncts, - eqJoinPredicates, errMsg); + getHashLookupJoinConjuncts(analyzer, outer, inner, eqJoinConjuncts, + eqJoinPredicates, errMsg, innerRef.getJoinOp()); if (eqJoinPredicates.isEmpty()) { // only inner join can change to cross join