Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

materialize scannode slot for case 'select count* from select xxxx fr… #39

Merged
merged 1 commit into from
Aug 21, 2017
Merged

materialize scannode slot for case 'select count* from select xxxx fr… #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions fe/src/com/baidu/palo/analysis/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -1543,4 +1544,53 @@ public boolean isOuterJoined(TupleId tid) {
public boolean containSubquery() {
return globalState.containsSubquery;
}

/**
* Mark slots that are being referenced by the plan tree itself or by the outputExprs exprs as materialized. If the
* latter is null, mark all slots in planRoot's tupleIds() as being referenced. All aggregate slots are
* materialized.
* <p/>
* TODO: instead of materializing everything produced by the plan root, derived referenced slots from destination
* fragment and add a materialization node if not all output is needed by destination fragment TODO 2: should the
* materialization decision be cost-based?
*/
public void markRefdSlots(Analyzer analyzer, PlanNode planRoot,
List<Expr> outputExprs, AnalyticInfo analyticInfo) {
if (planRoot == null) {
return;
}
List<SlotId> refdIdList = Lists.newArrayList();
planRoot.getMaterializedIds(analyzer, refdIdList);
if (outputExprs != null) {
Expr.getIds(outputExprs, null, refdIdList);
}

HashSet<SlotId> refdIds = Sets.newHashSet(refdIdList);
for (TupleDescriptor tupleDesc : analyzer.getDescTbl().getTupleDescs()) {
for (SlotDescriptor slotDesc : tupleDesc.getSlots()) {
if (refdIds.contains(slotDesc.getId())) {
slotDesc.setIsMaterialized(true);
}
}
}
if (analyticInfo != null) {
ArrayList<SlotDescriptor> list = analyticInfo.getOutputTupleDesc().getSlots();

for (SlotDescriptor slotDesc : list) {
if (refdIds.contains(slotDesc.getId())) {
slotDesc.setIsMaterialized(true);
}
}
}
if (outputExprs == null) {
// mark all slots in planRoot.getTupleIds() as materialized
ArrayList<TupleId> tids = planRoot.getTupleIds();
for (TupleId tid : tids) {
TupleDescriptor tupleDesc = analyzer.getDescTbl().getTupleDesc(tid);
for (SlotDescriptor slotDesc : tupleDesc.getSlots()) {
slotDesc.setIsMaterialized(true);
}
}
}
}
}
51 changes: 1 addition & 50 deletions fe/src/com/baidu/palo/planner/Planner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,55 +107,6 @@ private void setResultExprScale(Analyzer analyzer, ArrayList<Expr> outputExprs)
}
}

/**
* Mark slots that are being referenced by the plan tree itself or by the outputExprs exprs as materialized. If the
* latter is null, mark all slots in planRoot's tupleIds() as being referenced. All aggregate slots are
* materialized.
* <p/>
* TODO: instead of materializing everything produced by the plan root, derived referenced slots from destination
* fragment and add a materialization node if not all output is needed by destination fragment TODO 2: should the
* materialization decision be cost-based?
*/
private void markRefdSlots(Analyzer analyzer, PlanNode planRoot,
List<Expr> outputExprs, AnalyticInfo analyticInfo) {
if (planRoot == null) {
return;
}
List<SlotId> refdIdList = Lists.newArrayList();
planRoot.getMaterializedIds(analyzer, refdIdList);
if (outputExprs != null) {
Expr.getIds(outputExprs, null, refdIdList);
}

HashSet<SlotId> refdIds = Sets.newHashSet(refdIdList);
for (TupleDescriptor tupleDesc : analyzer.getDescTbl().getTupleDescs()) {
for (SlotDescriptor slotDesc : tupleDesc.getSlots()) {
if (refdIds.contains(slotDesc.getId())) {
slotDesc.setIsMaterialized(true);
}
}
}
if (analyticInfo != null) {
ArrayList<SlotDescriptor> list = analyticInfo.getOutputTupleDesc().getSlots();

for (SlotDescriptor slotDesc : list) {
if (refdIds.contains(slotDesc.getId())) {
slotDesc.setIsMaterialized(true);
}
}
}
if (outputExprs == null) {
// mark all slots in planRoot.getTupleIds() as materialized
ArrayList<TupleId> tids = planRoot.getTupleIds();
for (TupleId tid : tids) {
TupleDescriptor tupleDesc = analyzer.getDescTbl().getTupleDesc(tid);
for (SlotDescriptor slotDesc : tupleDesc.getSlots()) {
slotDesc.setIsMaterialized(true);
}
}
}
}

/**
* Return combined explain string for all plan fragments.
*/
Expand Down Expand Up @@ -201,7 +152,7 @@ public void createPlanFragments(StatementBase statment, Analyzer analyzer, TQuer
}

// compute referenced slots before calling computeMemLayout()
markRefdSlots(analyzer, singleNodePlan, resultExprs, null);
analyzer.markRefdSlots(analyzer, singleNodePlan, resultExprs, null);

setResultExprScale(analyzer, queryStmt.getResultExprs());

Expand Down
9 changes: 3 additions & 6 deletions fe/src/com/baidu/palo/planner/SingleNodePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,26 +651,23 @@ private PlanNode createSelectPlan(SelectStmt selectStmt, Analyzer analyzer, long
// add aggregate node here
AggregateInfo aggInfo = selectStmt.getAggInfo();

/*

// for case: select count(*) from (select col from table) t
// for simple, we just materialize sub tree if has count star
if (aggInfo != null) {
for (FunctionCallExpr aggExpr : aggInfo.getAggregateExprs()) {
if (aggExpr.isCountStar()) {
LOG.debug("count(*) to {}", root.debugString());
markRefdSlots(analyzer, root, null, null);
analyzer.markRefdSlots(analyzer, root, null, null);
break;
}
}
for (Expr groupExpr : aggInfo.getGroupingExprs()) {
if (groupExpr.isConstant()) {
LOG.debug("count(distinct 1) to {}", root.debugString());
markRefdSlots(analyzer, root, null, null);
analyzer.markRefdSlots(analyzer, root, null, null);
break;
}
}
}
*/

turnOffPreAgg(aggInfo, selectStmt, analyzer, root);

Expand Down