Skip to content

Commit

Permalink
enhance DataTrait uniform, add const value to uniform description
Browse files Browse the repository at this point in the history
  • Loading branch information
@feiniaofeiafei
feiniaofeiafei committed Nov 11, 2024
1 parent 3303c04 commit 801a894
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 21 deletions.
140 changes: 130 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,21 @@

package org.apache.doris.nereids.properties;

import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.util.ImmutableEqualSet;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.org.glassfish.jersey.internal.guava.Maps;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand All @@ -47,15 +52,15 @@ public class DataTrait {

public static final DataTrait EMPTY_TRAIT
= new DataTrait(new NestedSet().toImmutable(),
new NestedSet().toImmutable(), new ImmutableSet.Builder<FdItem>().build(),
new UniformDescription().toImmutable(), new ImmutableSet.Builder<FdItem>().build(),
ImmutableEqualSet.empty(), new FuncDepsDG.Builder().build());
private final NestedSet uniqueSet;
private final NestedSet uniformSet;
private final UniformDescription uniformSet;
private final ImmutableSet<FdItem> fdItems;
private final ImmutableEqualSet<Slot> equalSet;
private final FuncDepsDG fdDg;

private DataTrait(NestedSet uniqueSet, NestedSet uniformSet, ImmutableSet<FdItem> fdItems,
private DataTrait(NestedSet uniqueSet, UniformDescription uniformSet, ImmutableSet<FdItem> fdItems,
ImmutableEqualSet<Slot> equalSet, FuncDepsDG fdDg) {
this.uniqueSet = uniqueSet;
this.uniformSet = uniformSet;
Expand Down Expand Up @@ -90,6 +95,10 @@ public boolean isUniform(Set<Slot> slotSet) {
&& uniformSet.slots.containsAll(slotSet);
}

public boolean isUniformAndHasConstValue(Slot slot) {
return uniformSet.slotUniformValue.containsKey(slot);
}

public boolean isUniqueAndNotNull(Slot slot) {
return !slot.nullable() && isUnique(slot);
}
Expand All @@ -102,11 +111,16 @@ public boolean isUniqueAndNotNull(Set<Slot> slotSet) {
}

public boolean isUniformAndNotNull(Slot slot) {
return !slot.nullable() && isUniform(slot);
return isUniformAndHasConstValue(slot) || (!slot.nullable() && isUniform(slot));
}

public boolean isUniformAndNotNull(ImmutableSet<Slot> slotSet) {
return slotSet.stream().noneMatch(Slot::nullable) && isUniform(slotSet);
for (Slot slot : slotSet) {
if (!isUniformAndNotNull(slot)) {
return false;
}
}
return true;
}

public boolean isNullSafeEqual(Slot l, Slot r) {
Expand Down Expand Up @@ -144,21 +158,21 @@ public String toString() {
*/
public static class Builder {
private final NestedSet uniqueSet;
private final NestedSet uniformSet;
private final UniformDescription uniformSet;
private ImmutableSet<FdItem> fdItems;
private final ImmutableEqualSet.Builder<Slot> equalSetBuilder;
private final FuncDepsDG.Builder fdDgBuilder;

public Builder() {
uniqueSet = new NestedSet();
uniformSet = new NestedSet();
uniformSet = new UniformDescription();
fdItems = new ImmutableSet.Builder<FdItem>().build();
equalSetBuilder = new ImmutableEqualSet.Builder<>();
fdDgBuilder = new FuncDepsDG.Builder();
}

public Builder(DataTrait other) {
this.uniformSet = new NestedSet(other.uniformSet);
this.uniformSet = new UniformDescription(other.uniformSet);
this.uniqueSet = new NestedSet(other.uniqueSet);
this.fdItems = ImmutableSet.copyOf(other.fdItems);
equalSetBuilder = new ImmutableEqualSet.Builder<>(other.equalSet);
Expand All @@ -173,6 +187,11 @@ public void addUniformSlot(DataTrait dataTrait) {
uniformSet.add(dataTrait.uniformSet);
}

public void addUniformSlotAndLiteral(Slot slot, Expression literal) {
uniformSet.add(slot);
uniformSet.slotUniformValue.put(slot, literal);
}

public void addUniqueSlot(Slot slot) {
uniqueSet.add(slot);
}
Expand Down Expand Up @@ -261,8 +280,23 @@ public void addUniqueByEqualSet(Set<Slot> equalSet) {
* if there is a uniform slot in the equivalence set, then all slots of an equivalence set are uniform
*/
public void addUniformByEqualSet(Set<Slot> equalSet) {
if (uniformSet.isIntersect(uniformSet.slots, equalSet)) {
uniformSet.slots.addAll(equalSet);
if (Collections.disjoint(uniformSet.slots, equalSet)) {
return;
}
uniformSet.add(equalSet);
List<Slot> intersectionList = uniformSet.slotUniformValue.keySet().stream()
.filter(equalSet::contains)
.collect(Collectors.toList());
Set<Slot> intersectionSet = new HashSet<>(intersectionList);
if (intersectionList.isEmpty()) {
return;
}
for (Slot equal : equalSet) {
if (intersectionSet.contains(equal)) {
continue;
}
uniformSet.slotUniformValue.put(equal,
uniformSet.slotUniformValue.get(intersectionList.iterator().next()));
}
}

Expand Down Expand Up @@ -450,4 +484,90 @@ public NestedSet toImmutable() {
return new NestedSet(ImmutableSet.copyOf(slots), ImmutableSet.copyOf(slotSets));
}
}

static class UniformDescription {
Set<Slot> slots;
Map<Slot, Expression> slotUniformValue;

public UniformDescription() {
slots = new HashSet<>();
slotUniformValue = new LinkedHashMap<>();
}

public UniformDescription(UniformDescription ud) {
slots = new HashSet<>(ud.slots);
slotUniformValue = new LinkedHashMap<>(ud.slotUniformValue);
}

public UniformDescription(Set<Slot> slots, Map<Slot, Expression> slotUniformValue) {
this.slots = slots;
this.slotUniformValue = slotUniformValue;
}

public UniformDescription toImmutable() {
return new UniformDescription(ImmutableSet.copyOf(slots), ImmutableMap.copyOf(slotUniformValue));
}

public boolean isEmpty() {
return slots.isEmpty();
}

public boolean contains(Slot slot) {
return slots.contains(slot);
}

public void add(Slot slot) {
slots.add(slot);
}

public void add(Set<Slot> s) {
slots.addAll(s);
}

public void add(UniformDescription ud) {
slots.addAll(ud.slots);
slotUniformValue.putAll(ud.slotUniformValue);
}

public void removeNotContain(Set<Slot> slotSet) {
if (slotSet.isEmpty()) {
return;
}
Set<Slot> newSlots = Sets.newHashSetWithExpectedSize(slots.size());
for (Slot slot : slots) {
if (slotSet.contains(slot)) {
newSlots.add(slot);
}
}
this.slots = newSlots;

Map<Slot, Expression> newSlotUniformValue = Maps.newHashMapWithExpectedSize(slotUniformValue.size());
for (Map.Entry<Slot, Expression> entry : slotUniformValue.entrySet()) {
if (slotSet.contains(entry.getKey())) {
newSlotUniformValue.put(entry.getKey(), entry.getValue());
}
}
this.slotUniformValue = newSlotUniformValue;
}

public void replace(Map<Slot, Slot> replaceMap) {
Set<Slot> newSlots = Sets.newHashSetWithExpectedSize(slots.size());
for (Slot s : slots) {
newSlots.add(replaceMap.getOrDefault(s, s));
}
slots = newSlots;

Map<Slot, Expression> newSlotUniformValue = Maps.newHashMapWithExpectedSize(slotUniformValue.size());
for (Map.Entry<Slot, Expression> entry : slotUniformValue.entrySet()) {
Slot newKey = replaceMap.getOrDefault(entry.getKey(), entry.getKey());
newSlotUniformValue.put(newKey, entry.getValue());
}
slotUniformValue = newSlotUniformValue;
}

@Override
public String toString() {
return "{" + slots + "}";
}
}
}
7 changes: 4 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
Expand Down Expand Up @@ -154,9 +155,9 @@ public void computeUnique(Builder builder) {
@Override
public void computeUniform(Builder builder) {
for (Expression e : getConjuncts()) {
Set<Slot> uniformSlots = ExpressionUtils.extractUniformSlot(e);
for (Slot slot : uniformSlots) {
builder.addUniformSlot(slot);
Map<Slot, Expression> uniformSlots = ExpressionUtils.extractUniformSlot(e);
for (Map.Entry<Slot, Expression> entry : uniformSlots.entrySet()) {
builder.addUniformSlotAndLiteral(entry.getKey(), entry.getValue());
}
}
builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
Expand Down
7 changes: 4 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.google.common.collect.ImmutableSet;

import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
Expand Down Expand Up @@ -125,9 +126,9 @@ public void computeUnique(Builder builder) {
@Override
public void computeUniform(Builder builder) {
for (Expression e : getConjuncts()) {
Set<Slot> uniformSlots = ExpressionUtils.extractUniformSlot(e);
for (Slot slot : uniformSlots) {
builder.addUniformSlot(slot);
Map<Slot, Expression> uniformSlots = ExpressionUtils.extractUniformSlot(e);
for (Map.Entry<Slot, Expression> entry : uniformSlots.entrySet()) {
builder.addUniformSlotAndLiteral(entry.getKey(), entry.getValue());
}
}
builder.addUniformSlot(child(0).getLogicalProperties().getTrait());
Expand Down
11 changes: 6 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -736,15 +737,15 @@ public static <E> List<E> collectToList(Collection<? extends Expression> express
/**
* extract uniform slot for the given predicate, such as a = 1 and b = 2
*/
public static ImmutableSet<Slot> extractUniformSlot(Expression expression) {
ImmutableSet.Builder<Slot> builder = new ImmutableSet.Builder<>();
public static ImmutableMap<Slot, Expression> extractUniformSlot(Expression expression) {
ImmutableMap.Builder<Slot, Expression> builder = new ImmutableMap.Builder<>();
if (expression instanceof And) {
builder.addAll(extractUniformSlot(expression.child(0)));
builder.addAll(extractUniformSlot(expression.child(1)));
builder.putAll(extractUniformSlot(expression.child(0)));
builder.putAll(extractUniformSlot(expression.child(1)));
}
if (expression instanceof EqualTo) {
if (isInjective(expression.child(0)) && expression.child(1).isConstant()) {
builder.add((Slot) expression.child(0));
builder.put((Slot) expression.child(0), expression.child(1));
}
}
return builder.build();
Expand Down

0 comments on commit 801a894

Please sign in to comment.