-
Notifications
You must be signed in to change notification settings - Fork 25k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
SQL: Resolve attributes recursively for improved subquery support #69765
Changes from 5 commits
aba2f63
39e4cf1
015acb3
3b755be
f739e39
932c01c
fa49615
1cb642d
50ab4d1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,8 @@ | |
import java.util.Set; | ||
|
||
import static java.util.stream.Collectors.toList; | ||
import static org.elasticsearch.xpack.ql.TestUtils.fieldAttribute; | ||
import static org.elasticsearch.xpack.ql.TestUtils.of; | ||
import static org.hamcrest.Matchers.arrayContaining; | ||
import static org.hamcrest.Matchers.arrayWithSize; | ||
import static org.hamcrest.Matchers.contains; | ||
|
@@ -61,6 +63,30 @@ public void testAttributeMapWithSameAliasesCanResolveAttributes() { | |
assertTrue(newAttributeMap.get(param2.toAttribute()) == param2.child()); | ||
} | ||
|
||
public void testResolve() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd like to see more tests for dealing with: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ✓ Done. |
||
AttributeMap.Builder<Object> builder = AttributeMap.builder(); | ||
Attribute one = a("one"); | ||
Attribute two = fieldAttribute("two", DataTypes.INTEGER); | ||
Attribute three = fieldAttribute("three", DataTypes.INTEGER); | ||
Alias threeAlias = new Alias(Source.EMPTY, "three_alias", three); | ||
Alias threeAliasAlias = new Alias(Source.EMPTY, "three_alias_alias", threeAlias); | ||
builder.put(one, of("one")); | ||
builder.put(two, "two"); | ||
builder.put(three, of("three")); | ||
builder.put(threeAlias.toAttribute(), threeAlias.child()); | ||
builder.put(threeAliasAlias.toAttribute(), threeAliasAlias.child()); | ||
AttributeMap<Object> map = builder.build(); | ||
|
||
assertEquals(of("one"), map.resolve(one, null)); | ||
assertEquals("two", map.resolve(two, null)); | ||
assertEquals(of("three"), map.resolve(three, null)); | ||
assertEquals(of("three"), map.resolve(threeAlias, null)); | ||
assertEquals(of("three"), map.resolve(threeAliasAlias, null)); | ||
Attribute four = a("four"); | ||
assertEquals("not found", map.resolve(four, "not found")); | ||
assertNull(map.resolve(four, null)); | ||
} | ||
|
||
private Alias createIntParameterAlias(int index, int value) { | ||
Source source = new Source(1, index * 5, "?"); | ||
Literal literal = new Literal(source, value, DataTypes.INTEGER); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -316,10 +316,11 @@ private static boolean checkGroupByOrder(LogicalPlan p, Set<Failure> localFailur | |
Map<Expression, Node<?>> missing = new LinkedHashMap<>(); | ||
|
||
o.order().forEach(oe -> { | ||
Expression e = oe.child(); | ||
final Expression e = oe.child(); | ||
final Expression resolvedE = attributeRefs.resolve(e, e); | ||
|
||
// aggregates are allowed | ||
if (Functions.isAggregate(attributeRefs.getOrDefault(e, e))) { | ||
if (Functions.isAggregate(resolvedE)) { | ||
return; | ||
} | ||
|
||
|
@@ -340,8 +341,12 @@ private static boolean checkGroupByOrder(LogicalPlan p, Set<Failure> localFailur | |
// e.g.: if "GROUP BY f2(f1(field))" you can "ORDER BY f4(f3(f2(f1(field))))" | ||
// | ||
// Also, make sure to compare attributes directly | ||
if (e.anyMatch(expression -> Expressions.anyMatch(groupingAndMatchingAggregatesAliases, | ||
g -> expression.semanticEquals(expression instanceof Attribute ? Expressions.attribute(g) : g)))) { | ||
if (resolvedE.anyMatch(expression -> Expressions.anyMatch(groupingAndMatchingAggregatesAliases, | ||
g -> { | ||
Expression resolvedG = attributeRefs.resolve(g, g); | ||
resolvedG = expression instanceof Attribute ? Expressions.attribute(resolvedG) : resolvedG; | ||
return expression.semanticEquals(resolvedG); | ||
}))) { | ||
return; | ||
} | ||
|
||
|
@@ -406,7 +411,7 @@ private static boolean checkGroupByHavingHasOnlyAggs(Expression e, Set<Expressio | |
|
||
// resolve FunctionAttribute to backing functions | ||
if (e instanceof ReferenceAttribute) { | ||
e = attributeRefs.get(e); | ||
e = attributeRefs.resolve(e, null); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The null variant is used enough times to make a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ✓ Done. |
||
} | ||
|
||
// scalar functions can be a binary tree | ||
|
@@ -484,7 +489,7 @@ private static boolean onlyRawFields(Iterable<? extends Expression> expressions, | |
|
||
expressions.forEach(e -> e.forEachDown(c -> { | ||
if (c instanceof ReferenceAttribute) { | ||
c = attributeRefs.getOrDefault(c, c); | ||
c = attributeRefs.resolve(c, c); | ||
} | ||
if (c instanceof Function) { | ||
localFailures.add(fail(c, "No functions allowed (yet); encountered [{}]", c.sourceText())); | ||
|
@@ -579,7 +584,7 @@ private static boolean checkGroupMatch(Expression e, Node<?> source, List<Expres | |
|
||
// resolve FunctionAttribute to backing functions | ||
if (e instanceof ReferenceAttribute) { | ||
e = attributeRefs.get(e); | ||
e = attributeRefs.resolve(e, null); | ||
} | ||
|
||
// scalar functions can be a binary tree | ||
|
@@ -668,7 +673,7 @@ private static void checkFilterOnAggs(LogicalPlan p, Set<Failure> localFailures, | |
LogicalPlan filterChild = filter.child(); | ||
if (filterChild instanceof Aggregate == false) { | ||
filter.condition().forEachDown(Expression.class, e -> { | ||
if (Functions.isAggregate(attributeRefs.getOrDefault(e, e))) { | ||
if (Functions.isAggregate(attributeRefs.resolve(e, e))) { | ||
if (filterChild instanceof Project) { | ||
filter.condition().forEachDown(FieldAttribute.class, | ||
f -> localFailures.add(fail(e, "[{}] field must appear in the GROUP BY clause or in an aggregate function", | ||
|
@@ -690,7 +695,7 @@ private static void checkFilterOnGrouping(LogicalPlan p, Set<Failure> localFailu | |
if (p instanceof Filter) { | ||
Filter filter = (Filter) p; | ||
filter.condition().forEachDown(Expression.class, e -> { | ||
if (Functions.isGrouping(attributeRefs.getOrDefault(e, e))) { | ||
if (Functions.isGrouping(attributeRefs.resolve(e, e))) { | ||
localFailures | ||
.add(fail(e, "Cannot filter on grouping function [{}], use its argument instead", Expressions.name(e))); | ||
} | ||
|
@@ -717,7 +722,7 @@ private static void checkNestedUsedInGroupByOrHavingOrWhereOrOrderBy(LogicalPlan | |
} | ||
}; | ||
Consumer<Expression> checkForNested = e -> | ||
attributeRefs.getOrDefault(e, e).forEachUp(FieldAttribute.class, matchNested); | ||
attributeRefs.resolve(e, e).forEachUp(FieldAttribute.class, matchNested); | ||
Consumer<ScalarFunction> checkForNestedInFunction = f -> f.arguments().forEach( | ||
arg -> arg.forEachUp(FieldAttribute.class, matchNested)); | ||
|
||
|
@@ -739,7 +744,7 @@ private static void checkNestedUsedInGroupByOrHavingOrWhereOrOrderBy(LogicalPlan | |
|
||
// check in where (scalars not allowed) | ||
p.forEachDown(Filter.class, f -> f.condition().forEachUp(e -> | ||
attributeRefs.getOrDefault(e, e).forEachUp(ScalarFunction.class, sf -> { | ||
attributeRefs.resolve(e, e).forEachUp(ScalarFunction.class, sf -> { | ||
if (sf instanceof BinaryComparison == false && | ||
sf instanceof IsNull == false && | ||
sf instanceof IsNotNull == false && | ||
|
@@ -758,7 +763,7 @@ private static void checkNestedUsedInGroupByOrHavingOrWhereOrOrderBy(LogicalPlan | |
|
||
// check in order by (scalars not allowed) | ||
p.forEachDown(OrderBy.class, ob -> ob.order().forEach(o -> o.forEachUp(e -> | ||
attributeRefs.getOrDefault(e, e).forEachUp(ScalarFunction.class, checkForNestedInFunction) | ||
attributeRefs.resolve(e, e).forEachUp(ScalarFunction.class, checkForNestedInFunction) | ||
))); | ||
if (nested.isEmpty() == false) { | ||
localFailures.add( | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -222,8 +222,8 @@ public LogicalPlan apply(LogicalPlan plan) { | |
AttributeMap.Builder<Expression> builder = AttributeMap.builder(); | ||
// collect aliases | ||
plan.forEachExpressionUp(Alias.class, a -> builder.put(a.toAttribute(), a.child())); | ||
final Map<Attribute, Expression> collectRefs = builder.build(); | ||
java.util.function.Function<ReferenceAttribute, Expression> replaceReference = r -> collectRefs.getOrDefault(r, r); | ||
final AttributeMap<Expression> collectRefs = builder.build(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Noise. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
java.util.function.Function<ReferenceAttribute, Expression> replaceReference = r -> collectRefs.resolve(r, r); | ||
|
||
plan = plan.transformUp(p -> { | ||
// non attribute defining plans get their references removed | ||
|
@@ -300,7 +300,7 @@ static class PruneOrderByNestedFields extends OptimizerRule<Project> { | |
private void findNested(Expression exp, AttributeMap<Function> functions, Consumer<FieldAttribute> onFind) { | ||
exp.forEachUp(e -> { | ||
if (e instanceof ReferenceAttribute) { | ||
Function f = functions.get(e); | ||
Function f = functions.resolve(e, null); | ||
if (f != null) { | ||
findNested(f, functions, onFind); | ||
} | ||
|
@@ -578,7 +578,7 @@ private List<NamedExpression> combineProjections(List<? extends NamedExpression> | |
// replace any matching attribute with a lower alias (if there's a match) | ||
// but clean-up non-top aliases at the end | ||
for (NamedExpression ne : upper) { | ||
NamedExpression replacedExp = (NamedExpression) ne.transformUp(Attribute.class, a -> aliases.getOrDefault(a, a)); | ||
NamedExpression replacedExp = (NamedExpression) ne.transformUp(Attribute.class, a -> aliases.resolve(a, a)); | ||
replaced.add((NamedExpression) CleanAliases.trimNonTopLevelAliases(replacedExp)); | ||
} | ||
return replaced; | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As this method builds on other types methods in this class, it shouldn't have unchecked warnings nor used
NOT_FOUND
. Also not sure why thecandidate
is returned instead ofvalue
, should be the other way around.Further more this implementation looks buggy: if the map contains
(e,e)
calling resolve(e, default) would incorrectly returndefault
and note
.Further more there's no handling of cycles which leads to an infinite loop: (a, b), (b, a).
Below a suggestion to fix both issues:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good catch with the default value. I was thinking of adding the limitation on the number of lookups, but seemed we would need quite a huge bug in our AttributeMap population for it to happen with more than one hops in the cycle. Anyways, I added this defensive check.
The suggested code won't really work, because:
1.
resolveOrDefault(e, e)
will always returne
even if the map hase -> f
mapping.2. The loop will always terminate after the first lookup (
key == value == candidate
)Anyways, the updated code fixes the bug you found.