From 554105b3f00977fb3ce529f4e60d24534bfac8fe Mon Sep 17 00:00:00 2001 From: forestmvey Date: Mon, 8 May 2023 14:43:57 -0700 Subject: [PATCH] New implementation for Syntax option #1 Signed-off-by: forestmvey --- .../org/opensearch/sql/analysis/Analyzer.java | 24 ---- .../script/filter/FilterQueryBuilder.java | 37 ++++++ .../script/filter/lucene/LuceneQuery.java | 16 ++- .../script/filter/lucene/NestedQuery.java | 105 +++--------------- 4 files changed, 63 insertions(+), 119 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 29c0e4050a..15e82c5319 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -220,7 +220,6 @@ public LogicalPlan visitLimit(Limit node, AnalysisContext context) { public LogicalPlan visitFilter(Filter node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); Expression condition = expressionAnalyzer.analyze(node.getCondition(), context); - verifySupportsCondition(condition); ExpressionReferenceOptimizer optimizer = new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child); @@ -228,28 +227,6 @@ public LogicalPlan visitFilter(Filter node, AnalysisContext context) { return new LogicalFilter(child, optimized); } - /** - * Ensure NESTED function is not used in WHERE, GROUP BY, and HAVING clauses. - * Fallback to legacy engine. Can remove when support is added for NESTED function in WHERE, - * GROUP BY, ORDER BY, and HAVING clauses. - * @param condition : Filter condition - */ - private void verifySupportsCondition(Expression condition) { - if (condition instanceof FunctionExpression) { - if (((FunctionExpression) condition).getFunctionName().getFunctionName().equalsIgnoreCase( - BuiltinFunctionName.NESTED.name() - )) { - throw new SyntaxCheckException( - "Falling back to legacy engine. Nested function is not supported in WHERE," - + " GROUP BY, and HAVING clauses." - ); - } - ((FunctionExpression)condition).getArguments().stream() - .forEach(e -> verifySupportsCondition(e) - ); - } - } - /** * Build {@link LogicalRename}. */ @@ -300,7 +277,6 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) { for (UnresolvedExpression expr : node.getGroupExprList()) { NamedExpression resolvedExpr = namedExpressionAnalyzer.analyze(expr, context); - verifySupportsCondition(resolvedExpr.getDelegated()); groupbyBuilder.add(resolvedExpr); } ImmutableList groupBys = groupbyBuilder.build(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java index 9b37afa2d1..67461991a6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java @@ -25,6 +25,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; @@ -123,6 +124,10 @@ public QueryBuilder visitFunction(FunctionExpression func, Object context) { // example: WHERE nested(foo.bar, nested(zoo.blah, condition)) + if (func.getArguments().size() == 1) { // Syntax: nested(field | field, path) OPERATOR LITERAL + LuceneQuery query = luceneQueries.get(name); + return query.build(func); + } if (func.getArguments().size() > 1) { Expression secondArgument = func.getArguments().get(1); if (secondArgument instanceof FunctionExpression) { @@ -140,11 +145,43 @@ public QueryBuilder visitFunction(FunctionExpression func, Object context) { if (query != null && query.canSupport(func)) { return query.build(func); } + if (query != null && query.isNestedFunction(func)) { + QueryBuilder outerQuery = query.buildNested(func); + boolean hasPathParam = (((FunctionExpression)func.getArguments().get(0)).getArguments().size() == 2); + String pathStr = !hasPathParam ? + getNestedPathString((ReferenceExpression) ((FunctionExpression)func.getArguments().get(0)).getArguments().get(0)) : + ((FunctionExpression)func.getArguments().get(0)).getArguments().get(0).toString(); + NestedQuery innerQuery = (NestedQuery) luceneQueries.get(((FunctionExpression)func.getArguments().get(0)).getFunctionName()); + return innerQuery.adInnerQuery(outerQuery, pathStr); + } + + // Nested used in predicate expression with syntax 'WHERE nested(field | field, path) = ...' + if (func.getArguments().get(0) instanceof FunctionExpression + && ((FunctionExpression)func.getArguments().get(0)).getFunctionName().getFunctionName().equalsIgnoreCase(BuiltinFunctionName.NESTED.name())) { + LuceneQuery innerQuery = luceneQueries.get(((FunctionExpression)func.getArguments().get(0)).getFunctionName()); + return innerQuery.build(func); + // containsInnerNestedQuery() + // innerQuery.buildPredicateExpression() + } else if (query instanceof NestedQuery) { + // TODO Throw exception if does not have conditional parameter. + return query.build(func); + } return buildScriptQuery(func); } } } + private boolean funcArgsIsPredicateExpression(FunctionExpression func) { + func.getArguments().stream().forEach( + a -> { + if (a instanceof FunctionExpression) { + funcArgsIsPredicateExpression((FunctionExpression) a); + } + } + ); + return false; + } + private String getNestedPathString(ReferenceExpression field) { String ret = ""; for (int i = 0; i < field.getPaths().size() - 1; i++) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java index 2458d18ee9..b03f967b67 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java @@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableMap; +import java.sql.Ref; import java.util.ArrayList; import java.util.Map; import java.util.function.BiFunction; @@ -58,11 +59,10 @@ public boolean canSupport(FunctionExpression func) { && (func.getArguments().get(0) instanceof ReferenceExpression) && (func.getArguments().get(1) instanceof LiteralExpression || literalExpressionWrappedByCast(func)) - || isMultiParameterQuery(func) - || isNestedFunction(func); + || isMultiParameterQuery(func); } - private boolean isNestedFunction(FunctionExpression func) { + public boolean isNestedFunction(FunctionExpression func) { return ((func.getArguments().get(0) instanceof FunctionExpression && ((FunctionExpression)func.getArguments().get(0)).getFunctionName().getFunctionName().equalsIgnoreCase("nested")) || func.getFunctionName().getFunctionName().equalsIgnoreCase("nested")); @@ -110,6 +110,16 @@ public QueryBuilder build(FunctionExpression func) { return doBuild(ref.getAttr(), ref.type(), literalValue); } + public QueryBuilder buildNested(FunctionExpression func) { + FunctionExpression ref = (FunctionExpression) func.getArguments().get(0); + Expression expr = func.getArguments().get(1); + ExprValue literalValue = expr instanceof LiteralExpression ? expr + .valueOf() : cast((FunctionExpression) expr); + + ReferenceExpression funcExpr = (ReferenceExpression) ref.getArguments().get(0); + + return doBuild(funcExpr.getAttr(), ref.type(), literalValue); + } private ExprValue cast(FunctionExpression castFunction) { return castMap.get(castFunction.getFunctionName()).apply( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/NestedQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/NestedQuery.java index 6e33a45fd9..2f762e5483 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/NestedQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/NestedQuery.java @@ -28,23 +28,7 @@ public class NestedQuery extends LuceneQuery { @Override public QueryBuilder build(FunctionExpression func) { BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - // WHERE nested(message, message.info = '' AND comment.data = '') - if (func.getFunctionName().getFunctionName().equalsIgnoreCase(BuiltinFunctionName.NESTED.name())) { - switch (((FunctionExpression)func.getArguments().get(1)).getFunctionName().getFunctionName()) { - case "and": - applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::must, boolQuery); - break; - case "or": - applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::should, boolQuery); - break; - case "not": - applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::mustNot, boolQuery); - break; - default: - applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::filter, boolQuery); - } - // No recursion for operators needed - } else if (func.getArguments().get(0) instanceof ReferenceExpression) { // TODO can this be handled in just the else? + if (func.getArguments().get(0) instanceof ReferenceExpression) { // TODO can this be handled in just the else? applyInnerQuery(func, BoolQueryBuilder::filter, boolQuery); } else if (func.getArguments().get(0) instanceof FunctionExpression && ((FunctionExpression)func.getArguments().get(0)).getFunctionName().getFunctionName().equalsIgnoreCase("nested")) { // Is predicate expression @@ -53,85 +37,22 @@ public QueryBuilder build(FunctionExpression func) { return boolQuery; } - private QueryBuilder applyInnerQueryOrRecurse(FunctionExpression func, BiFunction accumulator, BoolQueryBuilder boolQuery) { - if (func.getFunctionName().getFunctionName().equalsIgnoreCase("nested")) { - ReferenceExpression nestedPath = (ReferenceExpression) func.getArguments().get(0); - ReferenceExpression nestedField = (ReferenceExpression) ((FunctionExpression)func.getArguments().get(1)).getArguments().get(0); - ExprValue literal = ((FunctionExpression)func.getArguments().get(1)).getArguments().get(1).valueOf(); - String fieldName = convertTextToKeyword(nestedField.toString(), nestedField.type()); - TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(literal)); - NestedQueryBuilder ret = QueryBuilders.nestedQuery(nestedPath.toString(), termQuery, ScoreMode.None); - return ret; - } else if (func.getArguments().get(0) instanceof ReferenceExpression) { - // innerNestedQuery - // nested(message, message.info = 'a' AND comment.data = 'ab' OR message.dayOfWeek = 1) - // nested(message, message.info = 'a' AND message.dayOfWeek = 1) - ReferenceExpression field = (ReferenceExpression)func.getArguments().get(0); - String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type? - TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf())); - NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None); - return nestedQueryBuilder; - } else if (func.getArguments().get(1) instanceof LiteralExpression) { // Syntax: 'WHERE nested(message.info) = 'a' - ReferenceExpression field = (ReferenceExpression)((FunctionExpression)func.getArguments().get(0)).getArguments().get(0); - String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type? - TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf())); - NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None); - return nestedQueryBuilder; - } else { // Syntax: recursion... - for (Expression arg : func.getArguments()) { - if (arg instanceof FunctionExpression) { - switch (((FunctionExpression)arg).getFunctionName().getFunctionName()) { - case "and": - accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::must, QueryBuilders.boolQuery())); - break; - case "or": - accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::should, QueryBuilders.boolQuery())); - break; - case "not": - accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::mustNot, QueryBuilders.boolQuery())); - break; - default: - accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::filter, QueryBuilders.boolQuery())); - } - } else { - // Throw exception? Shouldn't get here. - } - } - } - return boolQuery; + public QueryBuilder adInnerQuery(QueryBuilder builder, String path) { +// BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); + NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(path, builder, ScoreMode.None); + return nestedQueryBuilder; } private QueryBuilder applyInnerQuery(FunctionExpression func, BiFunction accumulator, BoolQueryBuilder boolQuery) { - if (func.getFunctionName().getFunctionName().equalsIgnoreCase("nested")) { // Not sure if we need this case anymore. - ReferenceExpression nestedPath = (ReferenceExpression) func.getArguments().get(0); - ReferenceExpression nestedField = (ReferenceExpression) ((FunctionExpression)func.getArguments().get(1)).getArguments().get(0); - ExprValue literal = ((FunctionExpression)func.getArguments().get(1)).getArguments().get(1).valueOf(); - String fieldName = convertTextToKeyword(nestedField.toString(), nestedField.type()); - TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(literal)); - NestedQueryBuilder ret = QueryBuilders.nestedQuery(nestedPath.toString(), termQuery, ScoreMode.None); - return accumulator.apply(boolQuery, ret); - } else if (func.getArguments().get(0) instanceof ReferenceExpression) { - // innerNestedQuery - // nested(message, message.info = 'a' AND comment.data = 'ab' OR message.dayOfWeek = 1) - ReferenceExpression field = (ReferenceExpression)func.getArguments().get(0); - String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type? - TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf())); - NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None); - return accumulator.apply(boolQuery, nestedQueryBuilder); - } else if (func.getArguments().get(1) instanceof LiteralExpression) { - // Syntax: 'WHERE nested(message.info) = 'a' - // nestedFunctionAsPredicateExpression - ReferenceExpression field = (ReferenceExpression)((FunctionExpression)func.getArguments().get(0)).getArguments().get(0); - String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type? - TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf())); - NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None); - return accumulator.apply(boolQuery, nestedQueryBuilder); - } else { - // Error? Shouldn't get here. - return null; - } + // Syntax: 'WHERE nested(message.info) = 'a' + // nestedFunctionAsPredicateExpression + ReferenceExpression field = (ReferenceExpression)((FunctionExpression)func.getArguments().get(0)).getArguments().get(0); + String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type? + TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf())); + NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None); + return accumulator.apply(boolQuery, nestedQueryBuilder); + // TODO add range query and others that may apply... } private String getNestedPathString(ReferenceExpression field) {