Skip to content

Commit

Permalink
EQL: Convert wildcards to LIKE in analyzer (elastic#51901)
Browse files Browse the repository at this point in the history
* EQL: Convert wildcard comparisons to Like
* EQL: Simplify wildcard handling, update tests
* EQL: Lint fixes for Optimizer.java
  • Loading branch information
rw-access authored Mar 6, 2020
1 parent cd5910b commit 51adeaa
Show file tree
Hide file tree
Showing 2 changed files with 168 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,23 @@

package org.elasticsearch.xpack.eql.optimizer;

import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ConstantFolding;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.OptimizerRule;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneFilters;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized;
import org.elasticsearch.xpack.ql.plan.logical.Filter;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.rule.RuleExecutor;

Expand All @@ -33,6 +42,7 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
new BooleanSimplification(),
new BooleanLiteralsOnTheRight(),
// needs to occur before BinaryComparison combinations
new ReplaceWildcards(),
new PropagateEquals(),
new CombineBinaryComparisons(),
// prune/elimination
Expand All @@ -45,4 +55,51 @@ protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {

return Arrays.asList(operators, label);
}


private static class ReplaceWildcards extends OptimizerRule<Filter> {

private static boolean isWildcard(Expression expr) {
if (expr.foldable()) {
Object value = expr.fold();
return value instanceof String && value.toString().contains("*");
}
return false;
}

private static LikePattern toLikePattern(String s) {
// pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself
char escape = 1;

// replace wildcards with % and escape special characters
String likeString = s.replace("%", escape + "%")
.replace("_", escape + "_")
.replace("*", "%");

return new LikePattern(likeString, escape);
}

@Override
protected LogicalPlan rule(Filter filter) {
return filter.transformExpressionsUp(e -> {
// expr == "wildcard*phrase" || expr != "wildcard*phrase"
if (e instanceof Equals || e instanceof NotEquals) {
BinaryComparison cmp = (BinaryComparison) e;

if (isWildcard(cmp.right())) {
String wcString = cmp.right().fold().toString();
Expression like = new Like(e.source(), cmp.left(), toLikePattern(wcString));

if (e instanceof NotEquals) {
like = new Not(e.source(), like);
}

e = like;
}
}

return e;
});
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.eql.optimizer;

import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.eql.analysis.Analyzer;
import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
import org.elasticsearch.xpack.eql.analysis.Verifier;
import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
import org.elasticsearch.xpack.eql.parser.EqlParser;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.index.EsIndex;
import org.elasticsearch.xpack.ql.index.IndexResolution;
import org.elasticsearch.xpack.ql.plan.logical.Filter;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
import org.elasticsearch.xpack.ql.type.EsField;
import org.elasticsearch.xpack.ql.type.TypesTests;

import java.util.Map;

public class OptimizerTests extends ESTestCase {


private static final String INDEX_NAME = "test";
private EqlParser parser = new EqlParser();
private IndexResolution index = loadIndexResolution("mapping-default.json");
private static Map<String, EsField> loadEqlMapping(String name) {
return TypesTests.loadMapping(name);
}

private IndexResolution loadIndexResolution(String name) {
return IndexResolution.valid(new EsIndex(INDEX_NAME, loadEqlMapping(name)));
}

private LogicalPlan accept(IndexResolution resolution, String eql) {
PreAnalyzer preAnalyzer = new PreAnalyzer();
Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier());
Optimizer optimizer = new Optimizer();
return optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution)));
}

private LogicalPlan accept(String eql) {
return accept(index, eql);
}


public void testEqualsWildcard() {
for (String q : new String[]{"foo where command_line == '* bar *'", "foo where '* bar *' == command_line"}) {
LogicalPlan plan = accept(q);
assertTrue(plan instanceof OrderBy);
plan = ((OrderBy) plan).child();
assertTrue(plan instanceof Filter);

Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Like);

Like like = (Like) condition.right();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* bar .*$");
assertEquals(like.pattern().asLuceneWildcard(), "* bar *");
assertEquals(like.pattern().asIndexNameWildcard(), "* bar *");
}
}

public void testNotEqualsWildcard() {
for (String q : new String[]{"foo where command_line != '* baz *'", "foo where '* baz *' != command_line"}) {

LogicalPlan plan = accept(q);
assertTrue(plan instanceof OrderBy);
plan = ((OrderBy) plan).child();
assertTrue(plan instanceof Filter);

Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Not);

Not not = (Not) condition.right();
Like like = (Like) not.field();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* baz .*$");
assertEquals(like.pattern().asLuceneWildcard(), "* baz *");
assertEquals(like.pattern().asIndexNameWildcard(), "* baz *");
}
}

public void testWildcardEscapes() {
LogicalPlan plan = accept("foo where command_line == '* %bar_ * \\\\ \\n \\r \\t'");
assertTrue(plan instanceof OrderBy);
plan = ((OrderBy) plan).child();
assertTrue(plan instanceof Filter);

Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Like);

Like like = (Like) condition.right();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* %bar_ .* \\\\ \n \r \t$");
assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t");
assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t");
}
}

0 comments on commit 51adeaa

Please sign in to comment.