EQL: Plug query params into the AstBuilder

x-pack/plugin/eql/build.gradle

@@ -27,6 +27,7 @@ dependencies {
   testCompile project(':test:framework')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
+  testCompile project(path: xpackModule('ql'), configuration: 'testArtifacts')
   testCompile project(path: ':modules:reindex', configuration: 'runtime')
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
   testCompile project(path: ':modules:analysis-common', configuration: 'runtime')

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java

@@ -26,6 +26,10 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FETCH_SIZE;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY;
 
 public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent {
 
@@ -34,10 +38,10 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
         false, true, false);
 
     private QueryBuilder query = null;
-    private String timestampField = "@timestamp";
-    private String eventTypeField = "event.category";
-    private String implicitJoinKeyField = "agent.id";
-    private int fetchSize = 50;
+    private String timestampField = FIELD_TIMESTAMP;
+    private String eventTypeField = FIELD_EVENT_TYPE;
+    private String implicitJoinKeyField = IMPLICIT_JOIN_KEY;
+    private int fetchSize = FETCH_SIZE;
     private SearchAfterBuilder searchAfterBuilder;
     private String rule;
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+public final class RequestDefaults {
+
+    private RequestDefaults() {}
+
+    public static final String FIELD_TIMESTAMP = "@timestamp";
+    public static final String FIELD_EVENT_TYPE = "event_type";
+    public static final String IMPLICIT_JOIN_KEY = "agent.id";
+
+    public static int FETCH_SIZE = 50;
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.analysis;
+
+import org.elasticsearch.xpack.ql.expression.Attribute;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+import org.elasticsearch.xpack.ql.type.InvalidMappedField;
+import org.elasticsearch.xpack.ql.type.UnsupportedEsField;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.stream.Collectors.toList;
+
+public final class AnalysisUtils {
+
+    private AnalysisUtils() {}
+
+    //
+    // Shared methods around the analyzer rules
+    //
+    static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList) {
+        return resolveAgainstList(u, attrList, false);
+    }
+
+    static Attribute resolveAgainstList(UnresolvedAttribute u, Collection<Attribute> attrList, boolean allowCompound) {
+        List<Attribute> matches = new ArrayList<>();
+
+        // first take into account the qualified version
+        boolean qualified = u.qualifier() != null;
+
+        for (Attribute attribute : attrList) {
+            if (!attribute.synthetic()) {
+                boolean match = qualified ? Objects.equals(u.qualifiedName(), attribute.qualifiedName()) :
+                // if the field is unqualified
+                // first check the names directly
+                        (Objects.equals(u.name(), attribute.name())
+                                // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
+                                || Objects.equals(u.name(), attribute.qualifiedName()));
+                if (match) {
+                    matches.add(attribute.withLocation(u.source()));
+                }
+            }
+        }
+
+        // none found
+        if (matches.isEmpty()) {
+            return null;
+        }
+
+        if (matches.size() == 1) {
+            return handleSpecialFields(u, matches.get(0), allowCompound);
+        }
+
+        return u.withUnresolvedMessage(
+                "Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); matches any of "
+                        + matches.stream().map(a -> "\"" + a.qualifier() + "\".\"" + a.name() + "\"").sorted().collect(toList()));
+    }
+
+    private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
+        // if it's a object/compound type, keep it unresolved with a nice error message
+        if (named instanceof FieldAttribute) {
+            FieldAttribute fa = (FieldAttribute) named;
+
+            // incompatible mappings
+            if (fa.field() instanceof InvalidMappedField) {
+                named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being "
+                        + ((InvalidMappedField) fa.field()).errorMessage());
+            }
+            // unsupported types
+            else if (DataTypes.isUnsupported(fa.dataType())) {
+                UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
+                if (unsupportedField.hasInherited()) {
+                    named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] with unsupported type ["
+                            + unsupportedField.getOriginalType() + "] " + "in hierarchy (field [" + unsupportedField.getInherited() + "])");
+                } else {
+                    named = u.withUnresolvedMessage(
+                            "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
+                }
+            }
+            // compound fields
+            else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
+                named = u.withUnresolvedMessage(
+                        "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields");
+            }
+        }
+        return named;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java

@@ -6,13 +6,20 @@
 
 package org.elasticsearch.xpack.eql.analysis;
 
+import org.elasticsearch.xpack.ql.expression.Attribute;
+import org.elasticsearch.xpack.ql.expression.NamedExpression;
+import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.rule.Rule;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList;
 
 public class Analyzer extends RuleExecutor<LogicalPlan> {
 
@@ -26,7 +33,8 @@ public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) {
 
     @Override
     protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
-        Batch resolution = new Batch("Resolution");
+        Batch resolution = new Batch("Resolution",
+                new ResolveRefs());
 
         return asList(resolution);
     }
@@ -42,4 +50,56 @@ private LogicalPlan verify(LogicalPlan plan) {
         }
         return plan;
     }
-}
+
+    private static class ResolveRefs extends AnalyzeRule<LogicalPlan> {
+
+        @Override
+        protected LogicalPlan rule(LogicalPlan plan) {
+            // if the children are not resolved, there's no way the node can be resolved
+            if (!plan.childrenResolved()) {
+                return plan;
+            }
+
+            // okay, there's a chance so let's get started
+            if (log.isTraceEnabled()) {
+                log.trace("Attempting to resolve {}", plan.nodeString());
+            }
+
+            return plan.transformExpressionsUp(e -> {
+                if (e instanceof UnresolvedAttribute) {
+                    UnresolvedAttribute u = (UnresolvedAttribute) e;
+                    List<Attribute> childrenOutput = new ArrayList<>();
+                    for (LogicalPlan child : plan.children()) {
+                        childrenOutput.addAll(child.output());
+                    }
+                    NamedExpression named = resolveAgainstList(u, childrenOutput);
+                    // if resolved, return it; otherwise keep it in place to be resolved later
+                    if (named != null) {
+                        if (log.isTraceEnabled()) {
+                            log.trace("Resolved {} to {}", u, named);
+                        }
+                        return named;
+                    }
+                }
+                return e;
+            });
+        }
+    }
+
+    abstract static class AnalyzeRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
+
+        // transformUp (post-order) - that is first children and then the node
+        // but with a twist; only if the tree is not resolved or analyzed
+        @Override
+        public final LogicalPlan apply(LogicalPlan plan) {
+            return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
+        }
+
+        @Override
+        protected abstract LogicalPlan rule(SubPlan plan);
+
+        protected boolean skipResolved() {
+            return true;
+        }
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.analysis;
+
+import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.ql.rule.Rule;
+
+public abstract class AnalyzerRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
+
+    // transformUp (post-order) - that is first children and then the node
+    // but with a twist; only if the tree is not resolved or analyzed
+    @Override
+    public final LogicalPlan apply(LogicalPlan plan) {
+        return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken());
+    }
+
+    @Override
+    protected abstract LogicalPlan rule(SubPlan plan);
+
+    protected boolean skipResolved() {
+        return true;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java

@@ -99,6 +99,8 @@ Collection<Failure> verify(LogicalPlan plan) {
                     });
                 });
             }
+
+            failures.addAll(localFailures);
         });
 
         return failures;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java

@@ -13,15 +13,14 @@
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.Verifier;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
+import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.planner.Planner;
 import org.elasticsearch.xpack.eql.session.Configuration;
 import org.elasticsearch.xpack.eql.session.EqlSession;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
 
-import java.util.List;
-
 import static org.elasticsearch.action.ActionListener.wrap;
 
 public class PlanExecutor {
@@ -53,7 +52,7 @@ private EqlSession newSession(Configuration cfg) {
         return new EqlSession(client, cfg, indexResolver, preAnalyzer, analyzer, optimizer, planner, this);
     }
 
-    public void eql(Configuration cfg, String eql, List<Object> params, ActionListener<Results> listener) {
-        newSession(cfg).eql(eql, params, wrap(listener::onResponse, listener::onFailure));
+    public void eql(Configuration cfg, String eql, ParserParams parserParams, ActionListener<Results> listener) {
+        newSession(cfg).eql(eql, parserParams, wrap(listener::onResponse, listener::onFailure));
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function;
+
+import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+
+public class EqlFunctionRegistry extends FunctionRegistry {
+
+    public EqlFunctionRegistry() {
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java

@@ -11,6 +11,10 @@
 
 public class AstBuilder extends LogicalPlanBuilder {
 
+    AstBuilder(ParserParams params) {
+        super(params);
+    }
+
     @Override
     public LogicalPlan visitSingleStatement(SingleStatementContext ctx) {
         return plan(ctx.statement());

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java

@@ -39,26 +39,33 @@ public class EqlParser {
 
     /**
      * Parses an EQL statement into execution plan
-     * @param eql - the EQL statement
      */
     public LogicalPlan createStatement(String eql) {
+        return createStatement(eql, new ParserParams());
+    }
+
+    public LogicalPlan createStatement(String eql, ParserParams params) {
         if (log.isDebugEnabled()) {
             log.debug("Parsing as statement: {}", eql);
         }
-        return invokeParser(eql, EqlBaseParser::singleStatement, AstBuilder::plan);
+        return invokeParser(eql, params, EqlBaseParser::singleStatement, AstBuilder::plan);
     }
 
     public Expression createExpression(String expression) {
+        return createExpression(expression, new ParserParams());
+    }
+
+    public Expression createExpression(String expression, ParserParams params) {
         if (log.isDebugEnabled()) {
             log.debug("Parsing as expression: {}", expression);
         }
 
-        return invokeParser(expression, EqlBaseParser::singleExpression, AstBuilder::expression);
+        return invokeParser(expression, params, EqlBaseParser::singleExpression, AstBuilder::expression);
     }
 
-    private <T> T invokeParser(String eql,
+    private <T> T invokeParser(String eql, ParserParams params,
             Function<EqlBaseParser, ParserRuleContext> parseFunction,
-                               BiFunction<AstBuilder, ParserRuleContext, T> visitor) {
+            BiFunction<AstBuilder, ParserRuleContext, T> visitor) {
         try {
             EqlBaseLexer lexer = new EqlBaseLexer(new ANTLRInputStream(eql));
 
@@ -94,7 +101,7 @@ private <T> T invokeParser(String eql,
                 log.info("Parse tree {} " + tree.toStringTree());
             }
 
-            return visitor.apply(new AstBuilder(), tree);
+            return visitor.apply(new AstBuilder(params), tree);
         } catch (StackOverflowError e) {
             throw new ParsingException("EQL statement is too large, " +
                 "causing stack overflow when generating the parsing tree: [{}]", eql);