EQL: Plug query params into the AstBuilder (elastic#51886)

As the eventType is customizable, plug that into the parser based on the given request.
costin · Feb 5, 2020 · 5b4a3a3 · 5b4a3a3
1 parent 374eca7
commit 5b4a3a3
Show file tree

Hide file tree

Showing 10 changed files with 136 additions and 33 deletions.
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java
@@ -26,6 +26,10 @@
 import java.util.function.Supplier;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FETCH_SIZE;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY;
 
 public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent {
 
@@ -34,10 +38,10 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
         false, true, false);
 
     private QueryBuilder query = null;
-    private String timestampField = "@timestamp";
-    private String eventTypeField = "event.category";
-    private String implicitJoinKeyField = "agent.id";
-    private int fetchSize = 50;
+    private String timestampField = FIELD_TIMESTAMP;
+    private String eventTypeField = FIELD_EVENT_TYPE;
+    private String implicitJoinKeyField = IMPLICIT_JOIN_KEY;
+    private int fetchSize = FETCH_SIZE;
     private SearchAfterBuilder searchAfterBuilder;
     private String rule;
 

diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+public final class RequestDefaults {
+
+    private RequestDefaults() {}
+
+    public static final String FIELD_TIMESTAMP = "@timestamp";
+    public static final String FIELD_EVENT_TYPE = "event_type";
+    public static final String IMPLICIT_JOIN_KEY = "agent.id";
+
+    public static int FETCH_SIZE = 50;
+}
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java
@@ -13,15 +13,14 @@
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.Verifier;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
+import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.planner.Planner;
 import org.elasticsearch.xpack.eql.session.Configuration;
 import org.elasticsearch.xpack.eql.session.EqlSession;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
 
-import java.util.List;
-
 import static org.elasticsearch.action.ActionListener.wrap;
 
 public class PlanExecutor {
@@ -53,7 +52,7 @@ private EqlSession newSession(Configuration cfg) {
         return new EqlSession(client, cfg, indexResolver, preAnalyzer, analyzer, optimizer, planner, this);
     }
 
-    public void eql(Configuration cfg, String eql, List<Object> params, ActionListener<Results> listener) {
-        newSession(cfg).eql(eql, params, wrap(listener::onResponse, listener::onFailure));
+    public void eql(Configuration cfg, String eql, ParserParams parserParams, ActionListener<Results> listener) {
+        newSession(cfg).eql(eql, parserParams, wrap(listener::onResponse, listener::onFailure));
     }
 }
diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java
@@ -11,6 +11,10 @@
 
 public class AstBuilder extends LogicalPlanBuilder {
 
+    AstBuilder(ParserParams params) {
+        super(params);
+    }
+
     @Override
     public LogicalPlan visitSingleStatement(SingleStatementContext ctx) {
         return plan(ctx.statement());

diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java
@@ -39,26 +39,33 @@ public class EqlParser {
 
     /**
      * Parses an EQL statement into execution plan
-     * @param eql - the EQL statement
      */
     public LogicalPlan createStatement(String eql) {
+        return createStatement(eql, new ParserParams());
+    }
+
+    public LogicalPlan createStatement(String eql, ParserParams params) {
         if (log.isDebugEnabled()) {
             log.debug("Parsing as statement: {}", eql);
         }
-        return invokeParser(eql, EqlBaseParser::singleStatement, AstBuilder::plan);
+        return invokeParser(eql, params, EqlBaseParser::singleStatement, AstBuilder::plan);
     }
 
     public Expression createExpression(String expression) {
+        return createExpression(expression, new ParserParams());
+    }
+
+    public Expression createExpression(String expression, ParserParams params) {
         if (log.isDebugEnabled()) {
             log.debug("Parsing as expression: {}", expression);
         }
 
-        return invokeParser(expression, EqlBaseParser::singleExpression, AstBuilder::expression);
+        return invokeParser(expression, params, EqlBaseParser::singleExpression, AstBuilder::expression);
     }
 
-    private <T> T invokeParser(String eql,
+    private <T> T invokeParser(String eql, ParserParams params,
             Function<EqlBaseParser, ParserRuleContext> parseFunction,
-                               BiFunction<AstBuilder, ParserRuleContext, T> visitor) {
+            BiFunction<AstBuilder, ParserRuleContext, T> visitor) {
         try {
             EqlBaseLexer lexer = new EqlBaseLexer(new ANTLRInputStream(eql));
 
@@ -94,7 +101,7 @@ private <T> T invokeParser(String eql,
                 log.info("Parse tree {} " + tree.toStringTree());
             }
 
-            return visitor.apply(new AstBuilder(), tree);
+            return visitor.apply(new AstBuilder(params), tree);
         } catch (StackOverflowError e) {
             throw new ParsingException("EQL statement is too large, " +
                 "causing stack overflow when generating the parsing tree: [{}]", eql);

diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java
@@ -3,7 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
 package org.elasticsearch.xpack.eql.parser;
 
 import org.elasticsearch.xpack.ql.expression.Expression;
@@ -19,24 +18,26 @@
 
 public abstract class LogicalPlanBuilder extends ExpressionBuilder {
 
-    // TODO: these need to be made configurable
-    private static final String EVENT_TYPE = "event_type";
+    private final ParserParams params;
+
+    public LogicalPlanBuilder(ParserParams params) {
+        this.params = params;
+    }
 
     @Override
     public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) {
         Source source = source(ctx);
         Expression condition = expression(ctx.expression());
 
         if (ctx.event != null) {
-            Source eventTypeSource = source(ctx.event);
-            String eventTypeName = visitIdentifier(ctx.event);
-            Literal eventTypeValue = new Literal(eventTypeSource, eventTypeName, DataTypes.KEYWORD);
-
-            UnresolvedAttribute eventTypeField = new UnresolvedAttribute(eventTypeSource, EVENT_TYPE);
-            Expression eventTypeCheck = new Equals(eventTypeSource, eventTypeField, eventTypeValue);
+            Source eventSource = source(ctx.event);
+            String eventName = visitIdentifier(ctx.event);
+            Literal eventValue = new Literal(eventSource, eventName, DataTypes.KEYWORD);
 
-            condition = new And(source, eventTypeCheck, condition);
+            UnresolvedAttribute eventField = new UnresolvedAttribute(eventSource, params.fieldEventType());
+            Expression eventMatch = new Equals(eventSource, eventField, eventValue);
 
+            condition = new And(source, eventMatch, condition);
         }
 
         return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition);

diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.parser;
+
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP;
+import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY;
+
+public class ParserParams {
+
+    private String fieldEventType = FIELD_EVENT_TYPE;
+    private String fieldTimestamp = FIELD_TIMESTAMP;
+    private String implicitJoinKey = IMPLICIT_JOIN_KEY;
+    private List<Object> queryParams = emptyList();
+
+    public String fieldEventType() {
+        return fieldEventType;
+    }
+
+    public ParserParams fieldEventType(String fieldEventType) {
+        this.fieldEventType = fieldEventType;
+        return this;
+    }
+
+    public String fieldTimestamp() {
+        return fieldTimestamp;
+    }
+
+    public ParserParams fieldTimestamp(String fieldTimestamp) {
+        this.fieldTimestamp = fieldTimestamp;
+        return this;
+    }
+
+    public String implicitJoinKey() {
+        return implicitJoinKey;
+    }
+
+    public ParserParams implicitJoinKey(String implicitJoinKey) {
+        this.implicitJoinKey = implicitJoinKey;
+        return this;
+    }
+
+    public List<Object> params() {
+        return queryParams;
+    }
+
+    public ParserParams params(List<Object> params) {
+        this.queryParams = params;
+        return this;
+    }
+}
diff --git a/...plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/...plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.eql.action.EqlSearchRequest;
 import org.elasticsearch.xpack.eql.action.EqlSearchResponse;
 import org.elasticsearch.xpack.eql.execution.PlanExecutor;
+import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.session.Configuration;
 import org.elasticsearch.xpack.eql.session.Results;
 
@@ -63,8 +64,13 @@ public static void operation(PlanExecutor planExecutor, EqlSearchRequest request
         boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false;
         String clientId = null;
 
+        ParserParams params = new ParserParams()
+                .fieldEventType(request.eventTypeField())
+                .fieldTimestamp(request.timestampField())
+                .implicitJoinKey(request.implicitJoinKeyField());
+
         Configuration cfg = new Configuration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen, clientId);
-        //planExecutor.eql(cfg, request.rule(), emptyList(), wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure));
+        //planExecutor.eql(cfg, request.rule(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure));
         listener.onResponse(createResponse(null));
     }
 

diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java
@@ -14,13 +14,11 @@
 import org.elasticsearch.xpack.eql.execution.PlanExecutor;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
 import org.elasticsearch.xpack.eql.parser.EqlParser;
+import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan;
 import org.elasticsearch.xpack.eql.planner.Planner;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.ql.util.Check;
-
-import java.util.List;
 
 import static org.elasticsearch.action.ActionListener.wrap;
 
@@ -59,11 +57,11 @@ public Configuration configuration() {
         return configuration;
     }
 
-    public void eql(String eql, List<Object> params, ActionListener<Results> listener) {
+    public void eql(String eql, ParserParams params, ActionListener<Results> listener) {
         eqlExecutable(eql, params, wrap(e -> e.execute(this, listener), listener::onFailure));
     }
 
-    public void eqlExecutable(String eql, List<Object> params, ActionListener<PhysicalPlan> listener) {
+    public void eqlExecutable(String eql, ParserParams params, ActionListener<PhysicalPlan> listener) {
         try {
             physicalPlan(doParse(eql, params), listener);
         } catch (Exception ex) {
@@ -96,8 +94,7 @@ private <T> void preAnalyze(LogicalPlan parsed, ActionListener<LogicalPlan> list
         }, listener::onFailure));
     }
 
-    private LogicalPlan doParse(String eql, List<Object> params) {
-        Check.isTrue(params.isEmpty(), "Parameters were given despite being ignored - server bug");
-        return new EqlParser().createStatement(eql);
+    private LogicalPlan doParse(String eql, ParserParams params) {
+        return new EqlParser().createStatement(eql, params);
     }
 }
diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java
@@ -27,4 +27,13 @@ public void testEventQuery() {
 
         assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression));
     }
+
+    public void testParameterizedEventQuery() {
+        ParserParams params = new ParserParams().fieldEventType("myCustomEvent");
+        LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'", params);
+        Expression fullExpression = expr("myCustomEvent == 'process' and process_name == 'net.exe'");
+
+        assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression));
+    }
+
 }