From 5b4a3a3c07eacbc339cbd4c05a3621d056cc8d60 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 5 Feb 2020 15:07:55 +0200 Subject: [PATCH] EQL: Plug query params into the AstBuilder (#51886) As the eventType is customizable, plug that into the parser based on the given request. --- .../xpack/eql/action/EqlSearchRequest.java | 12 ++-- .../xpack/eql/action/RequestDefaults.java | 18 ++++++ .../xpack/eql/execution/PlanExecutor.java | 7 +-- .../xpack/eql/parser/AstBuilder.java | 4 ++ .../xpack/eql/parser/EqlParser.java | 19 ++++-- .../xpack/eql/parser/LogicalPlanBuilder.java | 21 +++---- .../xpack/eql/parser/ParserParams.java | 58 +++++++++++++++++++ .../eql/plugin/TransportEqlSearchAction.java | 8 ++- .../xpack/eql/session/EqlSession.java | 13 ++--- .../xpack/eql/parser/LogicalPlanTests.java | 9 +++ 10 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 00814783527b6..b78a398437f99 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -26,6 +26,10 @@ import java.util.function.Supplier; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FETCH_SIZE; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY; public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent { @@ -34,10 +38,10 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re false, true, false); private QueryBuilder query = null; - private String timestampField = "@timestamp"; - private String eventTypeField = "event.category"; - private String implicitJoinKeyField = "agent.id"; - private int fetchSize = 50; + private String timestampField = FIELD_TIMESTAMP; + private String eventTypeField = FIELD_EVENT_TYPE; + private String implicitJoinKeyField = IMPLICIT_JOIN_KEY; + private int fetchSize = FETCH_SIZE; private SearchAfterBuilder searchAfterBuilder; private String rule; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java new file mode 100644 index 0000000000000..f89b78cb7f3d8 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.action; + +public final class RequestDefaults { + + private RequestDefaults() {} + + public static final String FIELD_TIMESTAMP = "@timestamp"; + public static final String FIELD_EVENT_TYPE = "event_type"; + public static final String IMPLICIT_JOIN_KEY = "agent.id"; + + public static int FETCH_SIZE = 50; +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java index 39658b3acf226..5b9ccab611b14 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; import org.elasticsearch.xpack.eql.analysis.Verifier; import org.elasticsearch.xpack.eql.optimizer.Optimizer; +import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.planner.Planner; import org.elasticsearch.xpack.eql.session.Configuration; import org.elasticsearch.xpack.eql.session.EqlSession; @@ -20,8 +21,6 @@ import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.index.IndexResolver; -import java.util.List; - import static org.elasticsearch.action.ActionListener.wrap; public class PlanExecutor { @@ -53,7 +52,7 @@ private EqlSession newSession(Configuration cfg) { return new EqlSession(client, cfg, indexResolver, preAnalyzer, analyzer, optimizer, planner, this); } - public void eql(Configuration cfg, String eql, List params, ActionListener listener) { - newSession(cfg).eql(eql, params, wrap(listener::onResponse, listener::onFailure)); + public void eql(Configuration cfg, String eql, ParserParams parserParams, ActionListener listener) { + newSession(cfg).eql(eql, parserParams, wrap(listener::onResponse, listener::onFailure)); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java index 9867f757c5e2e..65c6c1a73e63d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java @@ -11,6 +11,10 @@ public class AstBuilder extends LogicalPlanBuilder { + AstBuilder(ParserParams params) { + super(params); + } + @Override public LogicalPlan visitSingleStatement(SingleStatementContext ctx) { return plan(ctx.statement()); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java index d524564250fdb..4896f3f15f737 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java @@ -39,26 +39,33 @@ public class EqlParser { /** * Parses an EQL statement into execution plan - * @param eql - the EQL statement */ public LogicalPlan createStatement(String eql) { + return createStatement(eql, new ParserParams()); + } + + public LogicalPlan createStatement(String eql, ParserParams params) { if (log.isDebugEnabled()) { log.debug("Parsing as statement: {}", eql); } - return invokeParser(eql, EqlBaseParser::singleStatement, AstBuilder::plan); + return invokeParser(eql, params, EqlBaseParser::singleStatement, AstBuilder::plan); } public Expression createExpression(String expression) { + return createExpression(expression, new ParserParams()); + } + + public Expression createExpression(String expression, ParserParams params) { if (log.isDebugEnabled()) { log.debug("Parsing as expression: {}", expression); } - return invokeParser(expression, EqlBaseParser::singleExpression, AstBuilder::expression); + return invokeParser(expression, params, EqlBaseParser::singleExpression, AstBuilder::expression); } - private T invokeParser(String eql, + private T invokeParser(String eql, ParserParams params, Function parseFunction, - BiFunction visitor) { + BiFunction visitor) { try { EqlBaseLexer lexer = new EqlBaseLexer(new ANTLRInputStream(eql)); @@ -94,7 +101,7 @@ private T invokeParser(String eql, log.info("Parse tree {} " + tree.toStringTree()); } - return visitor.apply(new AstBuilder(), tree); + return visitor.apply(new AstBuilder(params), tree); } catch (StackOverflowError e) { throw new ParsingException("EQL statement is too large, " + "causing stack overflow when generating the parsing tree: [{}]", eql); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java index 3662a1d7c351b..718175282625d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.xpack.eql.parser; import org.elasticsearch.xpack.ql.expression.Expression; @@ -19,8 +18,11 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { - // TODO: these need to be made configurable - private static final String EVENT_TYPE = "event_type"; + private final ParserParams params; + + public LogicalPlanBuilder(ParserParams params) { + this.params = params; + } @Override public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { @@ -28,15 +30,14 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { Expression condition = expression(ctx.expression()); if (ctx.event != null) { - Source eventTypeSource = source(ctx.event); - String eventTypeName = visitIdentifier(ctx.event); - Literal eventTypeValue = new Literal(eventTypeSource, eventTypeName, DataTypes.KEYWORD); - - UnresolvedAttribute eventTypeField = new UnresolvedAttribute(eventTypeSource, EVENT_TYPE); - Expression eventTypeCheck = new Equals(eventTypeSource, eventTypeField, eventTypeValue); + Source eventSource = source(ctx.event); + String eventName = visitIdentifier(ctx.event); + Literal eventValue = new Literal(eventSource, eventName, DataTypes.KEYWORD); - condition = new And(source, eventTypeCheck, condition); + UnresolvedAttribute eventField = new UnresolvedAttribute(eventSource, params.fieldEventType()); + Expression eventMatch = new Equals(eventSource, eventField, eventValue); + condition = new And(source, eventMatch, condition); } return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java new file mode 100644 index 0000000000000..1f6cf19f808f2 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.parser; + +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY; + +public class ParserParams { + + private String fieldEventType = FIELD_EVENT_TYPE; + private String fieldTimestamp = FIELD_TIMESTAMP; + private String implicitJoinKey = IMPLICIT_JOIN_KEY; + private List queryParams = emptyList(); + + public String fieldEventType() { + return fieldEventType; + } + + public ParserParams fieldEventType(String fieldEventType) { + this.fieldEventType = fieldEventType; + return this; + } + + public String fieldTimestamp() { + return fieldTimestamp; + } + + public ParserParams fieldTimestamp(String fieldTimestamp) { + this.fieldTimestamp = fieldTimestamp; + return this; + } + + public String implicitJoinKey() { + return implicitJoinKey; + } + + public ParserParams implicitJoinKey(String implicitJoinKey) { + this.implicitJoinKey = implicitJoinKey; + return this; + } + + public List params() { + return queryParams; + } + + public ParserParams params(List params) { + this.queryParams = params; + return this; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 960c3c93c5a1d..32f72540d0ea1 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.eql.action.EqlSearchRequest; import org.elasticsearch.xpack.eql.action.EqlSearchResponse; import org.elasticsearch.xpack.eql.execution.PlanExecutor; +import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.session.Configuration; import org.elasticsearch.xpack.eql.session.Results; @@ -63,8 +64,13 @@ public static void operation(PlanExecutor planExecutor, EqlSearchRequest request boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false; String clientId = null; + ParserParams params = new ParserParams() + .fieldEventType(request.eventTypeField()) + .fieldTimestamp(request.timestampField()) + .implicitJoinKey(request.implicitJoinKeyField()); + Configuration cfg = new Configuration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen, clientId); - //planExecutor.eql(cfg, request.rule(), emptyList(), wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure)); + //planExecutor.eql(cfg, request.rule(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure)); listener.onResponse(createResponse(null)); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index babc35fff181f..a89eaf00b52e3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -14,13 +14,11 @@ import org.elasticsearch.xpack.eql.execution.PlanExecutor; import org.elasticsearch.xpack.eql.optimizer.Optimizer; import org.elasticsearch.xpack.eql.parser.EqlParser; +import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.eql.planner.Planner; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.ql.util.Check; - -import java.util.List; import static org.elasticsearch.action.ActionListener.wrap; @@ -59,11 +57,11 @@ public Configuration configuration() { return configuration; } - public void eql(String eql, List params, ActionListener listener) { + public void eql(String eql, ParserParams params, ActionListener listener) { eqlExecutable(eql, params, wrap(e -> e.execute(this, listener), listener::onFailure)); } - public void eqlExecutable(String eql, List params, ActionListener listener) { + public void eqlExecutable(String eql, ParserParams params, ActionListener listener) { try { physicalPlan(doParse(eql, params), listener); } catch (Exception ex) { @@ -96,8 +94,7 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list }, listener::onFailure)); } - private LogicalPlan doParse(String eql, List params) { - Check.isTrue(params.isEmpty(), "Parameters were given despite being ignored - server bug"); - return new EqlParser().createStatement(eql); + private LogicalPlan doParse(String eql, ParserParams params) { + return new EqlParser().createStatement(eql, params); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java index 3ee6955813648..4b909ab7bdec3 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java @@ -27,4 +27,13 @@ public void testEventQuery() { assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression)); } + + public void testParameterizedEventQuery() { + ParserParams params = new ParserParams().fieldEventType("myCustomEvent"); + LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'", params); + Expression fullExpression = expr("myCustomEvent == 'process' and process_name == 'net.exe'"); + + assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression)); + } + }