From 14893bf26f0f739118284184f790c31a86f5b4a7 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Mon, 3 Feb 2020 22:39:07 +0200 Subject: [PATCH 1/2] EQL: Add field resolution and verification Add basic field resolution inside the Analyzer and a basic Verifier to check for any unresolved fields --- x-pack/plugin/eql/build.gradle | 1 + .../xpack/eql/analysis/AnalysisUtils.java | 97 +++++++++++++++++++ .../xpack/eql/analysis/Analyzer.java | 64 +++++++++++- .../xpack/eql/analysis/AnalyzerRule.java | 27 ++++++ .../xpack/eql/analysis/Verifier.java | 2 + .../function/EqlFunctionRegistry.java | 15 +++ .../xpack/eql/parser/LogicalPlanBuilder.java | 10 +- .../xpack/eql/session/Configuration.java | 3 +- .../xpack/eql/session/EqlSession.java | 4 +- .../elasticsearch/xpack/eql/EqlTestUtils.java | 34 +++++++ .../xpack/eql/analysis/VerifierTests.java | 65 +++++++++++++ .../xpack/eql/parser/LogicalPlanTests.java | 11 +-- .../src/test/resources/mapping-default.json | 55 +++++++++++ .../ql/plan/logical/UnresolvedRelation.java | 6 +- .../xpack/ql/type/TypesTests.java | 13 ++- 15 files changed, 381 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java create mode 100644 x-pack/plugin/eql/src/test/resources/mapping-default.json diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index 47b66cf7151a6..65ec91f6ca83a 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -27,6 +27,7 @@ dependencies { testCompile project(':test:framework') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('security'), configuration: 'testArtifacts') + testCompile project(path: xpackModule('ql'), configuration: 'testArtifacts') testCompile project(path: ':modules:reindex', configuration: 'runtime') testCompile project(path: ':modules:parent-join', configuration: 'runtime') testCompile project(path: ':modules:analysis-common', configuration: 'runtime') diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java new file mode 100644 index 0000000000000..51764b139c2c5 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalysisUtils.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.analysis; + +import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.type.InvalidMappedField; +import org.elasticsearch.xpack.ql.type.UnsupportedEsField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import static java.util.stream.Collectors.toList; + +public final class AnalysisUtils { + + private AnalysisUtils() {} + + // + // Shared methods around the analyzer rules + // + static Attribute resolveAgainstList(UnresolvedAttribute u, Collection attrList) { + return resolveAgainstList(u, attrList, false); + } + + static Attribute resolveAgainstList(UnresolvedAttribute u, Collection attrList, boolean allowCompound) { + List matches = new ArrayList<>(); + + // first take into account the qualified version + boolean qualified = u.qualifier() != null; + + for (Attribute attribute : attrList) { + if (!attribute.synthetic()) { + boolean match = qualified ? Objects.equals(u.qualifiedName(), attribute.qualifiedName()) : + // if the field is unqualified + // first check the names directly + (Objects.equals(u.name(), attribute.name()) + // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields + || Objects.equals(u.name(), attribute.qualifiedName())); + if (match) { + matches.add(attribute.withLocation(u.source())); + } + } + } + + // none found + if (matches.isEmpty()) { + return null; + } + + if (matches.size() == 1) { + return handleSpecialFields(u, matches.get(0), allowCompound); + } + + return u.withUnresolvedMessage( + "Reference [" + u.qualifiedName() + "] is ambiguous (to disambiguate use quotes or qualifiers); matches any of " + + matches.stream().map(a -> "\"" + a.qualifier() + "\".\"" + a.name() + "\"").sorted().collect(toList())); + } + + private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) { + // if it's a object/compound type, keep it unresolved with a nice error message + if (named instanceof FieldAttribute) { + FieldAttribute fa = (FieldAttribute) named; + + // incompatible mappings + if (fa.field() instanceof InvalidMappedField) { + named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being " + + ((InvalidMappedField) fa.field()).errorMessage()); + } + // unsupported types + else if (DataTypes.isUnsupported(fa.dataType())) { + UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field(); + if (unsupportedField.hasInherited()) { + named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] with unsupported type [" + + unsupportedField.getOriginalType() + "] " + "in hierarchy (field [" + unsupportedField.getInherited() + "])"); + } else { + named = u.withUnresolvedMessage( + "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]"); + } + } + // compound fields + else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) { + named = u.withUnresolvedMessage( + "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields"); + } + } + return named; + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java index eb741da145e30..9f18db3c9202b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java @@ -6,13 +6,20 @@ package org.elasticsearch.xpack.eql.analysis; +import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.expression.NamedExpression; +import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.rule.Rule; import org.elasticsearch.xpack.ql.rule.RuleExecutor; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import static java.util.Arrays.asList; +import static org.elasticsearch.xpack.eql.analysis.AnalysisUtils.resolveAgainstList; public class Analyzer extends RuleExecutor { @@ -26,7 +33,8 @@ public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) { @Override protected Iterable.Batch> batches() { - Batch resolution = new Batch("Resolution"); + Batch resolution = new Batch("Resolution", + new ResolveRefs()); return asList(resolution); } @@ -42,4 +50,56 @@ private LogicalPlan verify(LogicalPlan plan) { } return plan; } -} + + private static class ResolveRefs extends AnalyzeRule { + + @Override + protected LogicalPlan rule(LogicalPlan plan) { + // if the children are not resolved, there's no way the node can be resolved + if (!plan.childrenResolved()) { + return plan; + } + + // okay, there's a chance so let's get started + if (log.isTraceEnabled()) { + log.trace("Attempting to resolve {}", plan.nodeString()); + } + + return plan.transformExpressionsUp(e -> { + if (e instanceof UnresolvedAttribute) { + UnresolvedAttribute u = (UnresolvedAttribute) e; + List childrenOutput = new ArrayList<>(); + for (LogicalPlan child : plan.children()) { + childrenOutput.addAll(child.output()); + } + NamedExpression named = resolveAgainstList(u, childrenOutput); + // if resolved, return it; otherwise keep it in place to be resolved later + if (named != null) { + if (log.isTraceEnabled()) { + log.trace("Resolved {} to {}", u, named); + } + return named; + } + } + return e; + }); + } + } + + abstract static class AnalyzeRule extends Rule { + + // transformUp (post-order) - that is first children and then the node + // but with a twist; only if the tree is not resolved or analyzed + @Override + public final LogicalPlan apply(LogicalPlan plan) { + return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken()); + } + + @Override + protected abstract LogicalPlan rule(SubPlan plan); + + protected boolean skipResolved() { + return true; + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java new file mode 100644 index 0000000000000..27352e39a2a05 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/AnalyzerRule.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.analysis; + +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.rule.Rule; + +public abstract class AnalyzerRule extends Rule { + + // transformUp (post-order) - that is first children and then the node + // but with a twist; only if the tree is not resolved or analyzed + @Override + public final LogicalPlan apply(LogicalPlan plan) { + return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken()); + } + + @Override + protected abstract LogicalPlan rule(SubPlan plan); + + protected boolean skipResolved() { + return true; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java index 51dfb6a3d971c..071bdb0cb0029 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Verifier.java @@ -99,6 +99,8 @@ Collection verify(LogicalPlan plan) { }); }); } + + failures.addAll(localFailures); }); return failures; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java new file mode 100644 index 0000000000000..a219d4482d0a0 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function; + +import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; + +public class EqlFunctionRegistry extends FunctionRegistry { + + public EqlFunctionRegistry() { + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java index d87bf06c62855..3662a1d7c351b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java @@ -11,20 +11,16 @@ import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.ql.index.EsIndex; -import org.elasticsearch.xpack.ql.plan.logical.EsRelation; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import static java.util.Collections.emptyMap; - public abstract class LogicalPlanBuilder extends ExpressionBuilder { // TODO: these need to be made configurable - private static final String EVENT_TYPE = "event.category"; - private static final EsIndex esIndex = new EsIndex("", emptyMap()); + private static final String EVENT_TYPE = "event_type"; @Override public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { @@ -43,6 +39,6 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { } - return new Filter(source(ctx), new EsRelation(Source.EMPTY, esIndex, false), condition); + return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Configuration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Configuration.java index c0bfbf389e0f7..d1aff0749e448 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Configuration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Configuration.java @@ -23,8 +23,7 @@ public class Configuration extends org.elasticsearch.xpack.ql.session.Configurat private QueryBuilder filter; public Configuration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, - TimeValue requestTimeout, - boolean includeFrozen, String clientId) { + TimeValue requestTimeout, boolean includeFrozen, String clientId) { super(zi, username, clusterName); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index 20cbd3cf98fe8..babc35fff181f 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; import org.elasticsearch.xpack.eql.execution.PlanExecutor; import org.elasticsearch.xpack.eql.optimizer.Optimizer; +import org.elasticsearch.xpack.eql.parser.EqlParser; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.eql.planner.Planner; import org.elasticsearch.xpack.ql.index.IndexResolver; @@ -97,7 +98,6 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list private LogicalPlan doParse(String eql, List params) { Check.isTrue(params.isEmpty(), "Parameters were given despite being ignored - server bug"); - //LogicalPlan plan = new EqlParser().createStatement(eql); - throw new UnsupportedOperationException(); + return new EqlParser().createStatement(eql); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java new file mode 100644 index 0000000000000..dba73070690db --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.session.Configuration; + +import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong; +import static org.elasticsearch.test.ESTestCase.randomZone; + +public final class EqlTestUtils { + + private EqlTestUtils() {} + + public static final Configuration TEST_CFG = new Configuration(new String[] { "none" }, org.elasticsearch.xpack.ql.util.DateUtils.UTC, + "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, ""); + + public static Configuration randomConfiguration() { + return new Configuration(new String[] {randomAlphaOfLength(16)}, + randomZone(), + randomAlphaOfLength(16), + randomAlphaOfLength(16), + null, + new TimeValue(randomNonNegativeLong()), + randomBoolean(), + randomAlphaOfLength(16)); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java new file mode 100644 index 0000000000000..64f5328b5d1f6 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.analysis; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; +import org.elasticsearch.xpack.eql.parser.EqlParser; +import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexResolution; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.type.EsField; +import org.elasticsearch.xpack.ql.type.TypesTests; + +import java.util.Map; + +public class VerifierTests extends ESTestCase { + + private EqlParser parser = new EqlParser(); + private IndexResolution index = IndexResolution.valid(new EsIndex("test", loadEqlMapping("mapping-default.json"))); + + private LogicalPlan accept(IndexResolution resolution, String eql) { + PreAnalyzer preAnalyzer = new PreAnalyzer(); + Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier()); + return analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution)); + } + + private LogicalPlan accept(String eql) { + return accept(index, eql); + } + + private String error(String sql) { + return error(index, sql); + } + + private String error(IndexResolution resolution, String eql) { + VerificationException e = expectThrows(VerificationException.class, () -> accept(resolution, eql)); + assertTrue(e.getMessage().startsWith("Found ")); + String header = "Found 1 problem(s)\nline "; + return e.getMessage().substring(header.length()); + } + + public void testBasicQuery() { + accept("foo where true"); + } + + public void testMissingColumn() { + assertEquals("1:11: Unknown column [xxx]", error("foo where xxx == 100")); + } + + public void testMisspelledColumn() { + assertEquals("1:11: Unknown column [md4], did you mean [md5]?", error("foo where md4 == 1")); + } + + public void testMisspelledColumnWithMultipleOptions() { + assertEquals("1:11: Unknown column [pib], did you mean any of [pid, ppid]?", error("foo where pib == 1")); + } + + + private static Map loadEqlMapping(String name) { + return TypesTests.loadMapping(name); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java index 157fd9fa4738c..3ee6955813648 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java @@ -4,19 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.xpack.eql.parser; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.index.EsIndex; -import org.elasticsearch.xpack.ql.plan.logical.EsRelation; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.ql.tree.Source; -import static java.util.Collections.emptyMap; - public class LogicalPlanTests extends ESTestCase { private final EqlParser parser = new EqlParser(); @@ -27,9 +23,8 @@ public Expression expr(String source) { public void testEventQuery() { LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'"); - Expression fullExpression = expr("event.category == 'process' and process_name == 'net.exe'"); - EsIndex esIndex = new EsIndex("", emptyMap()); + Expression fullExpression = expr("event_type == 'process' and process_name == 'net.exe'"); - assertEquals(fullQuery, new Filter(null, new EsRelation(Source.EMPTY, esIndex, false), fullExpression)); + assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression)); } } diff --git a/x-pack/plugin/eql/src/test/resources/mapping-default.json b/x-pack/plugin/eql/src/test/resources/mapping-default.json new file mode 100644 index 0000000000000..2c87024f7f09a --- /dev/null +++ b/x-pack/plugin/eql/src/test/resources/mapping-default.json @@ -0,0 +1,55 @@ +{ + "properties" : { + "command_line" : { + "type" : "keyword" + }, + "event_type" : { + "type" : "keyword" + }, + "md5" : { + "type" : "keyword" + }, + "parent_process_name": { + "type" : "keyword" + }, + "parent_process_path": { + "type" : "keyword" + }, + "pid" : { + "type" : "long" + }, + "ppid" : { + "type" : "long" + }, + "process_name": { + "type" : "keyword" + }, + "process_path": { + "type" : "keyword" + }, + "subtype" : { + "type" : "keyword" + }, + "timestamp" : { + "type" : "date" + }, + "user" : { + "type" : "keyword" + }, + "user_name" : { + "type" : "keyword" + }, + "user_domain": { + "type" : "keyword" + }, + "hostname" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/UnresolvedRelation.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/UnresolvedRelation.java index e0c569af21b1b..41d7555358177 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/UnresolvedRelation.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/UnresolvedRelation.java @@ -89,11 +89,11 @@ public boolean equals(Object obj) { } UnresolvedRelation other = (UnresolvedRelation) obj; - return source().equals(other.source()) - && table.equals(other.table) + return Objects.equals(source(), other.source()) + && Objects.equals(table, other.table) && Objects.equals(alias, other.alias) && Objects.equals(frozen, other.frozen) - && unresolvedMsg.equals(other.unresolvedMsg); + && Objects.equals(unresolvedMsg, other.unresolvedMsg); } @Override diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java index 8057c1cd48348..9958d00b02929 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.io.InputStream; import java.util.Map; @@ -195,9 +196,17 @@ public static Map loadMapping(DataTypeRegistry registry, String } public static Map loadMapping(DataTypeRegistry registry, String name, Boolean ordered) { - boolean order = ordered != null ? ordered.booleanValue() : randomBoolean(); InputStream stream = TypesTests.class.getResourceAsStream("/" + name); assertNotNull("Could not find mapping resource:" + name, stream); - return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, stream, order)); + return loadMapping(registry, stream, ordered); + } + + public static Map loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered) { + boolean order = ordered != null ? ordered.booleanValue() : randomBoolean(); + try (InputStream in = stream) { + return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order)); + } catch (IOException ex) { + throw new RuntimeException(ex); + } } } \ No newline at end of file From 55cdd80b5f7f59fa06b0582f1defa320e3c21110 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 4 Feb 2020 21:31:45 +0200 Subject: [PATCH 2/2] EQL: Plug query params into the AstBuilder As the eventType is customizable, plug that into the parser based on the given request. --- .../xpack/eql/action/EqlSearchRequest.java | 12 ++-- .../xpack/eql/action/RequestDefaults.java | 18 ++++++ .../xpack/eql/execution/PlanExecutor.java | 7 +-- .../xpack/eql/parser/AstBuilder.java | 4 ++ .../xpack/eql/parser/EqlParser.java | 19 ++++-- .../xpack/eql/parser/LogicalPlanBuilder.java | 21 +++---- .../xpack/eql/parser/ParserParams.java | 58 +++++++++++++++++++ .../eql/plugin/TransportEqlSearchAction.java | 8 ++- .../xpack/eql/session/EqlSession.java | 13 ++--- .../xpack/eql/parser/LogicalPlanTests.java | 9 +++ 10 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 00814783527b6..b78a398437f99 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -26,6 +26,10 @@ import java.util.function.Supplier; import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FETCH_SIZE; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY; public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent { @@ -34,10 +38,10 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re false, true, false); private QueryBuilder query = null; - private String timestampField = "@timestamp"; - private String eventTypeField = "event.category"; - private String implicitJoinKeyField = "agent.id"; - private int fetchSize = 50; + private String timestampField = FIELD_TIMESTAMP; + private String eventTypeField = FIELD_EVENT_TYPE; + private String implicitJoinKeyField = IMPLICIT_JOIN_KEY; + private int fetchSize = FETCH_SIZE; private SearchAfterBuilder searchAfterBuilder; private String rule; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java new file mode 100644 index 0000000000000..f89b78cb7f3d8 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.action; + +public final class RequestDefaults { + + private RequestDefaults() {} + + public static final String FIELD_TIMESTAMP = "@timestamp"; + public static final String FIELD_EVENT_TYPE = "event_type"; + public static final String IMPLICIT_JOIN_KEY = "agent.id"; + + public static int FETCH_SIZE = 50; +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java index 39658b3acf226..5b9ccab611b14 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; import org.elasticsearch.xpack.eql.analysis.Verifier; import org.elasticsearch.xpack.eql.optimizer.Optimizer; +import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.planner.Planner; import org.elasticsearch.xpack.eql.session.Configuration; import org.elasticsearch.xpack.eql.session.EqlSession; @@ -20,8 +21,6 @@ import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.index.IndexResolver; -import java.util.List; - import static org.elasticsearch.action.ActionListener.wrap; public class PlanExecutor { @@ -53,7 +52,7 @@ private EqlSession newSession(Configuration cfg) { return new EqlSession(client, cfg, indexResolver, preAnalyzer, analyzer, optimizer, planner, this); } - public void eql(Configuration cfg, String eql, List params, ActionListener listener) { - newSession(cfg).eql(eql, params, wrap(listener::onResponse, listener::onFailure)); + public void eql(Configuration cfg, String eql, ParserParams parserParams, ActionListener listener) { + newSession(cfg).eql(eql, parserParams, wrap(listener::onResponse, listener::onFailure)); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java index 9867f757c5e2e..65c6c1a73e63d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AstBuilder.java @@ -11,6 +11,10 @@ public class AstBuilder extends LogicalPlanBuilder { + AstBuilder(ParserParams params) { + super(params); + } + @Override public LogicalPlan visitSingleStatement(SingleStatementContext ctx) { return plan(ctx.statement()); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java index d524564250fdb..4896f3f15f737 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java @@ -39,26 +39,33 @@ public class EqlParser { /** * Parses an EQL statement into execution plan - * @param eql - the EQL statement */ public LogicalPlan createStatement(String eql) { + return createStatement(eql, new ParserParams()); + } + + public LogicalPlan createStatement(String eql, ParserParams params) { if (log.isDebugEnabled()) { log.debug("Parsing as statement: {}", eql); } - return invokeParser(eql, EqlBaseParser::singleStatement, AstBuilder::plan); + return invokeParser(eql, params, EqlBaseParser::singleStatement, AstBuilder::plan); } public Expression createExpression(String expression) { + return createExpression(expression, new ParserParams()); + } + + public Expression createExpression(String expression, ParserParams params) { if (log.isDebugEnabled()) { log.debug("Parsing as expression: {}", expression); } - return invokeParser(expression, EqlBaseParser::singleExpression, AstBuilder::expression); + return invokeParser(expression, params, EqlBaseParser::singleExpression, AstBuilder::expression); } - private T invokeParser(String eql, + private T invokeParser(String eql, ParserParams params, Function parseFunction, - BiFunction visitor) { + BiFunction visitor) { try { EqlBaseLexer lexer = new EqlBaseLexer(new ANTLRInputStream(eql)); @@ -94,7 +101,7 @@ private T invokeParser(String eql, log.info("Parse tree {} " + tree.toStringTree()); } - return visitor.apply(new AstBuilder(), tree); + return visitor.apply(new AstBuilder(params), tree); } catch (StackOverflowError e) { throw new ParsingException("EQL statement is too large, " + "causing stack overflow when generating the parsing tree: [{}]", eql); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java index 3662a1d7c351b..718175282625d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.xpack.eql.parser; import org.elasticsearch.xpack.ql.expression.Expression; @@ -19,8 +18,11 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { - // TODO: these need to be made configurable - private static final String EVENT_TYPE = "event_type"; + private final ParserParams params; + + public LogicalPlanBuilder(ParserParams params) { + this.params = params; + } @Override public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { @@ -28,15 +30,14 @@ public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { Expression condition = expression(ctx.expression()); if (ctx.event != null) { - Source eventTypeSource = source(ctx.event); - String eventTypeName = visitIdentifier(ctx.event); - Literal eventTypeValue = new Literal(eventTypeSource, eventTypeName, DataTypes.KEYWORD); - - UnresolvedAttribute eventTypeField = new UnresolvedAttribute(eventTypeSource, EVENT_TYPE); - Expression eventTypeCheck = new Equals(eventTypeSource, eventTypeField, eventTypeValue); + Source eventSource = source(ctx.event); + String eventName = visitIdentifier(ctx.event); + Literal eventValue = new Literal(eventSource, eventName, DataTypes.KEYWORD); - condition = new And(source, eventTypeCheck, condition); + UnresolvedAttribute eventField = new UnresolvedAttribute(eventSource, params.fieldEventType()); + Expression eventMatch = new Equals(eventSource, eventField, eventValue); + condition = new And(source, eventMatch, condition); } return new Filter(source(ctx), new UnresolvedRelation(Source.EMPTY, null, "", false, ""), condition); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java new file mode 100644 index 0000000000000..1f6cf19f808f2 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.parser; + +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_TYPE; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP; +import static org.elasticsearch.xpack.eql.action.RequestDefaults.IMPLICIT_JOIN_KEY; + +public class ParserParams { + + private String fieldEventType = FIELD_EVENT_TYPE; + private String fieldTimestamp = FIELD_TIMESTAMP; + private String implicitJoinKey = IMPLICIT_JOIN_KEY; + private List queryParams = emptyList(); + + public String fieldEventType() { + return fieldEventType; + } + + public ParserParams fieldEventType(String fieldEventType) { + this.fieldEventType = fieldEventType; + return this; + } + + public String fieldTimestamp() { + return fieldTimestamp; + } + + public ParserParams fieldTimestamp(String fieldTimestamp) { + this.fieldTimestamp = fieldTimestamp; + return this; + } + + public String implicitJoinKey() { + return implicitJoinKey; + } + + public ParserParams implicitJoinKey(String implicitJoinKey) { + this.implicitJoinKey = implicitJoinKey; + return this; + } + + public List params() { + return queryParams; + } + + public ParserParams params(List params) { + this.queryParams = params; + return this; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 960c3c93c5a1d..32f72540d0ea1 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.eql.action.EqlSearchRequest; import org.elasticsearch.xpack.eql.action.EqlSearchResponse; import org.elasticsearch.xpack.eql.execution.PlanExecutor; +import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.session.Configuration; import org.elasticsearch.xpack.eql.session.Results; @@ -63,8 +64,13 @@ public static void operation(PlanExecutor planExecutor, EqlSearchRequest request boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false; String clientId = null; + ParserParams params = new ParserParams() + .fieldEventType(request.eventTypeField()) + .fieldTimestamp(request.timestampField()) + .implicitJoinKey(request.implicitJoinKeyField()); + Configuration cfg = new Configuration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen, clientId); - //planExecutor.eql(cfg, request.rule(), emptyList(), wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure)); + //planExecutor.eql(cfg, request.rule(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure)); listener.onResponse(createResponse(null)); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index babc35fff181f..a89eaf00b52e3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -14,13 +14,11 @@ import org.elasticsearch.xpack.eql.execution.PlanExecutor; import org.elasticsearch.xpack.eql.optimizer.Optimizer; import org.elasticsearch.xpack.eql.parser.EqlParser; +import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.eql.planner.Planner; import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.ql.util.Check; - -import java.util.List; import static org.elasticsearch.action.ActionListener.wrap; @@ -59,11 +57,11 @@ public Configuration configuration() { return configuration; } - public void eql(String eql, List params, ActionListener listener) { + public void eql(String eql, ParserParams params, ActionListener listener) { eqlExecutable(eql, params, wrap(e -> e.execute(this, listener), listener::onFailure)); } - public void eqlExecutable(String eql, List params, ActionListener listener) { + public void eqlExecutable(String eql, ParserParams params, ActionListener listener) { try { physicalPlan(doParse(eql, params), listener); } catch (Exception ex) { @@ -96,8 +94,7 @@ private void preAnalyze(LogicalPlan parsed, ActionListener list }, listener::onFailure)); } - private LogicalPlan doParse(String eql, List params) { - Check.isTrue(params.isEmpty(), "Parameters were given despite being ignored - server bug"); - return new EqlParser().createStatement(eql); + private LogicalPlan doParse(String eql, ParserParams params) { + return new EqlParser().createStatement(eql, params); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java index 3ee6955813648..4b909ab7bdec3 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java @@ -27,4 +27,13 @@ public void testEventQuery() { assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression)); } + + public void testParameterizedEventQuery() { + ParserParams params = new ParserParams().fieldEventType("myCustomEvent"); + LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'", params); + Expression fullExpression = expr("myCustomEvent == 'process' and process_name == 'net.exe'"); + + assertEquals(fullQuery, new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression)); + } + }