From f58680bad33d5ce4139157a69a4d9f5f286bc3c4 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Thu, 19 Mar 2020 16:58:54 +0200 Subject: [PATCH] EQL: Add Substring function with Python semantics (#53688) Does not reuse substring from SQL due to the difference in semantics and the accepted arguments. Currently it is missing full integration tests as, due to the usage of scripting, requires an actual integration test against a proper cluster (and likely its own QA project). --- x-pack/plugin/eql/build.gradle | 6 - .../xpack/eql/analysis/Analyzer.java | 43 ++++-- .../xpack/eql/execution/PlanExecutor.java | 3 +- .../function/EqlFunctionRegistry.java | 20 +++ .../function/scalar/string/StringUtils.java | 50 +++++++ .../function/scalar/string/Substring.java | 129 ++++++++++++++++++ .../scalar/string/SubstringFunctionPipe.java | 111 +++++++++++++++ .../string/SubstringFunctionProcessor.java | 108 +++++++++++++++ .../whitelist/InternalEqlScriptUtils.java | 24 ++++ .../eql/plugin/EqlPainlessExtension.java | 41 ++++++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/eql/plugin/eql_whitelist.txt | 53 +++++++ .../eql/action/AbstractEqlIntegTestCase.java | 4 +- .../xpack/eql/action/EqlActionIT.java | 3 +- .../xpack/eql/analysis/VerifierTests.java | 4 +- .../scalar/string/StringUtilsTests.java | 75 ++++++++++ .../xpack/eql/planner/QueryFolderOkTests.java | 12 ++ .../expression/function/FunctionRegistry.java | 8 +- .../whitelist/InternalQlScriptUtils.java | 26 +++- .../ql/expression/gen/script/Scripts.java | 10 +- .../scalar/string/StringFunctionUtils.java | 14 +- .../whitelist/InternalSqlScriptUtils.java | 28 ---- 22 files changed, 704 insertions(+), 69 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java create mode 100644 x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index 2d720f170d83c..97713aa8f7ee2 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -46,12 +46,6 @@ dependencies { // TOML parser for EqlActionIT tests testCompile 'io.ous:jtoml:2.0.0' - - // JSON parser for tests input data - testCompile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" - testCompile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - testCompile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" - } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java index 82ce55ac74f3d..aecd531984606 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java @@ -10,9 +10,11 @@ import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.ql.expression.function.Function; +import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; +import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.ql.rule.Rule; import org.elasticsearch.xpack.ql.rule.RuleExecutor; import java.util.ArrayList; @@ -35,7 +37,8 @@ public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) { @Override protected Iterable.Batch> batches() { Batch resolution = new Batch("Resolution", - new ResolveRefs()); + new ResolveRefs(), + new ResolveFunctions()); return asList(resolution); } @@ -52,7 +55,7 @@ private LogicalPlan verify(LogicalPlan plan) { return plan; } - private static class ResolveRefs extends AnalyzeRule { + private static class ResolveRefs extends AnalyzerRule { @Override protected LogicalPlan rule(LogicalPlan plan) { @@ -87,20 +90,34 @@ protected LogicalPlan rule(LogicalPlan plan) { } } - abstract static class AnalyzeRule extends Rule { + private class ResolveFunctions extends AnalyzerRule { - // transformUp (post-order) - that is first children and then the node - // but with a twist; only if the tree is not resolved or analyzed @Override - public final LogicalPlan apply(LogicalPlan plan) { - return plan.transformUp(t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t), typeToken()); - } + protected LogicalPlan rule(LogicalPlan plan) { + return plan.transformExpressionsUp(e -> { + if (e instanceof UnresolvedFunction) { + UnresolvedFunction uf = (UnresolvedFunction) e; - @Override - protected abstract LogicalPlan rule(SubPlan plan); + if (uf.analyzed()) { + return uf; + } + + String name = uf.name(); - protected boolean skipResolved() { - return true; + if (uf.childrenResolved() == false) { + return uf; + } + + String functionName = functionRegistry.resolveAlias(name); + if (functionRegistry.functionExists(functionName) == false) { + return uf.missing(functionName, functionRegistry.listFunctions()); + } + FunctionDefinition def = functionRegistry.resolveFunction(functionName); + Function f = uf.buildResolved(null, def); + return f; + } + return e; + }); } } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java index 650d5b750a8a0..cc6ba5d018f6a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.eql.analysis.Analyzer; import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; import org.elasticsearch.xpack.eql.analysis.Verifier; +import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; import org.elasticsearch.xpack.eql.optimizer.Optimizer; import org.elasticsearch.xpack.eql.parser.ParserParams; import org.elasticsearch.xpack.eql.planner.Planner; @@ -44,7 +45,7 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe this.writableRegistry = writeableRegistry; this.indexResolver = indexResolver; - this.functionRegistry = null; + this.functionRegistry = new EqlFunctionRegistry(); this.metrics = new Metrics(); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index a219d4482d0a0..119e12fa9f39c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,10 +6,30 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring; +import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; +import java.util.Locale; + public class EqlFunctionRegistry extends FunctionRegistry { public EqlFunctionRegistry() { + super(functions()); + } + + private static FunctionDefinition[][] functions() { + return new FunctionDefinition[][] { + // Scalar functions + // String + new FunctionDefinition[] { + def(Substring.class, Substring::new, "substring"), + }, + }; + } + + @Override + protected String normalize(String name) { + return name.toLowerCase(Locale.ROOT); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java new file mode 100644 index 0000000000000..11657872c7f69 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.Strings; + +import static org.elasticsearch.common.Strings.hasLength; + +final class StringUtils { + + private StringUtils() {} + + /** + * Returns a substring using the Python slice semantics, meaning + * start and end can be negative + */ + static String substringSlice(String string, int start, int end) { + if (hasLength(string) == false) { + return string; + } + + int length = string.length(); + + // handle first negative values + if (start < 0) { + start += length; + } + if (start < 0) { + start = 0; + } + if (end < 0) { + end += length; + } + if (end < 0) { + end = 0; + } else if (end > length) { + end = length; + } + + if (start >= end) { + return org.elasticsearch.xpack.ql.util.StringUtils.EMPTY; + } + + return Strings.substring(string, start, end); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java new file mode 100644 index 0000000000000..5828a74532522 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Substring.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor.doProcess; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isInteger; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; +import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; + +/** + * EQL specific substring function - similar to the one in Python. + * Note this is different than the one in SQL. + */ +public class Substring extends ScalarFunction implements OptionalArgument { + + private final Expression source, start, end; + + public Substring(Source source, Expression src, Expression start, Expression end) { + super(source, Arrays.asList(src, start, end != null ? end : new Literal(source, null, DataTypes.NULL))); + this.source = src; + this.start = start; + this.end = arguments().get(2); + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution sourceResolution = isStringAndExact(source, sourceText(), ParamOrdinal.FIRST); + if (sourceResolution.unresolved()) { + return sourceResolution; + } + + TypeResolution startResolution = isInteger(start, sourceText(), ParamOrdinal.SECOND); + if (startResolution.unresolved()) { + return startResolution; + } + + return isInteger(end, sourceText(), ParamOrdinal.THIRD); + } + + @Override + protected Pipe makePipe() { + return new SubstringFunctionPipe(source(), this, Expressions.pipe(source), Expressions.pipe(start), Expressions.pipe(end)); + } + + @Override + public boolean foldable() { + return source.foldable() && start.foldable() && end.foldable(); + } + + @Override + public Object fold() { + return doProcess(source.fold(), start.fold(), end.fold()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Substring::new, source, start, end); + } + + @Override + public ScriptTemplate asScript() { + ScriptTemplate sourceScript = asScript(source); + ScriptTemplate startScript = asScript(start); + ScriptTemplate endScript = asScript(end); + + return asScriptFrom(sourceScript, startScript, endScript); + } + + protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate startScript, ScriptTemplate endScript) { + return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s,%s)"), + "substring", + sourceScript.template(), + startScript.template(), + endScript.template()), + paramsBuilder() + .script(sourceScript.params()) + .script(startScript.params()) + .script(endScript.params()) + .build(), dataType()); + } + + @Override + public ScriptTemplate scriptWithField(FieldAttribute field) { + return new ScriptTemplate(processScript("doc[{}].value"), + paramsBuilder().variable(field.exactAttribute().name()).build(), + dataType()); + } + + @Override + public DataType dataType() { + return DataTypes.KEYWORD; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 3) { + throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]"); + } + + return new Substring(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java new file mode 100644 index 0000000000000..a6959432edb0e --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionPipe.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class SubstringFunctionPipe extends Pipe { + + private final Pipe source, start, end; + + public SubstringFunctionPipe(Source source, Expression expression, Pipe src, Pipe start, Pipe end) { + super(source, expression, Arrays.asList(src, start, end)); + this.source = src; + this.start = start; + this.end = end; + } + + @Override + public final Pipe replaceChildren(List newChildren) { + if (newChildren.size() != 3) { + throw new IllegalArgumentException("expected [3] children but received [" + newChildren.size() + "]"); + } + return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + public final Pipe resolveAttributes(AttributeResolver resolver) { + Pipe newSource = source.resolveAttributes(resolver); + Pipe newStart = start.resolveAttributes(resolver); + Pipe newEnd = end.resolveAttributes(resolver); + if (newSource == source && newStart == start && newEnd == end) { + return this; + } + return replaceChildren(newSource, newStart, newEnd); + } + + @Override + public boolean supportedByAggsOnlyQuery() { + return source.supportedByAggsOnlyQuery() && start.supportedByAggsOnlyQuery() && end.supportedByAggsOnlyQuery(); + } + + @Override + public boolean resolved() { + return source.resolved() && start.resolved() && end.resolved(); + } + + protected Pipe replaceChildren(Pipe newSource, Pipe newStart, Pipe newEnd) { + return new SubstringFunctionPipe(source(), expression(), newSource, newStart, newEnd); + } + + @Override + public final void collectFields(QlSourceBuilder sourceBuilder) { + source.collectFields(sourceBuilder); + start.collectFields(sourceBuilder); + end.collectFields(sourceBuilder); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, SubstringFunctionPipe::new, expression(), source, start, end); + } + + @Override + public SubstringFunctionProcessor asProcessor() { + return new SubstringFunctionProcessor(source.asProcessor(), start.asProcessor(), end.asProcessor()); + } + + public Pipe src() { + return source; + } + + public Pipe start() { + return start; + } + + public Pipe end() { + return end; + } + + @Override + public int hashCode() { + return Objects.hash(source, start, end); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + SubstringFunctionPipe other = (SubstringFunctionPipe) obj; + return Objects.equals(source, other.source) + && Objects.equals(start, other.start) + && Objects.equals(end, other.end); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java new file mode 100644 index 0000000000000..c4820232e3f26 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/SubstringFunctionProcessor.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; + +import java.io.IOException; +import java.util.Objects; + +public class SubstringFunctionProcessor implements Processor { + + public static final String NAME = "ssub"; + + private final Processor source, start, end; + + public SubstringFunctionProcessor(Processor source, Processor start, Processor end) { + this.source = source; + this.start = start; + this.end = end; + } + + public SubstringFunctionProcessor(StreamInput in) throws IOException { + source = in.readNamedWriteable(Processor.class); + start = in.readNamedWriteable(Processor.class); + end = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(source); + out.writeNamedWriteable(start); + out.writeNamedWriteable(end); + } + + @Override + public Object process(Object input) { + return doProcess(source.process(input), start.process(input), end.process(input)); + } + + public static Object doProcess(Object source, Object start, Object end) { + if (source == null) { + return null; + } + if (!(source instanceof String || source instanceof Character)) { + throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source); + } + if (start == null) { + return source; + } + if ((start instanceof Number) == false) { + throw new EqlIllegalArgumentException("A number is required; received [{}]", start); + } + if (end != null && (end instanceof Number) == false) { + throw new EqlIllegalArgumentException("A number is required; received [{}]", end); + } + + String str = source.toString(); + int startIndex = ((Number) start).intValue(); + int endIndex = end == null ? str.length() : ((Number) end).intValue(); + + return StringUtils.substringSlice(str, startIndex, endIndex); + } + + protected Processor source() { + return source; + } + + protected Processor start() { + return start; + } + + protected Processor end() { + return end; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + SubstringFunctionProcessor other = (SubstringFunctionProcessor) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(start(), other.start()) + && Objects.equals(end(), other.end()); + } + + @Override + public int hashCode() { + return Objects.hash(source(), start(), end()); + } + + + @Override + public String getWriteableName() { + return NAME; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java new file mode 100644 index 0000000000000..959334a73d8b0 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist; + +import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor; +import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils; + +/* + * Whitelisted class for EQL scripts. + * Acts as a registry of the various static methods used internally by the scalar functions + * (to simplify the whitelist definition). + */ +public class InternalEqlScriptUtils extends InternalQlScriptUtils { + + InternalEqlScriptUtils() {} + + public static String substring(String s, Number start, Number end) { + return (String) SubstringFunctionProcessor.doProcess(s, start, end); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java new file mode 100644 index 0000000000000..011aefd56a855 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPainlessExtension.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.BucketAggregationSelectorScript; +import org.elasticsearch.script.FieldScript; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.NumberSortScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.StringSortScript; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; + +public class EqlPainlessExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(EqlPainlessExtension.class, "eql_whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + Map, List> whitelist = new HashMap<>(); + List list = singletonList(WHITELIST); + whitelist.put(FilterScript.CONTEXT, list); + whitelist.put(AggregationScript.CONTEXT, list); + whitelist.put(FieldScript.CONTEXT, list); + whitelist.put(NumberSortScript.CONTEXT, list); + whitelist.put(StringSortScript.CONTEXT, list); + whitelist.put(BucketAggregationSelectorScript.CONTEXT, list); + return whitelist; + } +} diff --git a/x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..70e279c04909a --- /dev/null +++ b/x-pack/plugin/eql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.eql.plugin.EqlPainlessExtension \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt new file mode 100644 index 0000000000000..8fb4a57958299 --- /dev/null +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -0,0 +1,53 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# This file contains a whitelist for EQL specific utilities and classes available inside EQL scripting + +#### Classes + +class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils { +} + +class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils { +# +# Utilities +# + def docValue(java.util.Map, String) + boolean nullSafeFilter(Boolean) + double nullSafeSortNumeric(Number) + String nullSafeSortString(Object) + +# +# Comparison +# + Boolean eq(Object, Object) + Boolean nulleq(Object, Object) + Boolean neq(Object, Object) + Boolean lt(Object, Object) + Boolean lte(Object, Object) + Boolean gt(Object, Object) + Boolean gte(Object, Object) + Boolean in(Object, java.util.List) + +# +# Logical +# + Boolean and(Boolean, Boolean) + Boolean or(Boolean, Boolean) + Boolean not(Boolean) + Boolean isNull(Object) + Boolean isNotNull(Object) + +# +# Regex +# + Boolean regex(String, String) + +# +# ASCII Functions +# + String substring(String, Number, Number) +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java index a7f1cf5099766..386c0e337b310 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/AbstractEqlIntegTestCase.java @@ -13,8 +13,8 @@ import org.elasticsearch.xpack.eql.plugin.EqlPlugin; import java.util.Collection; -import java.util.Collections; +import static java.util.Collections.singletonList; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; @ESIntegTestCase.ClusterScope(scope = SUITE, numDataNodes = 0, numClientNodes = 0, maxNumDataNodes = 0) @@ -35,7 +35,7 @@ protected Settings nodeSettings(int nodeOrdinal) { @Override protected Collection> nodePlugins() { - return Collections.singletonList(LocalStateEqlXPackPlugin.class); + return singletonList(LocalStateEqlXPackPlugin.class); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java index cf313acd1a1b6..b2e5c2c89abe1 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlActionIT.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.eql.action; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.fasterxml.jackson.databind.ObjectMapper; + import org.elasticsearch.Build; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -43,7 +43,6 @@ public static void checkForSnapshot() { @SuppressWarnings("unchecked") public void setUpData() throws Exception { // Insert test data - ObjectMapper mapper = new ObjectMapper(); BulkRequestBuilder bulkBuilder = client().prepareBulk(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, EqlActionIT.class.getResourceAsStream("/test_data.json"))) { List list = parser.list(); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index e6d398b2ea30f..1570184ced1e0 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -131,8 +131,6 @@ public void testFunctionParsingUnknown() { // Test the known EQL functions that are not supported public void testFunctionVerificationUnknown() { - assertEquals("1:26: Unknown function [substring]", - error("foo where user_domain == substring('abcdfeg', 0, 5)")); assertEquals("1:25: Unknown function [endsWith]", error("file where opcode=0 and endsWith(file_name, 'loREr.exe')")); assertEquals("1:25: Unknown function [startsWith]", @@ -143,7 +141,7 @@ public void testFunctionVerificationUnknown() { error("file where opcode=0 and indexOf(file_name, 'plore') == 2")); assertEquals("1:15: Unknown function [add]", error("process where add(serial_event_id, 0) == 1")); - assertEquals("1:15: Unknown function [subtract]", + assertEquals("1:15: Unknown function [subtract], did you mean [substring]?", error("process where subtract(serial_event_id, -5) == 6")); assertEquals("1:15: Unknown function [multiply]", error("process where multiply(6, serial_event_id) == 30")); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java new file mode 100644 index 0000000000000..0abf9c5eb755d --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.StringUtils.substringSlice; + +public class StringUtilsTests extends ESTestCase { + + public void testSubstringSlicePositive() { + String str = randomAlphaOfLength(10); + assertEquals(str.substring(1, 7), substringSlice(str, 1, 7)); + } + + public void testSubstringSliceNegative() { + String str = randomAlphaOfLength(10); + assertEquals(str.substring(5, 9), substringSlice(str, -5, -1)); + } + + public void testSubstringSliceNegativeOverLength() { + String str = randomAlphaOfLength(10); + assertEquals("", substringSlice(str, -15, -11)); + } + + public void testSubstringSlicePositiveOverLength() { + String str = randomAlphaOfLength(10); + assertEquals("", substringSlice(str, 11, 14)); + } + + public void testSubstringHigherEndThanStartNegative() { + String str = randomAlphaOfLength(10); + assertEquals("", substringSlice(str, -20, -11)); + } + + public void testSubstringRandomSlicePositive() { + String str = randomAlphaOfLength(10); + int start = randomInt(5); + int end = start + randomInt(3); + assertEquals(str.substring(start, end), substringSlice(str, start, end)); + } + + public void testSubstringRandomSliceNegative() { + String str = randomAlphaOfLength(10); + int end = 1 + randomInt(3); + int start = end + randomInt(5); + assertEquals(str.substring(10 - start, 10 - end), substringSlice(str, -start, -end)); + } + + public void testStartNegativeHigherThanLength() { + String str = randomAlphaOfLength(10); + int start = 10 + randomInt(10); + assertEquals(str.substring(0, 10 - 1), substringSlice(str, -start, -1)); + } + + public void testEndHigherThanLength() { + String str = randomAlphaOfLength(10); + int end = 10 + randomInt(10); + assertEquals(str, substringSlice(str, 0, end)); + } + + public void testSubstringRandomSliceSameStartEnd() { + String str = randomAlphaOfLength(10); + int start = randomInt(); + assertEquals("", substringSlice(str, start, start)); + } + + public void testNullValue() { + assertNull(substringSlice(null, 0, 0)); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java index acbe328a4740a..b774ff916414c 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.planner; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; @@ -59,6 +60,17 @@ public class QueryFolderOkTests extends AbstractQueryFolderTestCase { "\"term\":{\"opcode\":{\"value\":3", } }, + {"substringFunction", "process where substring(file_name, -4) == '.exe'", + new Object[]{ + "{\"script\":{\"source\":\"" + + "InternalSqlScriptUtils.nullSafeFilter(" + + "InternalSqlScriptUtils.eq(" + + "InternalSqlScriptUtils.substring(" + + "InternalSqlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", + "\"params\":{\"v0\":\"file_name.keyword\",\"v1\":-4,\"v2\":null,\"v3\":\".exe\"}" + + } + } }; private final String name; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 04ecf703908d9..6b90a59289f07 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -86,8 +86,12 @@ public FunctionDefinition resolveFunction(String functionName) { return def; } + protected String normalize(String name) { + return name.toUpperCase(Locale.ROOT); + } + public String resolveAlias(String alias) { - String upperCase = alias.toUpperCase(Locale.ROOT); + String upperCase = normalize(alias); return aliases.getOrDefault(upperCase, upperCase); } @@ -102,7 +106,7 @@ public Collection listFunctions() { public Collection listFunctions(String pattern) { // It is worth double checking if we need this copy. These are immutable anyway. - Pattern p = Strings.hasText(pattern) ? Pattern.compile(pattern.toUpperCase(Locale.ROOT)) : null; + Pattern p = Strings.hasText(pattern) ? Pattern.compile(normalize(pattern)) : null; return defs.entrySet().stream() .filter(e -> p == null || p.matcher(e.getKey()).matches()) .map(e -> new FunctionDefinition(e.getKey(), emptyList(), diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java index 26c4bea1cf592..1f63bd7ac34c4 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java @@ -9,12 +9,16 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation; import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor; +import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.ql.util.StringUtils; +import java.util.List; import java.util.Map; -public abstract class InternalQlScriptUtils { +public class InternalQlScriptUtils { // // Utilities @@ -79,6 +83,10 @@ public static Boolean gte(Object left, Object right) { return BinaryComparisonOperation.GTE.apply(left, right); } + public static Boolean in(Object value, List values) { + return InProcessor.apply(value, values); + } + public static Boolean and(Boolean left, Boolean right) { return BinaryLogicOperation.AND.apply(left, right); } @@ -90,4 +98,20 @@ public static Boolean or(Boolean left, Boolean right) { public static Boolean not(Boolean expression) { return NotProcessor.apply(expression); } + + public static Boolean isNull(Object expression) { + return CheckNullOperation.IS_NULL.apply(expression); + } + + public static Boolean isNotNull(Object expression) { + return CheckNullOperation.IS_NOT_NULL.apply(expression); + } + + // + // Regex + // + public static Boolean regex(String value, String pattern) { + // TODO: this needs to be improved to avoid creating the pattern on every call + return RegexOperation.match(value, pattern); + } } \ No newline at end of file diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java index a743500337f45..337957089e6dd 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java @@ -25,6 +25,8 @@ public final class Scripts { public static final String DOC_VALUE = "doc[{}].value"; + public static final String QL_SCRIPTS = "{ql}"; + public static final String EQL_SCRIPTS = "{eql}"; public static final String SQL_SCRIPTS = "{sql}"; public static final String PARAM = "{}"; // FIXME: this needs to be either renamed (drop Sql) or find a pluggable approach (through ScriptWeaver) @@ -33,7 +35,9 @@ public final class Scripts { private Scripts() {} static final Map FORMATTING_PATTERNS = unmodifiableMap(Stream.of( - new SimpleEntry<>(DOC_VALUE, SQL_SCRIPTS + ".docValue(doc,{})"), + new SimpleEntry<>(DOC_VALUE, QL_SCRIPTS + ".docValue(doc,{})"), + new SimpleEntry<>(QL_SCRIPTS, INTERNAL_SCRIPT_UTILS), + new SimpleEntry<>(EQL_SCRIPTS, INTERNAL_SCRIPT_UTILS), new SimpleEntry<>(SQL_SCRIPTS, INTERNAL_SCRIPT_UTILS), new SimpleEntry<>(PARAM, "params.%s")) .collect(toMap(e -> Pattern.compile(e.getKey(), Pattern.LITERAL), Map.Entry::getValue, (a, b) -> a, LinkedHashMap::new))); @@ -56,7 +60,7 @@ public static String formatTemplate(String template) { public static ScriptTemplate nullSafeFilter(ScriptTemplate script) { return new ScriptTemplate(formatTemplate( - format(Locale.ROOT, "{sql}.nullSafeFilter(%s)", script.template())), + format(Locale.ROOT, "{ql}.nullSafeFilter(%s)", script.template())), script.params(), DataTypes.BOOLEAN); } @@ -64,7 +68,7 @@ public static ScriptTemplate nullSafeFilter(ScriptTemplate script) { public static ScriptTemplate nullSafeSort(ScriptTemplate script) { String methodName = script.outputType().isNumeric() ? "nullSafeSortNumeric" : "nullSafeSortString"; return new ScriptTemplate(formatTemplate( - format(Locale.ROOT, "{sql}.%s(%s)", methodName, script.template())), + format(Locale.ROOT, "{ql}.%s(%s)", methodName, script.template())), script.params(), script.outputType()); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java index 7fc55770bd3f8..d09e3ee317c2d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.sql.expression.function.scalar.string; +import static org.elasticsearch.common.Strings.hasLength; + abstract class StringFunctionUtils { /** @@ -20,11 +22,13 @@ static String substring(String s, int start, int length) { return s; } - if (start < 0) + if (start < 0) { start = 0; + } - if (start + 1 > s.length() || length < 0) + if (start + 1 > s.length() || length < 0) { return ""; + } return (start + length > s.length()) ? s.substring(start) : s.substring(start, start + length); } @@ -66,10 +70,4 @@ static String trimLeadingWhitespaces(String s) { } return sb.toString(); } - - private static boolean hasLength(String s) { - return (s != null && s.length() > 0); - } - - } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java index 82a41fb2024da..d303c9fa03c01 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -9,10 +9,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.JodaCompatibleZonedDateTime; import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils; -import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor; -import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexProcessor.RegexOperation; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateAddProcessor; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateDiffProcessor; @@ -66,23 +63,6 @@ public class InternalSqlScriptUtils extends InternalQlScriptUtils { InternalSqlScriptUtils() {} - - // - // Logical - // - - public static Boolean isNull(Object expression) { - return CheckNullOperation.IS_NULL.apply(expression); - } - - public static Boolean isNotNull(Object expression) { - return CheckNullOperation.IS_NOT_NULL.apply(expression); - } - - public static Boolean in(Object value, List values) { - return InProcessor.apply(value, values); - } - // // Conditional // @@ -106,14 +86,6 @@ public static Object nullif(Object left, Object right) { return NullIfProcessor.apply(left, right); } - // - // Regex - // - public static Boolean regex(String value, String pattern) { - // TODO: this needs to be improved to avoid creating the pattern on every call - return RegexOperation.match(value, pattern); - } - // // Math //