Skip to content

Commit

Permalink
EQL: Add string function (#54470)
Browse files Browse the repository at this point in the history
* EQL: Add string() function
* EQL: Reorder queryfolder_tests
* EQL: Add test queries
* EQL: Fix InternalEqlScriptUtils.string and test case
* EQL: Fix testStringFunctionWithText error message
* EQL: Flatten ToStringFunctionPipe.equals
* EQL: Reorder painless whitelist
* EQL: Address feedback and remove string(null) handling
* EQL: Move string(pid) test over
* EQL: Rename source -> value
  • Loading branch information
rw-access committed Apr 10, 2020
1 parent d14ed34 commit 96a903b
Show file tree
Hide file tree
Showing 9 changed files with 296 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,7 @@ expected_event_ids = [95]
query = '''
file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
'''

[[queries]]
query = 'process where string(serial_event_id) = "1"'
expected_event_ids = [1]
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContains;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
Expand All @@ -37,6 +38,7 @@ private static FunctionDefinition[][] functions() {
def(IndexOf.class, IndexOf::new, "indexof"),
def(Length.class, Length::new, "length"),
def(StartsWith.class, StartsWith::new, "startswith"),
def(ToString.class, ToString::new, "string"),
def(StringContains.class, StringContains::new, "stringcontains"),
def(Substring.class, Substring::new, "substring"),
def(Wildcard.class, Wildcard::new, "wildcard"),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.eql.expression.function.scalar.string;

import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;

import java.util.Collections;
import java.util.List;
import java.util.Locale;

import static java.lang.String.format;
import static org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor.doProcess;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isExact;
import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;

/**
* EQL specific string function that wraps object.toString.
*/
public class ToString extends ScalarFunction {

private final Expression value;

public ToString(Source source, Expression src) {
super(source, Collections.singletonList(src));
this.value = src;
}

@Override
protected TypeResolution resolveType() {
if (!childrenResolved()) {
return new TypeResolution("Unresolved children");
}

return isExact(value, sourceText(), ParamOrdinal.DEFAULT);
}

@Override
protected Pipe makePipe() {
return new ToStringFunctionPipe(source(), this, Expressions.pipe(value));
}

@Override
public boolean foldable() {
return value.foldable();
}

@Override
public Object fold() {
return doProcess(value.fold());
}

@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, ToString::new, value);
}

@Override
public ScriptTemplate asScript() {
ScriptTemplate sourceScript = asScript(value);

return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s)"),
"string",
sourceScript.template()),
paramsBuilder()
.script(sourceScript.params())
.build(), dataType());
}

@Override
public ScriptTemplate scriptWithField(FieldAttribute field) {
return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
paramsBuilder().variable(field.exactAttribute().name()).build(),
dataType());
}

@Override
public DataType dataType() {
return DataTypes.KEYWORD;
}

@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() != 1) {
throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
}

return new ToString(source(), newChildren.get(0));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;

import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;

import java.util.Collections;
import java.util.List;
import java.util.Objects;

public class ToStringFunctionPipe extends Pipe {

private final Pipe source;

public ToStringFunctionPipe(Source source, Expression expression, Pipe src) {
super(source, expression, Collections.singletonList(src));
this.source = src;
}

@Override
public final Pipe replaceChildren(List<Pipe> newChildren) {
if (newChildren.size() != 1) {
throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
}
return new ToStringFunctionPipe(source(), expression(), newChildren.get(0));
}

@Override
public final Pipe resolveAttributes(AttributeResolver resolver) {
Pipe newSource = source.resolveAttributes(resolver);
if (newSource == source) {
return this;
}
return replaceChildren(Collections.singletonList(newSource));
}

@Override
public boolean supportedByAggsOnlyQuery() {
return source.supportedByAggsOnlyQuery();
}

@Override
public boolean resolved() {
return source.resolved();
}

@Override
public final void collectFields(QlSourceBuilder sourceBuilder) {
source.collectFields(sourceBuilder);
}

@Override
protected NodeInfo<ToStringFunctionPipe> info() {
return NodeInfo.create(this, ToStringFunctionPipe::new, expression(), source);
}

@Override
public ToStringFunctionProcessor asProcessor() {
return new ToStringFunctionProcessor(source.asProcessor());
}

public Pipe src() {
return source;
}

@Override
public int hashCode() {
return Objects.hash(source);
}

@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}

if (obj == null || getClass() != obj.getClass()) {
return false;
}

return Objects.equals(source, ((ToStringFunctionPipe) obj).source);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;

import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;

import java.io.IOException;
import java.util.Objects;

public class ToStringFunctionProcessor implements Processor {

public static final String NAME = "sstr";

private final Processor source;

public ToStringFunctionProcessor(Processor source) {
this.source = source;
}

public ToStringFunctionProcessor(StreamInput in) throws IOException {
source = in.readNamedWriteable(Processor.class);
}

@Override
public final void writeTo(StreamOutput out) throws IOException {
out.writeNamedWriteable(source);
}

@Override
public Object process(Object input) {
return doProcess(source.process(input));
}

public static Object doProcess(Object source) {
return source == null ? null : source.toString();
}

protected Processor source() {
return source;
}

@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}

if (obj == null || getClass() != obj.getClass()) {
return false;
}

ToStringFunctionProcessor other = (ToStringFunctionProcessor) obj;
return Objects.equals(source(), other.source());
}

@Override
public int hashCode() {
return Objects.hash(source());
}


@Override
public String getWriteableName() {
return NAME;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContainsFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor;
import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;

/*
Expand Down Expand Up @@ -44,6 +45,10 @@ public static Boolean startsWith(String s, String pattern) {
return (Boolean) StartsWithFunctionProcessor.doProcess(s, pattern);
}

public static String string(Object s) {
return (String) ToStringFunctionProcessor.doProcess(s);
}

public static Boolean stringContains(String string, String substring) {
return (Boolean) StringContainsFunctionProcessor.doProcess(string, substring);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl

#
# Utilities
#
#
def docValue(java.util.Map, String)
boolean nullSafeFilter(Boolean)
double nullSafeSortNumeric(Number)
Expand Down Expand Up @@ -54,12 +54,13 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE

#
# ASCII Functions
#
#
String between(String, String, String, Boolean, Boolean)
Boolean endsWith(String, String)
Integer indexOf(String, String, Number)
Integer length(String)
Boolean startsWith(String, String)
String string(Object)
Boolean stringContains(String, String)
String substring(String, Number, Number)
}
Original file line number Diff line number Diff line change
Expand Up @@ -337,4 +337,12 @@ public void testMultiField() {
accept(idxr, "foo where multi_field_nested.end_date == ''");
accept(idxr, "foo where multi_field_nested.start_date == 'bar'");
}

public void testStringFunctionWithText() {
final IndexResolution idxr = loadIndexResolution("mapping-multi-field.json");
assertEquals("1:15: [string(multi_field.english)] cannot operate on field " +
"of data type [text]: No keyword/multi-field defined exact matches for [english]; " +
"define one or use MATCH/QUERY instead",
error(idxr, "process where string(multi_field.english) == 'foo'"));
}
}
7 changes: 7 additions & 0 deletions x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,13 @@ InternalQlScriptUtils.docValue(doc,params.v0),params.v1))"
"params":{"v0":"process_name","v1":"foo"}
;

stringFunction
process where string(pid) == "123";
"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
InternalEqlScriptUtils.string(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
"params":{"v0":"pid","v1":"123"}
;

indexOfFunction
process where indexOf(user_name, 'A', 2) > 0
;
Expand Down

0 comments on commit 96a903b

Please sign in to comment.