From e36802b973eb896e74c7905c60d64bf74c7f07a5 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 1 Feb 2021 16:05:37 -0500 Subject: [PATCH] Add grok and dissect methods to runtime fields (backport of #68088) (#68332) This adds a `grok` and a `dissect` method to runtime fields which returns a `Matcher` style object you can use to get the matched patterns. A fairly simple script to extract the "verb" from an apache log line with `grok` would look like this: ``` String verb = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.verb; if (verb != null) { emit(verb); } ``` And `dissect` would look like: ``` String verb = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.verb; if (verb != null) { emit(verb); } ``` We'll work later to get it down to a clean looking one liner, but for now, this'll do. The `grok` and `dissect` methods are special in that they only run at script compile time. You can't pass non-constants to them. They'll produce compile errors if you send in a bad pattern. This is nice because they can be expensive to "compile" and there are many other optimizations we can make when the patterns are available up front. Closes #67825 --- .../org/elasticsearch/dissect/DissectKey.java | 2 +- .../elasticsearch/dissect/DissectParser.java | 17 ++- .../dissect/DissectParserTests.java | 5 +- .../ingest/common/DissectProcessor.java | 2 +- .../annotation/CompileTimeOnlyAnnotation.java | 32 +++++ .../CompileTimeOnlyAnnotationParser.java | 45 +++++++ .../annotation/WhitelistAnnotationParser.java | 3 +- .../org/elasticsearch/painless/Compiler.java | 3 + .../lookup/PainlessInstanceBinding.java | 14 +- .../lookup/PainlessLookupBuilder.java | 37 ++++- ...faultConstantFoldingOptimizationPhase.java | 51 ++++++- .../phase/DefaultIRTreeToASMBytesPhase.java | 11 +- .../DefaultStaticConstantExtractionPhase.java | 86 ++++++++++++ .../painless/symbol/IRDecorations.java | 9 ++ .../painless/BaseClassTests.java | 12 +- .../elasticsearch/painless/BindingsTests.java | 126 ++++++++++++++++++ .../org/elasticsearch/painless/Debugger.java | 7 +- .../spi/org.elasticsearch.painless.test | 3 + x-pack/plugin/runtime-fields/build.gradle | 2 + .../xpack/runtimefields/RuntimeFields.java | 64 +++++++++ .../mapper/AbstractFieldScript.java | 2 +- .../mapper/BooleanFieldScript.java | 10 -- .../runtimefields/mapper/DateFieldScript.java | 8 -- .../mapper/DoubleFieldScript.java | 10 -- .../mapper/GeoPointFieldScript.java | 11 +- .../runtimefields/mapper/IpFieldScript.java | 8 -- .../runtimefields/mapper/LongFieldScript.java | 8 -- .../mapper/NamedGroupExtractor.java | 124 +++++++++++++++++ .../RuntimeFieldsPainlessExtension.java | 58 +++++--- .../mapper/StringFieldScript.java | 9 -- .../runtimefields/mapper/common_whitelist.txt | 14 ++ .../AbstractScriptFieldTypeTestCase.java | 4 +- .../mapper/BooleanScriptFieldTypeTests.java | 2 +- .../mapper/DateScriptFieldTypeTests.java | 2 +- .../mapper/DoubleScriptFieldTypeTests.java | 2 +- .../mapper/DynamicRuntimeTests.java | 3 +- .../mapper/GeoPointScriptFieldTypeTests.java | 2 +- .../mapper/IpScriptFieldTypeTests.java | 2 +- .../mapper/KeywordScriptFieldTypeTests.java | 2 +- .../mapper/LongScriptFieldTypeTests.java | 2 +- .../test/runtime_fields/250_grok.yml | 126 ++++++++++++++++++ .../test/runtime_fields/260_dissect.yml | 76 +++++++++++ 42 files changed, 899 insertions(+), 117 deletions(-) create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/common_whitelist.txt create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/250_grok.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/260_dissect.yml diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java index 5459850c521f0..03a6823b9503f 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java @@ -102,7 +102,7 @@ public final class DissectKey { } if (name == null || (name.isEmpty() && !skip)) { - throw new DissectException.KeyParse(key, "The key name could be determined"); + throw new DissectException.KeyParse(key, "The key name could not be determined"); } } diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java index 9db4b5cd3cc89..96cbf45a61134 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java @@ -34,7 +34,8 @@ import java.util.stream.Collectors; /** - *

Splits (dissects) a string into its parts based on a pattern.

A dissect pattern is composed of a set of keys and delimiters. + * Splits (dissects) a string into its parts based on a pattern. + *

A dissect pattern is composed of a set of keys and delimiters. * For example the dissect pattern: 

%{a} %{b},%{c}
has 3 keys (a,b,c) and two delimiters (space and comma). This pattern will * match a string of the form: 
foo bar,baz
and will result a key/value pairing of 
a=foo, b=bar, and c=baz.
*

Matches are all or nothing. For example, the same pattern will NOT match 

foo bar baz
since all of the delimiters did not @@ -276,7 +277,19 @@ public Map parse(String inputString) { } Map results = dissectMatch.getResults(); - if (dissectMatch.isValid(results) == false) { + return dissectMatch.isValid(results) ? results : null; + } + + /** + *

Entry point to dissect a string into it's parts.

+ * + * @param inputString The string to dissect + * @return the key/value Map of the results + * @throws DissectException if unable to dissect a pair into it's parts. + */ + public Map forceParse(String inputString) { + Map results = parse(inputString); + if (results == null) { throw new DissectException.FindMatch(pattern, inputString); } return results; diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java index 731a387d801e0..a4aa66c305c61 100644 --- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java +++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java @@ -344,11 +344,12 @@ public void testJsonSpecification() throws Exception { } } - private DissectException assertFail(String pattern, String input){ - return expectThrows(DissectException.class, () -> new DissectParser(pattern, null).parse(input)); + private DissectException assertFail(String pattern, String input) { + return expectThrows(DissectException.class, () -> new DissectParser(pattern, null).forceParse(input)); } private void assertMiss(String pattern, String input) { + assertNull(new DissectParser(pattern, null).parse(input)); DissectException e = assertFail(pattern, input); assertThat(e.getMessage(), CoreMatchers.containsString("Unable to find match for dissect pattern")); assertThat(e.getMessage(), CoreMatchers.containsString(pattern)); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java index c9d5a3059d1fe..ddd922cc1f11f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java @@ -54,7 +54,7 @@ public IngestDocument execute(IngestDocument ingestDocument) { } else if (input == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot process it."); } - dissectParser.parse(input).forEach(ingestDocument::setFieldValue); + dissectParser.forceParse(input).forEach(ingestDocument::setFieldValue); return ingestDocument; } diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java new file mode 100644 index 0000000000000..1eb9370a7a7f6 --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotation.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi.annotation; + +/** + * Methods annotated with this must be run at compile time so their arguments + * must all be constant and they produce a constant. + */ +public class CompileTimeOnlyAnnotation { + public static final String NAME = "compile_time_only"; + + public static final CompileTimeOnlyAnnotation INSTANCE = new CompileTimeOnlyAnnotation(); + + private CompileTimeOnlyAnnotation() {} +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java new file mode 100644 index 0000000000000..df8b3df3446c6 --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/CompileTimeOnlyAnnotationParser.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi.annotation; + +import java.util.Map; + +/** + * Methods annotated with {@link CompileTimeOnlyAnnotation} must be run at + * compile time so their arguments must all be constant and they produce a + * constant. + */ +public class CompileTimeOnlyAnnotationParser implements WhitelistAnnotationParser { + + public static final CompileTimeOnlyAnnotationParser INSTANCE = new CompileTimeOnlyAnnotationParser(); + + private CompileTimeOnlyAnnotationParser() {} + + @Override + public Object parse(Map arguments) { + if (arguments.isEmpty() == false) { + throw new IllegalArgumentException( + "unexpected parameters for [@" + CompileTimeOnlyAnnotation.NAME + "] annotation, found " + arguments + ); + } + + return CompileTimeOnlyAnnotation.INSTANCE; + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java index 43acf061e062c..f210dc6251a10 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/annotation/WhitelistAnnotationParser.java @@ -36,7 +36,8 @@ public interface WhitelistAnnotationParser { new AbstractMap.SimpleEntry<>(NoImportAnnotation.NAME, NoImportAnnotationParser.INSTANCE), new AbstractMap.SimpleEntry<>(DeprecatedAnnotation.NAME, DeprecatedAnnotationParser.INSTANCE), new AbstractMap.SimpleEntry<>(NonDeterministicAnnotation.NAME, NonDeterministicAnnotationParser.INSTANCE), - new AbstractMap.SimpleEntry<>(InjectConstantAnnotation.NAME, InjectConstantAnnotationParser.INSTANCE) + new AbstractMap.SimpleEntry<>(InjectConstantAnnotation.NAME, InjectConstantAnnotationParser.INSTANCE), + new AbstractMap.SimpleEntry<>(CompileTimeOnlyAnnotation.NAME, CompileTimeOnlyAnnotationParser.INSTANCE) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) ); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index 29b47f9289bb7..52e4a91ee9c91 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -26,6 +26,7 @@ import org.elasticsearch.painless.node.SClass; import org.elasticsearch.painless.phase.DefaultConstantFoldingOptimizationPhase; import org.elasticsearch.painless.phase.DefaultIRTreeToASMBytesPhase; +import org.elasticsearch.painless.phase.DefaultStaticConstantExtractionPhase; import org.elasticsearch.painless.phase.DefaultStringConcatenationOptimizationPhase; import org.elasticsearch.painless.phase.DocFieldsPhase; import org.elasticsearch.painless.phase.PainlessSemanticAnalysisPhase; @@ -227,6 +228,7 @@ ScriptScope compile(Loader loader, String name, String source, CompilerSettings ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); new DefaultStringConcatenationOptimizationPhase().visitClass(classNode, null); new DefaultConstantFoldingOptimizationPhase().visitClass(classNode, null); + new DefaultStaticConstantExtractionPhase().visitClass(classNode, scriptScope); new DefaultIRTreeToASMBytesPhase().visitScript(classNode); byte[] bytes = classNode.getBytes(); @@ -263,6 +265,7 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); new DefaultStringConcatenationOptimizationPhase().visitClass(classNode, null); new DefaultConstantFoldingOptimizationPhase().visitClass(classNode, null); + new DefaultStaticConstantExtractionPhase().visitClass(classNode, scriptScope); classNode.setDebugStream(debugStream); new DefaultIRTreeToASMBytesPhase().visitScript(classNode); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java index 6952a3f05fb64..bd4640eb3d5c3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessInstanceBinding.java @@ -21,6 +21,7 @@ import java.lang.reflect.Method; import java.util.List; +import java.util.Map; import java.util.Objects; public class PainlessInstanceBinding { @@ -30,13 +31,21 @@ public class PainlessInstanceBinding { public final Class returnType; public final List> typeParameters; + public final Map, Object> annotations; - PainlessInstanceBinding(Object targetInstance, Method javaMethod, Class returnType, List> typeParameters) { + PainlessInstanceBinding( + Object targetInstance, + Method javaMethod, + Class returnType, + List> typeParameters, + Map, Object> annotations + ) { this.targetInstance = targetInstance; this.javaMethod = javaMethod; this.returnType = returnType; this.typeParameters = typeParameters; + this.annotations = annotations; } @Override @@ -54,7 +63,8 @@ public boolean equals(Object object) { return targetInstance == that.targetInstance && Objects.equals(javaMethod, that.javaMethod) && Objects.equals(returnType, that.returnType) && - Objects.equals(typeParameters, that.typeParameters); + Objects.equals(typeParameters, that.typeParameters) && + Objects.equals(annotations, that.annotations); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index 517c742b3d6ea..d0324986c1c1c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -30,6 +30,7 @@ import org.elasticsearch.painless.spi.WhitelistField; import org.elasticsearch.painless.spi.WhitelistInstanceBinding; import org.elasticsearch.painless.spi.WhitelistMethod; +import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation; import org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation; import org.elasticsearch.painless.spi.annotation.NoImportAnnotation; import org.objectweb.asm.ClassWriter; @@ -174,7 +175,8 @@ public static PainlessLookup buildFromWhitelists(List whitelists) { origin = whitelistInstanceBinding.origin; painlessLookupBuilder.addPainlessInstanceBinding( whitelistInstanceBinding.targetInstance, whitelistInstanceBinding.methodName, - whitelistInstanceBinding.returnCanonicalTypeName, whitelistInstanceBinding.canonicalTypeNameParameters); + whitelistInstanceBinding.returnCanonicalTypeName, whitelistInstanceBinding.canonicalTypeNameParameters, + whitelistInstanceBinding.painlessAnnotations); } } } catch (Exception exception) { @@ -393,6 +395,10 @@ public void addPainlessConstructor(Class targetClass, List> typePara "[[" + targetCanonicalClassName + "], " + typesToCanonicalTypeNames(typeParameters) + "]", iae); } + if (annotations.containsKey(CompileTimeOnlyAnnotation.class)) { + throw new IllegalArgumentException("constructors can't have @" + CompileTimeOnlyAnnotation.NAME); + } + MethodType methodType = methodHandle.type(); String painlessConstructorKey = buildPainlessConstructorKey(typeParametersSize); @@ -574,6 +580,10 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, } } + if (annotations.containsKey(CompileTimeOnlyAnnotation.class)) { + throw new IllegalArgumentException("regular methods can't have @" + CompileTimeOnlyAnnotation.NAME); + } + MethodType methodType = methodHandle.type(); boolean isStatic = augmentedClass == null && Modifier.isStatic(javaMethod.getModifiers()); String painlessMethodKey = buildPainlessMethodKey(methodName, typeParametersSize); @@ -989,6 +999,10 @@ public void addPainlessClassBinding(Class targetClass, String methodName, Cla "invalid method name [" + methodName + "] for class binding [" + targetCanonicalClassName + "]."); } + if (annotations.containsKey(CompileTimeOnlyAnnotation.class)) { + throw new IllegalArgumentException("class bindings can't have @" + CompileTimeOnlyAnnotation.NAME); + } + Method[] javaMethods = targetClass.getMethods(); Method javaMethod = null; @@ -1079,7 +1093,8 @@ public void addPainlessClassBinding(Class targetClass, String methodName, Cla } public void addPainlessInstanceBinding(Object targetInstance, - String methodName, String returnCanonicalTypeName, List canonicalTypeNameParameters) { + String methodName, String returnCanonicalTypeName, List canonicalTypeNameParameters, + Map, Object> painlessAnnotations) { Objects.requireNonNull(targetInstance); Objects.requireNonNull(methodName); @@ -1108,10 +1123,16 @@ public void addPainlessInstanceBinding(Object targetInstance, "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); } - addPainlessInstanceBinding(targetInstance, methodName, returnType, typeParameters); + addPainlessInstanceBinding(targetInstance, methodName, returnType, typeParameters, painlessAnnotations); } - public void addPainlessInstanceBinding(Object targetInstance, String methodName, Class returnType, List> typeParameters) { + public void addPainlessInstanceBinding( + Object targetInstance, + String methodName, + Class returnType, + List> typeParameters, + Map, Object> painlessAnnotations + ) { Objects.requireNonNull(targetInstance); Objects.requireNonNull(methodName); Objects.requireNonNull(returnType); @@ -1189,7 +1210,7 @@ public void addPainlessInstanceBinding(Object targetInstance, String methodName, PainlessInstanceBinding existingPainlessInstanceBinding = painlessMethodKeysToPainlessInstanceBindings.get(painlessMethodKey); PainlessInstanceBinding newPainlessInstanceBinding = - new PainlessInstanceBinding(targetInstance, javaMethod, returnType, typeParameters); + new PainlessInstanceBinding(targetInstance, javaMethod, returnType, typeParameters, painlessAnnotations); if (existingPainlessInstanceBinding == null) { newPainlessInstanceBinding = painlessInstanceBindingCache.computeIfAbsent(newPainlessInstanceBinding, key -> key); @@ -1200,11 +1221,13 @@ public void addPainlessInstanceBinding(Object targetInstance, String methodName, "[[" + targetCanonicalClassName + "], " + "[" + methodName + "], " + "[" + typeToCanonicalTypeName(returnType) + "], " + - typesToCanonicalTypeNames(typeParameters) + "] and " + + typesToCanonicalTypeNames(typeParameters) + "], " + + painlessAnnotations + " and " + "[[" + targetCanonicalClassName + "], " + "[" + methodName + "], " + "[" + typeToCanonicalTypeName(existingPainlessInstanceBinding.returnType) + "], " + - typesToCanonicalTypeNames(existingPainlessInstanceBinding.typeParameters) + "]"); + typesToCanonicalTypeNames(existingPainlessInstanceBinding.typeParameters) + "], " + + existingPainlessInstanceBinding.annotations); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java index be6c2e681aca8..c3efa32bdb854 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultConstantFoldingOptimizationPhase.java @@ -21,8 +21,8 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.Operation; -import org.elasticsearch.painless.ir.BinaryMathNode; import org.elasticsearch.painless.ir.BinaryImplNode; +import org.elasticsearch.painless.ir.BinaryMathNode; import org.elasticsearch.painless.ir.BooleanNode; import org.elasticsearch.painless.ir.CastNode; import org.elasticsearch.painless.ir.ComparisonNode; @@ -66,13 +66,20 @@ import org.elasticsearch.painless.ir.ThrowNode; import org.elasticsearch.painless.ir.UnaryMathNode; import org.elasticsearch.painless.ir.WhileLoopNode; +import org.elasticsearch.painless.lookup.PainlessInstanceBinding; import org.elasticsearch.painless.lookup.PainlessLookupUtility; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation; import org.elasticsearch.painless.symbol.IRDecorations.IRDCast; import org.elasticsearch.painless.symbol.IRDecorations.IRDComparisonType; import org.elasticsearch.painless.symbol.IRDecorations.IRDConstant; import org.elasticsearch.painless.symbol.IRDecorations.IRDExpressionType; +import org.elasticsearch.painless.symbol.IRDecorations.IRDInstanceBinding; +import org.elasticsearch.painless.symbol.IRDecorations.IRDMethod; import org.elasticsearch.painless.symbol.IRDecorations.IRDOperation; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.function.Consumer; /** @@ -824,6 +831,48 @@ public void visitInvokeCallMember(InvokeCallMemberNode irInvokeCallMemberNode, C int j = i; irInvokeCallMemberNode.getArgumentNodes().get(i).visit(this, (e) -> irInvokeCallMemberNode.getArgumentNodes().set(j, e)); } + PainlessMethod method = irInvokeCallMemberNode.getDecorationValue(IRDMethod.class); + if (method != null && method.annotations.containsKey(CompileTimeOnlyAnnotation.class)) { + replaceCallWithConstant(irInvokeCallMemberNode, scope, method.javaMethod, null); + return; + } + PainlessInstanceBinding instanceBinding = irInvokeCallMemberNode.getDecorationValue(IRDInstanceBinding.class); + if (instanceBinding != null && instanceBinding.annotations.containsKey(CompileTimeOnlyAnnotation.class)) { + replaceCallWithConstant(irInvokeCallMemberNode, scope, instanceBinding.javaMethod, instanceBinding.targetInstance); + return; + } + } + + private void replaceCallWithConstant( + InvokeCallMemberNode irInvokeCallMemberNode, + Consumer scope, + Method javaMethod, + Object receiver + ) { + Object[] args = new Object[irInvokeCallMemberNode.getArgumentNodes().size()]; + for (int i = 0; i < irInvokeCallMemberNode.getArgumentNodes().size(); i++) { + ExpressionNode argNode = irInvokeCallMemberNode.getArgumentNodes().get(i); + if (argNode instanceof ConstantNode == false) { + // TODO find a better string to output + throw irInvokeCallMemberNode.getLocation() + .createError(new IllegalArgumentException("all arguments must be constant but the [" + (i + 1) + "] argument isn't")); + } + args[i] = ((ConstantNode) argNode).getDecorationValue(IRDConstant.class); + } + Object result; + try { + result = javaMethod.invoke(receiver, args); + } catch (IllegalAccessException | IllegalArgumentException e) { + throw irInvokeCallMemberNode.getLocation() + .createError(new IllegalArgumentException("error invoking [" + irInvokeCallMemberNode + "] at compile time", e)); + } catch (InvocationTargetException e) { + throw irInvokeCallMemberNode.getLocation() + .createError(new IllegalArgumentException("error invoking [" + irInvokeCallMemberNode + "] at compile time", e.getCause())); + } + ConstantNode replacement = new ConstantNode(irInvokeCallMemberNode.getLocation()); + replacement.attachDecoration(new IRDConstant(result)); + replacement.attachDecoration(irInvokeCallMemberNode.getDecoration(IRDExpressionType.class)); + scope.accept(replacement); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java index 3338d3240ef12..1abc5b4aa65b2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java @@ -121,10 +121,11 @@ import org.elasticsearch.painless.symbol.IRDecorations.IRDClassBinding; import org.elasticsearch.painless.symbol.IRDecorations.IRDComparisonType; import org.elasticsearch.painless.symbol.IRDecorations.IRDConstant; +import org.elasticsearch.painless.symbol.IRDecorations.IRDConstantFieldName; import org.elasticsearch.painless.symbol.IRDecorations.IRDConstructor; import org.elasticsearch.painless.symbol.IRDecorations.IRDDeclarationType; -import org.elasticsearch.painless.symbol.IRDecorations.IRDDepth; import org.elasticsearch.painless.symbol.IRDecorations.IRDDefReferenceEncoding; +import org.elasticsearch.painless.symbol.IRDecorations.IRDDepth; import org.elasticsearch.painless.symbol.IRDecorations.IRDExceptionType; import org.elasticsearch.painless.symbol.IRDecorations.IRDExpressionType; import org.elasticsearch.painless.symbol.IRDecorations.IRDField; @@ -1220,7 +1221,13 @@ public void visitConstant(ConstantNode irConstantNode, WriteScope writeScope) { else if (constant instanceof Byte) methodWriter.push((byte)constant); else if (constant instanceof Boolean) methodWriter.push((boolean)constant); else { - throw new IllegalStateException("unexpected constant [" + constant + "]"); + /* + * The constant doesn't properly fit into the constant pool so + * we should have made a static field for it. + */ + String fieldName = irConstantNode.getDecorationValue(IRDConstantFieldName.class); + Type asmFieldType = MethodWriter.getType(irConstantNode.getDecorationValue(IRDExpressionType.class)); + methodWriter.getStatic(CLASS_TYPE, fieldName, asmFieldType); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java new file mode 100644 index 0000000000000..4849e25c0a21b --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultStaticConstantExtractionPhase.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.phase; + +import org.elasticsearch.painless.ir.ClassNode; +import org.elasticsearch.painless.ir.ConstantNode; +import org.elasticsearch.painless.ir.FieldNode; +import org.elasticsearch.painless.symbol.IRDecorations.IRDConstant; +import org.elasticsearch.painless.symbol.IRDecorations.IRDConstantFieldName; +import org.elasticsearch.painless.symbol.IRDecorations.IRDExpressionType; +import org.elasticsearch.painless.symbol.IRDecorations.IRDFieldType; +import org.elasticsearch.painless.symbol.IRDecorations.IRDModifiers; +import org.elasticsearch.painless.symbol.IRDecorations.IRDName; +import org.elasticsearch.painless.symbol.ScriptScope; + +import java.lang.reflect.Modifier; + +/** + * Looks for {@link ConstantNode}s that can't be pushed into the constant pool + * and creates a {@code static} constant member that is injected using reflection + * on construction. + */ +public class DefaultStaticConstantExtractionPhase extends IRTreeBaseVisitor { + private ClassNode classNode; + + @Override + public void visitClass(ClassNode irClassNode, ScriptScope scope) { + this.classNode = irClassNode; + super.visitClass(irClassNode, scope); + } + + @Override + public void visitConstant(ConstantNode irConstantNode, ScriptScope scope) { + super.visitConstant(irConstantNode, scope); + Object constant = irConstantNode.getDecorationValue(IRDConstant.class); + if (constant instanceof String + || constant instanceof Double + || constant instanceof Float + || constant instanceof Long + || constant instanceof Integer + || constant instanceof Character + || constant instanceof Short + || constant instanceof Byte + || constant instanceof Boolean) { + /* + * Constant can be loaded into the constant pool so we let the byte + * code generation phase do that. + */ + return; + } + /* + * The constant *can't* be loaded into the constant pool so we make it + * a static constant and register the value with ScriptScope. The byte + * code generation will load the static constant. + */ + String fieldName = scope.getNextSyntheticName("constant"); + scope.addStaticConstant(fieldName, constant); + + FieldNode constantField = new FieldNode(irConstantNode.getLocation()); + constantField.attachDecoration(new IRDModifiers(Modifier.PUBLIC | Modifier.STATIC)); + constantField.attachDecoration(irConstantNode.getDecoration(IRDConstant.class)); + Class type = irConstantNode.getDecorationValue(IRDExpressionType.class); + constantField.attachDecoration(new IRDFieldType(type)); + constantField.attachDecoration(new IRDName(fieldName)); + classNode.addFieldNode(constantField); + + irConstantNode.attachDecoration(new IRDConstantFieldName(fieldName)); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java index bb9c4caff8fd6..ab6ac4b670de8 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/IRDecorations.java @@ -148,6 +148,15 @@ public IRDConstant(Object value) { } } + /** + * describes the field name holding a constant value. + */ + public static class IRDConstantFieldName extends IRDecoration { + public IRDConstantFieldName(String value) { + super(value); + } + } + /** * describes the type for a declaration */ diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java index b43e6afcaf68b..63d19d30c5546 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java @@ -131,7 +131,7 @@ public void testNoArgs() throws Exception { scriptEngine.compile("testNoArgs3", "_score", NoArgs.CONTEXT, emptyMap())); assertEquals("cannot resolve symbol [_score]", e.getMessage()); - String debug = Debugger.toString(NoArgs.class, "int i = 0", new CompilerSettings()); + String debug = Debugger.toString(NoArgs.class, "int i = 0", new CompilerSettings(), Whitelist.BASE_WHITELISTS); assertThat(debug, containsString("ACONST_NULL")); assertThat(debug, containsString("ARETURN")); } @@ -317,7 +317,7 @@ public void testReturnsVoid() throws Exception { scriptEngine.compile("testReturnsVoid1", "map.remove('a')", ReturnsVoid.CONTEXT, emptyMap()).newInstance().execute(map); assertEquals(emptyMap(), map); - String debug = Debugger.toString(ReturnsVoid.class, "int i = 0", new CompilerSettings()); + String debug = Debugger.toString(ReturnsVoid.class, "int i = 0", new CompilerSettings(), Whitelist.BASE_WHITELISTS); // The important thing is that this contains the opcode for returning void assertThat(debug, containsString(" RETURN")); // We shouldn't contain any weird "default to null" logic @@ -358,7 +358,7 @@ public void testReturnsPrimitiveBoolean() throws Exception { scriptEngine.compile("testReturnsPrimitiveBoolean6", "true || false", ReturnsPrimitiveBoolean.CONTEXT, emptyMap()) .newInstance().execute()); - String debug = Debugger.toString(ReturnsPrimitiveBoolean.class, "false", new CompilerSettings()); + String debug = Debugger.toString(ReturnsPrimitiveBoolean.class, "false", new CompilerSettings(), Whitelist.BASE_WHITELISTS); assertThat(debug, containsString("ICONST_0")); // The important thing here is that we have the bytecode for returning an integer instead of an object. booleans are integers. assertThat(debug, containsString("IRETURN")); @@ -426,7 +426,7 @@ public void testReturnsPrimitiveInt() throws Exception { assertEquals(2, scriptEngine.compile("testReturnsPrimitiveInt7", "1 + 1", ReturnsPrimitiveInt.CONTEXT, emptyMap()).newInstance().execute()); - String debug = Debugger.toString(ReturnsPrimitiveInt.class, "1", new CompilerSettings()); + String debug = Debugger.toString(ReturnsPrimitiveInt.class, "1", new CompilerSettings(), Whitelist.BASE_WHITELISTS); assertThat(debug, containsString("ICONST_1")); // The important thing here is that we have the bytecode for returning an integer instead of an object assertThat(debug, containsString("IRETURN")); @@ -493,7 +493,7 @@ public void testReturnsPrimitiveFloat() throws Exception { "testReturnsPrimitiveFloat7", "def d = Double.valueOf(1.1); d", ReturnsPrimitiveFloat.CONTEXT, emptyMap()) .newInstance().execute()); - String debug = Debugger.toString(ReturnsPrimitiveFloat.class, "1f", new CompilerSettings()); + String debug = Debugger.toString(ReturnsPrimitiveFloat.class, "1f", new CompilerSettings(), Whitelist.BASE_WHITELISTS); assertThat(debug, containsString("FCONST_1")); // The important thing here is that we have the bytecode for returning a float instead of an object assertThat(debug, containsString("FRETURN")); @@ -556,7 +556,7 @@ public void testReturnsPrimitiveDouble() throws Exception { scriptEngine.compile("testReturnsPrimitiveDouble12", "1.1 + 6.7", ReturnsPrimitiveDouble.CONTEXT, emptyMap()) .newInstance().execute(), 0); - String debug = Debugger.toString(ReturnsPrimitiveDouble.class, "1", new CompilerSettings()); + String debug = Debugger.toString(ReturnsPrimitiveDouble.class, "1", new CompilerSettings(), Whitelist.BASE_WHITELISTS); // The important thing here is that we have the bytecode for returning a double instead of an object assertThat(debug, containsString("DRETURN")); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java index 171880abd7907..c4c49b3e1e19f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BindingsTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistInstanceBinding; import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation; import org.elasticsearch.script.ScriptContext; import java.util.ArrayList; @@ -29,8 +30,24 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; + public class BindingsTests extends ScriptTestCase { + public static int classMul(int i, int j) { + return i * j; + } + + public static int compileTimeBlowUp(int i, int j) { + throw new RuntimeException("Boom"); + } + + public static List fancyConstant(String thing1, String thing2) { + return org.elasticsearch.common.collect.List.of(thing1, thing2); + } + public static class BindingTestClass { public int state; @@ -83,6 +100,10 @@ public void setInstanceBindingValue(int value) { public int getInstanceBindingValue() { return value; } + + public int instanceMul(int i, int j) { + return i * j; + } } public abstract static class BindingsTestScript { @@ -106,9 +127,18 @@ protected Map, List> scriptContexts() { "setInstanceBindingValue", "void", Collections.singletonList("int"), Collections.emptyList()); WhitelistInstanceBinding setter = new WhitelistInstanceBinding("test", instanceBindingTestClass, "getInstanceBindingValue", "int", Collections.emptyList(), Collections.emptyList()); + WhitelistInstanceBinding mul = new WhitelistInstanceBinding( + "test", + instanceBindingTestClass, + "instanceMul", + "int", + org.elasticsearch.common.collect.List.of("int", "int"), + org.elasticsearch.common.collect.List.of(CompileTimeOnlyAnnotation.INSTANCE) + ); List instanceBindingsList = new ArrayList<>(); instanceBindingsList.add(getter); instanceBindingsList.add(setter); + instanceBindingsList.add(mul); Whitelist instanceBindingsWhitelist = new Whitelist(instanceBindingTestClass.getClass().getClassLoader(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), instanceBindingsList); whitelists.add(instanceBindingsWhitelist); @@ -180,4 +210,100 @@ public void testInstanceBinding() { executableScript = factory.newInstance(); assertEquals(8, executableScript.execute(-2, 6)); } + + public void testClassMethodCompileTimeOnly() { + String script = "classMul(2, 2)"; + BindingsTestScript.Factory factory = scriptEngine.compile(null, script, BindingsTestScript.CONTEXT, Collections.emptyMap()); + BindingsTestScript executableScript = factory.newInstance(); + assertEquals(4, executableScript.execute(1, 1)); + + assertThat( + Debugger.toString(BindingsTestScript.class, script, new CompilerSettings(), scriptContexts().get(BindingsTestScript.CONTEXT)), + containsString("ICONST_4") + ); + } + + public void testClassMethodCompileTimeOnlyVariableParams() { + Exception e = expectScriptThrows( + IllegalArgumentException.class, + () -> scriptEngine.compile(null, "def a = 2; classMul(2, a)", BindingsTestScript.CONTEXT, Collections.emptyMap()) + ); + assertThat(e.getMessage(), equalTo("all arguments must be constant but the [2] argument isn't")); + } + + public void testClassMethodCompileTimeOnlyThrows() { + Exception e = expectScriptThrows( + IllegalArgumentException.class, + () -> scriptEngine.compile(null, "compileTimeBlowUp(2, 2)", BindingsTestScript.CONTEXT, Collections.emptyMap()) + ); + assertThat(e.getMessage(), startsWith("error invoking")); + assertThat(e.getCause().getMessage(), equalTo("Boom")); + } + + public void testInstanceBindingCompileTimeOnly() { + String script = "instanceMul(2, 2)"; + BindingsTestScript.Factory factory = scriptEngine.compile(null, script, BindingsTestScript.CONTEXT, Collections.emptyMap()); + BindingsTestScript executableScript = factory.newInstance(); + assertEquals(4, executableScript.execute(1, 1)); + + assertThat( + Debugger.toString(BindingsTestScript.class, script, new CompilerSettings(), scriptContexts().get(BindingsTestScript.CONTEXT)), + containsString("ICONST_4") + ); + } + + public void testInstanceMethodCompileTimeOnlyVariableParams() { + Exception e = expectScriptThrows( + IllegalArgumentException.class, + () -> scriptEngine.compile(null, "def a = 2; instanceMul(a, 2)", BindingsTestScript.CONTEXT, Collections.emptyMap()) + ); + assertThat(e.getMessage(), equalTo("all arguments must be constant but the [1] argument isn't")); + } + + public void testCompileTimeOnlyParameterFoldedToConstant() { + String script = "classMul(1, 1 + 1)"; + BindingsTestScript.Factory factory = scriptEngine.compile(null, script, BindingsTestScript.CONTEXT, Collections.emptyMap()); + BindingsTestScript executableScript = factory.newInstance(); + assertEquals(2, executableScript.execute(1, 1)); + + assertThat( + Debugger.toString(BindingsTestScript.class, script, new CompilerSettings(), scriptContexts().get(BindingsTestScript.CONTEXT)), + containsString("ICONST_2") + ); + } + + /** + * Tests that {@code @compile_time_only} can return values that don't + * fit into the constant pool and we'll create them as static members. + */ + public void testCompileTimeOnlyResultOutsideConstantPool() { + String script = "fancyConstant('foo', 'bar').stream().mapToInt(String::length).sum()"; + BindingsTestScript.Factory factory = scriptEngine.compile(null, script, BindingsTestScript.CONTEXT, Collections.emptyMap()); + BindingsTestScript executableScript = factory.newInstance(); + assertEquals(6, executableScript.execute(1, 1)); + + String code = Debugger.toString( + BindingsTestScript.class, + script, + new CompilerSettings(), + scriptContexts().get(BindingsTestScript.CONTEXT) + ); + assertThat( + "We make a static field to hold the constant", + code, + containsString("public static synthetic Ljava/util/List; constant$synthetic$0") + ); + assertThat( + "We load the constant from the static field", + code, + containsString("GETSTATIC org/elasticsearch/painless/PainlessScript$Script.constant$synthetic$0 : Ljava/util/List;") + ); + /* + * It's kind of important that we use java.util.List above and *not* + * the specific subtype returned by java.util.List.of(). Doing that + * means that we don't have to whitelist the actual returned type, just + * the interface. The JVM ought to be able to optimize the invocation + * because the constant is effectively final. + */ + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java index a81bd5f97ea76..76bfd2542d146 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java @@ -26,22 +26,23 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.util.List; /** quick and dirty tools for debugging */ final class Debugger { /** compiles source to bytecode, and returns debugging output */ static String toString(final String source) { - return toString(PainlessTestScript.class, source, new CompilerSettings()); + return toString(PainlessTestScript.class, source, new CompilerSettings(), Whitelist.BASE_WHITELISTS); } /** compiles to bytecode, and returns debugging output */ - static String toString(Class iface, String source, CompilerSettings settings) { + static String toString(Class iface, String source, CompilerSettings settings, List whitelists) { StringWriter output = new StringWriter(); PrintWriter outputWriter = new PrintWriter(output); Textifier textifier = new Textifier(); try { - new Compiler(iface, null, null, PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS)) + new Compiler(iface, null, null, PainlessLookupBuilder.buildFromWhitelists(whitelists)) .compile("", source, settings, textifier); } catch (RuntimeException e) { textifier.print(outputWriter); diff --git a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test index 85493cad22d7e..28c032418f373 100644 --- a/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test +++ b/modules/lang-painless/src/test/resources/org/elasticsearch/painless/spi/org.elasticsearch.painless.test @@ -52,4 +52,7 @@ static_import { int addWithState(int, int, int, double) bound_to org.elasticsearch.painless.BindingsTests$BindingTestClass int addThisWithState(BindingsTests.BindingsTestScript, int, int, int, double) bound_to org.elasticsearch.painless.BindingsTests$ThisBindingTestClass int addEmptyThisWithState(BindingsTests.BindingsTestScript, int) bound_to org.elasticsearch.painless.BindingsTests$EmptyThisBindingTestClass + int classMul(int, int) from_class org.elasticsearch.painless.BindingsTests @compile_time_only + int compileTimeBlowUp(int, int) from_class org.elasticsearch.painless.BindingsTests @compile_time_only + List fancyConstant(String, String) from_class org.elasticsearch.painless.BindingsTests @compile_time_only } diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle index 7b72aa5723679..a03ec308685be 100644 --- a/x-pack/plugin/runtime-fields/build.gradle +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -11,6 +11,8 @@ archivesBaseName = 'x-pack-runtime-fields' dependencies { compileOnly project(":server") compileOnly project(':modules:lang-painless:spi') + api project(':libs:elasticsearch-grok') + api project(':libs:elasticsearch-dissect') compileOnly project(path: xpackModule('core'), configuration: 'default') } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 30880acb4bea4..092e09a7a5048 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,7 +6,17 @@ package org.elasticsearch.xpack.runtimefields; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DynamicRuntimeFieldsBuilder; @@ -18,7 +28,11 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.BooleanScriptFieldType; @@ -33,15 +47,43 @@ import org.elasticsearch.xpack.runtimefields.mapper.KeywordScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.LongFieldScript; import org.elasticsearch.xpack.runtimefields.mapper.LongScriptFieldType; +import org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor; +import org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor.GrokHelper; import org.elasticsearch.xpack.runtimefields.mapper.StringFieldScript; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { + static final Setting GROK_WATCHDOG_INTERVAL = Setting.timeSetting( + "runtime_fields.grok.watchdog.interval", + TimeValue.timeValueSeconds(1), + Setting.Property.NodeScope + ); + static final Setting GROK_WATCHDOG_MAX_EXECUTION_TIME = Setting.timeSetting( + "runtime_fields.grok.watchdog.max_execution_time", + TimeValue.timeValueSeconds(1), + Setting.Property.NodeScope + ); + + private final NamedGroupExtractor.GrokHelper grokHelper; + + public RuntimeFields(Settings settings) { + grokHelper = new NamedGroupExtractor.GrokHelper( + GROK_WATCHDOG_INTERVAL.get(settings), + GROK_WATCHDOG_MAX_EXECUTION_TIME.get(settings) + ); + } + + @Override + public List> getSettings() { + return org.elasticsearch.common.collect.List.of(GROK_WATCHDOG_INTERVAL, GROK_WATCHDOG_MAX_EXECUTION_TIME); + } + @Override public Map getRuntimeFieldTypes() { return org.elasticsearch.common.collect.Map.ofEntries( @@ -76,4 +118,26 @@ public DynamicRuntimeFieldsBuilder getDynamicRuntimeFieldsBuilder() { public Collection createGuiceModules() { return Collections.singletonList(b -> XPackPlugin.bindFeatureSet(b, RuntimeFieldsFeatureSet.class)); } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + grokHelper.finishInitializing(threadPool); + return org.elasticsearch.common.collect.List.of(); + } + + public GrokHelper grokHelper() { + return grokHelper; + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractFieldScript.java index bdb9647404640..99ff5c3808ea8 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractFieldScript.java @@ -35,7 +35,7 @@ public abstract class AbstractFieldScript { */ static final int MAX_VALUES = 100; - public static ScriptContext newContext(String name, Class factoryClass) { + static ScriptContext newContext(String name, Class factoryClass) { return new ScriptContext<>( name + "_script_field", factoryClass, diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanFieldScript.java index 0220331fe8347..0adc49a097968 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanFieldScript.java @@ -8,26 +8,16 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.Booleans; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; -import java.util.Collections; -import java.util.List; import java.util.Map; public abstract class BooleanFieldScript extends AbstractFieldScript { public static final ScriptContext CONTEXT = newContext("boolean_script_field", Factory.class); - static List whitelist() { - return Collections.singletonList( - WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "boolean_whitelist.txt") - ); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateFieldScript.java index c83a9133b8f99..33a18f48ee66b 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateFieldScript.java @@ -8,24 +8,16 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; -import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Objects; public abstract class DateFieldScript extends AbstractLongFieldScript { public static final ScriptContext CONTEXT = newContext("date", Factory.class); - static List whitelist() { - return Collections.singletonList(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "date_whitelist.txt")); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleFieldScript.java index fc42f1634680d..6e5ba9ad3a335 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleFieldScript.java @@ -8,25 +8,15 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.ArrayUtil; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; -import java.util.Collections; -import java.util.List; import java.util.Map; public abstract class DoubleFieldScript extends AbstractFieldScript { public static final ScriptContext CONTEXT = newContext("double_script_field", Factory.class); - static List whitelist() { - return Collections.singletonList( - WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt") - ); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java index b14e184e458ee..41b1a7f157b3c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointFieldScript.java @@ -6,18 +6,15 @@ package org.elasticsearch.xpack.runtimefields.mapper; +import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; -import org.apache.lucene.document.LatLonDocValuesField; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -31,12 +28,6 @@ public abstract class GeoPointFieldScript extends AbstractLongFieldScript { public static final ScriptContext CONTEXT = newContext("geo_point_script_field", Factory.class); - static List whitelist() { - return Collections.singletonList( - WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "geo_point_whitelist.txt") - ); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/IpFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/IpFieldScript.java index 07a3085c45de8..215a931dbc973 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/IpFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/IpFieldScript.java @@ -12,8 +12,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.index.mapper.IpFieldMapper; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; @@ -21,8 +19,6 @@ import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; -import java.util.Collections; -import java.util.List; import java.util.Map; /** @@ -42,10 +38,6 @@ public abstract class IpFieldScript extends AbstractFieldScript { public static final ScriptContext CONTEXT = newContext("ip_script_field", Factory.class); - static List whitelist() { - return Collections.singletonList(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "ip_whitelist.txt")); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/LongFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/LongFieldScript.java index 0f89a815471e6..93baebe9f8af8 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/LongFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/LongFieldScript.java @@ -8,23 +8,15 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; -import java.util.Collections; -import java.util.List; import java.util.Map; public abstract class LongFieldScript extends AbstractLongFieldScript { public static final ScriptContext CONTEXT = newContext("long_script_field", Factory.class); - static List whitelist() { - return Collections.singletonList(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java new file mode 100644 index 0000000000000..dc84cf370c029 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.LazyInitializable; +import org.elasticsearch.dissect.DissectParser; +import org.elasticsearch.grok.Grok; +import org.elasticsearch.grok.MatcherWatchdog; +import org.elasticsearch.threadpool.ThreadPool; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +/** + * Extracts named groups from grok and dissect. Think of it kind of like + * {@link Pattern} but for grok and dissect. + */ +public interface NamedGroupExtractor { + /** + * Returns a {@link Map} containing all named capture groups if the + * string matches or {@code null} if it doesn't. + */ + Map extract(String in); + + /** + * Create a {@link NamedGroupExtractor} that runs {@link DissectParser} + * with the default {@code appendSeparator}. + */ + static NamedGroupExtractor dissect(String pattern) { + return dissect(pattern, null); + } + + /** + * Create a {@link NamedGroupExtractor} that runs {@link DissectParser}. + */ + static NamedGroupExtractor dissect(String pattern, String appendSeparator) { + DissectParser dissect = new DissectParser(pattern, appendSeparator); + return new NamedGroupExtractor() { + @Override + public Map extract(String in) { + return dissect.parse(in); + } + }; + } + + /** + * Builds {@link NamedGroupExtractor}s from grok patterns. + */ + class GrokHelper { + private final SetOnce threadPoolContainer = new SetOnce<>(); + private final Supplier watchdogSupplier; + + public GrokHelper(TimeValue interval, TimeValue maxExecutionTime) { + this.watchdogSupplier = new LazyInitializable(() -> { + ThreadPool threadPool = threadPoolContainer.get(); + if (threadPool == null) { + throw new IllegalStateException("missing call to finishInitializing"); + } + return MatcherWatchdog.newInstance( + interval.millis(), + maxExecutionTime.millis(), + threadPool::relativeTimeInMillis, + (delay, command) -> threadPool.schedule(command, TimeValue.timeValueMillis(delay), ThreadPool.Names.GENERIC) + ); + })::getOrCompute; + } + + /** + * Finish initializing. This is split from the ctor because we need an + * instance of this class to feed into painless before the + * {@link ThreadPool} is ready. + */ + public void finishInitializing(ThreadPool threadPool) { + threadPoolContainer.set(threadPool); + } + + public NamedGroupExtractor grok(String pattern) { + MatcherWatchdog watchdog = watchdogSupplier.get(); + /* + * Build the grok pattern in a PrivilegedAction so it can load + * things from the classpath. + */ + Grok grok = AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Grok run() { + try { + // Try to collect warnings up front and refuse to compile the expression if there are any + List warnings = new ArrayList<>(); + new Grok(Grok.BUILTIN_PATTERNS, pattern, watchdog, warnings::add).match("__nomatch__"); + if (false == warnings.isEmpty()) { + throw new IllegalArgumentException("emitted warnings: " + warnings); + } + + return new Grok( + Grok.BUILTIN_PATTERNS, + pattern, + watchdog, + w -> { throw new IllegalArgumentException("grok [" + pattern + "] emitted a warning: " + w); } + ); + } catch (RuntimeException e) { + throw new IllegalArgumentException("error compiling grok pattern [" + pattern + "]: " + e.getMessage(), e); + } + } + }); + return new NamedGroupExtractor() { + @Override + public Map extract(String in) { + return grok.captures(in); + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java index 1e8a6b4618419..3894ce98923e5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldsPainlessExtension.java @@ -8,29 +8,57 @@ import org.elasticsearch.painless.spi.PainlessExtension; import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistInstanceBinding; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.xpack.runtimefields.RuntimeFields; import java.util.List; import java.util.Map; public class RuntimeFieldsPainlessExtension implements PainlessExtension { + private final Whitelist commonWhitelist = WhitelistLoader.loadFromResourceFiles(AbstractFieldScript.class, "common_whitelist.txt"); + + private final Whitelist grokWhitelist; + + public RuntimeFieldsPainlessExtension(RuntimeFields plugin) { + grokWhitelist = new Whitelist( + commonWhitelist.classLoader, + org.elasticsearch.common.collect.List.of(), + org.elasticsearch.common.collect.List.of(), + org.elasticsearch.common.collect.List.of(), + org.elasticsearch.common.collect.List.of( + new WhitelistInstanceBinding( + AbstractFieldScript.class.getCanonicalName(), + plugin.grokHelper(), + "grok", + NamedGroupExtractor.class.getName(), + org.elasticsearch.common.collect.List.of(String.class.getName()), + org.elasticsearch.common.collect.List.of(CompileTimeOnlyAnnotation.INSTANCE) + ) + ) + ); + } + + private List load(String path) { + return org.elasticsearch.common.collect.List.of( + commonWhitelist, + grokWhitelist, + WhitelistLoader.loadFromResourceFiles(AbstractFieldScript.class, path) + ); + } + @Override public Map, List> getContextWhitelists() { - return org.elasticsearch.common.collect.Map.of( - BooleanFieldScript.CONTEXT, - BooleanFieldScript.whitelist(), - DateFieldScript.CONTEXT, - DateFieldScript.whitelist(), - DoubleFieldScript.CONTEXT, - DoubleFieldScript.whitelist(), - GeoPointFieldScript.CONTEXT, - GeoPointFieldScript.whitelist(), - IpFieldScript.CONTEXT, - IpFieldScript.whitelist(), - LongFieldScript.CONTEXT, - LongFieldScript.whitelist(), - StringFieldScript.CONTEXT, - StringFieldScript.whitelist() + return org.elasticsearch.common.collect.Map.ofEntries( + org.elasticsearch.common.collect.Map.entry(BooleanFieldScript.CONTEXT, load("boolean_whitelist.txt")), + org.elasticsearch.common.collect.Map.entry(DateFieldScript.CONTEXT, load("date_whitelist.txt")), + org.elasticsearch.common.collect.Map.entry(DoubleFieldScript.CONTEXT, load("double_whitelist.txt")), + org.elasticsearch.common.collect.Map.entry(GeoPointFieldScript.CONTEXT, load("geo_point_whitelist.txt")), + org.elasticsearch.common.collect.Map.entry(IpFieldScript.CONTEXT, load("ip_whitelist.txt")), + org.elasticsearch.common.collect.Map.entry(LongFieldScript.CONTEXT, load("long_whitelist.txt")), + org.elasticsearch.common.collect.Map.entry(StringFieldScript.CONTEXT, load("string_whitelist.txt")) ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/StringFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/StringFieldScript.java index 6b1ccb8f75810..f7bfe9255f3c3 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/StringFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/StringFieldScript.java @@ -7,14 +7,11 @@ package org.elasticsearch.xpack.runtimefields.mapper; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.SearchLookup; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,12 +24,6 @@ public abstract class StringFieldScript extends AbstractFieldScript { public static final ScriptContext CONTEXT = newContext("string_script_field", Factory.class); - static List whitelist() { - return Collections.singletonList( - WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt") - ); - } - @SuppressWarnings("unused") public static final String[] PARAMETERS = {}; diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/common_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/common_whitelist.txt new file mode 100644 index 0000000000000..9c14de5946968 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/mapper/common_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +class org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor @no_import { + Map extract(String); +} + +static_import { + org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor dissect(String) from_class org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor @compile_time_only + org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor dissect(String, String) from_class org.elasticsearch.xpack.runtimefields.mapper.NamedGroupExtractor @compile_time_only +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java index d377a85792cf9..7a6ef3082f06d 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldTypeTestCase.java @@ -313,7 +313,7 @@ protected final void minimalMapping(XContentBuilder b) throws IOException { @Override protected Collection getPlugins() { - return org.elasticsearch.common.collect.List.of(new RuntimeFields(), new TestScriptPlugin()); + return org.elasticsearch.common.collect.List.of(new RuntimeFields(Settings.EMPTY), new TestScriptPlugin()); } private static class TestScriptPlugin extends Plugin implements ScriptPlugin { @@ -347,7 +347,7 @@ protected Object buildScriptFactory(ScriptContext context) { } public Set> getSupportedContexts() { - return org.elasticsearch.common.collect.Set.copyOf(new RuntimeFields().getContexts()); + return org.elasticsearch.common.collect.Set.copyOf(new RuntimeFields(Settings.EMPTY).getContexts()); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanScriptFieldTypeTests.java index fcda3cd830692..696536304a287 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/BooleanScriptFieldTypeTests.java @@ -515,7 +515,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { BooleanFieldScript.Factory factory = scriptService.compile(script, BooleanFieldScript.CONTEXT); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldTypeTests.java index e69697c61091c..fe470d4684736 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldTypeTests.java @@ -615,7 +615,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { DateFieldScript.Factory factory = scriptService.compile(script, DateFieldScript.CONTEXT); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleScriptFieldTypeTests.java index 9882e77ca24e5..dfe313fa57f39 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DoubleScriptFieldTypeTests.java @@ -326,7 +326,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { DoubleFieldScript.Factory factory = scriptService.compile(script, DoubleFieldScript.CONTEXT); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DynamicRuntimeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DynamicRuntimeTests.java index 17a261fcde74d..898ccfd9e7ce9 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DynamicRuntimeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/DynamicRuntimeTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.runtimefields.mapper; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ObjectMapper; @@ -21,7 +22,7 @@ public class DynamicRuntimeTests extends MapperServiceTestCase { @Override protected Collection getPlugins() { - return Collections.singletonList(new RuntimeFields()); + return Collections.singletonList(new RuntimeFields(Settings.EMPTY)); } public void testDynamicLeafFields() throws IOException { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java index b4e9353fe5579..7ca704bc27d75 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/GeoPointScriptFieldTypeTests.java @@ -292,7 +292,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { GeoPointFieldScript.Factory factory = scriptService.compile(script, GeoPointFieldScript.CONTEXT); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/IpScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/IpScriptFieldTypeTests.java index 39013cbc998c8..b6192180d216d 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/IpScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/IpScriptFieldTypeTests.java @@ -369,7 +369,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { IpFieldScript.Factory factory = scriptService.compile(script, IpFieldScript.CONTEXT); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/KeywordScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/KeywordScriptFieldTypeTests.java index f267bb2f52d9a..fbe4012b089c2 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/KeywordScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/KeywordScriptFieldTypeTests.java @@ -448,7 +448,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { StringFieldScript.Factory factory = scriptService.compile(script, StringFieldScript.CONTEXT); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/LongScriptFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/LongScriptFieldTypeTests.java index 037c8d3e63b5e..74e3e0f77a2ee 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/LongScriptFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/LongScriptFieldTypeTests.java @@ -379,7 +379,7 @@ public void execute() { }; ScriptModule scriptModule = new ScriptModule( Settings.EMPTY, - org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields()) + org.elasticsearch.common.collect.List.of(scriptPlugin, new RuntimeFields(Settings.EMPTY)) ); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { LongFieldScript.Factory factory = scriptService.compile(script, LongFieldScript.CONTEXT); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/250_grok.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/250_grok.yml new file mode 100644 index 0000000000000..6658853732047 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/250_grok.yml @@ -0,0 +1,126 @@ +--- +setup: + - do: + indices.create: + index: http_logs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + runtime: + http.clientip: + type: ip + script: + source: | + String clientip = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + http.verb: + type: keyword + script: + source: | + String verb = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.verb; + if (verb != null) { + emit(verb); + } + http.response: + type: long + script: + source: | + String response = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.response; + if (response != null) { + emit(Integer.parseInt(response)); + } + properties: + timestamp: + type: date + message: + type: wildcard + - do: + bulk: + index: http_logs + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [30/Apr/1998:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [30/Apr/1998:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [30/Apr/1998:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [30/Apr/1998:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:28-05:00", "message" : "not a valid apache log"} + +--- +fetch: + - do: + search: + index: http_logs + body: + sort: timestamp + fields: + - http.clientip + - http.verb + - http.response + - match: {hits.total.value: 7} + - match: {hits.hits.0.fields.http\.clientip: [40.135.0.0] } + - match: {hits.hits.0.fields.http\.verb: [GET] } + - match: {hits.hits.0.fields.http\.response: [200] } + - is_false: hits.hits.6.fields.http\.clientip + - is_false: hits.hits.6.fields.http\.verb + - is_false: hits.hits.6.fields.http\.response + +--- +mutable pattern: + - do: + catch: /all arguments must be constant but the \[1\] argument isn't/ + search: + index: http_logs + body: + runtime_mappings: + broken: + type: keyword + script: | + def clientip = grok(doc["type"].value).extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + +--- +syntax error in pattern: + - do: + catch: '/error compiling grok pattern \[.+\]: invalid group name <2134>/' + search: + index: http_logs + body: + runtime_mappings: + broken: + type: keyword + script: | + def clientip = grok('(?<2134>').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + +--- +warning in pattern: + - do: + catch: '/error compiling grok pattern \[.+\]: emitted warnings: \[.+]/' + search: + index: http_logs + body: + runtime_mappings: + broken: + type: keyword + script: | + def clientip = grok('\\o').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/260_dissect.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/260_dissect.yml new file mode 100644 index 0000000000000..d7fcffdf6c986 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/260_dissect.yml @@ -0,0 +1,76 @@ +setup: + - do: + indices.create: + index: http_logs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + runtime: + http.clientip: + type: ip + script: + source: | + String clientip = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + http.verb: + type: keyword + script: + source: | + String verb = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.verb; + if (verb != null) { + emit(verb); + } + http.response: + type: long + script: + source: | + String response = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) { + emit(Integer.parseInt(response)); + } + properties: + timestamp: + type: date + message: + type: wildcard + - do: + bulk: + index: http_logs + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [30/Apr/1998:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [30/Apr/1998:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [30/Apr/1998:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [30/Apr/1998:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:28-05:00", "message" : "not a valid apache log"} +--- +fetch: + - do: + search: + index: http_logs + body: + sort: timestamp + fields: + - http.clientip + - http.verb + - http.response + - match: {hits.total.value: 7} + - match: {hits.hits.0.fields.http\.clientip: [40.135.0.0] } + - match: {hits.hits.0.fields.http\.verb: [GET] } + - match: {hits.hits.0.fields.http\.response: [200] } + - is_false: hits.hits.6.fields.http\.clientip + - is_false: hits.hits.6.fields.http\.verb + - is_false: hits.hits.6.fields.http\.response