Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

INGEST: Make a few Processors callable by Painless (#32170) #32261

Merged
merged 1 commit into from
Jul 21, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions modules/ingest-common/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,17 @@
esplugin {
description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
classname 'org.elasticsearch.ingest.common.IngestCommonPlugin'
extendedPlugins = ['lang-painless']
}

dependencies {
compileOnly project(':modules:lang-painless')
compile project(':libs:grok')
}

compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"

integTestCluster {
module project(':modules:lang-painless')
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@ public final class BytesProcessor extends AbstractStringProcessor {
super(processorTag, field, ignoreMissing, targetField);
}

public static long apply(String value) {
return ByteSizeValue.parseBytesSizeValue(value, null, "Ingest Field").getBytes();
}

@Override
protected Long process(String value) {
return ByteSizeValue.parseBytesSizeValue(value, null, getField()).getBytes();
return apply(value);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,11 @@ boolean isAddToRoot() {
return addToRoot;
}

@Override
public void execute(IngestDocument document) throws Exception {
Object fieldValue = document.getFieldValue(field, Object.class);
BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString());
public static Object apply(Object fieldValue) {
BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString());
try (InputStream stream = bytesRef.streamInput();
XContentParser parser = JsonXContent.jsonXContent
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
XContentParser.Token token = parser.nextToken();
Object value = null;
if (token == XContentParser.Token.VALUE_NULL) {
Expand All @@ -91,20 +89,32 @@ public void execute(IngestDocument document) throws Exception {
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
throw new IllegalArgumentException("cannot read binary value");
}
if (addToRoot && (value instanceof Map)) {
for (Map.Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
document.setFieldValue(entry.getKey(), entry.getValue());
}
} else if (addToRoot) {
throw new IllegalArgumentException("cannot add non-map fields to root of document");
} else {
document.setFieldValue(targetField, value);
}
return value;
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}

public static void apply(Map<String, Object> ctx, String fieldName) {
Object value = apply(ctx.get(fieldName));
if (value instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) value;
ctx.putAll(map);
} else {
throw new IllegalArgumentException("cannot add non-map fields to root of document");
}
}

@Override
public void execute(IngestDocument document) throws Exception {
if (addToRoot) {
apply(document.getSourceAndMetadata(), field);
} else {
document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class)));
}
}

@Override
public String getType() {
return TYPE;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
super(processorTag, field, ignoreMissing, targetField);
}

public static String apply(String value) {
return value.toLowerCase(Locale.ROOT);
}

@Override
protected String process(String value) {
return value.toLowerCase(Locale.ROOT);
return apply(value);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.ingest.common;

import java.util.Map;

public final class Processors {

public static long bytes(String value) {
return BytesProcessor.apply(value);
}

public static String lowercase(String value) {
return LowercaseProcessor.apply(value);
}

public static String uppercase(String value) {
return UppercaseProcessor.apply(value);
}

public static Object json(Object fieldValue) {
return JsonProcessor.apply(fieldValue);
}

public static void json(Map<String, Object> ctx, String field) {
JsonProcessor.apply(ctx, field);
}

public static String urlDecode(String value) {
return URLDecodeProcessor.apply(value);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.ingest.common;

import org.elasticsearch.painless.spi.PainlessExtension;
import org.elasticsearch.painless.spi.Whitelist;
import org.elasticsearch.painless.spi.WhitelistLoader;
import org.elasticsearch.script.IngestScript;
import org.elasticsearch.script.ScriptContext;

import java.util.Collections;
import java.util.List;
import java.util.Map;

public class ProcessorsWhitelistExtension implements PainlessExtension {

private static final Whitelist WHITELIST =
WhitelistLoader.loadFromResourceFiles(ProcessorsWhitelistExtension.class, "processors_whitelist.txt");

@Override
public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
return Collections.singletonMap(IngestScript.CONTEXT, Collections.singletonList(WHITELIST));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,19 @@ public final class URLDecodeProcessor extends AbstractStringProcessor {
super(processorTag, field, ignoreMissing, targetField);
}

@Override
protected String process(String value) {
public static String apply(String value) {
try {
return URLDecoder.decode(value, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException("could not URL-decode field[" + getField() + "]", e);
throw new IllegalArgumentException("Could not URL-decode value.", e);
}
}

@Override
protected String process(String value) {
return apply(value);
}

@Override
public String getType() {
return TYPE;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,13 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
super(processorTag, field, ignoreMissing, targetField);
}

public static String apply(String value) {
return value.toUpperCase(Locale.ROOT);
}

@Override
protected String process(String value) {
return value.toUpperCase(Locale.ROOT);
return apply(value);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
org.elasticsearch.ingest.common.ProcessorsWhitelistExtension
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

# This file contains a whitelist of static processor methods that can be accessed from painless

class org.elasticsearch.ingest.common.Processors {
long bytes(String)
String lowercase(String)
String uppercase(String)
Object json(Object)
void json(Map, String)
String urlDecode(String)
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public void testTooLarge() {
Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
assertThat(exception.getMessage(),
CoreMatchers.equalTo("failed to parse setting [" + fieldName + "] with value [8912pb] as a size in bytes"));
CoreMatchers.equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes"));
assertThat(exception.getCause().getMessage(),
CoreMatchers.containsString("Values greater than 9223372036854775807 bytes are not supported"));
}
Expand Down Expand Up @@ -93,6 +93,6 @@ public void testFractional() throws Exception {
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L));
assertWarnings("Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " +
"[" + fieldName + "]");
"[Ingest Field]");
}
}
Loading