Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ingest: Enable Templated Fieldnames in Rename (#31690) #31896

Merged
merged 1 commit into from
Jul 9, 2018
Merged

Ingest: Enable Templated Fieldnames in Rename (#31690) #31896

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService));
processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService));
processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService));
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory());
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory(parameters.scriptService));
processors.put(RemoveProcessor.TYPE, new RemoveProcessor.Factory(parameters.scriptService));
processors.put(SplitProcessor.TYPE, new SplitProcessor.Factory());
processors.put(JoinProcessor.TYPE, new JoinProcessor.Factory());
Expand Down
43 changes: 29 additions & 14 deletions modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;

import java.util.Map;

Expand All @@ -33,22 +35,22 @@ public final class RenameProcessor extends AbstractProcessor {

public static final String TYPE = "rename";

private final String field;
private final String targetField;
private final TemplateScript.Factory field;
private final TemplateScript.Factory targetField;
private final boolean ignoreMissing;

RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
RenameProcessor(String tag, TemplateScript.Factory field, TemplateScript.Factory targetField, boolean ignoreMissing) {
super(tag);
this.field = field;
this.targetField = targetField;
this.ignoreMissing = ignoreMissing;
}

String getField() {
TemplateScript.Factory getField() {
return field;
}

String getTargetField() {
TemplateScript.Factory getTargetField() {
return targetField;
}

Expand All @@ -58,28 +60,30 @@ boolean isIgnoreMissing() {

@Override
public void execute(IngestDocument document) {
if (document.hasField(field, true) == false) {
String path = document.renderTemplate(field);
if (document.hasField(path, true) == false) {
if (ignoreMissing) {
return;
} else {
throw new IllegalArgumentException("field [" + field + "] doesn't exist");
throw new IllegalArgumentException("field [" + path + "] doesn't exist");
}
}
// We fail here if the target field point to an array slot that is out of range.
// If we didn't do this then we would fail if we set the value in the target_field
// and then on failure processors would not see that value we tried to rename as we already
// removed it.
if (document.hasField(targetField, true)) {
throw new IllegalArgumentException("field [" + targetField + "] already exists");
String target = document.renderTemplate(targetField);
if (document.hasField(target, true)) {
throw new IllegalArgumentException("field [" + target + "] already exists");
}

Object value = document.getFieldValue(field, Object.class);
document.removeField(field);
Object value = document.getFieldValue(path, Object.class);
document.removeField(path);
try {
document.setFieldValue(targetField, value);
document.setFieldValue(target, value);
} catch (Exception e) {
// setting the value back to the original field shouldn't as we just fetched it from that field:
document.setFieldValue(field, value);
document.setFieldValue(path, value);
throw e;
}
}
Expand All @@ -90,13 +94,24 @@ public String getType() {
}

public static final class Factory implements Processor.Factory {

private final ScriptService scriptService;

public Factory(ScriptService scriptService) {
this.scriptService = scriptService;
}

@Override
public RenameProcessor create(Map<String, Processor.Factory> registry, String processorTag,
Map<String, Object> config) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"field", field, scriptService);
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
TemplateScript.Factory targetFieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
"target_field", targetField, scriptService);
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
return new RenameProcessor(processorTag, fieldTemplate, targetFieldTemplate , ignoreMissing);
}
}
}
4 changes: 3 additions & 1 deletion ...ngest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;

import java.util.Collections;
Expand Down Expand Up @@ -86,7 +87,8 @@ public void testEscapeFields_valueField() throws Exception {
// so because foo is no branch field but a value field the `foo.bar` field can't be expanded
// into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
IngestDocument document = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"),
new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false);
processor.execute(document);
processor = new DotExpanderProcessor("_tag", null, "foo.bar");
processor.execute(document);
Expand Down
22 changes: 14 additions & 8 deletions ...est-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,44 +20,51 @@
package org.elasticsearch.ingest.common;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static org.hamcrest.CoreMatchers.equalTo;

public class RenameProcessorFactoryTests extends ESTestCase {

private RenameProcessor.Factory factory;

@Before
public void init() {
factory = new RenameProcessor.Factory(TestTemplateService.instance());
}

public void testCreate() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
config.put("target_field", "new_field");
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
}

public void testCreateWithIgnoreMissing() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
config.put("target_field", "new_field");
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, config);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
}

public void testCreateNoFieldPresent() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("target_field", "new_field");
try {
Expand All @@ -69,7 +76,6 @@ public void testCreateNoFieldPresent() throws Exception {
}

public void testCreateNoToPresent() throws Exception {
RenameProcessor.Factory factory = new RenameProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
try {
Expand Down
29 changes: 17 additions & 12 deletions ...les/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.ingest.TestTemplateService;
import org.elasticsearch.test.ESTestCase;

import java.util.ArrayList;
Expand All @@ -45,7 +46,7 @@ public void testRename() throws Exception {
do {
newFieldName = RandomDocumentPicks.randomFieldName(random());
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
}
Expand All @@ -63,7 +64,7 @@ public void testRenameArrayElement() throws Exception {
document.put("one", one);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);

Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "item", false);
Processor processor = createRenameProcessor("list.0", "item", false);
processor.execute(ingestDocument);
Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
assertThat(actualObject, instanceOf(List.class));
Expand All @@ -76,7 +77,7 @@ public void testRenameArrayElement() throws Exception {
assertThat(actualObject, instanceOf(String.class));
assertThat(actualObject, equalTo("item1"));

processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "list.3", false);
processor = createRenameProcessor("list.0", "list.3", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand All @@ -91,7 +92,7 @@ public void testRenameArrayElement() throws Exception {
public void testRenameNonExistingField() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
Processor processor = createRenameProcessor(fieldName,
RandomDocumentPicks.randomFieldName(random()), false);
try {
processor.execute(ingestDocument);
Expand All @@ -105,7 +106,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
String fieldName = RandomDocumentPicks.randomFieldName(random());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
Processor processor = createRenameProcessor(fieldName,
RandomDocumentPicks.randomFieldName(random()), true);
processor.execute(ingestDocument);
assertIngestDocument(originalIngestDocument, ingestDocument);
Expand All @@ -114,7 +115,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
public void testRenameNewFieldAlreadyExists() throws Exception {
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
Processor processor = new RenameProcessor(randomAlphaOfLength(10), RandomDocumentPicks.randomExistingFieldName(
Processor processor = createRenameProcessor(RandomDocumentPicks.randomExistingFieldName(
random(), ingestDocument), fieldName, false);
try {
processor.execute(ingestDocument);
Expand All @@ -129,7 +130,7 @@ public void testRenameExistingFieldNullValue() throws Exception {
String fieldName = RandomDocumentPicks.randomFieldName(random());
ingestDocument.setFieldValue(fieldName, null);
String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
processor.execute(ingestDocument);
assertThat(ingestDocument.hasField(fieldName), equalTo(false));
assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
Expand All @@ -149,7 +150,7 @@ public Object put(String key, Object value) {
source.put("list", Collections.singletonList("item"));

IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
Processor processor = createRenameProcessor("list", "new_field", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand All @@ -173,7 +174,7 @@ public Object remove(Object key) {
source.put("list", Collections.singletonList("item"));

IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
Processor processor = createRenameProcessor("list", "new_field", false);
try {
processor.execute(ingestDocument);
fail("processor execute should have failed");
Expand All @@ -188,22 +189,26 @@ public void testRenameLeafIntoBranch() throws Exception {
Map<String, Object> source = new HashMap<>();
source.put("foo", "bar");
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor1 = new RenameProcessor(randomAlphaOfLength(10), "foo", "foo.bar", false);
Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
processor1.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));

Processor processor2 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar", "foo.bar.baz", false);
Processor processor2 = createRenameProcessor( "foo.bar", "foo.bar.baz", false);
processor2.execute(ingestDocument);
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
Collections.singletonMap("baz", "bar"))));
assertThat(ingestDocument.getFieldValue("foo.bar", Map.class), equalTo(Collections.singletonMap("baz", "bar")));
assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));

// for fun lets try to restore it (which don't allow today)
Processor processor3 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar.baz", "foo", false);
Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false);
Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
assertThat(e.getMessage(), equalTo("field [foo] already exists"));
}

private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) {
return new RenameProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(field),
new TestTemplateService.MockTemplateScript.Factory(targetField), ignoreMissing);
}
}