Ingest: Enable Templated Fieldnames in Rename (#31690) (#31896)

* Ingest: Enable Templated Fieldnames in Rename

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

@@ -62,7 +62,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
         processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService));
         processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService));
         processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService));
-        processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory());
+        processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory(parameters.scriptService));
         processors.put(RemoveProcessor.TYPE, new RemoveProcessor.Factory(parameters.scriptService));
         processors.put(SplitProcessor.TYPE, new SplitProcessor.Factory());
         processors.put(JoinProcessor.TYPE, new JoinProcessor.Factory());

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java

@@ -23,6 +23,8 @@
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.TemplateScript;
 
 import java.util.Map;
 
@@ -33,22 +35,22 @@ public final class RenameProcessor extends AbstractProcessor {
 
     public static final String TYPE = "rename";
 
-    private final String field;
-    private final String targetField;
+    private final TemplateScript.Factory field;
+    private final TemplateScript.Factory targetField;
     private final boolean ignoreMissing;
 
-    RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) {
+    RenameProcessor(String tag, TemplateScript.Factory field, TemplateScript.Factory targetField, boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.ignoreMissing = ignoreMissing;
     }
 
-    String getField() {
+    TemplateScript.Factory getField() {
         return field;
     }
 
-    String getTargetField() {
+    TemplateScript.Factory getTargetField() {
         return targetField;
     }
 
@@ -58,28 +60,30 @@ boolean isIgnoreMissing() {
 
     @Override
     public void execute(IngestDocument document) {
-        if (document.hasField(field, true) == false) {
+        String path = document.renderTemplate(field);
+        if (document.hasField(path, true) == false) {
             if (ignoreMissing) {
                 return;
             } else {
-                throw new IllegalArgumentException("field [" + field + "] doesn't exist");
+                throw new IllegalArgumentException("field [" + path + "] doesn't exist");
             }
         }
         // We fail here if the target field point to an array slot that is out of range.
         // If we didn't do this then we would fail if we set the value in the target_field
         // and then on failure processors would not see that value we tried to rename as we already
         // removed it.
-        if (document.hasField(targetField, true)) {
-            throw new IllegalArgumentException("field [" + targetField + "] already exists");
+        String target = document.renderTemplate(targetField);
+        if (document.hasField(target, true)) {
+            throw new IllegalArgumentException("field [" + target + "] already exists");
         }
 
-        Object value = document.getFieldValue(field, Object.class);
-        document.removeField(field);
+        Object value = document.getFieldValue(path, Object.class);
+        document.removeField(path);
         try {
-            document.setFieldValue(targetField, value);
+            document.setFieldValue(target, value);
         } catch (Exception e) {
             // setting the value back to the original field shouldn't as we just fetched it from that field:
-            document.setFieldValue(field, value);
+            document.setFieldValue(path, value);
             throw e;
         }
     }
@@ -90,13 +94,24 @@ public String getType() {
     }
 
     public static final class Factory implements Processor.Factory {
+
+        private final ScriptService scriptService;
+
+        public Factory(ScriptService scriptService) {
+            this.scriptService = scriptService;
+        }
+
         @Override
         public RenameProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                                       Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
+                "field", field, scriptService);
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
+            TemplateScript.Factory targetFieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
+                "target_field", targetField, scriptService);
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            return new RenameProcessor(processorTag, field, targetField, ignoreMissing);
+            return new RenameProcessor(processorTag, fieldTemplate, targetFieldTemplate , ignoreMissing);
         }
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
@@ -86,7 +87,8 @@ public void testEscapeFields_valueField() throws Exception {
         // so because foo is no branch field but a value field the `foo.bar` field can't be expanded
         // into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
         IngestDocument document = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false);
+        Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"),
+            new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false);
         processor.execute(document);
         processor = new DotExpanderProcessor("_tag", null, "foo.bar");
         processor.execute(document);

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java

@@ -20,44 +20,51 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class RenameProcessorFactoryTests extends ESTestCase {
 
+    private RenameProcessor.Factory factory;
+
+    @Before
+    public void init() {
+        factory = new RenameProcessor.Factory(TestTemplateService.instance());
+    }
+
     public void testCreate() throws Exception {
-        RenameProcessor.Factory factory = new RenameProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         config.put("target_field", "new_field");
         String processorTag = randomAlphaOfLength(10);
         RenameProcessor renameProcessor = factory.create(null, processorTag, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
-        assertThat(renameProcessor.getField(), equalTo("old_field"));
-        assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
+        assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
         assertThat(renameProcessor.isIgnoreMissing(), equalTo(false));
     }
 
     public void testCreateWithIgnoreMissing() throws Exception {
-        RenameProcessor.Factory factory = new RenameProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         config.put("target_field", "new_field");
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
         RenameProcessor renameProcessor = factory.create(null, processorTag, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
-        assertThat(renameProcessor.getField(), equalTo("old_field"));
-        assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
+        assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
         assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
     }
 
     public void testCreateNoFieldPresent() throws Exception {
-        RenameProcessor.Factory factory = new RenameProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("target_field", "new_field");
         try {
@@ -69,7 +76,6 @@ public void testCreateNoFieldPresent() throws Exception {
     }
 
     public void testCreateNoToPresent() throws Exception {
-        RenameProcessor.Factory factory = new RenameProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         try {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -45,7 +46,7 @@ public void testRename() throws Exception {
         do {
             newFieldName = RandomDocumentPicks.randomFieldName(random());
         } while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
+        Processor processor = createRenameProcessor(fieldName, newFieldName, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
     }
@@ -63,7 +64,7 @@ public void testRenameArrayElement() throws Exception {
         document.put("one", one);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "item", false);
+        Processor processor = createRenameProcessor("list.0", "item", false);
         processor.execute(ingestDocument);
         Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
         assertThat(actualObject, instanceOf(List.class));
@@ -76,7 +77,7 @@ public void testRenameArrayElement() throws Exception {
         assertThat(actualObject, instanceOf(String.class));
         assertThat(actualObject, equalTo("item1"));
 
-        processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "list.3", false);
+        processor = createRenameProcessor("list.0", "list.3", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -91,7 +92,7 @@ public void testRenameArrayElement() throws Exception {
     public void testRenameNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
+        Processor processor = createRenameProcessor(fieldName,
             RandomDocumentPicks.randomFieldName(random()), false);
         try {
             processor.execute(ingestDocument);
@@ -105,7 +106,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName,
+        Processor processor = createRenameProcessor(fieldName,
             RandomDocumentPicks.randomFieldName(random()), true);
         processor.execute(ingestDocument);
         assertIngestDocument(originalIngestDocument, ingestDocument);
@@ -114,7 +115,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
     public void testRenameNewFieldAlreadyExists() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), RandomDocumentPicks.randomExistingFieldName(
+        Processor processor = createRenameProcessor(RandomDocumentPicks.randomExistingFieldName(
                 random(), ingestDocument), fieldName, false);
         try {
             processor.execute(ingestDocument);
@@ -129,7 +130,7 @@ public void testRenameExistingFieldNullValue() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         ingestDocument.setFieldValue(fieldName, null);
         String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false);
+        Processor processor = createRenameProcessor(fieldName, newFieldName, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(fieldName), equalTo(false));
         assertThat(ingestDocument.hasField(newFieldName), equalTo(true));
@@ -149,7 +150,7 @@ public Object put(String key, Object value) {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
+        Processor processor = createRenameProcessor("list", "new_field", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -173,7 +174,7 @@ public Object remove(Object key) {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false);
+        Processor processor = createRenameProcessor("list", "new_field", false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -188,22 +189,26 @@ public void testRenameLeafIntoBranch() throws Exception {
         Map<String, Object> source = new HashMap<>();
         source.put("foo", "bar");
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor1 = new RenameProcessor(randomAlphaOfLength(10), "foo", "foo.bar", false);
+        Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
         processor1.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
 
-        Processor processor2 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar", "foo.bar.baz", false);
+        Processor processor2 = createRenameProcessor( "foo.bar", "foo.bar.baz", false);
         processor2.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar",
                 Collections.singletonMap("baz", "bar"))));
         assertThat(ingestDocument.getFieldValue("foo.bar", Map.class), equalTo(Collections.singletonMap("baz", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
 
         // for fun lets try to restore it (which don't allow today)
-        Processor processor3 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar.baz", "foo", false);
+        Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false);
         Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
         assertThat(e.getMessage(), equalTo("field [foo] already exists"));
     }
 
+    private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) {
+        return new RenameProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(field),
+            new TestTemplateService.MockTemplateScript.Factory(targetField), ignoreMissing);
+    }
 }