Ingest: Add ignore_missing option to RemoveProc

Added `ignore_missing` setting to the RemoveProcessor to fix elastic#23086
original-brownbear · Jun 29, 2018 · 262a771 · 262a771
1 parent 8fa0629
commit 262a771
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 8 deletions.
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
@@ -1740,8 +1740,9 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown
 .Remove Options
 [options="header"]
 |======
-| Name      | Required  | Default  | Description
-| `field`   | yes       | -        | Fields to be removed
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | Fields to be removed
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
 |======
 
 Here is an example to remove a single field:

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
@@ -39,10 +39,12 @@ public final class RemoveProcessor extends AbstractProcessor {
     public static final String TYPE = "remove";
 
     private final List<TemplateScript.Factory> fields;
+    private final boolean ignoreMissing;
 
-    RemoveProcessor(String tag, List<TemplateScript.Factory> fields) {
+    RemoveProcessor(String tag, List<TemplateScript.Factory> fields, boolean ignoreMissing) {
         super(tag);
         this.fields = new ArrayList<>(fields);
+        this.ignoreMissing = ignoreMissing;
     }
 
     public List<TemplateScript.Factory> getFields() {
@@ -51,7 +53,16 @@ public List<TemplateScript.Factory> getFields() {
 
     @Override
     public void execute(IngestDocument document) {
-       fields.forEach(document::removeField);
+        if (ignoreMissing) {
+            fields.forEach(field -> {
+                String path = document.renderTemplate(field);
+                if (document.hasField(path)) {
+                    document.removeField(path);
+                }
+            });
+        } else {
+            fields.forEach(document::removeField);
+        }
     }
 
     @Override
@@ -81,7 +92,8 @@ public RemoveProcessor create(Map<String, Processor.Factory> registry, String pr
             final List<TemplateScript.Factory> compiledTemplates = fields.stream()
                 .map(f -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", f, scriptService))
                 .collect(Collectors.toList());
-            return new RemoveProcessor(processorTag, compiledTemplates);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new RemoveProcessor(processorTag, compiledTemplates, ignoreMissing);
         }
     }
 }

diff --git a/...les/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/...les/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
@@ -27,6 +27,7 @@
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -37,21 +38,34 @@ public void testRemoveFields() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
         Processor processor = new RemoveProcessor(randomAlphaOfLength(10),
-            Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)));
+            Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)), false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(field), equalTo(false));
     }
 
     public void testRemoveNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = new RemoveProcessor(randomAlphaOfLength(10),
-            Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(fieldName)));
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        String processorTag = randomAlphaOfLength(10);
+        Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config);
         try {
             processor.execute(ingestDocument);
             fail("remove field should have failed");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
         }
     }
+
+    public void testIgnoreMissing() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", fieldName);
+        config.put("ignore_missing", true);
+        String processorTag = randomAlphaOfLength(10);
+        Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config);
+        processor.execute(ingestDocument);
+    }
 }