Updated Logstash attributes mapper to use template pattern

Signed-off-by: Asif Sohail Mohammed <[email protected]>
opensearch-project · Nov 17, 2021 · 8126cc3 · 8126cc3
1 parent 54f8ce7
commit 8126cc3
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 45 deletions.
diff --git a/...a/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java b/...a/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java
@@ -0,0 +1,54 @@
+package org.opensearch.dataprepper.logstash.mapping;
+
+import org.opensearch.dataprepper.logstash.model.LogstashAttribute;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.List;
+import java.util.Map;
+import java.util.LinkedHashMap;
+
+public abstract class AbstractLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper {
+    private static final Logger LOG = LoggerFactory.getLogger(AbstractLogstashPluginAttributesMapper.class);
+
+    @Override
+    public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) {
+
+        Objects.requireNonNull(logstashAttributes);
+        Objects.requireNonNull(logstashAttributesMappings);
+        Objects.requireNonNull(logstashAttributesMappings.getMappedAttributeNames());
+        Objects.requireNonNull(logstashAttributesMappings.getAdditionalAttributes());
+
+        final Map<String, Object> pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes());
+        final Map<String, String> mappedAttributeNames = logstashAttributesMappings.getMappedAttributeNames();
+
+        Collection<String> customMappedAttributeNames = getCustomMappedAttributeNames();
+
+        logstashAttributes
+                .stream()
+                .filter(logstashAttribute -> !customMappedAttributeNames.contains(logstashAttribute.getAttributeName()))
+                .forEach(logstashAttribute -> {
+            if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) {
+                pluginSettings.put(
+                        mappedAttributeNames.get(logstashAttribute.getAttributeName()),
+                        logstashAttribute.getAttributeValue().getValue()
+                );
+            }
+            else {
+                LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName());
+            }
+        });
+
+        if (!customMappedAttributeNames.isEmpty()) {
+            mapCustomAttributes(logstashAttributes, logstashAttributesMappings, pluginSettings);
+        }
+
+        return pluginSettings;
+    }
+
+    abstract void mapCustomAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map<String, Object> pluginSettings);
+
+    abstract Collection<String> getCustomMappedAttributeNames();
+}
diff --git a/...va/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java b/...va/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java
@@ -6,40 +6,23 @@
 package org.opensearch.dataprepper.logstash.mapping;
 
 import org.opensearch.dataprepper.logstash.model.LogstashAttribute;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.util.LinkedHashMap;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
-class DefaultLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper {
-    private static final Logger LOG = LoggerFactory.getLogger(DefaultLogstashPluginAttributesMapper.class);
+class DefaultLogstashPluginAttributesMapper extends AbstractLogstashPluginAttributesMapper {
 
     @Override
-    public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) {
-
-        Objects.requireNonNull(logstashAttributes);
-        Objects.requireNonNull(logstashAttributesMappings);
-        Objects.requireNonNull(logstashAttributesMappings.getMappedAttributeNames());
-        Objects.requireNonNull(logstashAttributesMappings.getAdditionalAttributes());
-
-        final Map<String, Object> pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes());
-
-        final Map<String, String> mappedAttributeNames = logstashAttributesMappings.getMappedAttributeNames();
-        logstashAttributes.forEach(logstashAttribute -> {
-            if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) {
-                pluginSettings.put(
-                        mappedAttributeNames.get(logstashAttribute.getAttributeName()),
-                        logstashAttribute.getAttributeValue().getValue()
-                );
-            }
-            else {
-                LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName());
-            }
-        });
-
-        return pluginSettings;
+    void mapCustomAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map<String, Object> pluginSettings) {
+        // No custom attributes to map
     }
+
+    @Override
+    Collection<String> getCustomMappedAttributeNames() {
+        return Collections.emptyList();
+    }
+
+
 }
diff --git a/.../java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java b/.../java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java
@@ -10,38 +10,29 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.Collection;
+import java.util.Arrays;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.stream.Collectors;
 
-class GrokLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper {
+class GrokLogstashPluginAttributesMapper extends AbstractLogstashPluginAttributesMapper {
     protected static final String LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME = "match";
     protected static final String LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME = "pattern_definitions";
     private static final Logger LOG = LoggerFactory.getLogger(GrokLogstashPluginAttributesMapper.class);
 
     @SuppressWarnings("unchecked")
     @Override
-    public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) {
-        final Map<String, Object> pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes());
-
+    void mapCustomAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map<String, Object> pluginSettings) {
         final List<LogstashAttribute> matchAttributes = new ArrayList<>();
         final Map<String, String> patternDefinitions = new HashMap<>();
         logstashAttributes.forEach(logstashAttribute -> {
             if (logstashAttribute.getAttributeName().equals(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME)) {
                 matchAttributes.add(logstashAttribute);
             } else if (logstashAttribute.getAttributeName().equals(LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME)) {
                 patternDefinitions.putAll((Map<String, String>) logstashAttribute.getAttributeValue().getValue());
-            } else if (logstashAttributesMappings.getMappedAttributeNames().containsKey(logstashAttribute.getAttributeName())) {
-                pluginSettings.put(
-                        logstashAttributesMappings.getMappedAttributeNames().get(logstashAttribute.getAttributeName()),
-                        logstashAttribute.getAttributeValue().getValue()
-                );
-            }
-            else {
-                LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName());
             }
         });
 
@@ -59,8 +50,11 @@ public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashA
                     patternDefinitions
             );
         }
+    }
 
-        return pluginSettings;
+    @Override
+    Collection<String> getCustomMappedAttributeNames() {
+        return Arrays.asList(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME);
     }
 
     @SuppressWarnings("unchecked")