Updated Logstash attributes mapper to use template pattern (#617)

* Updated Logstash attributes mapper to use template pattern Signed-off-by: Asif Sohail Mohammed <[email protected]> Co-authored-by: Asif Sohail Mohammed <[email protected]>
opensearch-project · Nov 17, 2021 · 5f5cb41 · 5f5cb41
1 parent 169365c
commit 5f5cb41
Show file tree

Hide file tree

Showing 5 changed files with 154 additions and 48 deletions.
diff --git a/...a/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java b/...a/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java
@@ -0,0 +1,75 @@
+package org.opensearch.dataprepper.logstash.mapping;
+
+import org.opensearch.dataprepper.logstash.model.LogstashAttribute;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.List;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.HashSet;
+
+/**
+ * An abstract class which is responsible for mapping basic attributes
+ *
+ * @since 1.2
+ */
+public abstract class AbstractLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper {
+    private static final Logger LOG = LoggerFactory.getLogger(AbstractLogstashPluginAttributesMapper.class);
+
+    @Override
+    public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) {
+
+        Objects.requireNonNull(logstashAttributes);
+        Objects.requireNonNull(logstashAttributesMappings);
+        Objects.requireNonNull(logstashAttributesMappings.getMappedAttributeNames());
+        Objects.requireNonNull(logstashAttributesMappings.getAdditionalAttributes());
+
+        final Map<String, Object> pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes());
+        final Map<String, String> mappedAttributeNames = logstashAttributesMappings.getMappedAttributeNames();
+
+        Collection<String> customMappedAttributeNames = getCustomMappedAttributeNames();
+
+        logstashAttributes
+                .stream()
+                .filter(logstashAttribute -> !customMappedAttributeNames.contains(logstashAttribute.getAttributeName()))
+                .forEach(logstashAttribute -> {
+            if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) {
+                pluginSettings.put(
+                        mappedAttributeNames.get(logstashAttribute.getAttributeName()),
+                        logstashAttribute.getAttributeValue().getValue()
+                );
+            }
+            else {
+                LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName());
+            }
+        });
+
+        if (!customMappedAttributeNames.isEmpty()) {
+            mapCustomAttributes(logstashAttributes, logstashAttributesMappings, pluginSettings);
+        }
+
+        return pluginSettings;
+    }
+
+    /**
+     * Map custom logstashAttributes from a Logstash plugin.
+     *
+     * @param logstashAttributes All the Logstash logstashAttributes for the plugin
+     * @param logstashAttributesMappings The mappings for this Logstash plugin
+     * @param pluginSettings A map of Data Prepper basic plugin settings.
+     *
+     * @since 1.2
+     */
+    protected abstract void mapCustomAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map<String, Object> pluginSettings);
+
+    /**
+     * Get custom logstashAttributes names from a Logstash plugin.
+     *
+     * @return A set of custom attributes
+     * @since 1.2
+     */
+    protected abstract HashSet<String> getCustomMappedAttributeNames();
+}
diff --git a/...va/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java b/...va/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java
@@ -6,40 +6,21 @@
 package org.opensearch.dataprepper.logstash.mapping;
 
 import org.opensearch.dataprepper.logstash.model.LogstashAttribute;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.HashSet;
 
-class DefaultLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper {
-    private static final Logger LOG = LoggerFactory.getLogger(DefaultLogstashPluginAttributesMapper.class);
+class DefaultLogstashPluginAttributesMapper extends AbstractLogstashPluginAttributesMapper {
 
     @Override
-    public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) {
-
-        Objects.requireNonNull(logstashAttributes);
-        Objects.requireNonNull(logstashAttributesMappings);
-        Objects.requireNonNull(logstashAttributesMappings.getMappedAttributeNames());
-        Objects.requireNonNull(logstashAttributesMappings.getAdditionalAttributes());
-
-        final Map<String, Object> pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes());
-
-        final Map<String, String> mappedAttributeNames = logstashAttributesMappings.getMappedAttributeNames();
-        logstashAttributes.forEach(logstashAttribute -> {
-            if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) {
-                pluginSettings.put(
-                        mappedAttributeNames.get(logstashAttribute.getAttributeName()),
-                        logstashAttribute.getAttributeValue().getValue()
-                );
-            }
-            else {
-                LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName());
-            }
-        });
+    protected void mapCustomAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map<String, Object> pluginSettings) {
+        // No custom attributes to map
+    }
 
-        return pluginSettings;
+    @Override
+    protected HashSet<String> getCustomMappedAttributeNames() {
+        return new HashSet<>();
     }
+
 }
diff --git a/.../java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java b/.../java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java
@@ -10,38 +10,29 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.stream.Collectors;
+import java.util.Arrays;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.HashMap;
+import java.util.HashSet;
 
-class GrokLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper {
+class GrokLogstashPluginAttributesMapper extends AbstractLogstashPluginAttributesMapper {
     protected static final String LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME = "match";
     protected static final String LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME = "pattern_definitions";
     private static final Logger LOG = LoggerFactory.getLogger(GrokLogstashPluginAttributesMapper.class);
 
     @SuppressWarnings("unchecked")
     @Override
-    public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) {
-        final Map<String, Object> pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes());
-
+    protected void mapCustomAttributes(List<LogstashAttribute> logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map<String, Object> pluginSettings) {
         final List<LogstashAttribute> matchAttributes = new ArrayList<>();
         final Map<String, String> patternDefinitions = new HashMap<>();
         logstashAttributes.forEach(logstashAttribute -> {
             if (logstashAttribute.getAttributeName().equals(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME)) {
                 matchAttributes.add(logstashAttribute);
             } else if (logstashAttribute.getAttributeName().equals(LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME)) {
                 patternDefinitions.putAll((Map<String, String>) logstashAttribute.getAttributeValue().getValue());
-            } else if (logstashAttributesMappings.getMappedAttributeNames().containsKey(logstashAttribute.getAttributeName())) {
-                pluginSettings.put(
-                        logstashAttributesMappings.getMappedAttributeNames().get(logstashAttribute.getAttributeName()),
-                        logstashAttribute.getAttributeValue().getValue()
-                );
-            }
-            else {
-                LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName());
             }
         });
 
@@ -59,8 +50,11 @@ public Map<String, Object> mapAttributes(final List<LogstashAttribute> logstashA
                     patternDefinitions
             );
         }
+    }
 
-        return pluginSettings;
+    @Override
+    protected HashSet<String> getCustomMappedAttributeNames() {
+        return new HashSet<>(Arrays.asList(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME));
     }
 
     @SuppressWarnings("unchecked")

diff --git a/...g/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapperTest.java b/...g/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapperTest.java
@@ -0,0 +1,59 @@
+package org.opensearch.dataprepper.logstash.mapping;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.opensearch.dataprepper.logstash.model.LogstashAttribute;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import java.util.HashSet;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.never;
+
+class AbstractLogstashPluginAttributesMapperTest {
+
+    private List<LogstashAttribute> logstashAttributes;
+    private LogstashAttributesMappings mappings;
+    private Map<String, Object> pluginSettings;
+
+    @BeforeEach
+    void setUp() {
+        final LogstashAttribute logstashAttribute = mock(LogstashAttribute.class);
+
+        logstashAttributes = Collections.singletonList(logstashAttribute);
+        mappings = mock(LogstashAttributesMappings.class);
+        pluginSettings = new LinkedHashMap<>(mappings.getAdditionalAttributes());
+    }
+
+    @Test
+    void mapAttributes_with_no_custom_attributes_does_not_invoke_mapCustomAttributes_Test() {
+        AbstractLogstashPluginAttributesMapper abstractLogstashPluginAttributesMapper = Mockito
+                .spy(AbstractLogstashPluginAttributesMapper.class);
+
+        when(abstractLogstashPluginAttributesMapper.getCustomMappedAttributeNames()).thenReturn(new HashSet<>());
+
+        Map<String, Object> pluginSettings = abstractLogstashPluginAttributesMapper.mapAttributes(logstashAttributes, mappings);
+        verify(abstractLogstashPluginAttributesMapper, never()).mapCustomAttributes(
+                logstashAttributes, mappings, pluginSettings);
+    }
+
+
+    @Test
+    void mapAttributes_with_custom_attributes_invokes_mapCustomAttributes_Test() {
+        AbstractLogstashPluginAttributesMapper abstractLogstashPluginAttributesMapper = Mockito
+                .spy(AbstractLogstashPluginAttributesMapper.class);
+
+        when(abstractLogstashPluginAttributesMapper.getCustomMappedAttributeNames())
+                .thenReturn(new HashSet<>(Collections.singletonList("customAttribute")));
+
+        Map<String, Object> pluginSettings = abstractLogstashPluginAttributesMapper.mapAttributes(logstashAttributes, mappings);
+        verify(abstractLogstashPluginAttributesMapper).mapCustomAttributes(logstashAttributes, mappings, pluginSettings);
+    }
+
+}
diff --git a/...t/java/org/opensearch/dataprepper/logstash/parser/ModelConvertingLogstashVisitorTest.java b/...t/java/org/opensearch/dataprepper/logstash/parser/ModelConvertingLogstashVisitorTest.java
@@ -111,9 +111,6 @@ void visit_plugin_section_test() {
         List<LogstashPlugin> actualPluginSections = (List<LogstashPlugin>) logstashVisitor.visitPlugin_section(pluginSectionMock);
         List<LogstashPlugin> expectedPluginSections = TestDataProvider.pluginSectionData();
 
-        System.out.println(actualPluginSections);
-        System.out.println(expectedPluginSections);
-
         assertThat(actualPluginSections.size(), equalTo(expectedPluginSections.size()));
         for (int i = 0; i < expectedPluginSections.size(); i++)
             assertThat(actualPluginSections.get(i).getPluginName(), equalTo(expectedPluginSections.get(i).getPluginName()));