From 5f5cb41f1e26a7d5f01af42d94a056b9c2c600dc Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Wed, 17 Nov 2021 16:27:44 -0600 Subject: [PATCH] Updated Logstash attributes mapper to use template pattern (#617) * Updated Logstash attributes mapper to use template pattern Signed-off-by: Asif Sohail Mohammed Co-authored-by: Asif Sohail Mohammed --- ...bstractLogstashPluginAttributesMapper.java | 75 +++++++++++++++++++ ...DefaultLogstashPluginAttributesMapper.java | 37 +++------ .../GrokLogstashPluginAttributesMapper.java | 28 +++---- ...actLogstashPluginAttributesMapperTest.java | 59 +++++++++++++++ .../ModelConvertingLogstashVisitorTest.java | 3 - 5 files changed, 154 insertions(+), 48 deletions(-) create mode 100644 data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java create mode 100644 data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapperTest.java diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java new file mode 100644 index 0000000000..5c1c9d0550 --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java @@ -0,0 +1,75 @@ +package org.opensearch.dataprepper.logstash.mapping; + +import org.opensearch.dataprepper.logstash.model.LogstashAttribute; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Objects; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.HashSet; + +/** + * An abstract class which is responsible for mapping basic attributes + * + * @since 1.2 + */ +public abstract class AbstractLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper { + private static final Logger LOG = LoggerFactory.getLogger(AbstractLogstashPluginAttributesMapper.class); + + @Override + public Map mapAttributes(final List logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) { + + Objects.requireNonNull(logstashAttributes); + Objects.requireNonNull(logstashAttributesMappings); + Objects.requireNonNull(logstashAttributesMappings.getMappedAttributeNames()); + Objects.requireNonNull(logstashAttributesMappings.getAdditionalAttributes()); + + final Map pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes()); + final Map mappedAttributeNames = logstashAttributesMappings.getMappedAttributeNames(); + + Collection customMappedAttributeNames = getCustomMappedAttributeNames(); + + logstashAttributes + .stream() + .filter(logstashAttribute -> !customMappedAttributeNames.contains(logstashAttribute.getAttributeName())) + .forEach(logstashAttribute -> { + if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) { + pluginSettings.put( + mappedAttributeNames.get(logstashAttribute.getAttributeName()), + logstashAttribute.getAttributeValue().getValue() + ); + } + else { + LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName()); + } + }); + + if (!customMappedAttributeNames.isEmpty()) { + mapCustomAttributes(logstashAttributes, logstashAttributesMappings, pluginSettings); + } + + return pluginSettings; + } + + /** + * Map custom logstashAttributes from a Logstash plugin. + * + * @param logstashAttributes All the Logstash logstashAttributes for the plugin + * @param logstashAttributesMappings The mappings for this Logstash plugin + * @param pluginSettings A map of Data Prepper basic plugin settings. + * + * @since 1.2 + */ + protected abstract void mapCustomAttributes(List logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map pluginSettings); + + /** + * Get custom logstashAttributes names from a Logstash plugin. + * + * @return A set of custom attributes + * @since 1.2 + */ + protected abstract HashSet getCustomMappedAttributeNames(); +} diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java index 3b74a3bbce..457dc10735 100644 --- a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapper.java @@ -6,40 +6,21 @@ package org.opensearch.dataprepper.logstash.mapping; import org.opensearch.dataprepper.logstash.model.LogstashAttribute; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.HashSet; -class DefaultLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper { - private static final Logger LOG = LoggerFactory.getLogger(DefaultLogstashPluginAttributesMapper.class); +class DefaultLogstashPluginAttributesMapper extends AbstractLogstashPluginAttributesMapper { @Override - public Map mapAttributes(final List logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) { - - Objects.requireNonNull(logstashAttributes); - Objects.requireNonNull(logstashAttributesMappings); - Objects.requireNonNull(logstashAttributesMappings.getMappedAttributeNames()); - Objects.requireNonNull(logstashAttributesMappings.getAdditionalAttributes()); - - final Map pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes()); - - final Map mappedAttributeNames = logstashAttributesMappings.getMappedAttributeNames(); - logstashAttributes.forEach(logstashAttribute -> { - if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) { - pluginSettings.put( - mappedAttributeNames.get(logstashAttribute.getAttributeName()), - logstashAttribute.getAttributeValue().getValue() - ); - } - else { - LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName()); - } - }); + protected void mapCustomAttributes(List logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map pluginSettings) { + // No custom attributes to map + } - return pluginSettings; + @Override + protected HashSet getCustomMappedAttributeNames() { + return new HashSet<>(); } + } diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java index 7ad694dcf5..5e4cd1c03c 100644 --- a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/GrokLogstashPluginAttributesMapper.java @@ -10,23 +10,22 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; +import java.util.stream.Collectors; +import java.util.Arrays; import java.util.List; +import java.util.ArrayList; import java.util.Map; -import java.util.stream.Collectors; +import java.util.HashMap; +import java.util.HashSet; -class GrokLogstashPluginAttributesMapper implements LogstashPluginAttributesMapper { +class GrokLogstashPluginAttributesMapper extends AbstractLogstashPluginAttributesMapper { protected static final String LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME = "match"; protected static final String LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME = "pattern_definitions"; private static final Logger LOG = LoggerFactory.getLogger(GrokLogstashPluginAttributesMapper.class); @SuppressWarnings("unchecked") @Override - public Map mapAttributes(final List logstashAttributes, final LogstashAttributesMappings logstashAttributesMappings) { - final Map pluginSettings = new LinkedHashMap<>(logstashAttributesMappings.getAdditionalAttributes()); - + protected void mapCustomAttributes(List logstashAttributes, LogstashAttributesMappings logstashAttributesMappings, Map pluginSettings) { final List matchAttributes = new ArrayList<>(); final Map patternDefinitions = new HashMap<>(); logstashAttributes.forEach(logstashAttribute -> { @@ -34,14 +33,6 @@ public Map mapAttributes(final List logstashA matchAttributes.add(logstashAttribute); } else if (logstashAttribute.getAttributeName().equals(LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME)) { patternDefinitions.putAll((Map) logstashAttribute.getAttributeValue().getValue()); - } else if (logstashAttributesMappings.getMappedAttributeNames().containsKey(logstashAttribute.getAttributeName())) { - pluginSettings.put( - logstashAttributesMappings.getMappedAttributeNames().get(logstashAttribute.getAttributeName()), - logstashAttribute.getAttributeValue().getValue() - ); - } - else { - LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName()); } }); @@ -59,8 +50,11 @@ public Map mapAttributes(final List logstashA patternDefinitions ); } + } - return pluginSettings; + @Override + protected HashSet getCustomMappedAttributeNames() { + return new HashSet<>(Arrays.asList(LOGSTASH_GROK_MATCH_ATTRIBUTE_NAME, LOGSTASH_GROK_PATTERN_DEFINITIONS_ATTRIBUTE_NAME)); } @SuppressWarnings("unchecked") diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapperTest.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapperTest.java new file mode 100644 index 0000000000..2e1e4baafb --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapperTest.java @@ -0,0 +1,59 @@ +package org.opensearch.dataprepper.logstash.mapping; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.opensearch.dataprepper.logstash.model.LogstashAttribute; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.HashSet; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.never; + +class AbstractLogstashPluginAttributesMapperTest { + + private List logstashAttributes; + private LogstashAttributesMappings mappings; + private Map pluginSettings; + + @BeforeEach + void setUp() { + final LogstashAttribute logstashAttribute = mock(LogstashAttribute.class); + + logstashAttributes = Collections.singletonList(logstashAttribute); + mappings = mock(LogstashAttributesMappings.class); + pluginSettings = new LinkedHashMap<>(mappings.getAdditionalAttributes()); + } + + @Test + void mapAttributes_with_no_custom_attributes_does_not_invoke_mapCustomAttributes_Test() { + AbstractLogstashPluginAttributesMapper abstractLogstashPluginAttributesMapper = Mockito + .spy(AbstractLogstashPluginAttributesMapper.class); + + when(abstractLogstashPluginAttributesMapper.getCustomMappedAttributeNames()).thenReturn(new HashSet<>()); + + Map pluginSettings = abstractLogstashPluginAttributesMapper.mapAttributes(logstashAttributes, mappings); + verify(abstractLogstashPluginAttributesMapper, never()).mapCustomAttributes( + logstashAttributes, mappings, pluginSettings); + } + + + @Test + void mapAttributes_with_custom_attributes_invokes_mapCustomAttributes_Test() { + AbstractLogstashPluginAttributesMapper abstractLogstashPluginAttributesMapper = Mockito + .spy(AbstractLogstashPluginAttributesMapper.class); + + when(abstractLogstashPluginAttributesMapper.getCustomMappedAttributeNames()) + .thenReturn(new HashSet<>(Collections.singletonList("customAttribute"))); + + Map pluginSettings = abstractLogstashPluginAttributesMapper.mapAttributes(logstashAttributes, mappings); + verify(abstractLogstashPluginAttributesMapper).mapCustomAttributes(logstashAttributes, mappings, pluginSettings); + } + +} diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/ModelConvertingLogstashVisitorTest.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/ModelConvertingLogstashVisitorTest.java index c5c5980c99..dcd9cd50a5 100644 --- a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/ModelConvertingLogstashVisitorTest.java +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/ModelConvertingLogstashVisitorTest.java @@ -111,9 +111,6 @@ void visit_plugin_section_test() { List actualPluginSections = (List) logstashVisitor.visitPlugin_section(pluginSectionMock); List expectedPluginSections = TestDataProvider.pluginSectionData(); - System.out.println(actualPluginSections); - System.out.println(expectedPluginSections); - assertThat(actualPluginSections.size(), equalTo(expectedPluginSections.size())); for (int i = 0; i < expectedPluginSections.size(); i++) assertThat(actualPluginSections.get(i).getPluginName(), equalTo(expectedPluginSections.get(i).getPluginName()));