diff --git a/build.gradle b/build.gradle index ab34ff88d3..1ce0378408 100644 --- a/build.gradle +++ b/build.gradle @@ -45,6 +45,7 @@ subprojects { sourceCompatibility = '1.8' spotless { java { + targetExclude 'build/generated-src/antlr/**' // TODO: enrich format rules removeUnusedImports() } diff --git a/config/checkstyle/checkstyle-suppressions.xml b/config/checkstyle/checkstyle-suppressions.xml new file mode 100644 index 0000000000..28bea50b3e --- /dev/null +++ b/config/checkstyle/checkstyle-suppressions.xml @@ -0,0 +1,9 @@ + + + + + + + \ No newline at end of file diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml index aa0ad55f3f..dda71724f4 100644 --- a/config/checkstyle/checkstyle.xml +++ b/config/checkstyle/checkstyle.xml @@ -6,6 +6,10 @@ + + + + diff --git a/data-prepper-logstash-configuration/build.gradle b/data-prepper-logstash-configuration/build.gradle new file mode 100644 index 0000000000..4189d3ffba --- /dev/null +++ b/data-prepper-logstash-configuration/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'java' + id 'antlr' + id 'idea' +} + +repositories { + mavenCentral() +} + +dependencies { + antlr "org.antlr:antlr4:4.9.2" + testImplementation "org.hamcrest:hamcrest:2.2" + testImplementation "org.mockito:mockito-inline:${versionMap.mockito}" + testImplementation platform("org.junit:junit-bom:${versionMap.junitJupiter}") +} + +generateGrammarSource { + maxHeapSize = "128m" + arguments += ['-listener', '-visitor'] + outputDirectory = new File("build/generated-src/antlr/main/org/opensearch/dataprepper/logstash/".toString()) +} +compileJava.dependsOn generateGrammarSource diff --git a/data-prepper-logstash-configuration/src/main/antlr/Logstash.g4 b/data-prepper-logstash-configuration/src/main/antlr/Logstash.g4 new file mode 100644 index 0000000000..5d7ca79271 --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/antlr/Logstash.g4 @@ -0,0 +1,141 @@ +/* +* ANTLR grammar file for parsing Logstash configurations +*/ +grammar Logstash; + +@header { + package org.opensearch.dataprepper.logstash; +} +/* +* Parser Rules +*/ +config: filler plugin_section filler (filler plugin_section)* filler; + +filler: (COMMENT | WS | NEWLINE)*; + +plugin_section: plugin_type filler '{' + filler (branch_or_plugin filler)* + '}'; + +plugin_type: ('input' | 'filter' | 'output'); + +branch_or_plugin: branch | plugin; + +plugin: + name filler '{' + filler + attributes + filler + '}'; + +attributes:( attribute (filler attribute)*)?; + +attribute: name filler '=>' filler value; + +name: BAREWORD | STRING; + +value: plugin | BAREWORD | STRING | NUMBER | array | hash; + +branch: r_if (filler else_if)* (filler r_else)?; + +r_if: 'if' filler condition filler '{' filler (branch_or_plugin filler)* '}'; + +else_if: 'else' filler 'if' filler condition filler '{' filler ( branch_or_plugin filler)* '}'; + +r_else: 'else' filler '{' filler (branch_or_plugin filler)* '}'; + +condition: expression (filler boolean_operator filler expression)*; + +expression: + ( + ('(' filler condition filler ')') + | negative_expression + | in_expression + | not_in_expression + | compare_expression + | regexp_expression + | rvalue + ); + +array: + '[' + filler + ( + value (filler ',' filler value)* + )? + filler + ']'; + +hash: + '{' + filler + hashentries? + filler + '}'; + +hashentries: hashentry (WS hashentry)*; + +hashentry: hashname filler '=>' filler value; + +hashname: BAREWORD | STRING | NUMBER; + +boolean_operator: ('and' | 'or' | 'xor' | 'nand'); + +negative_expression: + ( + ('!' filler '(' filler condition filler ')') + | ('!' filler selector) + ); + +in_expression: rvalue filler in_operator filler rvalue; + +not_in_expression: rvalue filler not_in_operator filler rvalue; + +rvalue: STRING | NUMBER | selector | array | method_call | regexp; + +regexp: '/' ('\\' | ~'/' .)*? '/'; + +selector: selector_element+; + +compare_expression: rvalue filler compare_operator filler rvalue; + +regexp_expression: rvalue filler regexp_operator filler (STRING | regexp); + +selector_element: '[' ~( '[' | ']' | ',' )+ ']'; + +in_operator: 'in'; + +not_in_operator: 'not' filler 'in'; + +method_call: + BAREWORD filler '(' filler + ( + rvalue ( filler ',' filler rvalue )* + )? + filler ')'; + +compare_operator: ('==' | '!=' | '<=' | '>=' | '<' | '>') ; + +regexp_operator: ('=~' | '!~'); + +/* +* Lexer Rules +*/ + +COMMENT: (WS? '#' ~('\r'|'\n')*)+; + +NEWLINE: ('\r'? '\n' | '\r')+ -> skip; + +WS: ( NEWLINE | ' ' | '\t')+; + +fragment DIGIT: [0-9]; + +NUMBER: '-'? DIGIT+ ('.' DIGIT*)?; + +BAREWORD: [a-zA-Z0-9_]+; + +STRING: DOUBLE_QUOTED_STRING | SINGLE_QUOTED_STRING; + +fragment DOUBLE_QUOTED_STRING : ('"' ( '\\"' | . )*? '"'); + +fragment SINGLE_QUOTED_STRING : ('\'' ('\'' | . )*? '\''); \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashConfigurationException.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashConfigurationException.java new file mode 100644 index 0000000000..51f9bc5822 --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashConfigurationException.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.logstash.exception; + +/** + * Exception for Logstash configuration converter + * + * @since 1.2 + */ +public class LogstashConfigurationException extends RuntimeException { + + public LogstashConfigurationException(String errorMessage) { + super(errorMessage); + } +} diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashGrammarException.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashGrammarException.java new file mode 100644 index 0000000000..e135d53741 --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashGrammarException.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.logstash.exception; + +/** + * Exception thrown when ANTLR fails to parse the Logstash configuration + * + * @since 1.2 + */ +public class LogstashGrammarException extends LogstashParsingException { + + public LogstashGrammarException(String errorMessage) { + super(errorMessage); + } +} \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashParsingException.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashParsingException.java new file mode 100644 index 0000000000..cd34b6c3ce --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/exception/LogstashParsingException.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.logstash.exception; + +/** + * Exception thrown when ANTLR visitor is unable to convert Logstash configuration into Logstash model objects + * + * @since 1.2 + */ +public class LogstashParsingException extends LogstashConfigurationException { + + public LogstashParsingException(String errorMessage) { + super(errorMessage); + } +} \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/model/LogstashPluginType.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/model/LogstashPluginType.java index c519c4e8f9..1a6bd55c83 100644 --- a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/model/LogstashPluginType.java +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/model/LogstashPluginType.java @@ -1,12 +1,35 @@ package org.opensearch.dataprepper.logstash.model; +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + /** * Types of plugins in Logstash configuration * * @since 1.2 */ public enum LogstashPluginType { - INPUT, - FILTER, - OUTPUT + INPUT("input"), + FILTER("filter"), + OUTPUT("output"); + + private final String value; + + private static final Map VALUES_MAP = Arrays.stream(LogstashPluginType.values()) + .collect(Collectors.toMap(LogstashPluginType::toString, Function.identity())); + + LogstashPluginType(final String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + + public static LogstashPluginType getByValue(final String value) { + return VALUES_MAP.get(value.toLowerCase()); + } } diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitor.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitor.java new file mode 100644 index 0000000000..9cc6f4c8dc --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitor.java @@ -0,0 +1,148 @@ +package org.opensearch.dataprepper.logstash.parser; + +import org.antlr.v4.runtime.RuleContext; +import org.opensearch.dataprepper.logstash.exception.LogstashParsingException; +import org.opensearch.dataprepper.logstash.LogstashBaseVisitor; +import org.opensearch.dataprepper.logstash.LogstashParser; +import org.opensearch.dataprepper.logstash.model.LogstashConfiguration; +import org.opensearch.dataprepper.logstash.model.LogstashPlugin; +import org.opensearch.dataprepper.logstash.model.LogstashAttribute; +import org.opensearch.dataprepper.logstash.model.LogstashAttributeValue; +import org.opensearch.dataprepper.logstash.model.LogstashPluginType; +import org.opensearch.dataprepper.logstash.model.LogstashValueType; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.stream.Collectors; + +/** + * Class to populate Logstash configuration model objects using ANTLR library classes and generated code. + * + * @since 1.2 + */ +@SuppressWarnings("rawtypes") +class LogstashVisitor extends LogstashBaseVisitor { + + @Override + public Object visitConfig(final LogstashParser.ConfigContext configContext) { + final Map> pluginSections = new LinkedHashMap<>(); + + configContext.plugin_section().forEach(pluginSection -> { + final String pluginType = pluginSection.plugin_type().getText(); + if (!Arrays.asList(new String[]{"input", "filter", "output"}).contains(pluginType)) + throw new LogstashParsingException("only input, filter and output plugin sections are supported."); + final LogstashPluginType logstashPluginType = LogstashPluginType.getByValue(pluginType); + final List logstashPluginList = (List) visitPlugin_section(pluginSection); + pluginSections.put(logstashPluginType, logstashPluginList); + }); + + return LogstashConfiguration.builder() + .pluginSections(pluginSections) + .build(); + } + + @Override + public Object visitPlugin_section(final LogstashParser.Plugin_sectionContext pluginSectionContext) { + + return pluginSectionContext.branch_or_plugin().stream() + .map(this::visitBranch_or_plugin) + .collect(Collectors.toList()); + } + + @Override + public Object visitBranch_or_plugin(final LogstashParser.Branch_or_pluginContext branchOrPluginContext) { + + if (branchOrPluginContext.getChild(0) instanceof LogstashParser.PluginContext) + return visitPlugin(branchOrPluginContext.plugin()); + else + throw new LogstashParsingException("conditionals are not supported"); + } + + @Override + public Object visitPlugin(final LogstashParser.PluginContext pluginContext) { + final String pluginName = pluginContext.name().getText(); + final List logstashAttributeList = pluginContext.attributes().attribute().stream() + .map(attribute -> (LogstashAttribute) visitAttribute(attribute)) + .collect(Collectors.toList()); + + return LogstashPlugin.builder() + .pluginName(pluginName) + .attributes(logstashAttributeList) + .build(); + } + + @Override + public Object visitAttribute(final LogstashParser.AttributeContext attributeContext) { + LogstashValueType logstashValueType = null; + Object value = null; + + if (attributeContext.value().getChild(0) instanceof LogstashParser.ArrayContext) { + logstashValueType = LogstashValueType.ARRAY; + value = visitArray(attributeContext.value().array()); + } + else if (attributeContext.value().getChild(0) instanceof LogstashParser.HashContext) { + logstashValueType = LogstashValueType.HASH; + value = visitHash(attributeContext.value().hash()); + } + else if (attributeContext.value().getChild(0) instanceof LogstashParser.PluginContext) { + throw new LogstashParsingException("plugins are not supported in an attribute"); + } + else if (attributeContext.value().NUMBER() != null && attributeContext.value().getText().equals(attributeContext.value().NUMBER().toString())) { + logstashValueType = LogstashValueType.NUMBER; + value = Double.valueOf(attributeContext.value().getText()); + } + else if (attributeContext.value().BAREWORD() != null && attributeContext.value().getText().equals(attributeContext.value().BAREWORD().toString())) { + logstashValueType = LogstashValueType.BAREWORD; + value = attributeContext.value().getText(); + } + else if (attributeContext.value().STRING() != null && attributeContext.value().getText().equals(attributeContext.value().STRING().toString())) { + logstashValueType = LogstashValueType.STRING; + value = attributeContext.value().getText().replaceAll("^\"|\"$|^'|'$", ""); + } + + final LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder() + .attributeValueType(logstashValueType) + .value(value) + .build(); + + return LogstashAttribute.builder() + .attributeName(attributeContext.name().getText()) + .attributeValue(logstashAttributeValue) + .build(); + } + + @Override + public Object visitArray(final LogstashParser.ArrayContext arrayContext) { + return arrayContext.value().stream() + .map(RuleContext::getText) + .collect(Collectors.toList()); + } + + @Override + public Object visitHash(final LogstashParser.HashContext hashContext) { + return visitHashentries(hashContext.hashentries()); + } + + @Override + public Object visitHashentries(final LogstashParser.HashentriesContext hashentriesContext) { + final Map hashEntries = new LinkedHashMap<>(); + + hashentriesContext.hashentry().forEach(hashentryContext -> { + final String key = hashentryContext.hashname().getText(); + final Object value = visitHashentry(hashentryContext); + hashEntries.put(key, value); + }); + + return hashEntries; + } + + @Override + public Object visitHashentry(final LogstashParser.HashentryContext hashentryContext) { + if (hashentryContext.value().getChild(0) instanceof LogstashParser.ArrayContext) + return visitArray(hashentryContext.value().array()); + + return hashentryContext.value().getText(); + } +} \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitorTest.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitorTest.java new file mode 100644 index 0000000000..a8e6011f13 --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitorTest.java @@ -0,0 +1,476 @@ +package org.opensearch.dataprepper.logstash.parser; + +import org.antlr.v4.runtime.tree.TerminalNodeImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.mock; + +import org.mockito.Mock; +import org.mockito.Mockito; +import org.opensearch.dataprepper.logstash.LogstashParser; +import org.opensearch.dataprepper.logstash.exception.LogstashParsingException; +import org.opensearch.dataprepper.logstash.model.LogstashConfiguration; +import org.opensearch.dataprepper.logstash.model.LogstashPlugin; +import org.opensearch.dataprepper.logstash.model.LogstashPluginType; +import org.opensearch.dataprepper.logstash.model.LogstashAttribute; + +import java.util.Collections; +import java.util.Arrays; +import java.util.List; +import java.util.LinkedList; + +class LogstashVisitorTest { + + private static LogstashVisitor logstashVisitor; + + @BeforeEach + void createObjectUnderTest() { + logstashVisitor = spy(new LogstashVisitor()); + } + + @Mock + private LogstashParser.ConfigContext configContextMock = mock(LogstashParser.ConfigContext.class); + @Mock + private LogstashParser.Plugin_sectionContext pluginSectionMock = mock(LogstashParser.Plugin_sectionContext.class); + @Mock + private LogstashParser.Plugin_typeContext pluginTypeContextMock = mock(LogstashParser.Plugin_typeContext.class); + @Mock + private LogstashParser.Branch_or_pluginContext branchOrPluginContextMock = mock(LogstashParser.Branch_or_pluginContext.class); + @Mock + private LogstashParser.BranchContext branchContextMock = mock(LogstashParser.BranchContext.class); + @Mock + private LogstashParser.PluginContext pluginContextMock = mock(LogstashParser.PluginContext.class); + @Mock + private LogstashParser.NameContext nameContextMock = mock(LogstashParser.NameContext.class); + @Mock + private LogstashParser.AttributesContext attributesContextMock = mock(LogstashParser.AttributesContext.class); + @Mock + private LogstashParser.AttributeContext attributeContextMock = mock(LogstashParser.AttributeContext.class); + @Mock + private LogstashParser.ValueContext valueContextMock = mock(LogstashParser.ValueContext.class); + @Mock + private LogstashParser.ArrayContext arrayContextMock = mock(LogstashParser.ArrayContext.class); + @Mock + private LogstashParser.ValueContext arrayValueContextMock1 = mock(LogstashParser.ValueContext.class); + @Mock + private LogstashParser.ValueContext arrayValueContextMock2 = mock(LogstashParser.ValueContext.class); + @Mock + private LogstashParser.HashContext hashContextMock = mock(LogstashParser.HashContext.class); + @Mock + private LogstashParser.HashentriesContext hashentriesContextMock = mock(LogstashParser.HashentriesContext.class); + @Mock + private TerminalNodeImpl terminalNodeMock = mock(TerminalNodeImpl.class); + @Mock + private LogstashParser.HashentryContext hashentryContextMock = mock(LogstashParser.HashentryContext.class); + @Mock + private LogstashParser.HashnameContext hashnameContextMock = mock(LogstashParser.HashnameContext.class); + + + @Test + void visit_config_return_logstash_configuration_object_test() { + given(configContextMock.plugin_section()).willReturn(Collections.singletonList(pluginSectionMock)); + given(pluginSectionMock.plugin_type()).willReturn(pluginTypeContextMock); + given(pluginTypeContextMock.getText()).willReturn("input"); + given(pluginSectionMock.branch_or_plugin()).willReturn(Collections.singletonList(branchOrPluginContextMock)); + + Mockito.doReturn(TestDataProvider.pluginWithOneArrayContextAttributeData()).when(logstashVisitor).visitBranch_or_plugin(branchOrPluginContextMock); + + LogstashConfiguration actualLogstashConfiguration = (LogstashConfiguration) logstashVisitor.visitConfig(configContextMock); + LogstashConfiguration expectedLogstashConfiguration = TestDataProvider.configData(); + + assertThat(actualLogstashConfiguration.getPluginSection(LogstashPluginType.INPUT).size(), + equalTo(expectedLogstashConfiguration.getPluginSection(LogstashPluginType.INPUT).size())); + Mockito.verify(logstashVisitor, Mockito.times(1)).visitPlugin_section(pluginSectionMock); + } + + @Test + void visit_plugin_section_test() { + given(pluginSectionMock.plugin_type()).willReturn(pluginTypeContextMock); + given(pluginTypeContextMock.getText()).willReturn("input"); + + List branch_or_pluginContextList = new LinkedList<>( + Collections.singletonList(branchOrPluginContextMock) + ); + given(pluginSectionMock.branch_or_plugin()).willReturn(branch_or_pluginContextList); + given(branchOrPluginContextMock.getChild(0)).willReturn(pluginContextMock); + given(branchOrPluginContextMock.plugin()).willReturn(pluginContextMock); + + Mockito.doReturn(TestDataProvider.pluginWithOneArrayContextAttributeData()).when(logstashVisitor).visitPlugin(pluginContextMock); + + List actualPluginSections = (List) logstashVisitor.visitPlugin_section(pluginSectionMock); + List expectedPluginSections = TestDataProvider.pluginSectionData(); + + System.out.println(actualPluginSections); + System.out.println(expectedPluginSections); + + assertThat(actualPluginSections.size(), equalTo(expectedPluginSections.size())); + for (int i = 0; i < expectedPluginSections.size(); i++) + assertThat(actualPluginSections.get(i).getPluginName(), equalTo(expectedPluginSections.get(i).getPluginName())); + } + + @Test + void visit_config_with_unsupported_section_name_throws_logstash_parsing_exception_test() { + given(configContextMock.plugin_section()).willReturn(Collections.singletonList(pluginSectionMock)); + given(pluginSectionMock.plugin_type()).willReturn(pluginTypeContextMock); + given(pluginTypeContextMock.getText()).willReturn("inputt"); + + Exception exception = assertThrows(LogstashParsingException.class, () -> + logstashVisitor.visitConfig(configContextMock)); + + String expectedMessage = "only input, filter and output plugin sections are supported."; + String actualMessage = exception.getMessage(); + + assertThat(expectedMessage, equalTo(actualMessage)); + } + + @Test + void visit_branch_or_plugin_with_branch_throws_logstash_parsing_exception_test() { + given(branchOrPluginContextMock.getChild(0)).willReturn(branchContextMock); + + Exception exception = assertThrows(LogstashParsingException.class, () -> + logstashVisitor.visitBranch_or_plugin(branchOrPluginContextMock)); + + String expectedMessage = "conditionals are not supported"; + String actualMessage = exception.getMessage(); + + assertThat(expectedMessage, equalTo(actualMessage)); + } + + @Test + void visit_branch_or_plugin_returns_logstash_plugin_test() { + given(branchOrPluginContextMock.getChild(0)).willReturn(pluginContextMock); + given(branchOrPluginContextMock.plugin()).willReturn(pluginContextMock); + given(pluginContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(pluginContextMock.attributes()).willReturn(attributesContextMock); + + List attributeContexts = new LinkedList<>(Collections.singletonList(attributeContextMock)); + given(pluginContextMock.attributes()).willReturn(attributesContextMock); + given(attributesContextMock.attribute()).willReturn(attributeContexts); + + Mockito.doReturn(TestDataProvider.attributeWithArrayTypeValueData()).when(logstashVisitor).visitAttribute(attributeContextMock); + + LogstashPlugin actualLogstashPlugin = (LogstashPlugin) logstashVisitor.visitBranch_or_plugin(branchOrPluginContextMock); + LogstashPlugin expectedLogstashPlugin = TestDataProvider.pluginWithOneArrayContextAttributeData(); + + assertThat(actualLogstashPlugin.getPluginName(), equalTo(expectedLogstashPlugin.getPluginName())); + for (int i = 0; i < actualLogstashPlugin.getAttributes().size(); i++) { + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeName(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeName())); + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeValue().getValue(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeValue().getValue())); + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeValue().getAttributeValueType())); + } + + } + + @Test + void visit_plugin_with_no_attribute_test() { + given(pluginContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(pluginContextMock.attributes()).willReturn(attributesContextMock); + + Mockito.doReturn(TestDataProvider.arrayData()).when(logstashVisitor).visitArray(arrayContextMock); + + LogstashPlugin actualLogstashPlugin = (LogstashPlugin) logstashVisitor.visitPlugin(pluginContextMock); + LogstashPlugin expectedLogstashPlugin = TestDataProvider.pluginWithNoAttributeData(); + + assertThat(actualLogstashPlugin.getPluginName(), equalTo(expectedLogstashPlugin.getPluginName())); + assertThat(actualLogstashPlugin.getAttributes().size(), equalTo(expectedLogstashPlugin.getAttributes().size())); + } + + @Test + void visit_plugin_with_one_array_context_attribute_test() { + given(pluginContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(pluginContextMock.attributes()).willReturn(attributesContextMock); + + List attributeContexts = new LinkedList<>(Collections.singletonList(attributeContextMock)); + + given(attributesContextMock.attribute()).willReturn(attributeContexts); + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getChild(0)).willReturn(arrayContextMock); + given(valueContextMock.array()).willReturn(arrayContextMock); + + Mockito.doReturn(TestDataProvider.arrayData()).when(logstashVisitor).visitArray(arrayContextMock); + + LogstashPlugin actualLogstashPlugin = (LogstashPlugin) logstashVisitor.visitPlugin(pluginContextMock); + LogstashPlugin expectedLogstashPlugin = TestDataProvider.pluginWithOneArrayContextAttributeData(); + + assertThat(actualLogstashPlugin.getPluginName(), equalTo(expectedLogstashPlugin.getPluginName())); + assertThat(actualLogstashPlugin.getAttributes().size(), equalTo(expectedLogstashPlugin.getAttributes().size())); + + for (int i = 0; i < actualLogstashPlugin.getAttributes().size(); i++) { + + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeName(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeName())); + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeValue().getValue(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeValue().getValue())); + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeValue().getAttributeValueType())); + } + } + + @Test + void visit_plugin_with_more_than_one_array_context_attribute_test() { + given(pluginContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(pluginContextMock.attributes()).willReturn(attributesContextMock); + + List attributeContexts = new LinkedList<>(); + attributeContexts.add(attributeContextMock); + attributeContexts.add(attributeContextMock); + given(attributesContextMock.attribute()).willReturn(attributeContexts); + + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getChild(0)).willReturn(arrayContextMock); + given(valueContextMock.array()).willReturn(arrayContextMock); + + Mockito.doReturn(TestDataProvider.arrayData()).when(logstashVisitor).visitArray(arrayContextMock); + + LogstashPlugin actualLogstashPlugin = (LogstashPlugin) logstashVisitor.visitPlugin(pluginContextMock); + LogstashPlugin expectedLogstashPlugin = TestDataProvider.pluginWithMorThanOneArrayContextAttributeData(); + + assertThat(actualLogstashPlugin.getPluginName(), equalTo(expectedLogstashPlugin.getPluginName())); + assertThat(actualLogstashPlugin.getAttributes().size(), equalTo(expectedLogstashPlugin.getAttributes().size())); + + for (int i = 0; i < actualLogstashPlugin.getAttributes().size(); i++) { + + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeName(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeName())); + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeValue().getValue(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeValue().getValue())); + assertThat(actualLogstashPlugin.getAttributes().get(i).getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashPlugin.getAttributes().get(i).getAttributeValue().getAttributeValueType())); + } + } + + @Test + void visit_attribute_with_value_type_array_returns_correct_logstash_attribute_test() { + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getChild(0)).willReturn(arrayContextMock); + given(valueContextMock.array()).willReturn(arrayContextMock); + + List valueContextList = new LinkedList<>(Arrays.asList(arrayValueContextMock1, arrayValueContextMock2)); + given(arrayContextMock.value()).willReturn(valueContextList); + given(arrayValueContextMock1.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(arrayValueContextMock2.getText()).willReturn(TestDataProvider.RANDOM_STRING_2); + + LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock); + LogstashAttribute expectedLogstashAttribute = TestDataProvider.attributeWithArrayTypeValueData(); + + assertThat(actualLogstashAttribute.getAttributeName(), + equalTo(expectedLogstashAttribute.getAttributeName())); + assertThat(actualLogstashAttribute.getAttributeValue().getValue(), + equalTo(expectedLogstashAttribute.getAttributeValue().getValue())); + assertThat(actualLogstashAttribute.getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashAttribute.getAttributeValue().getAttributeValueType())); + } + + @Test + void visit_attribute_with_value_type_hash_returns_correct_logstash_attribute_test() { + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getChild(0)).willReturn(hashContextMock); + given(valueContextMock.hash()).willReturn(hashContextMock); + given(hashContextMock.hashentries()).willReturn(hashentriesContextMock); + + Mockito.doReturn(TestDataProvider.hashEntriesStringData()).when(logstashVisitor).visitHashentries(hashentriesContextMock); + + LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock); + LogstashAttribute expectedLogstashAttribute = TestDataProvider.attributeWithHashTypeValueData(); + + assertThat(actualLogstashAttribute.getAttributeName(), + equalTo(expectedLogstashAttribute.getAttributeName())); + assertThat(actualLogstashAttribute.getAttributeValue().getValue(), + equalTo(expectedLogstashAttribute.getAttributeValue().getValue())); + assertThat(actualLogstashAttribute.getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashAttribute.getAttributeValue().getAttributeValueType())); + } + + @Test + void visit_attribute_with_value_type_plugin_throws_logstash_parsing_exception_test() { + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getChild(0)).willReturn(pluginContextMock); + + Exception exception = assertThrows(LogstashParsingException.class, () -> logstashVisitor.visitAttribute(attributeContextMock)); + + String expectedMessage = "plugins are not supported in an attribute"; + String actualMessage = exception.getMessage(); + + assertThat(expectedMessage, equalTo(actualMessage)); + } + + @Test + void visit_attribute_with_value_type_number_returns_correct_logstash_attribute_test() { + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.NUMBER()).willReturn(terminalNodeMock); + given(valueContextMock.getText()).willReturn(TestDataProvider.RANDOM_VALUE); + given(valueContextMock.NUMBER().toString()).willReturn(TestDataProvider.RANDOM_VALUE); + + LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock); + LogstashAttribute expectedLogstashAttribute = TestDataProvider.attributeWithNumberTypeValueData(); + + assertThat(actualLogstashAttribute.getAttributeName(), + equalTo(expectedLogstashAttribute.getAttributeName())); + assertThat(actualLogstashAttribute.getAttributeValue().getValue(), + equalTo(expectedLogstashAttribute.getAttributeValue().getValue())); + assertThat(actualLogstashAttribute.getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashAttribute.getAttributeValue().getAttributeValueType())); + } + + @Test + void visit_attribute_with_value_type_bare_word_returns_correct_logstash_attribute_test() { + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.BAREWORD()).willReturn(terminalNodeMock); + given(valueContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_2); + given(valueContextMock.BAREWORD().toString()).willReturn(TestDataProvider.RANDOM_STRING_2); + + LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock); + LogstashAttribute expectedLogstashAttribute = TestDataProvider.attributeWithBareWordTypeValueData(); + + + assertThat(actualLogstashAttribute.getAttributeName(), + equalTo(expectedLogstashAttribute.getAttributeName())); + assertThat(actualLogstashAttribute.getAttributeValue().getValue(), + equalTo(expectedLogstashAttribute.getAttributeValue().getValue())); + assertThat(actualLogstashAttribute.getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashAttribute.getAttributeValue().getAttributeValueType())); + + } + + @Test + void visit_attribute_with_value_type_string_returns_correct_logstash_attribute_test() { + given(attributeContextMock.name()).willReturn(nameContextMock); + given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(attributeContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.STRING()).willReturn(terminalNodeMock); + given(valueContextMock.getText()).willReturn("\"" + TestDataProvider.RANDOM_STRING_2 + "\""); + given(valueContextMock.STRING().toString()).willReturn("\"" + TestDataProvider.RANDOM_STRING_2 + "\""); + + LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock); + LogstashAttribute expectedLogstashAttribute = TestDataProvider.attributeWithStringTypeValueData(); + + assertThat(actualLogstashAttribute.getAttributeName(), + equalTo(expectedLogstashAttribute.getAttributeName())); + assertThat(actualLogstashAttribute.getAttributeValue().getValue(), + equalTo(expectedLogstashAttribute.getAttributeValue().getValue())); + assertThat(actualLogstashAttribute.getAttributeValue().getAttributeValueType(), + equalTo(expectedLogstashAttribute.getAttributeValue().getAttributeValueType())); + + } + + @Test + void visit_array_with_empty_array_returns_empty_list_test() { + given(arrayContextMock.value()).willReturn(Collections.emptyList()); + Object actualList = logstashVisitor.visitArray(arrayContextMock); + + assertThat(actualList, equalTo(Collections.emptyList())); + } + + @Test + void visit_array_with_singleton_array_returns_list_df_size_one_test() { + given(valueContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(arrayContextMock.value()).willReturn(Collections.singletonList(valueContextMock)); + + Object actualList = logstashVisitor.visitArray(arrayContextMock); + + assertThat(actualList, equalTo(Collections.singletonList(TestDataProvider.RANDOM_STRING_1))); + } + + @Test + void visit_array_with_array_of_size_more_than_one_returns_list_of_size_more_than_one_test() { + List valueContextList = new LinkedList<>(Arrays.asList(arrayValueContextMock1, arrayValueContextMock2)); + + given(arrayContextMock.value()).willReturn(valueContextList); + given(arrayValueContextMock1.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(arrayValueContextMock2.getText()).willReturn(TestDataProvider.RANDOM_STRING_2); + + Object actualList = logstashVisitor.visitArray(arrayContextMock); + + assertThat(actualList, equalTo(TestDataProvider.arrayData())); + } + + @Test + void visit_hash_entries_with_string_value_returns_map_of_hash_entries_test() { + List hashentryContextList = new LinkedList<>( + Collections.singletonList(hashentryContextMock)); + + given(hashentriesContextMock.hashentry()).willReturn(hashentryContextList); + given(hashentryContextMock.hashname()).willReturn(hashnameContextMock); + given(hashnameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(hashentryContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_2); + + Object actualMap = logstashVisitor.visitHashentries(hashentriesContextMock); + + assertThat(actualMap, equalTo(TestDataProvider.hashEntriesStringData())); + } + + @Test + void visit_hash_entries_with_array_value_returns_map_of_hash_entries_test() { + List hashentryContextList = new LinkedList<>( + Collections.singletonList(hashentryContextMock)); + + given(hashentriesContextMock.hashentry()).willReturn(hashentryContextList); + given(hashentryContextMock.hashname()).willReturn(hashnameContextMock); + given(hashnameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(hashentryContextMock.value()).willReturn(valueContextMock); + given(hashentryContextMock.value().getChild(0)).willReturn(arrayContextMock); + given(valueContextMock.array()).willReturn(arrayContextMock); + + Mockito.doReturn(TestDataProvider.arrayData()).when(logstashVisitor).visitArray(arrayContextMock); + + Object actualMap = logstashVisitor.visitHashentries(hashentriesContextMock); + Object expectedMap = TestDataProvider.hashEntriesArrayData(); + + + assertThat(actualMap, equalTo(expectedMap)); + } + + @Test + void visit_hash_entry_with_value_type_string_returns_string_object_test() { + given(hashentryContextMock.value()).willReturn(valueContextMock); + given(valueContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_2); + + Object actualValue = logstashVisitor.visitHashentry(hashentryContextMock); + + assertThat(actualValue, equalTo(TestDataProvider.hashEntryStringData())); + } + + @Test + void visit_hash_entry_with_array_value_type_returns_list_test() { + final List linkedListValuesMock = new LinkedList<>( + Arrays.asList(arrayValueContextMock1, arrayValueContextMock2) + ); + + given(hashentryContextMock.value()).willReturn(valueContextMock); + given(hashentryContextMock.value().getChild(0)).willReturn(arrayContextMock); + given(valueContextMock.array()).willReturn(arrayContextMock); + given(arrayContextMock.value()).willReturn(linkedListValuesMock); + given(arrayValueContextMock1.getText()).willReturn(TestDataProvider.RANDOM_STRING_1); + given(arrayValueContextMock2.getText()).willReturn(TestDataProvider.RANDOM_STRING_2); + + Object actualList = logstashVisitor.visitHashentry(hashentryContextMock); + + assertThat(actualList, equalTo(TestDataProvider.hashEntryArrayData())); + } + +} \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/TestDataProvider.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/TestDataProvider.java new file mode 100644 index 0000000000..4354865a85 --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/parser/TestDataProvider.java @@ -0,0 +1,132 @@ +package org.opensearch.dataprepper.logstash.parser; + +import org.opensearch.dataprepper.logstash.model.LogstashConfiguration; +import org.opensearch.dataprepper.logstash.model.LogstashPluginType; +import org.opensearch.dataprepper.logstash.model.LogstashPlugin; +import org.opensearch.dataprepper.logstash.model.LogstashAttribute; +import org.opensearch.dataprepper.logstash.model.LogstashAttributeValue; +import org.opensearch.dataprepper.logstash.model.LogstashValueType; + +import java.util.UUID; +import java.util.Random; +import java.util.Collections; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.LinkedList; +import java.util.HashMap; +import java.util.LinkedHashMap; + + +class TestDataProvider { + static final String RANDOM_STRING_1 = UUID.randomUUID().toString(); + static final String RANDOM_STRING_2 = UUID.randomUUID().toString(); + static final String RANDOM_VALUE = String.valueOf(new Random().nextInt(1000)); + + public static LogstashConfiguration configData() { + List pluginContextList = new LinkedList<>(Collections.singletonList(pluginWithOneArrayContextAttributeData())); + Map> pluginSections = new LinkedHashMap<>(); + pluginSections.put(LogstashPluginType.INPUT, pluginContextList); + + return LogstashConfiguration.builder().pluginSections(pluginSections).build(); + } + + public static List pluginSectionData() { + + List pluginContextList = new LinkedList<>(Collections.singletonList(pluginWithOneArrayContextAttributeData())); + Map> pluginSections = new LinkedHashMap<>(); + pluginSections.put(LogstashPluginType.INPUT, pluginContextList); + + return pluginContextList; + } + + public static LogstashPlugin pluginWithNoAttributeData() { + List logstashAttributeList = new LinkedList<>(); + + return LogstashPlugin.builder().pluginName(TestDataProvider.RANDOM_STRING_1).attributes(logstashAttributeList).build(); + } + + public static LogstashPlugin pluginWithOneArrayContextAttributeData() { + List logstashAttributeList = new LinkedList<>(); + logstashAttributeList.add(TestDataProvider.attributeWithArrayTypeValueData()); + + return LogstashPlugin.builder().pluginName(TestDataProvider.RANDOM_STRING_1).attributes(logstashAttributeList).build(); + } + + public static LogstashPlugin pluginWithMorThanOneArrayContextAttributeData() { + List logstashAttributeList = new LinkedList<>(); + logstashAttributeList.add(TestDataProvider.attributeWithArrayTypeValueData()); + logstashAttributeList.add(TestDataProvider.attributeWithArrayTypeValueData()); + + return LogstashPlugin.builder().pluginName(TestDataProvider.RANDOM_STRING_1).attributes(logstashAttributeList).build(); + } + + public static LogstashAttribute attributeWithArrayTypeValueData() { + List values = new LinkedList<>(Arrays.asList(TestDataProvider.RANDOM_STRING_1, TestDataProvider.RANDOM_STRING_2)); + + LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder(). + attributeValueType(LogstashValueType.ARRAY).value(values).build(); + + return LogstashAttribute.builder() + .attributeName(TestDataProvider.RANDOM_STRING_1).attributeValue(logstashAttributeValue).build(); + } + + public static LogstashAttribute attributeWithHashTypeValueData() { + Map values = new LinkedHashMap<>(); + values.put(TestDataProvider.RANDOM_STRING_1, TestDataProvider.RANDOM_STRING_2); + + LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder(). + attributeValueType(LogstashValueType.HASH).value(values).build(); + + return LogstashAttribute.builder() + .attributeName(TestDataProvider.RANDOM_STRING_1).attributeValue(logstashAttributeValue).build(); + } + + public static LogstashAttribute attributeWithNumberTypeValueData() { + LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder(). + attributeValueType(LogstashValueType.NUMBER).value(Double.valueOf(TestDataProvider.RANDOM_VALUE)).build(); + return LogstashAttribute.builder() + .attributeName(TestDataProvider.RANDOM_STRING_1).attributeValue(logstashAttributeValue).build(); + } + + public static LogstashAttribute attributeWithBareWordTypeValueData() { + LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder(). + attributeValueType(LogstashValueType.BAREWORD).value(RANDOM_STRING_2).build(); + return LogstashAttribute.builder() + .attributeName(RANDOM_STRING_1).attributeValue(logstashAttributeValue).build(); + } + + public static LogstashAttribute attributeWithStringTypeValueData() { + LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder(). + attributeValueType(LogstashValueType.STRING).value(RANDOM_STRING_2).build(); + return LogstashAttribute.builder() + .attributeName(RANDOM_STRING_1).attributeValue(logstashAttributeValue).build(); + } + + public static List arrayData() { + return Arrays.asList(RANDOM_STRING_1, RANDOM_STRING_2); + } + + public static Map hashEntriesArrayData() { + Map hashentry = new HashMap<>(); + hashentry.put(RANDOM_STRING_1, new LinkedList<>(Arrays.asList(RANDOM_STRING_1, RANDOM_STRING_2))); + + return hashentry; + } + + public static Map hashEntriesStringData() { + Map hashentry = new HashMap<>(); + hashentry.put(RANDOM_STRING_1, RANDOM_STRING_2); + + return hashentry; + } + + public static List hashEntryArrayData() { + return Arrays.asList(RANDOM_STRING_1, RANDOM_STRING_2); + } + + public static String hashEntryStringData() { + return RANDOM_STRING_2; + } + +} \ No newline at end of file