Skip to content

Commit

Permalink
Fix issues with extra quotes in Logstash converted YAML files. (#587)
Browse files Browse the repository at this point in the history 
Fix issues with extra quotes in Logstash converted YAML files.

Signed-off-by: David Venable <[email protected]>
  • Loading branch information
@dlvenable
dlvenable authored Nov 13, 2021
1 parent fbc59b8 commit 2c7b58a
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 7 deletions.
15 changes: 10 additions & 5 deletions ...nfiguration/src/main/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public Object visitBranch_or_plugin(final LogstashParser.Branch_or_pluginContext

@Override
public Object visitPlugin(final LogstashParser.PluginContext pluginContext) {
final String pluginName = pluginContext.name().getText();
final String pluginName = normalizeText(pluginContext.name().getText());
final List<LogstashAttribute> logstashAttributeList = pluginContext.attributes().attribute().stream()
.map(attribute -> (LogstashAttribute) visitAttribute(attribute))
.filter(Objects::nonNull)
Expand Down Expand Up @@ -104,7 +104,7 @@ else if (attributeContext.value().BAREWORD() != null && attributeContext.value()
}
else if (attributeContext.value().STRING() != null && attributeContext.value().getText().equals(attributeContext.value().STRING().toString())) {
logstashValueType = LogstashValueType.STRING;
value = attributeContext.value().getText().replaceAll("^\"|\"$|^'|'$", "");
value = normalizeText(attributeContext.value().getText());
}

final LogstashAttributeValue logstashAttributeValue = LogstashAttributeValue.builder()
Expand All @@ -113,7 +113,7 @@ else if (attributeContext.value().STRING() != null && attributeContext.value().g
.build();

return LogstashAttribute.builder()
.attributeName(attributeContext.name().getText())
.attributeName(normalizeText(attributeContext.name().getText()))
.attributeValue(logstashAttributeValue)
.build();
}
Expand All @@ -122,6 +122,7 @@ else if (attributeContext.value().STRING() != null && attributeContext.value().g
public Object visitArray(final LogstashParser.ArrayContext arrayContext) {
return arrayContext.value().stream()
.map(RuleContext::getText)
.map(this::normalizeText)
.collect(Collectors.toList());
}

Expand All @@ -135,7 +136,7 @@ public Object visitHashentries(final LogstashParser.HashentriesContext hashentri
final Map<String, Object> hashEntries = new LinkedHashMap<>();

hashentriesContext.hashentry().forEach(hashentryContext -> {
final String key = hashentryContext.hashname().getText();
final String key = normalizeText(hashentryContext.hashname().getText());
final Object value = visitHashentry(hashentryContext);
hashEntries.put(key, value);
});
Expand All @@ -148,6 +149,10 @@ public Object visitHashentry(final LogstashParser.HashentryContext hashentryCont
if (hashentryContext.value().getChild(0) instanceof LogstashParser.ArrayContext)
return visitArray(hashentryContext.value().array());

return hashentryContext.value().getText();
return normalizeText(hashentryContext.value().getText());
}

private String normalizeText(final String unNormalizedLogstashText) {
return unNormalizedLogstashText.replaceAll("^\"|\"$|^'|'$", "");
}
}
72 changes: 70 additions & 2 deletions ...uration/src/test/java/org/opensearch/dataprepper/logstash/parser/LogstashVisitorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ void createObjectUnderTest() {
@Mock
private LogstashParser.HashnameContext hashnameContextMock = mock(LogstashParser.HashnameContext.class);

private static String wrapInQuotes(final String inputString) {
return "\"" + inputString + "\"";
}

@Test
void visit_config_return_logstash_configuration_object_test() {
Expand Down Expand Up @@ -276,6 +279,17 @@ void visit_plugin_with_more_than_one_array_context_attribute_test() {
}
}

@Test
void visit_plugin_with_quoted_pluginName_returns_it_unquoted() {
given(pluginContextMock.name()).willReturn(nameContextMock);
given(nameContextMock.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_1));
given(pluginContextMock.attributes()).willReturn(attributesContextMock);

final LogstashPlugin actualLogstashPlugin = (LogstashPlugin) logstashVisitor.visitPlugin(pluginContextMock);

assertThat(actualLogstashPlugin.getPluginName(), equalTo(TestDataProvider.RANDOM_STRING_1));
}

@Test
void visit_attribute_without_a_value_returns_null() {
assertThat(logstashVisitor.visitAttribute(attributeContextMock),
Expand Down Expand Up @@ -389,8 +403,8 @@ void visit_attribute_with_value_type_string_returns_correct_logstash_attribute_t
given(nameContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_1);
given(attributeContextMock.value()).willReturn(valueContextMock);
given(valueContextMock.STRING()).willReturn(terminalNodeMock);
given(valueContextMock.getText()).willReturn("\"" + TestDataProvider.RANDOM_STRING_2 + "\"");
given(valueContextMock.STRING().toString()).willReturn("\"" + TestDataProvider.RANDOM_STRING_2 + "\"");
given(valueContextMock.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_2));
given(valueContextMock.STRING().toString()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_2));

LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock);
LogstashAttribute expectedLogstashAttribute = TestDataProvider.attributeWithStringTypeValueData();
Expand All @@ -404,6 +418,21 @@ void visit_attribute_with_value_type_string_returns_correct_logstash_attribute_t

}

@Test
void visitAttribute_with_quoted_attribute_name_returns_unquoted() {
given(attributeContextMock.name()).willReturn(nameContextMock);
given(nameContextMock.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_1));
given(attributeContextMock.value()).willReturn(valueContextMock);
given(valueContextMock.STRING()).willReturn(terminalNodeMock);
given(valueContextMock.getText()).willReturn(TestDataProvider.RANDOM_STRING_2);
given(valueContextMock.STRING().toString()).willReturn(TestDataProvider.RANDOM_STRING_2);

final LogstashAttribute actualLogstashAttribute = (LogstashAttribute) logstashVisitor.visitAttribute(attributeContextMock);

assertThat(actualLogstashAttribute.getAttributeName(),
equalTo(TestDataProvider.RANDOM_STRING_1));
}

@Test
void visit_array_with_empty_array_returns_empty_list_test() {
given(arrayContextMock.value()).willReturn(Collections.emptyList());
Expand Down Expand Up @@ -435,6 +464,19 @@ void visit_array_with_array_of_size_more_than_one_returns_list_of_size_more_than
assertThat(actualList, equalTo(TestDataProvider.arrayData()));
}

@Test
void visitArray_with_with_quoted_strings_returns_them_unquoted() {
List<LogstashParser.ValueContext> valueContextList = new LinkedList<>(Arrays.asList(arrayValueContextMock1, arrayValueContextMock2));

given(arrayContextMock.value()).willReturn(valueContextList);
given(arrayValueContextMock1.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_1));
given(arrayValueContextMock2.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_2));

Object actualList = logstashVisitor.visitArray(arrayContextMock);

assertThat(actualList, equalTo(TestDataProvider.arrayData()));
}

@Test
void visit_hash_entries_with_string_value_returns_map_of_hash_entries_test() {
List<LogstashParser.HashentryContext> hashentryContextList = new LinkedList<>(
Expand All @@ -451,6 +493,22 @@ void visit_hash_entries_with_string_value_returns_map_of_hash_entries_test() {
assertThat(actualMap, equalTo(TestDataProvider.hashEntriesStringData()));
}

@Test
void visit_hash_entries_with_quoted_values_returns_unquoted() {
List<LogstashParser.HashentryContext> hashentryContextList = new LinkedList<>(
Collections.singletonList(hashentryContextMock));

given(hashentriesContextMock.hashentry()).willReturn(hashentryContextList);
given(hashentryContextMock.hashname()).willReturn(hashnameContextMock);
given(hashnameContextMock.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_1));
given(hashentryContextMock.value()).willReturn(valueContextMock);
given(valueContextMock.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_2));

Object actualMap = logstashVisitor.visitHashentries(hashentriesContextMock);

assertThat(actualMap, equalTo(TestDataProvider.hashEntriesStringData()));
}

@Test
void visit_hash_entries_with_array_value_returns_map_of_hash_entries_test() {
List<LogstashParser.HashentryContext> hashentryContextList = new LinkedList<>(
Expand Down Expand Up @@ -482,6 +540,16 @@ void visit_hash_entry_with_value_type_string_returns_string_object_test() {
assertThat(actualValue, equalTo(TestDataProvider.hashEntryStringData()));
}

@Test
void visit_hash_entry_with_quoted_text_returns_it_unquoted() {
given(hashentryContextMock.value()).willReturn(valueContextMock);
given(valueContextMock.getText()).willReturn(wrapInQuotes(TestDataProvider.RANDOM_STRING_2));

Object actualValue = logstashVisitor.visitHashentry(hashentryContextMock);

assertThat(actualValue, equalTo(TestDataProvider.hashEntryStringData()));
}

@Test
void visit_hash_entry_with_array_value_type_returns_list_test() {
final List<LogstashParser.ValueContext> linkedListValuesMock = new LinkedList<>(
Expand Down

0 comments on commit 2c7b58a

Please sign in to comment.