From a429acc024567b244f9991cd08ba588c38960960 Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Thu, 16 Dec 2021 12:48:15 -0600 Subject: [PATCH 1/2] Added support for negating boolean logstash attribute value Signed-off-by: Asif Sohail Mohammed --- ...bstractLogstashPluginAttributesMapper.java | 27 ++++++++++++------- .../logstash/mapping/opensearch.mapping.yaml | 7 +++++ ...ultLogstashPluginAttributesMapperTest.java | 19 ++++++++++++- 3 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java index 5c1c9d0550..555c30de1f 100644 --- a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java @@ -36,15 +36,24 @@ public Map mapAttributes(final List logstashA .stream() .filter(logstashAttribute -> !customMappedAttributeNames.contains(logstashAttribute.getAttributeName())) .forEach(logstashAttribute -> { - if (mappedAttributeNames.containsKey(logstashAttribute.getAttributeName())) { - pluginSettings.put( - mappedAttributeNames.get(logstashAttribute.getAttributeName()), - logstashAttribute.getAttributeValue().getValue() - ); - } - else { - LOG.warn("Attribute name {} is not found in mapping file.", logstashAttribute.getAttributeName()); - } + final String logstashAttributeName = logstashAttribute.getAttributeName(); + + if (mappedAttributeNames.containsKey(logstashAttributeName)) { + if (mappedAttributeNames.get(logstashAttributeName).startsWith("!")) { + pluginSettings.put( + mappedAttributeNames.get(logstashAttributeName).substring(1), + !(Boolean) logstashAttribute.getAttributeValue().getValue() + ); + } + else { + pluginSettings.put( + mappedAttributeNames.get(logstashAttributeName), + logstashAttribute.getAttributeValue().getValue()); + } + } + else { + LOG.warn("Attribute name {} is not found in mapping file.", logstashAttributeName); + } }); if (!customMappedAttributeNames.isEmpty()) { diff --git a/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml b/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml new file mode 100644 index 0000000000..c4e170efb3 --- /dev/null +++ b/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml @@ -0,0 +1,7 @@ +pluginName: opensearch +mappedAttributeNames: + hosts: hosts + user: username + password: password + index: index + ssl_certificate_verification: !insecure \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java index 516f6161cc..9fbd974b9d 100644 --- a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java @@ -28,6 +28,7 @@ class DefaultLogstashPluginAttributesMapperTest { private String logstashAttributeName; private String value; private List logstashAttributes; + private LogstashAttributeValue logstashAttributeValue; private LogstashAttributesMappings mappings; @BeforeEach @@ -35,7 +36,7 @@ void setUp() { value = UUID.randomUUID().toString(); logstashAttributeName = UUID.randomUUID().toString(); final LogstashAttribute logstashAttribute = mock(LogstashAttribute.class); - final LogstashAttributeValue logstashAttributeValue = mock(LogstashAttributeValue.class); + logstashAttributeValue = mock(LogstashAttributeValue.class); when(logstashAttributeValue.getValue()).thenReturn(value); when(logstashAttribute.getAttributeName()).thenReturn(logstashAttributeName); when(logstashAttribute.getAttributeValue()).thenReturn(logstashAttributeValue); @@ -114,4 +115,20 @@ void mapAttributes_sets_additional_attributes_to_those_values() { assertThat(actualPluginSettings, hasKey(additionalAttributeName)); assertThat(actualPluginSettings.get(additionalAttributeName), equalTo(additionalAttributeValue)); } + + @Test + void mapAttributes_with_negation_expression_negates_boolean_value() { + final String dataPrepperAttribute = "!".concat(UUID.randomUUID().toString()); + boolean value = true; + + when(logstashAttributeValue.getValue()).thenReturn(value); + when(mappings.getMappedAttributeNames()).thenReturn(Collections.singletonMap(logstashAttributeName, dataPrepperAttribute)); + + final Map actualPluginSettings = + createObjectUnderTest().mapAttributes(logstashAttributes, mappings); + + assertThat(actualPluginSettings, notNullValue()); + assertThat(actualPluginSettings.size(), equalTo(1)); + assertThat(actualPluginSettings.get(dataPrepperAttribute.substring(1)), equalTo(!value)); + } } \ No newline at end of file From 84eaa84a05fbc9b01658d53ddd042cd1ca4eb0ae Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Tue, 21 Dec 2021 14:04:38 -0600 Subject: [PATCH 2/2] Updated support for negating boolean logstash attribute value Signed-off-by: Asif Sohail Mohammed --- ...AbstractLogstashPluginAttributesMapper.java | 10 ++++------ .../logstash/mapping/opensearch.mapping.yaml | 2 +- ...aultLogstashPluginAttributesMapperTest.java | 15 ++++++++------- ...-logstash-opensearch-insecure.expected.yaml | 18 ++++++++++++++++++ .../log-ingest-to-logstash-opensearch.conf | 18 ++++++++++++++++++ ...ingest-to-logstash-opensearch.expected.yaml | 18 ++++++++++++++++++ ...ngest-to-logtstash-opensearch-insecure.conf | 18 ++++++++++++++++++ 7 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch-insecure.expected.yaml create mode 100644 data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.conf create mode 100644 data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.expected.yaml create mode 100644 data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logtstash-opensearch-insecure.conf diff --git a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java index 555c30de1f..fae7e26050 100644 --- a/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java +++ b/data-prepper-logstash-configuration/src/main/java/org/opensearch/dataprepper/logstash/mapping/AbstractLogstashPluginAttributesMapper.java @@ -37,18 +37,16 @@ public Map mapAttributes(final List logstashA .filter(logstashAttribute -> !customMappedAttributeNames.contains(logstashAttribute.getAttributeName())) .forEach(logstashAttribute -> { final String logstashAttributeName = logstashAttribute.getAttributeName(); + final String dataPrepperAttributeName = mappedAttributeNames.get(logstashAttributeName); if (mappedAttributeNames.containsKey(logstashAttributeName)) { - if (mappedAttributeNames.get(logstashAttributeName).startsWith("!")) { + if (dataPrepperAttributeName.startsWith("!") && logstashAttribute.getAttributeValue().getValue() instanceof Boolean) { pluginSettings.put( - mappedAttributeNames.get(logstashAttributeName).substring(1), - !(Boolean) logstashAttribute.getAttributeValue().getValue() + dataPrepperAttributeName.substring(1), !(Boolean) logstashAttribute.getAttributeValue().getValue() ); } else { - pluginSettings.put( - mappedAttributeNames.get(logstashAttributeName), - logstashAttribute.getAttributeValue().getValue()); + pluginSettings.put(dataPrepperAttributeName, logstashAttribute.getAttributeValue().getValue()); } } else { diff --git a/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml b/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml index c4e170efb3..5f0fec03c0 100644 --- a/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml +++ b/data-prepper-logstash-configuration/src/main/resources/org/opensearch/dataprepper/logstash/mapping/opensearch.mapping.yaml @@ -4,4 +4,4 @@ mappedAttributeNames: user: username password: password index: index - ssl_certificate_verification: !insecure \ No newline at end of file + ssl_certificate_verification: "!insecure" \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java index 9fbd974b9d..bf0619dd39 100644 --- a/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java +++ b/data-prepper-logstash-configuration/src/test/java/org/opensearch/dataprepper/logstash/mapping/DefaultLogstashPluginAttributesMapperTest.java @@ -7,6 +7,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.opensearch.dataprepper.logstash.model.LogstashAttribute; import org.opensearch.dataprepper.logstash.model.LogstashAttributeValue; @@ -116,19 +118,18 @@ void mapAttributes_sets_additional_attributes_to_those_values() { assertThat(actualPluginSettings.get(additionalAttributeName), equalTo(additionalAttributeValue)); } - @Test - void mapAttributes_with_negation_expression_negates_boolean_value() { + @ParameterizedTest + @ValueSource(booleans = { true, false } ) + void mapAttributes_with_negation_expression_negates_boolean_value(boolean inputValue) { final String dataPrepperAttribute = "!".concat(UUID.randomUUID().toString()); - boolean value = true; - when(logstashAttributeValue.getValue()).thenReturn(value); + when(logstashAttributeValue.getValue()).thenReturn(inputValue); when(mappings.getMappedAttributeNames()).thenReturn(Collections.singletonMap(logstashAttributeName, dataPrepperAttribute)); - final Map actualPluginSettings = - createObjectUnderTest().mapAttributes(logstashAttributes, mappings); + final Map actualPluginSettings = createObjectUnderTest().mapAttributes(logstashAttributes, mappings); assertThat(actualPluginSettings, notNullValue()); assertThat(actualPluginSettings.size(), equalTo(1)); - assertThat(actualPluginSettings.get(dataPrepperAttribute.substring(1)), equalTo(!value)); + assertThat(actualPluginSettings.get(dataPrepperAttribute.substring(1)), equalTo(!inputValue)); } } \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch-insecure.expected.yaml b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch-insecure.expected.yaml new file mode 100644 index 0000000000..4050cab6bb --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch-insecure.expected.yaml @@ -0,0 +1,18 @@ +logstash-converted-pipeline: + source: + http: + max_connection_count: 500 + request_timeout: 10000 + processor: + - grok: + match: + log: + - "%{COMBINEDAPACHELOG}" + sink: + - opensearch: + hosts: + - "fakedomain.us-east-1.es.amazonaws.com" + username: "myuser" + password: "mypassword" + index: "my-index" + insecure: true diff --git a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.conf b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.conf new file mode 100644 index 0000000000..acad6f6cd3 --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.conf @@ -0,0 +1,18 @@ +input { + http { + } +} +filter { + grok { + match => {"log" => "%{COMBINEDAPACHELOG}"} + } +} +output { + opensearch { + hosts => ["fakedomain.us-east-1.es.amazonaws.com"] + user => myuser + password => mypassword + index => "my-index" + ssl_certificate_verification => true + } +} \ No newline at end of file diff --git a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.expected.yaml b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.expected.yaml new file mode 100644 index 0000000000..618dddeaf0 --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logstash-opensearch.expected.yaml @@ -0,0 +1,18 @@ +logstash-converted-pipeline: + source: + http: + max_connection_count: 500 + request_timeout: 10000 + processor: + - grok: + match: + log: + - "%{COMBINEDAPACHELOG}" + sink: + - opensearch: + hosts: + - "fakedomain.us-east-1.es.amazonaws.com" + username: "myuser" + password: "mypassword" + index: "my-index" + insecure: false diff --git a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logtstash-opensearch-insecure.conf b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logtstash-opensearch-insecure.conf new file mode 100644 index 0000000000..4c08eb5da0 --- /dev/null +++ b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-logtstash-opensearch-insecure.conf @@ -0,0 +1,18 @@ +input { + http { + } +} +filter { + grok { + match => {"log" => "%{COMBINEDAPACHELOG}"} + } +} +output { + opensearch { + hosts => ["fakedomain.us-east-1.es.amazonaws.com"] + user => myuser + password => mypassword + index => "my-index" + ssl_certificate_verification => false + } +} \ No newline at end of file