From c972c4bd588c24846e38e0879e42b57e70410e3b Mon Sep 17 00:00:00 2001 From: graytaylor0 Date: Tue, 21 Sep 2021 11:44:18 -0500 Subject: [PATCH 1/3] Grok Prepper Configuration and Boilerplate Signed-off-by: graytaylor0 --- .../model/configuration/PluginSetting.java | 82 ++++++ .../configuration/PluginSettingsTests.java | 240 ++++++++++++++++++ .../grok-prepper/build.gradle | 37 +++ .../plugins/prepper/grok/GrokPrepper.java | 66 +++++ .../prepper/grok/GrokPrepperConfig.java | 130 ++++++++++ .../prepper/grok/GrokPrepperConfigTests.java | 160 ++++++++++++ .../prepper/grok/GrokPrepperTests.java | 56 ++++ settings.gradle | 1 + 8 files changed, 772 insertions(+) create mode 100644 data-prepper-plugins/grok-prepper/build.gradle create mode 100644 data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java create mode 100644 data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java create mode 100644 data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java create mode 100644 data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java index 4ba6040582..8f7a84a48a 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java @@ -11,6 +11,7 @@ package com.amazon.dataprepper.model.configuration; +import java.util.List; import java.util.Map; public class PluginSetting { @@ -136,6 +137,87 @@ public String getStringOrDefault(final String attribute, final String defaultVal throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); } + /** + * Returns the value of the specified List, or {@code defaultValue} if this settings contains no value for + * the attribute. + * + * @param attribute name of the attribute + * @param defaultValue default value for the setting + * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for + * the attribute + */ + @SuppressWarnings("unchecked") + public List getStringListOrDefault(final String attribute, final List defaultValue) { + Object object = getAttributeOrDefault(attribute, defaultValue); + if (object == null) { + return null; + } else if (object instanceof List) { + ((List) object).stream().filter(o -> !(o instanceof String)).forEach(o -> { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + }); + return (List) object; + } + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + + /** + * Returns the value of the specified Map object, or {@code defaultValue} if this settings contains no value for + * the attribute. + * + * @param attribute name of the attribute + * @param defaultValue default value for the setting + * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for + * the attribute + */ + @SuppressWarnings("unchecked") + public Map getStringMapOrDefault(final String attribute, final Map defaultValue) { + Object object = getAttributeOrDefault(attribute, defaultValue); + if (object == null) { + return null; + } else if (object instanceof Map) { + ((Map) object).forEach((key, value) -> { + if (!(key instanceof String) || !(value instanceof String)) { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + }); + return (Map) object; + } + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + + /** + * Returns the value of the specified Map>, or {@code defaultValue} if this settings contains no value for + * the attribute. + * + * @param attribute name of the attribute + * @param defaultValue default value for the setting + * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for + * the attribute + */ + @SuppressWarnings("unchecked") + public Map> getStringListMapOrDefault(final String attribute, final Map> defaultValue) { + Object object = getAttributeOrDefault(attribute, defaultValue); + if (object == null) { + return null; + } else if (object instanceof Map) { + ((Map) object).forEach((key, value) -> { + if (key instanceof String) { + if (value instanceof List){ + ((List) value).stream().filter(o -> !(o instanceof String)).forEach(o -> { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + }); + } else { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + } else { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + }); + return (Map>) object; + } + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + /** * Returns the value of the specified attribute as boolean, or {@code defaultValue} if this settings contains no * value for the attribute. If the value is null, null will be returned. diff --git a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java index f0d39fa6be..8d60f24d91 100644 --- a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java +++ b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java @@ -13,9 +13,11 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; +import org.junit.Before; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; @@ -40,12 +42,55 @@ public class PluginSettingsTests { private static final long TEST_LONG_DEFAULT_VALUE = 1000L; private static final long TEST_LONG_VALUE = TEST_LONG_DEFAULT_VALUE + 1; + private static final List TEST_STRINGLIST_DEFAULT_VALUE = new ArrayList<>(); + private static final List TEST_STRINGLIST_VALUE = new ArrayList<>(); + + private static final Map TEST_STRINGMAP_DEFAULT_VALUE = new HashMap<>(); + private static final Map TEST_STRINGMAP_VALUE = new HashMap<>(); + + private static final Map> TEST_STRINGLISTMAP_DEFAULT_VALUE = new HashMap<>(); + private static final Map> TEST_STRINGLISTMAP_VALUE = new HashMap<>(); + private static final String TEST_INT_ATTRIBUTE = "int-attribute"; private static final String TEST_STRING_ATTRIBUTE = "string-attribute"; + private static final String TEST_STRINGLIST_ATTRIBUTE = "string-list"; + private static final String TEST_STRINGMAP_ATTRIBUTE = "string-map"; + private static final String TEST_STRINGLISTMAP_ATTRIBUTE = "stringlist-map"; private static final String TEST_BOOL_ATTRIBUTE = "bool-attribute"; private static final String TEST_LONG_ATTRIBUTE = "long-attribute"; private static final String NOT_PRESENT_ATTRIBUTE = "not-present"; + @Before + public void setup() { + TEST_STRINGLIST_DEFAULT_VALUE.add("default"); + + TEST_STRINGLIST_VALUE.add("value1"); + TEST_STRINGLIST_VALUE.add("value2"); + TEST_STRINGLIST_VALUE.add("value3"); + + TEST_STRINGMAP_DEFAULT_VALUE.put("default_key", "default_value"); + + TEST_STRINGMAP_VALUE.put("key1", "value1"); + TEST_STRINGMAP_VALUE.put("key2", "value2"); + TEST_STRINGMAP_VALUE.put("key3", "value3"); + + final List TEST_STRINGLISTMAP_DEFAULT_LISTVALUE = new ArrayList<>(); + TEST_STRINGLISTMAP_DEFAULT_LISTVALUE.add("default_list"); + + TEST_STRINGLISTMAP_DEFAULT_VALUE.put("default_key", TEST_STRINGLISTMAP_DEFAULT_LISTVALUE); + + final int NUM_LISTS = 3; + + for (int i = 0; i < NUM_LISTS; i++) { + final List TEST_STRINGLISTMAP_VALUE_LIST = new ArrayList<>(); + TEST_STRINGLISTMAP_VALUE_LIST.add("value_1" + String.valueOf(i)); + TEST_STRINGLISTMAP_VALUE_LIST.add("value_2" + String.valueOf(i)); + TEST_STRINGLISTMAP_VALUE_LIST.add("value_3" + String.valueOf(i)); + + TEST_STRINGLISTMAP_VALUE.put("key_" + String.valueOf(i), TEST_STRINGLISTMAP_VALUE_LIST); + } + + } @Test public void testPluginSetting() { @@ -104,6 +149,30 @@ public void testGetStringOrDefault() { is(equalTo(TEST_STRING_VALUE))); } + @Test + public void testGetStringListOrDefault() { + final Map TEST_SETTINGS = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_VALUE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS); + + assertThat(pluginSetting.getStringListOrDefault(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE), is(equalTo(TEST_STRINGLIST_VALUE))); + } + + @Test + public void testGetStringMapOrDefault() { + final Map TEST_SETTINGS = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_VALUE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS); + + assertThat(pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE), is(equalTo(TEST_STRINGMAP_VALUE))); + } + + @Test + public void testGetStringListMapOrDefault() { + final Map TEST_SETTINGS = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_VALUE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS); + + assertThat(pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE), is(equalTo(TEST_STRINGLISTMAP_VALUE))); + } + @Test public void testGetBooleanOrDefault() { final Map TEST_SETTINGS = ImmutableMap.of(TEST_BOOL_ATTRIBUTE, TEST_BOOL_VALUE); @@ -179,6 +248,45 @@ public void testGetStringOrDefault_AsNull() { assertThat(pluginSetting.getStringOrDefault(TEST_STRING_NULL_ATTRIBUTE, TEST_STRING_DEFAULT_VALUE), nullValue()); } + /** + * Request attributes are present with null values, expect nulls to be returned + */ + @Test + public void testGetStringListOrDefault_AsNull() { + final String TEST_STRINGLIST_NULL_ATTRIBUTE = "stringlist-null-attribute"; + final Map TEST_SETTINGS_AS_NULL = new HashMap<>(); + TEST_SETTINGS_AS_NULL.put(TEST_STRINGLIST_NULL_ATTRIBUTE, null); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL); + + assertThat(pluginSetting.getStringListOrDefault(TEST_STRINGLIST_NULL_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE), nullValue()); + } + + /** + * Request attributes are present with null values, expect nulls to be returned + */ + @Test + public void testGetStringMapOrDefault_AsNull() { + final String TEST_STRINGMAP_NULL_ATTRIBUTE = "stringmap-null-attribute"; + final Map TEST_SETTINGS_AS_NULL = new HashMap<>(); + TEST_SETTINGS_AS_NULL.put(TEST_STRINGMAP_NULL_ATTRIBUTE, null); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL); + + assertThat(pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_NULL_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE), nullValue()); + } + + /** + * Request attributes are present with null values, expect nulls to be returned + */ + @Test + public void testGetStringListMapOrDefault_AsNull() { + final String TEST_STRINGLISTMAP_NULL_ATTRIBUTE = "stringlistmap-null-attribute"; + final Map TEST_SETTINGS_AS_NULL = new HashMap<>(); + TEST_SETTINGS_AS_NULL.put(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, null); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL); + + assertThat(pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE), nullValue()); + } + /** * Request attributes are present with null values, expect nulls to be returned */ @@ -258,6 +366,39 @@ public void testGetStringOrDefault_NotPresent() { is(equalTo(TEST_STRING_DEFAULT_VALUE))); } + /** + * Requested attributes are not present, expect default values to be returned + */ + @Test + public void testGetStringListOrDefault_NotPresent() { + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); + + assertThat(pluginSetting.getStringListOrDefault(NOT_PRESENT_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE), + is(equalTo(TEST_STRINGLIST_DEFAULT_VALUE))); + } + + /** + * Requested attributes are not present, expect default values to be returned + */ + @Test + public void testGetStringMapOrDefault_NotPresent() { + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); + + assertThat(pluginSetting.getStringMapOrDefault(NOT_PRESENT_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE), + is(equalTo(TEST_STRINGMAP_DEFAULT_VALUE))); + } + + /** + * Requested attributes are not present, expect default values to be returned + */ + @Test + public void testGetStringListMapOrDefault_NotPresent() { + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); + + assertThat(pluginSetting.getStringListMapOrDefault(NOT_PRESENT_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE), + is(equalTo(TEST_STRINGLISTMAP_DEFAULT_VALUE))); + } + /** * Requested attributes are not present, expect default values to be returned */ @@ -297,6 +438,105 @@ public void testGetStringOrDefault_UnsupportedType() { assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringOrDefault(TEST_STRING_ATTRIBUTE, TEST_STRING_DEFAULT_VALUE)); } + @Test + public void testGetStringListOrDefault_UnsupportedType() { + final String UNSUPPORTED_TYPE = "not-stringlist"; + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListOrDefault(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE)); + } + + @Test + public void testGetStringListOrDefault_UnsupportedListType() { + final List UNSUPPORTED_TYPE = new ArrayList<>(); + UNSUPPORTED_TYPE.add(1); + UNSUPPORTED_TYPE.add(2); + UNSUPPORTED_TYPE.add(3); + + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListOrDefault(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE)); + } + + @Test + public void testGetStringMapOrDefault_UnsupportedType() { + final String UNSUPPORTED_TYPE = "not-stringmap"; + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE)); + } + + + @Test + public void testGetStringMapOrDefault_UnsupportedMapValueType() { + final Map UNSUPPORTED_TYPE = new HashMap<>(); + UNSUPPORTED_TYPE.put("key1", 1); + UNSUPPORTED_TYPE.put("key2", 2); + UNSUPPORTED_TYPE.put("key3", 3); + + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE)); + } + + @Test + public void testGetStringListMapOrDefault_UnsupportedType() { + final String UNSUPPORTED_TYPE = "not-stringmap"; + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + } + + @Test + public void testGetStringListMapOrDefault_UnsupportedMapValueType() { + final Map UNSUPPORTED_TYPE = new HashMap<>(); + UNSUPPORTED_TYPE.put("key1", "value1"); + UNSUPPORTED_TYPE.put("key2", "value2"); + UNSUPPORTED_TYPE.put("key3", "value3"); + + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + } + + @Test + public void testGetStringListMapOrDefault_UnsupportedMapKeyType() { + final Map> UNSUPPORTED_TYPE = new HashMap<>(); + final List STRING_LIST_VALUE = new ArrayList<>(); + STRING_LIST_VALUE.add("value1"); + + UNSUPPORTED_TYPE.put(1, STRING_LIST_VALUE); + UNSUPPORTED_TYPE.put(2, STRING_LIST_VALUE); + UNSUPPORTED_TYPE.put(3, STRING_LIST_VALUE); + + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + } + + @Test + public void testGetStringListMapOrDefault_UnsupportedMapValueListType() { + final Map> UNSUPPORTED_TYPE = new HashMap<>(); + final List INT_LIST_VALUE = new ArrayList<>(); + INT_LIST_VALUE.add(1); + + UNSUPPORTED_TYPE.put("value1", INT_LIST_VALUE); + UNSUPPORTED_TYPE.put("value2", INT_LIST_VALUE); + UNSUPPORTED_TYPE.put("value3", INT_LIST_VALUE); + + final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); + final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); + + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + } + @Test public void testGetBooleanOrDefault_UnsupportedType() { final Object UNSUPPORTED_TYPE = new ArrayList<>(); diff --git a/data-prepper-plugins/grok-prepper/build.gradle b/data-prepper-plugins/grok-prepper/build.gradle new file mode 100644 index 0000000000..6698656bb7 --- /dev/null +++ b/data-prepper-plugins/grok-prepper/build.gradle @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +plugins { + id 'java' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') + testImplementation project(':data-prepper-api').sourceSets.test.output + implementation "io.krakens:java-grok:0.1.9" + testImplementation "org.hamcrest:hamcrest:2.2" +} + +jacocoTestCoverageVerification { + dependsOn jacocoTestReport + violationRules { + rule { + limit { + minimum = 1.0 + } + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java new file mode 100644 index 0000000000..7a9ae627e7 --- /dev/null +++ b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package com.amazon.dataprepper.plugins.prepper.grok; + + +import com.amazon.dataprepper.model.PluginType; +import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; + +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.record.Record; +import com.amazon.dataprepper.model.prepper.AbstractPrepper; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + + +@DataPrepperPlugin(name = "grok_prepper", type = PluginType.PREPPER) +public class GrokPrepper extends AbstractPrepper, Record> { + + private static final Logger LOG = LoggerFactory.getLogger(GrokPrepper.class); + + private final GrokPrepperConfig grokPrepperConfig; + + public GrokPrepper(final PluginSetting pluginSetting) { + super(pluginSetting); + grokPrepperConfig = GrokPrepperConfig.buildConfig(pluginSetting); + } + + /** + * execute the prepper logic which could potentially modify the incoming record. The level to which the record has + * been modified depends on the implementation + * + * @param records Input records that will be modified/processed + * @return Record modified output records + */ + @Override + public Collection> doExecute(Collection> records) { + return records; + } + + @Override + public void prepareForShutdown() { + + } + + @Override + public boolean isReadyForShutdown() { + return true; + } + + @Override + public void shutdown() { + + } +} diff --git a/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java new file mode 100644 index 0000000000..463798c5eb --- /dev/null +++ b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java @@ -0,0 +1,130 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package com.amazon.dataprepper.plugins.prepper.grok; + +import com.amazon.dataprepper.model.configuration.PluginSetting; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GrokPrepperConfig { + static final String BREAK_ON_MATCH = "break_on_match"; + static final String KEEP_EMPTY_CAPTURES = "keep_empty_captures"; + static final String MATCH = "match"; + static final String NAMED_CAPTURES_ONLY = "named_captures_only"; + static final String OVERWRITE= "overwrite"; + static final String PATTERN_DEFINITIONS = "pattern_definitions"; + static final String PATTERNS_DIR = "patterns_dir"; + static final String PATTERNS_FILES_GLOB = "patterns_files_glob"; + static final String TIMEOUT_MILLIS = "timeout_millis"; + static final String TARGET = "target"; + + static final boolean DEFAULT_BREAK_ON_MATCH = true; + static final boolean DEFAULT_KEEP_EMPTY_CAPTURES = false; + static final Map> DEFAULT_MATCH = new HashMap<>(); + static final boolean DEFAULT_NAMED_CAPTURES_ONLY = true; + static final List DEFAULT_OVERWRITE = new ArrayList<>(); + static final List DEFAULT_PATTERNS_DIR = new ArrayList<>(); + static final String DEFAULT_PATTERNS_FILES_GLOB = ".*"; + static final Map DEFAULT_PATTERN_DEFINITIONS = new HashMap<>(); + static final int DEFAULT_TIMEOUT_MILLIS = 30000; + static final String DEFAULT_TARGET = null; + + private final boolean breakOnMatch; + private final boolean keepEmptyCaptures; + private final Map> match; + private final boolean namedCapturesOnly; + private final List overwrite; + private final List patternsDir; + private final String patternsFilesGlob; + private final Map patternDefinitions; + private final int timeoutMillis; + private final String target; + + private GrokPrepperConfig(final boolean breakOnMatch, + final boolean keepEmptyCaptures, + final Map> match, + final boolean namedCapturesOnly, + final List overwrite, + final List patternsDir, + final String patternsFilesGlob, + final Map patternDefinitions, + final int timeoutMillis, + final String target) { + + this.breakOnMatch = breakOnMatch; + this.keepEmptyCaptures = keepEmptyCaptures; + this.match = match; + this.namedCapturesOnly = namedCapturesOnly; + this.overwrite = overwrite; + this.patternsDir = patternsDir; + this.patternsFilesGlob = patternsFilesGlob; + this.patternDefinitions = patternDefinitions; + this.timeoutMillis = timeoutMillis; + this.target = target; + } + + public static GrokPrepperConfig buildConfig(final PluginSetting pluginSetting) { + return new GrokPrepperConfig(pluginSetting.getBooleanOrDefault(BREAK_ON_MATCH, DEFAULT_BREAK_ON_MATCH), + pluginSetting.getBooleanOrDefault(KEEP_EMPTY_CAPTURES, DEFAULT_KEEP_EMPTY_CAPTURES), + pluginSetting.getStringListMapOrDefault(MATCH, DEFAULT_MATCH), + pluginSetting.getBooleanOrDefault(NAMED_CAPTURES_ONLY, DEFAULT_NAMED_CAPTURES_ONLY), + pluginSetting.getStringListOrDefault(OVERWRITE, DEFAULT_OVERWRITE), + pluginSetting.getStringListOrDefault(PATTERNS_DIR, DEFAULT_PATTERNS_DIR), + pluginSetting.getStringOrDefault(PATTERNS_FILES_GLOB, DEFAULT_PATTERNS_FILES_GLOB), + pluginSetting.getStringMapOrDefault(PATTERN_DEFINITIONS, DEFAULT_PATTERN_DEFINITIONS), + pluginSetting.getIntegerOrDefault(TIMEOUT_MILLIS, DEFAULT_TIMEOUT_MILLIS), + pluginSetting.getStringOrDefault(TARGET, DEFAULT_TARGET)); + } + + public boolean isBreakOnMatch() { + return breakOnMatch; + } + + public boolean isKeepEmptyCaptures() { + return keepEmptyCaptures; + } + + public Map> getMatch() { + return match; + } + + public boolean isNamedCapturesOnly() { + return namedCapturesOnly; + } + + public List getOverwrite() { + return overwrite; + } + + public List getPatternsDir() { + return patternsDir; + } + + public String getPatternsFilesGlob() { + return patternsFilesGlob; + } + + public Map getPatternDefinitions() { + return patternDefinitions; + } + + public int getTimeoutMillis() { + return timeoutMillis; + } + + public String getTarget() { + return target; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java new file mode 100644 index 0000000000..3e41c811c7 --- /dev/null +++ b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package com.amazon.dataprepper.plugins.prepper.grok; + +import com.amazon.dataprepper.model.configuration.PluginSetting; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_BREAK_ON_MATCH; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_MATCH; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_KEEP_EMPTY_CAPTURES; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_NAMED_CAPTURES_ONLY; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERN_DEFINITIONS; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERNS_DIR; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERNS_FILES_GLOB; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_TARGET; +import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_OVERWRITE; + +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +public class GrokPrepperConfigTests { + private static final String PLUGIN_NAME = "grok_prepper"; + + private static final Map> TEST_MATCH = new HashMap<>(); + private static final List TEST_OVERWRITE = new ArrayList<>(); + private static final List TEST_PATTERNS_DIR = new ArrayList<>(); + private static final String TEST_PATTERNS_FILES_GLOB = ".*pat"; + private static final Map TEST_PATTERN_DEFINITIONS = new HashMap<>(); + private static final int TEST_TIMEOUT_MILLIS = 100000; + private static final String TEST_TARGET = "test_target"; + + private static final Map TEST_INVALID_MATCH = new HashMap<>(); + + @Before + public void setUp() { + final List log_patterns = new ArrayList<>(); + log_patterns.add("%{SYNTAX:SEMANTIC}"); + log_patterns.add("%{SYNTAX_2:SEMANTIC_2}"); + + TEST_MATCH.put("message", Collections.singletonList("%{COMBINEDAPACHELOG}")); + TEST_MATCH.put("log",log_patterns); + + TEST_INVALID_MATCH.put("invalid_message_key", "invalid_message_value"); + + TEST_OVERWRITE.add("message"); + TEST_OVERWRITE.add("randomKey"); + + TEST_PATTERNS_DIR.add("/patterns"); + TEST_PATTERNS_DIR.add("patterns/second"); + + TEST_PATTERN_DEFINITIONS.put("JAVAFILE", "(?:[A-Za-z0-9_.-]+java)"); + TEST_PATTERN_DEFINITIONS.put("BEAGLE", "beagle"); + } + + @Test + public void testDefault() { + final GrokPrepperConfig grokPrepperConfig = GrokPrepperConfig.buildConfig(new PluginSetting(PLUGIN_NAME, null)); + + assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(DEFAULT_BREAK_ON_MATCH)); + assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(DEFAULT_KEEP_EMPTY_CAPTURES)); + assertThat(grokPrepperConfig.getMatch(), equalTo(DEFAULT_MATCH)); + assertThat(grokPrepperConfig.getOverwrite(), equalTo(DEFAULT_OVERWRITE)); + assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(DEFAULT_PATTERN_DEFINITIONS)); + assertThat(grokPrepperConfig.getPatternsDir(), equalTo(DEFAULT_PATTERNS_DIR)); + assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(DEFAULT_PATTERNS_FILES_GLOB)); + assertThat(grokPrepperConfig.getTarget(), equalTo(DEFAULT_TARGET)); + assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(DEFAULT_NAMED_CAPTURES_ONLY)); + assertThat(grokPrepperConfig.getTimeoutMillis(), equalTo(DEFAULT_TIMEOUT_MILLIS)); + } + + @Test + public void testValidConfig() { + final PluginSetting validPluginSetting = completePluginSettingForGrokPrepper( + false, + true, + TEST_MATCH, + false, + TEST_OVERWRITE, + TEST_PATTERNS_DIR, + TEST_PATTERNS_FILES_GLOB, + TEST_PATTERN_DEFINITIONS, + TEST_TIMEOUT_MILLIS, + TEST_TARGET); + + final GrokPrepperConfig grokPrepperConfig = GrokPrepperConfig.buildConfig(validPluginSetting); + + assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(false)); + assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(true)); + assertThat(grokPrepperConfig.getMatch(), equalTo(TEST_MATCH)); + assertThat(grokPrepperConfig.getOverwrite(), equalTo(TEST_OVERWRITE)); + assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(TEST_PATTERN_DEFINITIONS)); + assertThat(grokPrepperConfig.getPatternsDir(), equalTo(TEST_PATTERNS_DIR)); + assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(TEST_PATTERNS_FILES_GLOB)); + assertThat(grokPrepperConfig.getTarget(), equalTo(TEST_TARGET)); + assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(false)); + assertThat(grokPrepperConfig.getTimeoutMillis(), equalTo(TEST_TIMEOUT_MILLIS)); + } + + @Test + public void testInvalidConfig() { + final PluginSetting invalidPluginSetting = completePluginSettingForGrokPrepper( + false, + true, + TEST_MATCH, + false, + TEST_OVERWRITE, + TEST_PATTERNS_DIR, + TEST_PATTERNS_FILES_GLOB, + TEST_PATTERN_DEFINITIONS, + TEST_TIMEOUT_MILLIS, + TEST_TARGET); + + invalidPluginSetting.getSettings().put(GrokPrepperConfig.MATCH, TEST_INVALID_MATCH); + + assertThrows(IllegalArgumentException.class, () -> GrokPrepperConfig.buildConfig(invalidPluginSetting)); + } + + private PluginSetting completePluginSettingForGrokPrepper(final boolean breakOnMatch, + final boolean keepEmptyCaptures, + final Map> match, + final boolean namedCapturesOnly, + final List overwrite, + final List patternsDir, + final String patternsFilesGlob, + final Map patternDefinitions, + final int timeoutMillis, + final String target) { + final Map settings = new HashMap<>(); + settings.put(GrokPrepperConfig.BREAK_ON_MATCH, breakOnMatch); + settings.put(GrokPrepperConfig.NAMED_CAPTURES_ONLY, namedCapturesOnly); + settings.put(GrokPrepperConfig.MATCH, match); + settings.put(GrokPrepperConfig.KEEP_EMPTY_CAPTURES, keepEmptyCaptures); + settings.put(GrokPrepperConfig.OVERWRITE, overwrite); + settings.put(GrokPrepperConfig.PATTERNS_DIR, patternsDir); + settings.put(GrokPrepperConfig.PATTERN_DEFINITIONS, patternDefinitions); + settings.put(GrokPrepperConfig.PATTERNS_FILES_GLOB, patternsFilesGlob); + settings.put(GrokPrepperConfig.TIMEOUT_MILLIS, timeoutMillis); + settings.put(GrokPrepperConfig.TARGET, target); + + return new PluginSetting(PLUGIN_NAME, settings); + } +} diff --git a/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java new file mode 100644 index 0000000000..acb5b65463 --- /dev/null +++ b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package com.amazon.dataprepper.plugins.prepper.grok; + +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.record.Record; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.equalTo; + +import java.util.Collections; +import java.util.List; + +public class GrokPrepperTests { + + PluginSetting pluginSetting; + public GrokPrepper grokPrepper; + + @Before + public void setup() { + pluginSetting = new PluginSetting( + "grok_prepper", + null + ); + pluginSetting.setPipelineName("grokPipeline"); + grokPrepper = new GrokPrepper(pluginSetting); + } + + @After + public void tearDown() { + grokPrepper.shutdown(); + } + + @Test + public void testGrokPrepper() { + String testData = "{\"message\": \"127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] BEF25A72965 \\\"GET /apache_pb.gif HTTP/1.0\\\" 200 2326\"}"; + Record record = new Record<>(testData); + + List> grokkedRecords = (List>) grokPrepper.doExecute(Collections.singletonList(record)); + + assertThat(grokkedRecords.size(), equalTo(1)); + assertThat(grokkedRecords.get(0), equalTo(record)); + } +} diff --git a/settings.gradle b/settings.gradle index c7642a44fc..68c9865852 100644 --- a/settings.gradle +++ b/settings.gradle @@ -35,4 +35,5 @@ include 'data-prepper-plugins:otel-trace-group-prepper' include 'data-prepper-plugins:otel-trace-source' include 'data-prepper-plugins:peer-forwarder' include 'data-prepper-plugins:blocking-buffer' +include 'data-prepper-plugins:grok-prepper' From d3fed9dabe02ed3815b9d901731338c6847c8e8d Mon Sep 17 00:00:00 2001 From: graytaylor0 <33740195+graytaylor0@users.noreply.github.com> Date: Wed, 22 Sep 2021 10:35:12 -0500 Subject: [PATCH 2/3] Remove suppressed warnings and refactor String List check Signed-off-by: graytaylor0 <33740195+graytaylor0@users.noreply.github.com> --- .../model/configuration/PluginSetting.java | 15 ++++++++------- data-prepper-plugins/grok-prepper/README.md | 1 + data-prepper-plugins/grok-prepper/build.gradle | 5 ----- 3 files changed, 9 insertions(+), 12 deletions(-) create mode 100644 data-prepper-plugins/grok-prepper/README.md diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java index 8f7a84a48a..3d471c6058 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java @@ -146,14 +146,15 @@ public String getStringOrDefault(final String attribute, final String defaultVal * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for * the attribute */ - @SuppressWarnings("unchecked") public List getStringListOrDefault(final String attribute, final List defaultValue) { Object object = getAttributeOrDefault(attribute, defaultValue); if (object == null) { return null; } else if (object instanceof List) { - ((List) object).stream().filter(o -> !(o instanceof String)).forEach(o -> { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + ((List) object).forEach(o -> { + if (!(o instanceof String)) { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } }); return (List) object; } @@ -169,7 +170,6 @@ public List getStringListOrDefault(final String attribute, final List getStringMapOrDefault(final String attribute, final Map defaultValue) { Object object = getAttributeOrDefault(attribute, defaultValue); if (object == null) { @@ -194,7 +194,6 @@ public Map getStringMapOrDefault(final String attribute, final M * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for * the attribute */ - @SuppressWarnings("unchecked") public Map> getStringListMapOrDefault(final String attribute, final Map> defaultValue) { Object object = getAttributeOrDefault(attribute, defaultValue); if (object == null) { @@ -203,8 +202,10 @@ public Map> getStringListMapOrDefault(final String attribut ((Map) object).forEach((key, value) -> { if (key instanceof String) { if (value instanceof List){ - ((List) value).stream().filter(o -> !(o instanceof String)).forEach(o -> { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + ((List) value).forEach(val -> { + if (!(val instanceof String)) { + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } }); } else { throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); diff --git a/data-prepper-plugins/grok-prepper/README.md b/data-prepper-plugins/grok-prepper/README.md new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/data-prepper-plugins/grok-prepper/README.md @@ -0,0 +1 @@ + diff --git a/data-prepper-plugins/grok-prepper/build.gradle b/data-prepper-plugins/grok-prepper/build.gradle index 6698656bb7..8502bc73f2 100644 --- a/data-prepper-plugins/grok-prepper/build.gradle +++ b/data-prepper-plugins/grok-prepper/build.gradle @@ -12,12 +12,7 @@ plugins { id 'java' } -repositories { - mavenCentral() -} - dependencies { - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') testImplementation project(':data-prepper-api').sourceSets.test.output From 40af8048ef1bb25699446f7c98399173c2aa2633 Mon Sep 17 00:00:00 2001 From: graytaylor0 <33740195+graytaylor0@users.noreply.github.com> Date: Wed, 22 Sep 2021 19:10:43 -0500 Subject: [PATCH 3/3] Generics for PluginSetting and addressing other comments Signed-off-by: graytaylor0 <33740195+graytaylor0@users.noreply.github.com> --- .../model/configuration/PluginSetting.java | 90 +++++++++-------- .../configuration/PluginSettingsTests.java | 97 ++++++++----------- .../grok-prepper/build.gradle | 2 +- .../plugins/prepper/grok/GrokPrepper.java | 6 +- .../prepper/grok/GrokPrepperConfig.java | 14 +-- .../prepper/grok/GrokPrepperConfigTests.java | 22 ++--- .../prepper/grok/GrokPrepperTests.java | 24 +++-- 7 files changed, 121 insertions(+), 134 deletions(-) diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java index 3d471c6058..29a6469748 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java @@ -11,6 +11,7 @@ package com.amazon.dataprepper.model.configuration; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -142,23 +143,19 @@ public String getStringOrDefault(final String attribute, final String defaultVal * the attribute. * * @param attribute name of the attribute - * @param defaultValue default value for the setting + * @param type the type stored in the List * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for * the attribute */ - public List getStringListOrDefault(final String attribute, final List defaultValue) { - Object object = getAttributeOrDefault(attribute, defaultValue); + public List getTypedList(final String attribute, final Class type) { + Object object = getAttributeOrDefault(attribute, Collections.emptyList()); if (object == null) { return null; - } else if (object instanceof List) { - ((List) object).forEach(o -> { - if (!(o instanceof String)) { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); - } - }); - return (List) object; } - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + + checkObjectForListType(attribute, object, type); + + return (List) object; } /** @@ -166,23 +163,25 @@ public List getStringListOrDefault(final String attribute, final List getStringMapOrDefault(final String attribute, final Map defaultValue) { - Object object = getAttributeOrDefault(attribute, defaultValue); + public Map getTypedMap(final String attribute, final Class keyType, final Class valueType) { + Object object = getAttributeOrDefault(attribute, Collections.emptyMap()); if (object == null) { return null; - } else if (object instanceof Map) { - ((Map) object).forEach((key, value) -> { - if (!(key instanceof String) || !(value instanceof String)) { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); - } - }); - return (Map) object; } - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + + checkObjectType(attribute, object, Map.class); + + ((Map) object).forEach((key, value) -> { + checkObjectType(attribute, key, keyType); + checkObjectType(attribute, value, valueType); + }); + + return (Map) object; } /** @@ -190,33 +189,25 @@ public Map getStringMapOrDefault(final String attribute, final M * the attribute. * * @param attribute name of the attribute - * @param defaultValue default value for the setting + * @param keyType key type of the Map + * @param valueType value type stored in the List value of the Map * @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for * the attribute */ - public Map> getStringListMapOrDefault(final String attribute, final Map> defaultValue) { - Object object = getAttributeOrDefault(attribute, defaultValue); + public Map> getTypedListMap(final String attribute, final Class keyType, final Class valueType) { + Object object = getAttributeOrDefault(attribute, Collections.emptyMap()); if (object == null) { return null; - } else if (object instanceof Map) { - ((Map) object).forEach((key, value) -> { - if (key instanceof String) { - if (value instanceof List){ - ((List) value).forEach(val -> { - if (!(val instanceof String)) { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); - } - }); - } else { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); - } - } else { - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); - } - }); - return (Map>) object; } - throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + + checkObjectType(attribute, object, Map.class); + + ((Map) object).forEach((key, value) -> { + checkObjectType(attribute, key, keyType); + checkObjectForListType(attribute, value, valueType); + }); + + return (Map>) object; } /** @@ -263,4 +254,17 @@ public Long getLongOrDefault(final String attribute, final long defaultValue) { throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); } + private void checkObjectType(String attribute, Object object, Class type) { + if (!(type.isAssignableFrom(object.getClass()))){ + throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute)); + } + } + + private void checkObjectForListType(String attribute, Object object, Class type) { + checkObjectType(attribute, object, List.class); + + ((List) object).forEach(o -> { + checkObjectType(attribute, o, type); + }); + } } diff --git a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java index 8d60f24d91..49f6b253a9 100644 --- a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java +++ b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java @@ -16,6 +16,7 @@ import org.junit.Before; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,43 +43,31 @@ public class PluginSettingsTests { private static final long TEST_LONG_DEFAULT_VALUE = 1000L; private static final long TEST_LONG_VALUE = TEST_LONG_DEFAULT_VALUE + 1; - private static final List TEST_STRINGLIST_DEFAULT_VALUE = new ArrayList<>(); private static final List TEST_STRINGLIST_VALUE = new ArrayList<>(); - private static final Map TEST_STRINGMAP_DEFAULT_VALUE = new HashMap<>(); private static final Map TEST_STRINGMAP_VALUE = new HashMap<>(); - private static final Map> TEST_STRINGLISTMAP_DEFAULT_VALUE = new HashMap<>(); private static final Map> TEST_STRINGLISTMAP_VALUE = new HashMap<>(); private static final String TEST_INT_ATTRIBUTE = "int-attribute"; private static final String TEST_STRING_ATTRIBUTE = "string-attribute"; - private static final String TEST_STRINGLIST_ATTRIBUTE = "string-list"; - private static final String TEST_STRINGMAP_ATTRIBUTE = "string-map"; - private static final String TEST_STRINGLISTMAP_ATTRIBUTE = "stringlist-map"; + private static final String TEST_STRINGLIST_ATTRIBUTE = "list-attribute"; + private static final String TEST_STRINGMAP_ATTRIBUTE = "map-attribute"; + private static final String TEST_STRINGLISTMAP_ATTRIBUTE = "list-map-attribute"; private static final String TEST_BOOL_ATTRIBUTE = "bool-attribute"; private static final String TEST_LONG_ATTRIBUTE = "long-attribute"; private static final String NOT_PRESENT_ATTRIBUTE = "not-present"; @Before public void setup() { - TEST_STRINGLIST_DEFAULT_VALUE.add("default"); - TEST_STRINGLIST_VALUE.add("value1"); TEST_STRINGLIST_VALUE.add("value2"); TEST_STRINGLIST_VALUE.add("value3"); - TEST_STRINGMAP_DEFAULT_VALUE.put("default_key", "default_value"); - TEST_STRINGMAP_VALUE.put("key1", "value1"); TEST_STRINGMAP_VALUE.put("key2", "value2"); TEST_STRINGMAP_VALUE.put("key3", "value3"); - final List TEST_STRINGLISTMAP_DEFAULT_LISTVALUE = new ArrayList<>(); - TEST_STRINGLISTMAP_DEFAULT_LISTVALUE.add("default_list"); - - TEST_STRINGLISTMAP_DEFAULT_VALUE.put("default_key", TEST_STRINGLISTMAP_DEFAULT_LISTVALUE); - final int NUM_LISTS = 3; for (int i = 0; i < NUM_LISTS; i++) { @@ -150,27 +139,27 @@ public void testGetStringOrDefault() { } @Test - public void testGetStringListOrDefault() { + public void testGetTypedList() { final Map TEST_SETTINGS = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_VALUE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS); - assertThat(pluginSetting.getStringListOrDefault(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE), is(equalTo(TEST_STRINGLIST_VALUE))); + assertThat(pluginSetting.getTypedList(TEST_STRINGLIST_ATTRIBUTE, String.class), is(equalTo(TEST_STRINGLIST_VALUE))); } @Test - public void testGetStringMapOrDefault() { + public void testGetTypedMap() { final Map TEST_SETTINGS = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_VALUE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS); - assertThat(pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE), is(equalTo(TEST_STRINGMAP_VALUE))); + assertThat(pluginSetting.getTypedMap(TEST_STRINGMAP_ATTRIBUTE, String.class, String.class), is(equalTo(TEST_STRINGMAP_VALUE))); } @Test - public void testGetStringListMapOrDefault() { + public void testGetTypedListMap() { final Map TEST_SETTINGS = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_VALUE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS); - assertThat(pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE), is(equalTo(TEST_STRINGLISTMAP_VALUE))); + assertThat(pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class), is(equalTo(TEST_STRINGLISTMAP_VALUE))); } @Test @@ -252,39 +241,39 @@ public void testGetStringOrDefault_AsNull() { * Request attributes are present with null values, expect nulls to be returned */ @Test - public void testGetStringListOrDefault_AsNull() { - final String TEST_STRINGLIST_NULL_ATTRIBUTE = "stringlist-null-attribute"; + public void testGetTypedList_AsNull() { + final String TEST_STRINGLIST_NULL_ATTRIBUTE = "typedlist-null-attribute"; final Map TEST_SETTINGS_AS_NULL = new HashMap<>(); TEST_SETTINGS_AS_NULL.put(TEST_STRINGLIST_NULL_ATTRIBUTE, null); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL); - assertThat(pluginSetting.getStringListOrDefault(TEST_STRINGLIST_NULL_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE), nullValue()); + assertThat(pluginSetting.getTypedList(TEST_STRINGLIST_NULL_ATTRIBUTE, String.class), nullValue()); } /** * Request attributes are present with null values, expect nulls to be returned */ @Test - public void testGetStringMapOrDefault_AsNull() { - final String TEST_STRINGMAP_NULL_ATTRIBUTE = "stringmap-null-attribute"; + public void testGetTypedMap_AsNull() { + final String TEST_STRINGMAP_NULL_ATTRIBUTE = "typedgmap-null-attribute"; final Map TEST_SETTINGS_AS_NULL = new HashMap<>(); TEST_SETTINGS_AS_NULL.put(TEST_STRINGMAP_NULL_ATTRIBUTE, null); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL); - assertThat(pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_NULL_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE), nullValue()); + assertThat(pluginSetting.getTypedMap(TEST_STRINGMAP_NULL_ATTRIBUTE, String.class, String.class), nullValue()); } /** * Request attributes are present with null values, expect nulls to be returned */ @Test - public void testGetStringListMapOrDefault_AsNull() { - final String TEST_STRINGLISTMAP_NULL_ATTRIBUTE = "stringlistmap-null-attribute"; + public void testGetTypedListMap_AsNull() { + final String TEST_STRINGLISTMAP_NULL_ATTRIBUTE = "typedlistmap-null-attribute"; final Map TEST_SETTINGS_AS_NULL = new HashMap<>(); TEST_SETTINGS_AS_NULL.put(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, null); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL); - assertThat(pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE), nullValue()); + assertThat(pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, String.class, String.class), nullValue()); } /** @@ -370,11 +359,11 @@ public void testGetStringOrDefault_NotPresent() { * Requested attributes are not present, expect default values to be returned */ @Test - public void testGetStringListOrDefault_NotPresent() { + public void testGetTypedList_NotPresent() { final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); - assertThat(pluginSetting.getStringListOrDefault(NOT_PRESENT_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE), - is(equalTo(TEST_STRINGLIST_DEFAULT_VALUE))); + assertThat(pluginSetting.getTypedList(NOT_PRESENT_ATTRIBUTE, String.class), + is(equalTo(Collections.emptyList()))); } /** @@ -384,19 +373,19 @@ public void testGetStringListOrDefault_NotPresent() { public void testGetStringMapOrDefault_NotPresent() { final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); - assertThat(pluginSetting.getStringMapOrDefault(NOT_PRESENT_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE), - is(equalTo(TEST_STRINGMAP_DEFAULT_VALUE))); + assertThat(pluginSetting.getTypedMap(NOT_PRESENT_ATTRIBUTE, String.class, String.class), + is(equalTo(Collections.emptyMap()))); } /** * Requested attributes are not present, expect default values to be returned */ @Test - public void testGetStringListMapOrDefault_NotPresent() { + public void testGetTypedListMap_NotPresent() { final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null); - assertThat(pluginSetting.getStringListMapOrDefault(NOT_PRESENT_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE), - is(equalTo(TEST_STRINGLISTMAP_DEFAULT_VALUE))); + assertThat(pluginSetting.getTypedListMap(NOT_PRESENT_ATTRIBUTE, String.class, String.class), + is(equalTo(Collections.emptyMap()))); } /** @@ -439,16 +428,16 @@ public void testGetStringOrDefault_UnsupportedType() { } @Test - public void testGetStringListOrDefault_UnsupportedType() { + public void testGetTypedList_UnsupportedType() { final String UNSUPPORTED_TYPE = "not-stringlist"; final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListOrDefault(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedList(TEST_STRINGLIST_ATTRIBUTE, String.class)); } @Test - public void testGetStringListOrDefault_UnsupportedListType() { + public void testGetTypedList_UnsupportedListType() { final List UNSUPPORTED_TYPE = new ArrayList<>(); UNSUPPORTED_TYPE.add(1); UNSUPPORTED_TYPE.add(2); @@ -457,21 +446,21 @@ public void testGetStringListOrDefault_UnsupportedListType() { final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListOrDefault(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedList(TEST_STRINGLIST_ATTRIBUTE, String.class)); } @Test - public void testGetStringMapOrDefault_UnsupportedType() { + public void testGetTypedMap_UnsupportedType() { final String UNSUPPORTED_TYPE = "not-stringmap"; final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedMap(TEST_STRINGMAP_ATTRIBUTE, String.class, String.class)); } @Test - public void testGetStringMapOrDefault_UnsupportedMapValueType() { + public void testGetTypedMap_UnsupportedMapValueType() { final Map UNSUPPORTED_TYPE = new HashMap<>(); UNSUPPORTED_TYPE.put("key1", 1); UNSUPPORTED_TYPE.put("key2", 2); @@ -480,20 +469,20 @@ public void testGetStringMapOrDefault_UnsupportedMapValueType() { final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringMapOrDefault(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedMap(TEST_STRINGMAP_ATTRIBUTE, String.class, String.class)); } @Test - public void testGetStringListMapOrDefault_UnsupportedType() { + public void testGetTypedListMap_UnsupportedType() { final String UNSUPPORTED_TYPE = "not-stringmap"; final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class)); } @Test - public void testGetStringListMapOrDefault_UnsupportedMapValueType() { + public void testGetTypedListMap_UnsupportedMapValueType() { final Map UNSUPPORTED_TYPE = new HashMap<>(); UNSUPPORTED_TYPE.put("key1", "value1"); UNSUPPORTED_TYPE.put("key2", "value2"); @@ -502,11 +491,11 @@ public void testGetStringListMapOrDefault_UnsupportedMapValueType() { final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class)); } @Test - public void testGetStringListMapOrDefault_UnsupportedMapKeyType() { + public void testGetTypedListMap_UnsupportedMapKeyType() { final Map> UNSUPPORTED_TYPE = new HashMap<>(); final List STRING_LIST_VALUE = new ArrayList<>(); STRING_LIST_VALUE.add("value1"); @@ -518,11 +507,11 @@ public void testGetStringListMapOrDefault_UnsupportedMapKeyType() { final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class)); } @Test - public void testGetStringListMapOrDefault_UnsupportedMapValueListType() { + public void testGetTypedListMap_UnsupportedMapValueListType() { final Map> UNSUPPORTED_TYPE = new HashMap<>(); final List INT_LIST_VALUE = new ArrayList<>(); INT_LIST_VALUE.add(1); @@ -534,7 +523,7 @@ public void testGetStringListMapOrDefault_UnsupportedMapValueListType() { final Map TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE); final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE); - assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringListMapOrDefault(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_DEFAULT_VALUE)); + assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class)); } @Test diff --git a/data-prepper-plugins/grok-prepper/build.gradle b/data-prepper-plugins/grok-prepper/build.gradle index 8502bc73f2..331ed1e609 100644 --- a/data-prepper-plugins/grok-prepper/build.gradle +++ b/data-prepper-plugins/grok-prepper/build.gradle @@ -15,8 +15,8 @@ plugins { dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') - testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.krakens:java-grok:0.1.9" + implementation 'com.fasterxml.jackson.core:jackson-databind' testImplementation "org.hamcrest:hamcrest:2.2" } diff --git a/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java index 7a9ae627e7..569fcc1564 100644 --- a/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java +++ b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepper.java @@ -19,17 +19,17 @@ import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.model.prepper.AbstractPrepper; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collection; -@DataPrepperPlugin(name = "grok_prepper", type = PluginType.PREPPER) +@DataPrepperPlugin(name = "grok", type = PluginType.PREPPER) public class GrokPrepper extends AbstractPrepper, Record> { private static final Logger LOG = LoggerFactory.getLogger(GrokPrepper.class); - private final GrokPrepperConfig grokPrepperConfig; public GrokPrepper(final PluginSetting pluginSetting) { @@ -43,7 +43,7 @@ public GrokPrepper(final PluginSetting pluginSetting) { * * @param records Input records that will be modified/processed * @return Record modified output records - */ + */ @Override public Collection> doExecute(Collection> records) { return records; diff --git a/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java index 463798c5eb..6ebab7b372 100644 --- a/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java +++ b/data-prepper-plugins/grok-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfig.java @@ -13,8 +13,6 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,12 +30,8 @@ public class GrokPrepperConfig { static final boolean DEFAULT_BREAK_ON_MATCH = true; static final boolean DEFAULT_KEEP_EMPTY_CAPTURES = false; - static final Map> DEFAULT_MATCH = new HashMap<>(); static final boolean DEFAULT_NAMED_CAPTURES_ONLY = true; - static final List DEFAULT_OVERWRITE = new ArrayList<>(); - static final List DEFAULT_PATTERNS_DIR = new ArrayList<>(); static final String DEFAULT_PATTERNS_FILES_GLOB = ".*"; - static final Map DEFAULT_PATTERN_DEFINITIONS = new HashMap<>(); static final int DEFAULT_TIMEOUT_MILLIS = 30000; static final String DEFAULT_TARGET = null; @@ -78,12 +72,12 @@ private GrokPrepperConfig(final boolean breakOnMatch, public static GrokPrepperConfig buildConfig(final PluginSetting pluginSetting) { return new GrokPrepperConfig(pluginSetting.getBooleanOrDefault(BREAK_ON_MATCH, DEFAULT_BREAK_ON_MATCH), pluginSetting.getBooleanOrDefault(KEEP_EMPTY_CAPTURES, DEFAULT_KEEP_EMPTY_CAPTURES), - pluginSetting.getStringListMapOrDefault(MATCH, DEFAULT_MATCH), + pluginSetting.getTypedListMap(MATCH, String.class, String.class), pluginSetting.getBooleanOrDefault(NAMED_CAPTURES_ONLY, DEFAULT_NAMED_CAPTURES_ONLY), - pluginSetting.getStringListOrDefault(OVERWRITE, DEFAULT_OVERWRITE), - pluginSetting.getStringListOrDefault(PATTERNS_DIR, DEFAULT_PATTERNS_DIR), + pluginSetting.getTypedList(OVERWRITE, String.class), + pluginSetting.getTypedList(PATTERNS_DIR, String.class), pluginSetting.getStringOrDefault(PATTERNS_FILES_GLOB, DEFAULT_PATTERNS_FILES_GLOB), - pluginSetting.getStringMapOrDefault(PATTERN_DEFINITIONS, DEFAULT_PATTERN_DEFINITIONS), + pluginSetting.getTypedMap(PATTERN_DEFINITIONS, String.class, String.class), pluginSetting.getIntegerOrDefault(TIMEOUT_MILLIS, DEFAULT_TIMEOUT_MILLIS), pluginSetting.getStringOrDefault(TARGET, DEFAULT_TARGET)); } diff --git a/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java index 3e41c811c7..4540546b4c 100644 --- a/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java +++ b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperConfigTests.java @@ -13,18 +13,14 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_BREAK_ON_MATCH; -import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_MATCH; import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_KEEP_EMPTY_CAPTURES; import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_NAMED_CAPTURES_ONLY; -import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERN_DEFINITIONS; -import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERNS_DIR; import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_PATTERNS_FILES_GLOB; import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_TIMEOUT_MILLIS; import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_TARGET; -import static com.amazon.dataprepper.plugins.prepper.grok.GrokPrepperConfig.DEFAULT_OVERWRITE; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; @@ -37,7 +33,7 @@ import static org.junit.Assert.assertThrows; public class GrokPrepperConfigTests { - private static final String PLUGIN_NAME = "grok_prepper"; + private static final String PLUGIN_NAME = "grok"; private static final Map> TEST_MATCH = new HashMap<>(); private static final List TEST_OVERWRITE = new ArrayList<>(); @@ -49,8 +45,8 @@ public class GrokPrepperConfigTests { private static final Map TEST_INVALID_MATCH = new HashMap<>(); - @Before - public void setUp() { + @BeforeAll + public static void setUp() { final List log_patterns = new ArrayList<>(); log_patterns.add("%{SYNTAX:SEMANTIC}"); log_patterns.add("%{SYNTAX_2:SEMANTIC_2}"); @@ -76,10 +72,10 @@ public void testDefault() { assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(DEFAULT_BREAK_ON_MATCH)); assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(DEFAULT_KEEP_EMPTY_CAPTURES)); - assertThat(grokPrepperConfig.getMatch(), equalTo(DEFAULT_MATCH)); - assertThat(grokPrepperConfig.getOverwrite(), equalTo(DEFAULT_OVERWRITE)); - assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(DEFAULT_PATTERN_DEFINITIONS)); - assertThat(grokPrepperConfig.getPatternsDir(), equalTo(DEFAULT_PATTERNS_DIR)); + assertThat(grokPrepperConfig.getMatch(), equalTo(Collections.emptyMap())); + assertThat(grokPrepperConfig.getOverwrite(), equalTo(Collections.emptyList())); + assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(Collections.emptyMap())); + assertThat(grokPrepperConfig.getPatternsDir(), equalTo(Collections.emptyList())); assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(DEFAULT_PATTERNS_FILES_GLOB)); assertThat(grokPrepperConfig.getTarget(), equalTo(DEFAULT_TARGET)); assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(DEFAULT_NAMED_CAPTURES_ONLY)); diff --git a/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java index acb5b65463..e320381278 100644 --- a/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java +++ b/data-prepper-plugins/grok-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/grok/GrokPrepperTests.java @@ -11,34 +11,38 @@ package com.amazon.dataprepper.plugins.prepper.grok; -import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.equalTo; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; import java.util.Collections; import java.util.List; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +@ExtendWith(MockitoExtension.class) public class GrokPrepperTests { PluginSetting pluginSetting; - public GrokPrepper grokPrepper; + private GrokPrepper grokPrepper; - @Before + @BeforeEach public void setup() { pluginSetting = new PluginSetting( - "grok_prepper", + "grok", null ); pluginSetting.setPipelineName("grokPipeline"); grokPrepper = new GrokPrepper(pluginSetting); } - @After + @AfterEach public void tearDown() { grokPrepper.shutdown(); }