Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Grok Prepper Configuration and Boilerplate #302

Merged
merged 4 commits into from
Sep 27, 2021
Merged

Grok Prepper Configuration and Boilerplate #302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c972c4b
Grok Prepper Configuration and Boilerplate
graytaylor0 Sep 21, 2021
d3fed9d
Remove suppressed warnings and refactor String List check
graytaylor0 Sep 22, 2021
e491ac3
Merge remote-tracking branch 'upstream/main' into GrokPrepper
graytaylor0 Sep 22, 2021
40af804
Generics for PluginSetting and addressing other comments
graytaylor0 Sep 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions data-prepper-api/src/main/java/com/amazon/dataprepper/model/configuration/PluginSetting.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

package com.amazon.dataprepper.model.configuration;

import java.util.Collections;
import java.util.List;
import java.util.Map;

public class PluginSetting {
Expand Down Expand Up @@ -136,6 +138,78 @@ public String getStringOrDefault(final String attribute, final String defaultVal
throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute));
}

/**
* Returns the value of the specified List<String>, or {@code defaultValue} if this settings contains no value for
* the attribute.
*
* @param attribute name of the attribute
* @param type the type stored in the List
* @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for
* the attribute
*/
public <T> List<T> getTypedList(final String attribute, final Class<T> type) {
Object object = getAttributeOrDefault(attribute, Collections.emptyList());
if (object == null) {
return null;
}

checkObjectForListType(attribute, object, type);

return (List<T>) object;
}

/**
* Returns the value of the specified Map<String, String> object, or {@code defaultValue} if this settings contains no value for
* the attribute.
*
* @param attribute name of the attribute
* @param keyType key type of the Map
* @param valueType value type stored in the Map
* @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for
* the attribute
*/
public <K, V> Map<K, V> getTypedMap(final String attribute, final Class<K> keyType, final Class<V> valueType) {
Object object = getAttributeOrDefault(attribute, Collections.emptyMap());
if (object == null) {
return null;
}

checkObjectType(attribute, object, Map.class);

((Map<?, ?>) object).forEach((key, value) -> {
checkObjectType(attribute, key, keyType);
checkObjectType(attribute, value, valueType);
});

return (Map<K, V>) object;
}

/**
* Returns the value of the specified Map<String, List<String>>, or {@code defaultValue} if this settings contains no value for
* the attribute.
*
* @param attribute name of the attribute
* @param keyType key type of the Map
* @param valueType value type stored in the List value of the Map
* @return the value of the specified attribute, or {@code defaultValue} if this settings contains no value for
* the attribute
*/
public <K, V> Map<K, List<V>> getTypedListMap(final String attribute, final Class<K> keyType, final Class<V> valueType) {
Object object = getAttributeOrDefault(attribute, Collections.emptyMap());
if (object == null) {
return null;
}

checkObjectType(attribute, object, Map.class);

((Map<?, ?>) object).forEach((key, value) -> {
checkObjectType(attribute, key, keyType);
checkObjectForListType(attribute, value, valueType);
});

return (Map<K, List<V>>) object;
}

/**
* Returns the value of the specified attribute as boolean, or {@code defaultValue} if this settings contains no
* value for the attribute. If the value is null, null will be returned.
Expand Down Expand Up @@ -180,4 +254,17 @@ public Long getLongOrDefault(final String attribute, final long defaultValue) {
throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute));
}

private <T> void checkObjectType(String attribute, Object object, Class<T> type) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: final for method params line 257 & 263 to be consistent with other methods

if (!(type.isAssignableFrom(object.getClass()))){
throw new IllegalArgumentException(String.format(UNEXPECTED_ATTRIBUTE_TYPE_MSG, object.getClass(), attribute));
}
}

private <T> void checkObjectForListType(String attribute, Object object, Class<T> type) {
checkObjectType(attribute, object, List.class);

((List<?>) object).forEach(o -> {
checkObjectType(attribute, o, type);
});
}
}
229 changes: 229 additions & 0 deletions ...per-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@

import com.google.common.collect.ImmutableMap;
import org.junit.Test;
import org.junit.Before;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.hamcrest.CoreMatchers.equalTo;
Expand All @@ -40,12 +43,43 @@ public class PluginSettingsTests {
private static final long TEST_LONG_DEFAULT_VALUE = 1000L;
private static final long TEST_LONG_VALUE = TEST_LONG_DEFAULT_VALUE + 1;

private static final List<String> TEST_STRINGLIST_VALUE = new ArrayList<>();

private static final Map<String, String> TEST_STRINGMAP_VALUE = new HashMap<>();

private static final Map<String, List<String>> TEST_STRINGLISTMAP_VALUE = new HashMap<>();

private static final String TEST_INT_ATTRIBUTE = "int-attribute";
private static final String TEST_STRING_ATTRIBUTE = "string-attribute";
private static final String TEST_STRINGLIST_ATTRIBUTE = "list-attribute";
private static final String TEST_STRINGMAP_ATTRIBUTE = "map-attribute";
private static final String TEST_STRINGLISTMAP_ATTRIBUTE = "list-map-attribute";
private static final String TEST_BOOL_ATTRIBUTE = "bool-attribute";
private static final String TEST_LONG_ATTRIBUTE = "long-attribute";
private static final String NOT_PRESENT_ATTRIBUTE = "not-present";

@Before
public void setup() {
TEST_STRINGLIST_VALUE.add("value1");
TEST_STRINGLIST_VALUE.add("value2");
TEST_STRINGLIST_VALUE.add("value3");

TEST_STRINGMAP_VALUE.put("key1", "value1");
TEST_STRINGMAP_VALUE.put("key2", "value2");
TEST_STRINGMAP_VALUE.put("key3", "value3");

final int NUM_LISTS = 3;

for (int i = 0; i < NUM_LISTS; i++) {
final List<String> TEST_STRINGLISTMAP_VALUE_LIST = new ArrayList<>();
TEST_STRINGLISTMAP_VALUE_LIST.add("value_1" + String.valueOf(i));
TEST_STRINGLISTMAP_VALUE_LIST.add("value_2" + String.valueOf(i));
TEST_STRINGLISTMAP_VALUE_LIST.add("value_3" + String.valueOf(i));

TEST_STRINGLISTMAP_VALUE.put("key_" + String.valueOf(i), TEST_STRINGLISTMAP_VALUE_LIST);
}

}

@Test
public void testPluginSetting() {
Expand Down Expand Up @@ -104,6 +138,30 @@ public void testGetStringOrDefault() {
is(equalTo(TEST_STRING_VALUE)));
}

@Test
public void testGetTypedList() {
final Map<String, Object> TEST_SETTINGS = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, TEST_STRINGLIST_VALUE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS);

assertThat(pluginSetting.getTypedList(TEST_STRINGLIST_ATTRIBUTE, String.class), is(equalTo(TEST_STRINGLIST_VALUE)));
}

@Test
public void testGetTypedMap() {
final Map<String, Object> TEST_SETTINGS = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, TEST_STRINGMAP_VALUE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS);

assertThat(pluginSetting.getTypedMap(TEST_STRINGMAP_ATTRIBUTE, String.class, String.class), is(equalTo(TEST_STRINGMAP_VALUE)));
}

@Test
public void testGetTypedListMap() {
final Map<String, Object> TEST_SETTINGS = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, TEST_STRINGLISTMAP_VALUE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS);

assertThat(pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class), is(equalTo(TEST_STRINGLISTMAP_VALUE)));
}

@Test
public void testGetBooleanOrDefault() {
final Map<String, Object> TEST_SETTINGS = ImmutableMap.of(TEST_BOOL_ATTRIBUTE, TEST_BOOL_VALUE);
Expand Down Expand Up @@ -179,6 +237,45 @@ public void testGetStringOrDefault_AsNull() {
assertThat(pluginSetting.getStringOrDefault(TEST_STRING_NULL_ATTRIBUTE, TEST_STRING_DEFAULT_VALUE), nullValue());
}

/**
* Request attributes are present with null values, expect nulls to be returned
*/
@Test
public void testGetTypedList_AsNull() {
final String TEST_STRINGLIST_NULL_ATTRIBUTE = "typedlist-null-attribute";
final Map<String, Object> TEST_SETTINGS_AS_NULL = new HashMap<>();
TEST_SETTINGS_AS_NULL.put(TEST_STRINGLIST_NULL_ATTRIBUTE, null);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL);

assertThat(pluginSetting.getTypedList(TEST_STRINGLIST_NULL_ATTRIBUTE, String.class), nullValue());
}

/**
* Request attributes are present with null values, expect nulls to be returned
*/
@Test
public void testGetTypedMap_AsNull() {
final String TEST_STRINGMAP_NULL_ATTRIBUTE = "typedgmap-null-attribute";
final Map<String, Object> TEST_SETTINGS_AS_NULL = new HashMap<>();
TEST_SETTINGS_AS_NULL.put(TEST_STRINGMAP_NULL_ATTRIBUTE, null);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL);

assertThat(pluginSetting.getTypedMap(TEST_STRINGMAP_NULL_ATTRIBUTE, String.class, String.class), nullValue());
}

/**
* Request attributes are present with null values, expect nulls to be returned
*/
@Test
public void testGetTypedListMap_AsNull() {
final String TEST_STRINGLISTMAP_NULL_ATTRIBUTE = "typedlistmap-null-attribute";
final Map<String, Object> TEST_SETTINGS_AS_NULL = new HashMap<>();
TEST_SETTINGS_AS_NULL.put(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, null);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_AS_NULL);

assertThat(pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_NULL_ATTRIBUTE, String.class, String.class), nullValue());
}

/**
* Request attributes are present with null values, expect nulls to be returned
*/
Expand Down Expand Up @@ -258,6 +355,39 @@ public void testGetStringOrDefault_NotPresent() {
is(equalTo(TEST_STRING_DEFAULT_VALUE)));
}

/**
* Requested attributes are not present, expect default values to be returned
*/
@Test
public void testGetTypedList_NotPresent() {
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null);

assertThat(pluginSetting.getTypedList(NOT_PRESENT_ATTRIBUTE, String.class),
is(equalTo(Collections.emptyList())));
}

/**
* Requested attributes are not present, expect default values to be returned
*/
@Test
public void testGetStringMapOrDefault_NotPresent() {
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null);

assertThat(pluginSetting.getTypedMap(NOT_PRESENT_ATTRIBUTE, String.class, String.class),
is(equalTo(Collections.emptyMap())));
}

/**
* Requested attributes are not present, expect default values to be returned
*/
@Test
public void testGetTypedListMap_NotPresent() {
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, null);

assertThat(pluginSetting.getTypedListMap(NOT_PRESENT_ATTRIBUTE, String.class, String.class),
is(equalTo(Collections.emptyMap())));
}

/**
* Requested attributes are not present, expect default values to be returned
*/
Expand Down Expand Up @@ -297,6 +427,105 @@ public void testGetStringOrDefault_UnsupportedType() {
assertThrows(IllegalArgumentException.class, () -> pluginSetting.getStringOrDefault(TEST_STRING_ATTRIBUTE, TEST_STRING_DEFAULT_VALUE));
}

@Test
public void testGetTypedList_UnsupportedType() {
final String UNSUPPORTED_TYPE = "not-stringlist";
final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedList(TEST_STRINGLIST_ATTRIBUTE, String.class));
}

@Test
public void testGetTypedList_UnsupportedListType() {
final List<Integer> UNSUPPORTED_TYPE = new ArrayList<>();
UNSUPPORTED_TYPE.add(1);
UNSUPPORTED_TYPE.add(2);
UNSUPPORTED_TYPE.add(3);

final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLIST_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedList(TEST_STRINGLIST_ATTRIBUTE, String.class));
}

@Test
public void testGetTypedMap_UnsupportedType() {
final String UNSUPPORTED_TYPE = "not-stringmap";
final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedMap(TEST_STRINGMAP_ATTRIBUTE, String.class, String.class));
}


@Test
public void testGetTypedMap_UnsupportedMapValueType() {
final Map<String, Integer> UNSUPPORTED_TYPE = new HashMap<>();
UNSUPPORTED_TYPE.put("key1", 1);
UNSUPPORTED_TYPE.put("key2", 2);
UNSUPPORTED_TYPE.put("key3", 3);

final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGMAP_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedMap(TEST_STRINGMAP_ATTRIBUTE, String.class, String.class));
}

@Test
public void testGetTypedListMap_UnsupportedType() {
final String UNSUPPORTED_TYPE = "not-stringmap";
final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class));
}

@Test
public void testGetTypedListMap_UnsupportedMapValueType() {
final Map<String, String> UNSUPPORTED_TYPE = new HashMap<>();
UNSUPPORTED_TYPE.put("key1", "value1");
UNSUPPORTED_TYPE.put("key2", "value2");
UNSUPPORTED_TYPE.put("key3", "value3");

final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class));
}

@Test
public void testGetTypedListMap_UnsupportedMapKeyType() {
final Map<Integer, List<String>> UNSUPPORTED_TYPE = new HashMap<>();
final List<String> STRING_LIST_VALUE = new ArrayList<>();
STRING_LIST_VALUE.add("value1");

UNSUPPORTED_TYPE.put(1, STRING_LIST_VALUE);
UNSUPPORTED_TYPE.put(2, STRING_LIST_VALUE);
UNSUPPORTED_TYPE.put(3, STRING_LIST_VALUE);

final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class));
}

@Test
public void testGetTypedListMap_UnsupportedMapValueListType() {
final Map<String, List<Integer>> UNSUPPORTED_TYPE = new HashMap<>();
final List<Integer> INT_LIST_VALUE = new ArrayList<>();
INT_LIST_VALUE.add(1);

UNSUPPORTED_TYPE.put("value1", INT_LIST_VALUE);
UNSUPPORTED_TYPE.put("value2", INT_LIST_VALUE);
UNSUPPORTED_TYPE.put("value3", INT_LIST_VALUE);

final Map<String, Object> TEST_SETTINGS_WITH_UNSUPPORTED_TYPE = ImmutableMap.of(TEST_STRINGLISTMAP_ATTRIBUTE, UNSUPPORTED_TYPE);
final PluginSetting pluginSetting = new PluginSetting(TEST_PLUGIN_NAME, TEST_SETTINGS_WITH_UNSUPPORTED_TYPE);

assertThrows(IllegalArgumentException.class, () -> pluginSetting.getTypedListMap(TEST_STRINGLISTMAP_ATTRIBUTE, String.class, String.class));
}

@Test
public void testGetBooleanOrDefault_UnsupportedType() {
final Object UNSUPPORTED_TYPE = new ArrayList<>();
Expand Down
1 change: 1 addition & 0 deletions data-prepper-plugins/grok-prepper/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Loading