diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java index e5cd4a514a..17b7074fc1 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java @@ -16,6 +16,7 @@ import org.opensearch.dataprepper.typeconverter.TypeConverter; import java.util.Collection; +import java.util.List; import java.util.Objects; @DataPrepperPlugin(name = "convert_entry_type", pluginType = Processor.class, pluginConfigurationType = ConvertEntryTypeProcessorConfig.class) @@ -23,6 +24,7 @@ public class ConvertEntryTypeProcessor extends AbstractProcessor, private final String key; private final TypeConverter converter; private final String convertWhen; + private final List nullValues; private final ExpressionEvaluator expressionEvaluator; @@ -34,6 +36,8 @@ public ConvertEntryTypeProcessor(final PluginMetrics pluginMetrics, this.key = convertEntryTypeProcessorConfig.getKey(); this.converter = convertEntryTypeProcessorConfig.getType().getTargetConverter(); this.convertWhen = convertEntryTypeProcessorConfig.getConvertWhen(); + this.nullValues = convertEntryTypeProcessorConfig.getNullValues() + .orElse(List.of()); this.expressionEvaluator = expressionEvaluator; } @@ -49,7 +53,9 @@ public Collection> doExecute(final Collection> recor Object keyVal = recordEvent.get(key, Object.class); if (keyVal != null) { recordEvent.delete(key); - recordEvent.put(key, this.converter.convert(keyVal)); + if (!nullValues.contains(keyVal.toString())){ + recordEvent.put(key, this.converter.convert(keyVal)); + } } } return records; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java index c618e75d9d..983fe57fcf 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java @@ -8,6 +8,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.NotEmpty; +import java.util.List; +import java.util.Optional; + public class ConvertEntryTypeProcessorConfig { @JsonProperty("key") @NotEmpty @@ -19,6 +22,9 @@ public class ConvertEntryTypeProcessorConfig { @JsonProperty("convert_when") private String convertWhen; + @JsonProperty("null_values") + private List nullValues; + public String getKey() { return key; } @@ -28,4 +34,8 @@ public TargetType getType() { } public String getConvertWhen() { return convertWhen; } + + public Optional> getNullValues(){ + return Optional.ofNullable(nullValues); + } } diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor_NullValueTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor_NullValueTests.java new file mode 100644 index 0000000000..b156b2b3b6 --- /dev/null +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor_NullValueTests.java @@ -0,0 +1,120 @@ +package org.opensearch.dataprepper.plugins.processor.mutateevent; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.record.Record; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ConvertEntryTypeProcessor_NullValueTests { + + static final String TEST_KEY = UUID.randomUUID().toString(); + + @Mock + private PluginMetrics pluginMetrics; + + @Mock + private ConvertEntryTypeProcessorConfig mockConfig; + + @Mock + private ExpressionEvaluator expressionEvaluator; + + private ConvertEntryTypeProcessor nullValuesProcessor; + + @BeforeEach + private void setup() { + when(mockConfig.getKey()).thenReturn(TEST_KEY); + when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("integer")); + when(mockConfig.getConvertWhen()).thenReturn(null); + } + + private Event executeAndGetProcessedEvent(final Object testValue) { + final Record record = getMessage(UUID.randomUUID().toString(), TEST_KEY, testValue); + final List> processedRecords = (List>) nullValuesProcessor.doExecute(Collections.singletonList(record)); + assertThat(processedRecords.size(), equalTo(1)); + assertThat(processedRecords.get(0), notNullValue()); + Event event = processedRecords.get(0).getData(); + assertThat(event, notNullValue()); + return event; + } + + private Record getMessage(String message, String key, Object value) { + final Map testData = new HashMap(); + testData.put("message", message); + testData.put(key, value); + return buildRecordWithEvent(testData); + } + + static Record buildRecordWithEvent(final Map data) { + return new Record<>(JacksonEvent.builder() + .withData(data) + .withEventType("event") + .build()); + } + + @Test + void testNoNullValues() { + int testValue = 5432; + when(mockConfig.getNullValues()).thenReturn(Optional.empty()); + nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + Event event = executeAndGetProcessedEvent(testValue); + assertThat(event.get(TEST_KEY, Integer.class), equalTo(testValue)); + } + + @Test + void testEmptyListNullValues() { + int testValue = 5432; + when(mockConfig.getNullValues()).thenReturn(Optional.of(List.of())); + nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + Event event = executeAndGetProcessedEvent(testValue); + assertThat(event.get(TEST_KEY, Integer.class), equalTo(testValue)); + } + + @Test + void testOneElementNullValues() { + String testValue = "-"; + when(mockConfig.getNullValues()).thenReturn(Optional.of(List.of("-"))); + nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + Event event = executeAndGetProcessedEvent(testValue); + Object keyValue = event.get(TEST_KEY, Object.class); + assertThat(keyValue, nullValue()); + } + + @Test + void testMultipleElementNullValues() { + String testValue = "-"; + when(mockConfig.getNullValues()).thenReturn(Optional.of(List.of("-", "null"))); + + nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + Event event = executeAndGetProcessedEvent(testValue); + assertThat(event.get(TEST_KEY, Integer.class), nullValue()); + + testValue = "null"; + event = executeAndGetProcessedEvent(testValue); + assertThat(event.get(TEST_KEY, Integer.class), nullValue()); + + int testNumber = 5432; + event = executeAndGetProcessedEvent(testNumber); + assertThat(event.get(TEST_KEY, Integer.class), equalTo(testNumber)); + } + +}