GitHub-issue#2822: Define null characters in convert processor (opens…

…earch-project#2844)

GitHub-issue#2822: Define null characters in convert processor
Signed-off-by: Aidar Shaidullin <[email protected]>

Signed-off-by: saydar31 <[email protected]>

---------

Signed-off-by: saydar31 <[email protected]>
Co-authored-by: saydar31 <[email protected]>
Signed-off-by: Marcos_Gonzalez_Mayedo <[email protected]>

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java

@@ -16,13 +16,15 @@
 import org.opensearch.dataprepper.typeconverter.TypeConverter;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Objects;
 
 @DataPrepperPlugin(name = "convert_entry_type", pluginType = Processor.class, pluginConfigurationType = ConvertEntryTypeProcessorConfig.class)
 public class ConvertEntryTypeProcessor  extends AbstractProcessor<Record<Event>, Record<Event>> {
     private final String key;
     private final TypeConverter converter;
     private final String convertWhen;
+    private final List<String> nullValues;
 
     private final ExpressionEvaluator expressionEvaluator;
 
@@ -34,6 +36,8 @@ public ConvertEntryTypeProcessor(final PluginMetrics pluginMetrics,
         this.key = convertEntryTypeProcessorConfig.getKey();
         this.converter = convertEntryTypeProcessorConfig.getType().getTargetConverter();
         this.convertWhen = convertEntryTypeProcessorConfig.getConvertWhen();
+        this.nullValues = convertEntryTypeProcessorConfig.getNullValues()
+                .orElse(List.of());
         this.expressionEvaluator = expressionEvaluator;
     }
 
@@ -49,7 +53,9 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
             Object keyVal = recordEvent.get(key, Object.class);
             if (keyVal != null) {
                 recordEvent.delete(key);
-                recordEvent.put(key, this.converter.convert(keyVal));
+                if (!nullValues.contains(keyVal.toString())){
+                    recordEvent.put(key, this.converter.convert(keyVal));
+                }
             }
         }
         return records;

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java

@@ -8,6 +8,9 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import jakarta.validation.constraints.NotEmpty;
 
+import java.util.List;
+import java.util.Optional;
+
 public class ConvertEntryTypeProcessorConfig  {
     @JsonProperty("key")
     @NotEmpty
@@ -19,6 +22,9 @@ public class ConvertEntryTypeProcessorConfig  {
     @JsonProperty("convert_when")
     private String convertWhen;
 
+    @JsonProperty("null_values")
+    private List<String> nullValues;
+
     public String getKey() {
         return key;
     }
@@ -28,4 +34,8 @@ public TargetType getType() {
     }
 
     public String getConvertWhen() { return convertWhen; }
+
+    public Optional<List<String>> getNullValues(){
+        return Optional.ofNullable(nullValues);
+    }
 }

data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor_NullValueTests.java

@@ -0,0 +1,120 @@
+package org.opensearch.dataprepper.plugins.processor.mutateevent;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.dataprepper.expression.ExpressionEvaluator;
+import org.opensearch.dataprepper.metrics.PluginMetrics;
+import org.opensearch.dataprepper.model.event.Event;
+import org.opensearch.dataprepper.model.event.JacksonEvent;
+import org.opensearch.dataprepper.model.record.Record;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class ConvertEntryTypeProcessor_NullValueTests {
+
+    static final String TEST_KEY = UUID.randomUUID().toString();
+
+    @Mock
+    private PluginMetrics pluginMetrics;
+
+    @Mock
+    private ConvertEntryTypeProcessorConfig mockConfig;
+
+    @Mock
+    private ExpressionEvaluator expressionEvaluator;
+
+    private ConvertEntryTypeProcessor nullValuesProcessor;
+
+    @BeforeEach
+    private void setup() {
+        when(mockConfig.getKey()).thenReturn(TEST_KEY);
+        when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("integer"));
+        when(mockConfig.getConvertWhen()).thenReturn(null);
+    }
+
+    private Event executeAndGetProcessedEvent(final Object testValue) {
+        final Record<Event> record = getMessage(UUID.randomUUID().toString(), TEST_KEY, testValue);
+        final List<Record<Event>> processedRecords = (List<Record<Event>>) nullValuesProcessor.doExecute(Collections.singletonList(record));
+        assertThat(processedRecords.size(), equalTo(1));
+        assertThat(processedRecords.get(0), notNullValue());
+        Event event = processedRecords.get(0).getData();
+        assertThat(event, notNullValue());
+        return event;
+    }
+
+    private Record<Event> getMessage(String message, String key, Object value) {
+        final Map<String, Object> testData = new HashMap();
+        testData.put("message", message);
+        testData.put(key, value);
+        return buildRecordWithEvent(testData);
+    }
+
+    static Record<Event> buildRecordWithEvent(final Map<String, Object> data) {
+        return new Record<>(JacksonEvent.builder()
+                .withData(data)
+                .withEventType("event")
+                .build());
+    }
+
+    @Test
+    void testNoNullValues() {
+        int testValue = 5432;
+        when(mockConfig.getNullValues()).thenReturn(Optional.empty());
+        nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator);
+        Event event = executeAndGetProcessedEvent(testValue);
+        assertThat(event.get(TEST_KEY, Integer.class), equalTo(testValue));
+    }
+
+    @Test
+    void testEmptyListNullValues() {
+        int testValue = 5432;
+        when(mockConfig.getNullValues()).thenReturn(Optional.of(List.of()));
+        nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator);
+        Event event = executeAndGetProcessedEvent(testValue);
+        assertThat(event.get(TEST_KEY, Integer.class), equalTo(testValue));
+    }
+
+    @Test
+    void testOneElementNullValues() {
+        String testValue = "-";
+        when(mockConfig.getNullValues()).thenReturn(Optional.of(List.of("-")));
+        nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator);
+        Event event = executeAndGetProcessedEvent(testValue);
+        Object keyValue = event.get(TEST_KEY, Object.class);
+        assertThat(keyValue, nullValue());
+    }
+
+    @Test
+    void testMultipleElementNullValues() {
+        String testValue = "-";
+        when(mockConfig.getNullValues()).thenReturn(Optional.of(List.of("-", "null")));
+
+        nullValuesProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator);
+        Event event = executeAndGetProcessedEvent(testValue);
+        assertThat(event.get(TEST_KEY, Integer.class), nullValue());
+
+        testValue = "null";
+        event = executeAndGetProcessedEvent(testValue);
+        assertThat(event.get(TEST_KEY, Integer.class), nullValue());
+
+        int testNumber = 5432;
+        event = executeAndGetProcessedEvent(testNumber);
+        assertThat(event.get(TEST_KEY, Integer.class), equalTo(testNumber));
+    }
+
+}