From b7d3cc66ee116dca50eaaedc163e3f80a4021311 Mon Sep 17 00:00:00 2001 From: George Chen Date: Mon, 20 Nov 2023 15:15:38 -0600 Subject: [PATCH 1/2] TST: validate special data in opensearch sink Signed-off-by: George Chen --- .../sink/opensearch/OpenSearchSinkIT.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index a5237f21de..f3002c8c74 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -25,6 +25,7 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.opensearch.client.Request; @@ -99,6 +100,9 @@ import static org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.wipeAllTemplates; public class OpenSearchSinkIT { + private static final int LUCENE_CHAR_LENGTH_LIMIT = 32_766; + private static final String TEST_STRING_WITH_SPECIAL_CHARS = "Hello! Data-Prepper? #Example123"; + private static final String TEST_STRING_WITH_NON_LATIN_CHARS = "Привет,Γειά σας,こんにちは,你好"; private static final String PLUGIN_NAME = "opensearch"; private static final String PIPELINE_NAME = "integTestPipeline"; private static final String TEST_CUSTOM_INDEX_POLICY_FILE = "test-custom-index-policy-file.json"; @@ -969,6 +973,33 @@ public void testEventOutput() throws IOException, InterruptedException { sink.shutdown(); } + @ParameterizedTest + @MethodSource("getAttributeTestSpecialAndExtremeValues") + public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) throws IOException, InterruptedException { + final String testIndexAlias = "test-alias"; + final String testField = "value"; + final Map data = new HashMap<>(); + data.put(testField, testValue); + final Event testEvent = JacksonEvent.builder() + .withData(data) + .withEventType("event") + .build(); + + final List> testRecords = Collections.singletonList(new Record<>(testEvent)); + + final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, null); + final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); + sink.output(testRecords); + + final List> retSources = getSearchResponseDocSources(testIndexAlias); + final Map expectedContent = new HashMap<>(); + expectedContent.put(testField, testValue); + + assertThat(retSources.size(), equalTo(1)); + assertThat(retSources.containsAll(Arrays.asList(expectedContent)), equalTo(true)); + sink.shutdown(); + } + @ParameterizedTest @ValueSource(strings = {"info/ids/id", "id"}) public void testOpenSearchDocumentId(final String testDocumentIdField) throws IOException, InterruptedException { @@ -1436,4 +1467,25 @@ private void createV1IndexTemplate(final String templateName, final String index private static boolean isES6() { return DeclaredOpenSearchVersion.OPENDISTRO_0_10.compareTo(OpenSearchIntegrationHelper.getVersion()) >= 0; } + + private static Stream getAttributeTestSpecialAndExtremeValues() { + return Stream.of( + null, + Arguments.of(Long.MAX_VALUE), + Arguments.of(Long.MIN_VALUE), + Arguments.of(Integer.MAX_VALUE), + Arguments.of(Integer.MIN_VALUE), + Arguments.of(RandomStringUtils.randomAlphabetic(LUCENE_CHAR_LENGTH_LIMIT)), + Arguments.of(TEST_STRING_WITH_SPECIAL_CHARS), + Arguments.of(TEST_STRING_WITH_NON_LATIN_CHARS), + Arguments.of(Double.MIN_VALUE), + Arguments.of(-Double.MIN_VALUE), + Arguments.of((double) Float.MAX_VALUE), + Arguments.of((double) Float.MIN_VALUE), + Arguments.of((double) -Float.MAX_VALUE), + Arguments.of((double) -Float.MIN_VALUE), + Arguments.of(Boolean.TRUE), + Arguments.of(Boolean.FALSE) + ); + } } From 353c7c3c12f3a206d1ca248be8ce8e221447ba56 Mon Sep 17 00:00:00 2001 From: George Chen Date: Tue, 21 Nov 2023 11:39:23 -0600 Subject: [PATCH 2/2] MNT: assertion logic Signed-off-by: George Chen --- .../dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index f3002c8c74..ace3a35dac 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -996,7 +996,7 @@ public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) t expectedContent.put(testField, testValue); assertThat(retSources.size(), equalTo(1)); - assertThat(retSources.containsAll(Arrays.asList(expectedContent)), equalTo(true)); + assertThat(retSources.get(0), equalTo(expectedContent)); sink.shutdown(); }