From d2382708d185f64bb0d24876e92443da16706e00 Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Mon, 27 Nov 2023 10:49:02 -0600 Subject: [PATCH] Fix S3 scan failing tests (#3693) Signed-off-by: Asif Sohail Mohammed --- .../source/s3/S3ScanObjectWorkerIT.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java index 48c5862155..9ed910f33a 100644 --- a/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java +++ b/data-prepper-plugins/s3-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanObjectWorkerIT.java @@ -231,8 +231,8 @@ void parseS3Object_parquet_correctly_with_bucket_scan_and_loads_data_into_Buffer final String keyPrefix = "s3source/s3-scan/" + recordsGenerator.getFileExtension() + "/" + Instant.now().toEpochMilli(); final String buketOptionYaml = "name: " + bucket + "\n" + - "key_prefix:\n" + - " include:\n" + + "filter:\n" + + " include_prefix:\n" + " - " + keyPrefix + "\n" + " exclude_suffix:\n" + " - .csv\n" + @@ -247,7 +247,7 @@ void parseS3Object_parquet_correctly_with_bucket_scan_and_loads_data_into_Buffer final ScanOptions startTimeAndRangeScanOptions = new ScanOptions.Builder() .setBucketOption(objectMapper.readValue(buketOptionYaml, S3ScanBucketOption.class)) .setStartDateTime(LocalDateTime.now().minusDays(1)) - .setRange(Duration.parse("P2DT10M")) + .setEndDateTime(LocalDateTime.now().plus(Duration.ofMinutes(5))) .build(); final ScanObjectWorker objectUnderTest = createObjectUnderTest(recordsGenerator, @@ -279,8 +279,8 @@ void parseS3Object_correctly_with_bucket_scan_and_loads_data_into_Buffer( String keyPrefix = "s3source/s3-scan/" + recordsGenerator.getFileExtension() + "/" + Instant.now().toEpochMilli(); final String key = getKeyString(keyPrefix,recordsGenerator, shouldCompress); final String buketOptionYaml = "name: " + bucket + "\n" + - "key_prefix:\n" + - " include:\n" + + "filter:\n" + + " include_prefix:\n" + " - " + keyPrefix; scanOptions.setBucketOption(objectMapper.readValue(buketOptionYaml, S3ScanBucketOption.class)); @@ -320,10 +320,10 @@ void parseS3Object_correctly_with_bucket_scan_and_loads_data_into_Buffer_and_del when(s3SourceConfig.isDeleteS3ObjectsOnRead()).thenReturn(deleteS3Objects); String keyPrefix = "s3source/s3-scan/" + recordsGenerator.getFileExtension() + "/" + Instant.now().toEpochMilli(); - final String key = getKeyString(keyPrefix,recordsGenerator, shouldCompress); + final String key = getKeyString(keyPrefix, recordsGenerator, shouldCompress); final String buketOptionYaml = "name: " + bucket + "\n" + - "key_prefix:\n" + - " include:\n" + + "filter:\n" + + " include_prefix:\n" + " - " + keyPrefix; final ScanOptions.Builder startTimeAndEndTimeScanOptions = ScanOptions.builder() @@ -386,17 +386,17 @@ public Stream provideArguments(final ExtensionContext conte final List recordsToAccumulateList = List.of( 100); final List booleanList = List.of(Boolean.TRUE); - final ScanOptions.Builder startTimeAndRangeScanOptions = ScanOptions.builder() - .setStartDateTime(LocalDateTime.now()) - .setRange(Duration.parse("P2DT10H")); - final ScanOptions.Builder endTimeAndRangeScanOptions = ScanOptions.builder() - .setEndDateTime(LocalDateTime.now().plus(Duration.ofHours(1))) - .setRange(Duration.parse("P7DT10H")); + final ScanOptions.Builder startTimeScanOptions = ScanOptions.builder() + .setStartDateTime(LocalDateTime.now()); + final ScanOptions.Builder endTimeScanOptions = ScanOptions.builder() + .setEndDateTime(LocalDateTime.now().plus(Duration.ofHours(1))); final ScanOptions.Builder startTimeAndEndTimeScanOptions = ScanOptions.builder() .setStartDateTime(LocalDateTime.now().minus(Duration.ofMinutes(10))) .setEndDateTime(LocalDateTime.now().plus(Duration.ofHours(1))); + final ScanOptions.Builder rangeScanOptions = ScanOptions.builder() + .setRange(Duration.parse("P7DT10H")); - List scanOptions = List.of(startTimeAndRangeScanOptions,endTimeAndRangeScanOptions,startTimeAndEndTimeScanOptions); + List scanOptions = List.of(startTimeScanOptions, endTimeScanOptions, startTimeAndEndTimeScanOptions, rangeScanOptions); return recordsGenerators .stream() .flatMap(recordsGenerator -> numberOfRecordsList