diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index e2aa95aaf646..99055a262324 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -57,20 +57,15 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Random; -import java.util.Spliterator; -import java.util.Spliterators; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.StreamSupport; import org.joda.time.DateTime; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -1027,16 +1022,12 @@ protected void assertSameMessages(final List expected, .map(AirbyteMessage::getRecord) .peek(recordMessage -> recordMessage.setEmittedAt(null)) .map(recordMessage -> pruneAirbyteInternalFields ? safePrune(recordMessage) : recordMessage) - .map(AirbyteRecordMessage::getData) - .peek(this::sortDataFields) - .sorted(Comparator.comparing(JsonNode::toString)) + .map(recordMessage -> recordMessage.getData()) .collect(Collectors.toList()); final List actualProcessed = actual.stream() .map(recordMessage -> pruneAirbyteInternalFields ? safePrune(recordMessage) : recordMessage) - .map(AirbyteRecordMessage::getData) - .peek(this::sortDataFields) - .sorted(Comparator.comparing(JsonNode::toString)) + .map(recordMessage -> recordMessage.getData()) .collect(Collectors.toList()); assertSameData(expectedProcessed, actualProcessed); @@ -1074,21 +1065,6 @@ private void assertSameData(final List expected, final List } } - /** - * Method that will sort all fields by name and rewrite JsonNode in sorted order - * - * @param data - data node from AirbyteMessage - */ - private void sortDataFields(JsonNode data) { - var sortedFields = StreamSupport.stream(Spliterators.spliteratorUnknownSize(data.fields(), - Spliterator.ORDERED), false) - .sorted(Entry.comparingByKey(Comparator.comparing(String::toLowerCase))) - .collect(Collectors.toList()); - ((ObjectNode) data).removeAll(); - IntStream.range(0, sortedFields.size()) - .forEach(i -> ((ObjectNode) data).set(sortedFields.get(i).getKey(), sortedFields.get(i).getValue())); - } - // Allows subclasses to implement custom comparison asserts protected void assertSameValue(final JsonNode expectedValue, final JsonNode actualValue) { assertEquals(expectedValue, actualValue); diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json index a168a3338f09..3f7e6b03f910 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json @@ -139,21 +139,6 @@ } } } - }, - { - "name": "stream_with_dates", - "json_schema": { - "properties": { - "date": { - "type": "string", - "format": "date" - }, - "datetime": { - "type": "string", - "format": "date-time" - } - } - } } ] } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt index ad55b6ebe63b..5f2adf5a21b4 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt @@ -7,56 +7,12 @@ {"type": "RECORD", "record": {"stream": "reserved_keywords", "emitted_at": 1602637589000, "data": { "order" : "ascending" }}} {"type": "RECORD", "record": {"stream": "groups", "emitted_at": 1602637589000, "data": { "authorization" : "into" }}} {"type": "RECORD", "record": {"stream": "ProperCase", "emitted_at": 1602637589000, "data": { "ProperCase" : true }}} -{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : 101, "some_next_field" : "some_next_field_1", "some_field" : "some_field_1" }}} +{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : 101, "some_field" : "some_field_1", "some_next_field" : "some_next_field_1" }}} {"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589250, "data": { "some_id" : 102, "some_field" : "some_field_2" }}} {"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589300, "data": { "some_id" : 103, "some_next_field" : "some_next_field_3" }}} {"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589350, "data": { "some_id" : 104 }}} -{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "next_field_name" : "next_field_name_1", "some_id" : 201 }}} -{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "next_field_name" : "next_field_name_2", "some_id" : 202 }}} +{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "some_id" : 201, "next_field_name" : "next_field_name_1" }}} +{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "some_id" : 202, "next_field_name" : "next_field_name_2" }}} {"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589500, "data": { "some_id" : 203 }}} {"type": "RECORD", "record": {"stream": "stream_with_binary_data", "emitted_at": 1602637589500, "data": { "some_id" : 303, "binary_field_name":"dGVzdA==" }}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "2021-1-1"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "1/1/2021"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "20210101"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "2021-01-01"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "01/01/2021"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T09"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T0930"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 09:30"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021/01/01 09:30"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 09:30"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T09:30"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1 01:01:01"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021 01:01:01"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 09:30:26.123"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 09:30:26.123"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010,123"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010.123"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1 01:01:01 +1"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021 01:01:01 +1"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01-01:00"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01-01:00"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1T01:01:01 +1:00"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021T01:01:01 +1:00"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01 +0000"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01 +0000"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01Z"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01Z"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +1:00"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +1:00"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01 UTC"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01 UTC"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 UTC"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 UTC"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +1"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +1"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +0000"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +0000"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01+0000"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01+0000"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01+1"}}} -{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01+1"}}} {"type": "STATE", "state": { "data": {"start_date": "2020-09-02"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/format_reference.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/format_reference.txt deleted file mode 100644 index 017700c46295..000000000000 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/format_reference.txt +++ /dev/null @@ -1,46 +0,0 @@ -VALID FORMATS: - -"2021-1-1" -"1/1/2021" -"20210101" -"2021-01-01" -"01/01/2021" -"2021-01-01T09" -"20210101T0930" -"2021-01-01 09:30" -"2021/01/01 09:30" -"01/01/2021 09:30" -"20210101T093010" -"2021-01-01T09:30" -"2021-1-1 01:01:01" -"1/1/2021 01:01:01" -"2021-01-01 09:30:26.123" -"01/01/2021 09:30:26.123" -"20210101T093010,123" -"20210101T093010.123" -"2021-1-1 01:01:01 +1" -"1/1/2021 01:01:01 +1" -"2021-01-01T01:01:01-01:00" -"01/01/2021T01:01:01-01:00" -"2021-1-1T01:01:01 +1:00" -"1/1/2021T01:01:01 +1:00" -"2021-01-01 01:01:01" -"01/01/2021 01:01:01" -"2021-01-01 01:01:01 +0000" -"01/01/2021 01:01:01 +0000" -"2021-01-01T01:01:01Z" -"01/01/2021T01:01:01Z" -"2021-01-01T01:01:01 +1:00" -"01/01/2021T01:01:01 +1:00" -"2021-01-01 01:01:01 UTC" -"01/01/2021 01:01:01 UTC" -"2021-01-01T01:01:01 UTC" -"01/01/2021T01:01:01 UTC" -"2021-01-01T01:01:01 +1" -"01/01/2021T01:01:01 +1" -"2021-01-01T01:01:01 +0000" -"01/01/2021T01:01:01 +0000" -"2021-01-01T01:01:01+0000" -"01/01/2021T01:01:01+0000" -"2021-01-01T01:01:01+1" -"01/01/2021T01:01:01+1" \ No newline at end of file