Skip to content

Commit

Permalink
Revert "🐛 📝 DAT: verify that a destination is able to write any ISO86…
Browse files Browse the repository at this point in the history 
…01-compliant date string (#5814)" (#8484)

This reverts commit e4e95e3.
  • Loading branch information
@yurii-bidiuk
yurii-bidiuk authored Dec 3, 2021
1 parent 9270f32 commit c438e01
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 134 deletions.
28 changes: 2 additions & 26 deletions ...main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,15 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;
import org.joda.time.DateTime;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -1027,16 +1022,12 @@ protected void assertSameMessages(final List<AirbyteMessage> expected,
.map(AirbyteMessage::getRecord)
.peek(recordMessage -> recordMessage.setEmittedAt(null))
.map(recordMessage -> pruneAirbyteInternalFields ? safePrune(recordMessage) : recordMessage)
.map(AirbyteRecordMessage::getData)
.peek(this::sortDataFields)
.sorted(Comparator.comparing(JsonNode::toString))
.map(recordMessage -> recordMessage.getData())
.collect(Collectors.toList());

final List<JsonNode> actualProcessed = actual.stream()
.map(recordMessage -> pruneAirbyteInternalFields ? safePrune(recordMessage) : recordMessage)
.map(AirbyteRecordMessage::getData)
.peek(this::sortDataFields)
.sorted(Comparator.comparing(JsonNode::toString))
.map(recordMessage -> recordMessage.getData())
.collect(Collectors.toList());

assertSameData(expectedProcessed, actualProcessed);
Expand Down Expand Up @@ -1074,21 +1065,6 @@ private void assertSameData(final List<JsonNode> expected, final List<JsonNode>
}
}

/**
* Method that will sort all fields by name and rewrite JsonNode in sorted order
*
* @param data - data node from AirbyteMessage
*/
private void sortDataFields(JsonNode data) {
var sortedFields = StreamSupport.stream(Spliterators.spliteratorUnknownSize(data.fields(),
Spliterator.ORDERED), false)
.sorted(Entry.comparingByKey(Comparator.comparing(String::toLowerCase)))
.collect(Collectors.toList());
((ObjectNode) data).removeAll();
IntStream.range(0, sortedFields.size())
.forEach(i -> ((ObjectNode) data).set(sortedFields.get(i).getKey(), sortedFields.get(i).getValue()));
}

// Allows subclasses to implement custom comparison asserts
protected void assertSameValue(final JsonNode expectedValue, final JsonNode actualValue) {
assertEquals(expectedValue, actualValue);
Expand Down
15 changes: 0 additions & 15 deletions ...te-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,21 +139,6 @@
}
}
}
},
{
"name": "stream_with_dates",
"json_schema": {
"properties": {
"date": {
"type": "string",
"format": "date"
},
"datetime": {
"type": "string",
"format": "date-time"
}
}
}
}
]
}
50 changes: 3 additions & 47 deletions ...te-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,56 +7,12 @@
{"type": "RECORD", "record": {"stream": "reserved_keywords", "emitted_at": 1602637589000, "data": { "order" : "ascending" }}}
{"type": "RECORD", "record": {"stream": "groups", "emitted_at": 1602637589000, "data": { "authorization" : "into" }}}
{"type": "RECORD", "record": {"stream": "ProperCase", "emitted_at": 1602637589000, "data": { "ProperCase" : true }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : 101, "some_next_field" : "some_next_field_1", "some_field" : "some_field_1" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : 101, "some_field" : "some_field_1", "some_next_field" : "some_next_field_1" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589250, "data": { "some_id" : 102, "some_field" : "some_field_2" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589300, "data": { "some_id" : 103, "some_next_field" : "some_next_field_3" }}}
{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589350, "data": { "some_id" : 104 }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "next_field_name" : "next_field_name_1", "some_id" : 201 }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "next_field_name" : "next_field_name_2", "some_id" : 202 }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "some_id" : 201, "next_field_name" : "next_field_name_1" }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "some_id" : 202, "next_field_name" : "next_field_name_2" }}}
{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589500, "data": { "some_id" : 203 }}}
{"type": "RECORD", "record": {"stream": "stream_with_binary_data", "emitted_at": 1602637589500, "data": { "some_id" : 303, "binary_field_name":"dGVzdA==" }}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "2021-1-1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "1/1/2021"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "20210101"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "2021-01-01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "date" : "01/01/2021"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T09"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T0930"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021/01/01 09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T09:30"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 09:30:26.123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 09:30:26.123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010,123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "20210101T093010.123"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1 01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021 01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01-01:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01-01:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-1-1T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "1/1/2021T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01Z"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01Z"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +1:00"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01 01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021 01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 UTC"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01 +0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01+0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01+0000"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "2021-01-01T01:01:01+1"}}}
{"type": "RECORD", "record": {"stream": "stream_with_dates", "emitted_at": 1602637589000, "data": { "datetime" : "01/01/2021T01:01:01+1"}}}
{"type": "STATE", "state": { "data": {"start_date": "2020-09-02"}}}
46 changes: 0 additions & 46 deletions airbyte-integrations/bases/standard-destination-test/src/main/resources/format_reference.txt
View file

This file was deleted.

0 comments on commit c438e01

Please sign in to comment.