diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 3ed694a98256..decf0a189ce3 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -232,6 +232,15 @@ public static Map flatten(final JsonNode node) { mergeMaps(output, field, flatten(value)); } return output; + } else if (node.isArray()) { + final Map output = new HashMap<>(); + final int arrayLen = node.size(); + for (int i = 0; i < arrayLen; i++) { + final String field = String.format("[%d]", i); + final JsonNode value = node.get(i); + mergeMaps(output, field, flatten(value)); + } + return output; } else { final Object value; if (node.isBoolean()) { @@ -245,7 +254,7 @@ public static Map flatten(final JsonNode node) { } else if (node.isValueNode() && !node.isNull()) { value = node.asText(); } else { - // Fallback handling for e.g. arrays + // Fallback handling value = node.toString(); } return singletonMap(null, value); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java index 2c331417058f..176b28b43875 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java @@ -21,8 +21,11 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; class JsonsTest { @@ -251,6 +254,29 @@ void testGetEstimatedByteSize() { final JsonNode json = Jsons.deserialize("{\"string_key\":\"abc\",\"array_key\":[\"item1\", \"item2\"]}"); assertEquals(Jsons.toBytes(json).length, Jsons.getEstimatedByteSize(json)); } + + @Test + void testFlatten__noArrays() { + final JsonNode json = Jsons.deserialize("{ \"abc\": { \"def\": \"ghi\" }, \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc.def", "ghi" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(Jsons.flatten(json), expected); + } + + @Test + void testFlatten__withArrays() { + final JsonNode json = Jsons.deserialize("{ \"abc\": [{ \"def\": \"ghi\" }, { \"fed\": \"ihg\" }], \"jkl\": true, \"pqr\": 1 }"); + Map expected = Stream.of(new Object[][] { + { "abc.[0].def", "ghi" }, + { "abc.[1].fed", "ihg" }, + { "jkl", true }, + { "pqr", 1 }, + }).collect(Collectors.toMap(data -> (String) data[0], data -> data[1])); + assertEquals(Jsons.flatten(json), expected); + } private static class ToClass { diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 572d2277d3d8..4cd024817195 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -225,7 +225,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.37 + dockerImageTag: 0.3.38 documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift icon: redshift.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index d94534629fd7..a2c3fa302824 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3629,7 +3629,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.37" +- dockerImage: "airbyte/destination-redshift:0.3.38" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index a6bf71cbe1eb..4773167545b7 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.37 +LABEL io.airbyte.version=0.3.38 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index c3d67c16d0ef..a54a6a70024f 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -138,6 +138,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.38 | 2022-05-25 | [13069](https://github.com/airbytehq/airbyte/pull/13069) | Fixed array contents verification for SUPER type | | 0.3.37 | 2022-05-23 | [13090](https://github.com/airbytehq/airbyte/pull/13090) | Removed redshiftDataTmpTableMode. Some refactoring. | | 0.3.36 | 2022-05-23 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.3.35 | 2022-05-18 | [12940](https://github.com/airbytehq/airbyte/pull/12940) | Fixed maximum record size for SUPER type |