From 347861a2a9989b69d237b2fd8a032d1a10b66ca7 Mon Sep 17 00:00:00 2001 From: irynakruk Date: Fri, 22 Oct 2021 15:48:55 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Snowflake:=20Fixed=20pa?= =?UTF-8?q?rsing=20of=20extreme=20values=20for=20FLOAT=20and=20NUMBER=20da?= =?UTF-8?q?ta=20types=20(#7257)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixed parsing of extreme values for FLOAT and NUMBER data types --- .../e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../jdbc/test/JdbcSourceAcceptanceTest.java | 5 ++- .../connectors/source-snowflake/Dockerfile | 2 +- .../SnowflakeSource.java | 8 ++++- .../SnowflakeSourceOperations.java | 35 +++++++++++++++++++ .../SnowflakeJdbcSourceAcceptanceTest.java | 11 +++--- .../SnowflakeSourceAcceptanceTest.java | 6 ++-- .../sources/SnowflakeSourceDatatypeTest.java | 20 +++++------ docs/integrations/sources/snowflake.md | 1 + 10 files changed, 65 insertions(+), 27 deletions(-) create mode 100644 airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json index 031c66155961..411000da6023 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", "name": "Snowflake", "dockerRepository": "airbyte/source-snowflake", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 91fec2833fbc..1096397cea85 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -392,7 +392,7 @@ - sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 name: Snowflake dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake sourceType: database - sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index d8875b49227a..24246c42ee76 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -909,10 +909,9 @@ public void dropSchemas() throws SQLException { } private JsonNode convertIdBasedOnDatabase(final int idValue) { - if (getDriverClass().toLowerCase().contains("oracle")) { + final var driverClass = getDriverClass().toLowerCase(); + if (driverClass.contains("oracle") || driverClass.contains("snowflake")) { return Jsons.jsonNode(BigDecimal.valueOf(idValue)); - } else if (getDriverClass().toLowerCase().contains("snowflake")) { - return Jsons.jsonNode(Long.valueOf(idValue)); } else { return Jsons.jsonNode(idValue); } diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 5f7b64c4bd78..47435ff39edd 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index 45e9d44ffcbf..2bd067641e4b 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; @@ -20,7 +21,7 @@ public class SnowflakeSource extends AbstractJdbcSource implements Source { public static final String DRIVER_CLASS = "net.snowflake.client.jdbc.SnowflakeDriver"; public SnowflakeSource() { - super(DRIVER_CLASS, new SnowflakeJdbcStreamingQueryConfiguration()); + super(DRIVER_CLASS, new SnowflakeJdbcStreamingQueryConfiguration(), new SnowflakeSourceOperations()); } public static void main(final String[] args) throws Exception { @@ -52,4 +53,9 @@ public Set getExcludedInternalNameSpaces() { "INFORMATION_SCHEMA"); } + @Override + protected JdbcSourceOperations getSourceOperations() { + return new SnowflakeSourceOperations(); + } + } diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java new file mode 100644 index 000000000000..0affeee29d8c --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.snowflake; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.db.jdbc.JdbcSourceOperations; +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SnowflakeSourceOperations extends JdbcSourceOperations { + + @Override + protected void putDouble(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) { + try { + final double value = resultSet.getDouble(index); + node.put(columnName, value); + } catch (final SQLException e) { + node.put(columnName, (Double) null); + } + } + + @Override + protected void putBigInt(final ObjectNode node, final String columnName, final ResultSet resultSet, int index) { + try { + final var value = resultSet.getBigDecimal(index); + node.put(columnName, value); + } catch (final SQLException e) { + node.put(columnName, (BigDecimal) null); + } + } + +} diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java index 5748066ee2cf..da33400f7afa 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java @@ -11,6 +11,7 @@ import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.snowflake.SnowflakeSource; +import java.math.BigDecimal; import java.nio.file.Path; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -42,11 +43,11 @@ public void setup() throws Exception { COL_FIRST_NAME = "FIRST_NAME"; COL_LAST_NAME = "LAST_NAME"; COL_LAST_NAME_WITH_SPACE = "LAST NAME"; - ID_VALUE_1 = 1L; - ID_VALUE_2 = 2L; - ID_VALUE_3 = 3L; - ID_VALUE_4 = 4L; - ID_VALUE_5 = 5L; + ID_VALUE_1 = new BigDecimal(1); + ID_VALUE_2 = new BigDecimal(2); + ID_VALUE_3 = new BigDecimal(3); + ID_VALUE_4 = new BigDecimal(4); + ID_VALUE_5 = new BigDecimal(5); super.setup(); } diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java index 7c7aca2d2e27..515981840e4a 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java @@ -58,7 +58,7 @@ JsonNode getStaticConfig() { } @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) @@ -82,12 +82,12 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { } @Override - protected JsonNode getState() throws Exception { + protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } @Override - protected List getRegexTests() throws Exception { + protected List getRegexTests() { return Collections.emptyList(); } diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java index 0a94a6ffe873..98873ee7c82f 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java @@ -31,7 +31,7 @@ protected String getImageName() { } @Override - protected JsonNode getConfig() throws Exception { + protected JsonNode getConfig() { return config; } @@ -86,15 +86,12 @@ protected String getTestColumnName() { @Override protected void initTests() { - // TODO https://github.com/airbytehq/airbyte/issues/4316 - // should be tested with Snowflake extreme range -99999999999999999999999999999999999999 to - // +99999999999999999999999999999999999999 (inclusive) addDataTypeTestData( TestDataHolder.builder() .sourceType("NUMBER") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null", "9223372036854775807", "-9223372036854775808") - .addExpectedValues(null, "9223372036854775807", "-9223372036854775808") + .addInsertValues("null", "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999","9223372036854775807", "-9223372036854775808") + .addExpectedValues(null, "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999","9223372036854775807", "-9223372036854775808") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -107,15 +104,15 @@ protected void initTests() { TestDataHolder.builder() .sourceType("NUMERIC") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null", "9223372036854775807", "-9223372036854775808") - .addExpectedValues(null, "9223372036854775807", "-9223372036854775808") + .addInsertValues("null", "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999", "9223372036854775807", "-9223372036854775808") + .addExpectedValues(null, "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999", "9223372036854775807", "-9223372036854775808") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("BIGINT") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null", "9223372036854775807", "-9223372036854775808") - .addExpectedValues(null, "9223372036854775807", "-9223372036854775808") + .addInsertValues("null", "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999") + .addExpectedValues(null, "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -174,13 +171,12 @@ protected void initTests() { .addInsertValues("10e-308", "10e+307") .addExpectedValues("1.0E-307", "1.0E308") .build()); - // TODO should be fixed in scope of https://github.com/airbytehq/airbyte/issues/4316 addDataTypeTestData( TestDataHolder.builder() .sourceType("FLOAT") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("'NaN'", "'inf'", "'-inf'") - .addExpectedValues(null, null, null) + .addExpectedValues("NaN", "Infinity", "-Infinity") .build()); // Data Types for Text Strings diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 7fd16511b834..79097f57beb3 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -75,5 +75,6 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types | | 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |