🐛 Source Snowflake: Fixed parsing of extreme values for FLOAT and NUM…

…BER data types (airbytehq#7257)

* Fixed parsing of extreme values for FLOAT and NUMBER data types

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2",
   "name": "Snowflake",
   "dockerRepository": "airbyte/source-snowflake",
-  "dockerImageTag": "0.1.1",
+  "dockerImageTag": "0.1.2",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/snowflake"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -392,7 +392,7 @@
 - sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   name: Snowflake
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake
   sourceType: database
 - sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2

airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java

@@ -909,10 +909,9 @@ public void dropSchemas() throws SQLException {
   }
 
   private JsonNode convertIdBasedOnDatabase(final int idValue) {
-    if (getDriverClass().toLowerCase().contains("oracle")) {
+    final var driverClass = getDriverClass().toLowerCase();
+    if (driverClass.contains("oracle") || driverClass.contains("snowflake")) {
       return Jsons.jsonNode(BigDecimal.valueOf(idValue));
-    } else if (getDriverClass().toLowerCase().contains("snowflake")) {
-      return Jsons.jsonNode(Long.valueOf(idValue));
     } else {
       return Jsons.jsonNode(idValue);
     }

airbyte-integrations/connectors/source-snowflake/Dockerfile

@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-snowflake

airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java

@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
+import io.airbyte.db.jdbc.JdbcSourceOperations;
 import io.airbyte.integrations.base.IntegrationRunner;
 import io.airbyte.integrations.base.Source;
 import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
@@ -20,7 +21,7 @@ public class SnowflakeSource extends AbstractJdbcSource implements Source {
   public static final String DRIVER_CLASS = "net.snowflake.client.jdbc.SnowflakeDriver";
 
   public SnowflakeSource() {
-    super(DRIVER_CLASS, new SnowflakeJdbcStreamingQueryConfiguration());
+    super(DRIVER_CLASS, new SnowflakeJdbcStreamingQueryConfiguration(), new SnowflakeSourceOperations());
   }
 
   public static void main(final String[] args) throws Exception {
@@ -52,4 +53,9 @@ public Set<String> getExcludedInternalNameSpaces() {
         "INFORMATION_SCHEMA");
   }
 
+  @Override
+  protected JdbcSourceOperations getSourceOperations() {
+    return new SnowflakeSourceOperations();
+  }
+
 }

airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.source.snowflake;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.airbyte.db.jdbc.JdbcSourceOperations;
+import java.math.BigDecimal;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class SnowflakeSourceOperations extends JdbcSourceOperations {
+
+  @Override
+  protected void putDouble(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) {
+    try {
+      final double value = resultSet.getDouble(index);
+      node.put(columnName, value);
+    } catch (final SQLException e) {
+      node.put(columnName, (Double) null);
+    }
+  }
+
+  @Override
+  protected void putBigInt(final ObjectNode node, final String columnName, final ResultSet resultSet, int index) {
+    try {
+      final var value = resultSet.getBigDecimal(index);
+      node.put(columnName, value);
+    } catch (final SQLException e) {
+      node.put(columnName, (BigDecimal) null);
+    }
+  }
+
+}

airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java

@@ -11,6 +11,7 @@
 import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
 import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest;
 import io.airbyte.integrations.source.snowflake.SnowflakeSource;
+import java.math.BigDecimal;
 import java.nio.file.Path;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
@@ -42,11 +43,11 @@ public void setup() throws Exception {
     COL_FIRST_NAME = "FIRST_NAME";
     COL_LAST_NAME = "LAST_NAME";
     COL_LAST_NAME_WITH_SPACE = "LAST NAME";
-    ID_VALUE_1 = 1L;
-    ID_VALUE_2 = 2L;
-    ID_VALUE_3 = 3L;
-    ID_VALUE_4 = 4L;
-    ID_VALUE_5 = 5L;
+    ID_VALUE_1 = new BigDecimal(1);
+    ID_VALUE_2 = new BigDecimal(2);
+    ID_VALUE_3 = new BigDecimal(3);
+    ID_VALUE_4 = new BigDecimal(4);
+    ID_VALUE_5 = new BigDecimal(5);
 
     super.setup();
   }

airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java

@@ -58,7 +58,7 @@ JsonNode getStaticConfig() {
   }
 
   @Override
-  protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception {
+  protected ConfiguredAirbyteCatalog getConfiguredCatalog() {
     return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(
         new ConfiguredAirbyteStream()
             .withSyncMode(SyncMode.INCREMENTAL)
@@ -82,12 +82,12 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception {
   }
 
   @Override
-  protected JsonNode getState() throws Exception {
+  protected JsonNode getState() {
     return Jsons.jsonNode(new HashMap<>());
   }
 
   @Override
-  protected List<String> getRegexTests() throws Exception {
+  protected List<String> getRegexTests() {
     return Collections.emptyList();
   }
 

airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java

@@ -31,7 +31,7 @@ protected String getImageName() {
   }
 
   @Override
-  protected JsonNode getConfig() throws Exception {
+  protected JsonNode getConfig() {
     return config;
   }
 
@@ -86,15 +86,12 @@ protected String getTestColumnName() {
 
   @Override
   protected void initTests() {
-    // TODO https://github.com/airbytehq/airbyte/issues/4316
-    // should be tested with Snowflake extreme range -99999999999999999999999999999999999999 to
-    // +99999999999999999999999999999999999999 (inclusive)
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("NUMBER")
             .airbyteType(JsonSchemaPrimitive.NUMBER)
-            .addInsertValues("null", "9223372036854775807", "-9223372036854775808")
-            .addExpectedValues(null, "9223372036854775807", "-9223372036854775808")
+            .addInsertValues("null", "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999","9223372036854775807", "-9223372036854775808")
+            .addExpectedValues(null, "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999","9223372036854775807", "-9223372036854775808")
             .build());
     addDataTypeTestData(
         TestDataHolder.builder()
@@ -107,15 +104,15 @@ protected void initTests() {
         TestDataHolder.builder()
             .sourceType("NUMERIC")
             .airbyteType(JsonSchemaPrimitive.NUMBER)
-            .addInsertValues("null", "9223372036854775807", "-9223372036854775808")
-            .addExpectedValues(null, "9223372036854775807", "-9223372036854775808")
+            .addInsertValues("null", "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999", "9223372036854775807", "-9223372036854775808")
+            .addExpectedValues(null, "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999", "9223372036854775807", "-9223372036854775808")
             .build());
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("BIGINT")
             .airbyteType(JsonSchemaPrimitive.NUMBER)
-            .addInsertValues("null", "9223372036854775807", "-9223372036854775808")
-            .addExpectedValues(null, "9223372036854775807", "-9223372036854775808")
+            .addInsertValues("null", "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999")
+            .addExpectedValues(null, "99999999999999999999999999999999999999", "-99999999999999999999999999999999999999")
             .build());
     addDataTypeTestData(
         TestDataHolder.builder()
@@ -174,13 +171,12 @@ protected void initTests() {
             .addInsertValues("10e-308", "10e+307")
             .addExpectedValues("1.0E-307", "1.0E308")
             .build());
-    // TODO should be fixed in scope of https://github.com/airbytehq/airbyte/issues/4316
     addDataTypeTestData(
         TestDataHolder.builder()
             .sourceType("FLOAT")
             .airbyteType(JsonSchemaPrimitive.NUMBER)
             .addInsertValues("'NaN'", "'inf'", "'-inf'")
-            .addExpectedValues(null, null, null)
+            .addExpectedValues("NaN", "Infinity", "-Infinity")
             .build());
 
     // Data Types for Text Strings

docs/integrations/sources/snowflake.md

@@ -75,5 +75,6 @@ Your database user should now be ready for use with Airbyte.
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.2 | 2021-10-21 | [7257](https://github.com/airbytehq/airbyte/pull/7257) | Fixed parsing of extreme values for FLOAT and NUMBER data types |
 | 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |