🎉 Make credentials optional in BigQuery connector (issue #3657) (#3947)

* Make credentials optional for BigQuery connector * Bump destination-bigquery version 0.3.5 -> 0.3.6 Co-authored-by: Sabolc Franjo <[email protected]>
airbytehq · Jun 17, 2021 · c336bfb · c336bfb
1 parent f365c0c
commit c336bfb
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 14 deletions.
diff --git a/...esources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/...esources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json
@@ -2,6 +2,6 @@
   "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133",
   "name": "BigQuery",
   "dockerRepository": "airbyte/destination-bigquery",
-  "dockerImageTag": "0.3.5",
+  "dockerImageTag": "0.3.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -17,7 +17,7 @@
 - destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
   name: BigQuery
   dockerRepository: airbyte/destination-bigquery
-  dockerImageTag: 0.3.5
+  dockerImageTag: 0.3.6
   documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
 - destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   name: Snowflake

diff --git a/airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md b/airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md
@@ -2,3 +2,6 @@
 
 ## 0.3.4
 Added option to choose dataset location
+
+## 0.3.6
+Service account credentials are now optional. Default credentials will be used if the field is empty.
diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile
@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.5
+LABEL io.airbyte.version=0.3.6
 LABEL io.airbyte.name=airbyte/destination-bigquery
diff --git a/...query/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/...query/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java
@@ -69,6 +69,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.util.Objects.isNull;
+
 public class BigQueryDestination extends BaseConnector implements Destination {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestination.class);
@@ -130,24 +132,32 @@ private void createSchemaTable(BigQuery bigquery, String datasetId, String datas
 
   private BigQuery getBigQuery(JsonNode config) {
     final String projectId = config.get(CONFIG_PROJECT_ID).asText();
-    // handle the credentials json being passed as a json object or a json object already serialized as
-    // a string.
-    final String credentialsString =
-        config.get(CONFIG_CREDS).isObject() ? Jsons.serialize(config.get(CONFIG_CREDS)) : config.get(CONFIG_CREDS).asText();
-    try {
-      final ServiceAccountCredentials credentials = ServiceAccountCredentials
-          .fromStream(new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8)));
 
-      return BigQueryOptions.newBuilder()
+    try {
+      BigQueryOptions.Builder bigQueryBuilder = BigQueryOptions.newBuilder();
+      ServiceAccountCredentials credentials = null;
+      if (isUsingJsonCredentials(config)) {
+        // handle the credentials json being passed as a json object or a json object already serialized as
+        // a string.
+        final String credentialsString =
+            config.get(CONFIG_CREDS).isObject() ? Jsons.serialize(config.get(CONFIG_CREDS)) : config.get(CONFIG_CREDS).asText();
+        credentials = ServiceAccountCredentials
+            .fromStream(new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8)));
+      }
+      return bigQueryBuilder
           .setProjectId(projectId)
-          .setCredentials(credentials)
+          .setCredentials(!isNull(credentials) ? credentials : ServiceAccountCredentials.getApplicationDefault())
           .build()
           .getService();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  public static boolean isUsingJsonCredentials(JsonNode config) {
+    return config.has(CONFIG_CREDS) && !config.get(CONFIG_CREDS).asText().isEmpty();
+  }
+
   /**
    * Strategy:
    * <p>

diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json
@@ -8,7 +8,7 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "BigQuery Destination Spec",
     "type": "object",
-    "required": ["project_id", "dataset_id", "credentials_json"],
+    "required": ["project_id", "dataset_id"],
     "additionalProperties": false,
     "properties": {
       "project_id": {
@@ -58,7 +58,7 @@
       },
       "credentials_json": {
         "type": "string",
-        "description": "The contents of the JSON service account key. Check out the <a href=\"https://docs.airbyte.io/integrations/destinations/bigquery\">docs</a> if you need help generating this key.",
+        "description": "The contents of the JSON service account key. Check out the <a href=\"https://docs.airbyte.io/integrations/destinations/bigquery\">docs</a> if you need help generating this key. Default credentials will be used if this field is left empty.",
         "title": "Credentials JSON",
         "airbyte_secret": true
       }