From c336bfb2ca65cc33df1d233e0dd9e73420532fd5 Mon Sep 17 00:00:00 2001 From: sabifranjo Date: Fri, 18 Jun 2021 01:59:16 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Make=20credentials=20optional=20?= =?UTF-8?q?in=20BigQuery=20connector=20(issue=20#3657)=20(#3947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make credentials optional for BigQuery connector * Bump destination-bigquery version 0.3.5 -> 0.3.6 Co-authored-by: Sabolc Franjo --- .../22f6c74f-5699-40ff-833c-4a879ea40133.json | 2 +- .../seed/destination_definitions.yaml | 2 +- .../destination-bigquery/CHANGELOG.md | 3 ++ .../destination-bigquery/Dockerfile | 2 +- .../bigquery/BigQueryDestination.java | 28 +++++++++++++------ .../src/main/resources/spec.json | 4 +-- 6 files changed, 27 insertions(+), 14 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json index 5edd040ea834..68ed770b76d9 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", "name": "BigQuery", "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "0.3.5", + "dockerImageTag": "0.3.6", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 360c94c7ed5d..689218d03b2d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -17,7 +17,7 @@ - destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 name: BigQuery dockerRepository: airbyte/destination-bigquery - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.6 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery - destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba name: Snowflake diff --git a/airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md b/airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md index bab869ae8350..3e8d2bd40c46 100644 --- a/airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md +++ b/airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md @@ -2,3 +2,6 @@ ## 0.3.4 Added option to choose dataset location + +## 0.3.6 +Service account credentials are now optional. Default credentials will be used if the field is empty. \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 28da43c726e4..872d77ffe160 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.5 +LABEL io.airbyte.version=0.3.6 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 02e17cc4660c..0f71a0c0ad0e 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -69,6 +69,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.util.Objects.isNull; + public class BigQueryDestination extends BaseConnector implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestination.class); @@ -130,17 +132,21 @@ private void createSchemaTable(BigQuery bigquery, String datasetId, String datas private BigQuery getBigQuery(JsonNode config) { final String projectId = config.get(CONFIG_PROJECT_ID).asText(); - // handle the credentials json being passed as a json object or a json object already serialized as - // a string. - final String credentialsString = - config.get(CONFIG_CREDS).isObject() ? Jsons.serialize(config.get(CONFIG_CREDS)) : config.get(CONFIG_CREDS).asText(); - try { - final ServiceAccountCredentials credentials = ServiceAccountCredentials - .fromStream(new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8))); - return BigQueryOptions.newBuilder() + try { + BigQueryOptions.Builder bigQueryBuilder = BigQueryOptions.newBuilder(); + ServiceAccountCredentials credentials = null; + if (isUsingJsonCredentials(config)) { + // handle the credentials json being passed as a json object or a json object already serialized as + // a string. + final String credentialsString = + config.get(CONFIG_CREDS).isObject() ? Jsons.serialize(config.get(CONFIG_CREDS)) : config.get(CONFIG_CREDS).asText(); + credentials = ServiceAccountCredentials + .fromStream(new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8))); + } + return bigQueryBuilder .setProjectId(projectId) - .setCredentials(credentials) + .setCredentials(!isNull(credentials) ? credentials : ServiceAccountCredentials.getApplicationDefault()) .build() .getService(); } catch (IOException e) { @@ -148,6 +154,10 @@ private BigQuery getBigQuery(JsonNode config) { } } + public static boolean isUsingJsonCredentials(JsonNode config) { + return config.has(CONFIG_CREDS) && !config.get(CONFIG_CREDS).asText().isEmpty(); + } + /** * Strategy: *

diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index cc8fc4eede2b..23b20e245d6d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -8,7 +8,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "BigQuery Destination Spec", "type": "object", - "required": ["project_id", "dataset_id", "credentials_json"], + "required": ["project_id", "dataset_id"], "additionalProperties": false, "properties": { "project_id": { @@ -58,7 +58,7 @@ }, "credentials_json": { "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", + "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", "title": "Credentials JSON", "airbyte_secret": true }