From 7f7b220f03676a56f943dfaa12755a8e92a65389 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 1 Dec 2021 22:14:20 +0900 Subject: [PATCH 01/13] add Dataset ID parse method --- .../bigquery/BigQueryDestination.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 99a3568ec5f2..8ff18a74c42c 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -50,6 +50,8 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +67,8 @@ public class BigQueryDestination extends BaseConnector implements Destination { private final BigQuerySQLNameTransformer namingResolver; + private static final Pattern datasetIdPattern = Pattern.compile("^(([a-z]([a-z0-9\\-]*[a-z0-9])?):)?([a-zA-Z0-9_]+)$"); + public BigQueryDestination() { namingResolver = new BigQuerySQLNameTransformer(); } @@ -72,7 +76,7 @@ public BigQueryDestination() { @Override public AirbyteConnectionStatus check(final JsonNode config) { try { - final String datasetId = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + final String datasetId = getDatasetId(config); final String datasetLocation = BigQueryUtils.getDatasetLocation(config); final BigQuery bigquery = getBigQuery(config); final UploadingMethod uploadingMethod = getLoadingMethod(config); @@ -277,7 +281,7 @@ protected Schema getBigQuerySchema(final JsonNode jsonSchema) { } private static String getSchema(final JsonNode config, final ConfiguredAirbyteStream stream) { - final String defaultSchema = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + final String defaultSchema = getDatasetId(config); final String srcNamespace = stream.getStream().getNamespace(); if (srcNamespace == null) { return defaultSchema; @@ -324,6 +328,27 @@ private boolean isKeepFilesInGcs(final JsonNode config) { } } + public static String getDatasetId(final JsonNode config) { + String datasetId = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + Matcher matcher = datasetIdPattern.matcher(datasetId); + + if (matcher.matches()) { + if (!isNull(matcher.group(1))) { + final String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + if (!(projectId.equals(matcher.group(2)))) { + throw new IllegalArgumentException(String.format( + "Project ID included in Dataset ID must match Project ID field's value: Project ID is %s, but you specified %s in Dataset ID", + projectId, + matcher.group(2))); + } + } + return matcher.group(4); + } + throw new IllegalArgumentException(String.format( + "BigQuery Dataset ID format must match '[project-id:]dataset_id': %s", + datasetId)); + } + public enum UploadingMethod { STANDARD, GCS From 69b510d917d661edc7ae75b1225a689f0155dd97 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 1 Dec 2021 22:16:01 +0900 Subject: [PATCH 02/13] add BigQuery Destination unit test --- .../bigquery/BigQueryDestinationTest.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java new file mode 100644 index 000000000000..9ce5c3b8047f --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.*; + +public class BigQueryDestinationTest { + + private ImmutableMap.Builder configMapBuilder; + + @BeforeEach + public void init() { + configMapBuilder = ImmutableMap.builder() + .put(BigQueryConsts.CONFIG_CREDS, "test_secret") + .put(BigQueryConsts.CONFIG_DATASET_LOCATION, "US"); + } + + public static Stream validBigQueryIdProvider() { + return Stream.of( + Arguments.arguments("my-project", "my_dataset", "my_dataset"), + Arguments.arguments("my-project", "my-project:my_dataset", "my_dataset")); + } + + @ParameterizedTest + @MethodSource("validBigQueryIdProvider") + public void testGetDatasetIdSuccess(String projectId, String datasetId, String expected) throws Exception { + JsonNode config = Jsons.jsonNode(configMapBuilder + .put(BigQueryConsts.CONFIG_PROJECT_ID, projectId) + .put(BigQueryConsts.CONFIG_DATASET_ID, datasetId) + .build()); + + String actual = BigQueryDestination.getDatasetId(config); + + assertEquals(expected, actual); + } + + public static Stream invalidBigQueryIdProvider() { + return Stream.of( + Arguments.arguments("my-project", ":my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': :my_dataset"), + Arguments.arguments("my-project", "your-project:my_dataset", + "Project ID included in Dataset ID must match Project ID field's value: Project ID is my-project, but you specified your-project in Dataset ID")); + } + + @ParameterizedTest + @MethodSource("invalidBigQueryIdProvider") + public void testGetDatasetIdFail(String projectId, String datasetId, String expected) throws Exception { + JsonNode config = Jsons.jsonNode(configMapBuilder + .put(BigQueryConsts.CONFIG_PROJECT_ID, projectId) + .put(BigQueryConsts.CONFIG_DATASET_ID, datasetId) + .build()); + + Exception exception = assertThrows(IllegalArgumentException.class, () -> BigQueryDestination.getDatasetId(config)); + + assertEquals(expected, exception.getMessage()); + } + +} From 5d99d33179c97f15985a0fab9ba72087b2030865 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 1 Dec 2021 23:17:58 +0900 Subject: [PATCH 03/13] update change log --- docs/integrations/destinations/bigquery.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 99873e08b91c..9bd52708e652 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -155,6 +155,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.5.1 | 2021-12-02 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.5.0 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | | 0.4.1 | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers | | 0.4.0 | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option | From d3d9cda49d128f82231d241586ea65d3febd0d3b Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 22 Dec 2021 14:27:44 +0000 Subject: [PATCH 04/13] fit to the latest code base --- .../bigquery/BigQueryDestination.java | 34 +++---------------- .../destination/bigquery/BigQueryUtils.java | 27 ++++++++++++++- ...nationTest.java => BigQueryUtilsTest.java} | 6 ++-- 3 files changed, 34 insertions(+), 33 deletions(-) rename airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/{BigQueryDestinationTest.java => BigQueryUtilsTest.java} (93%) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 1f29190b9baa..e243a59ef9b7 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -39,8 +39,6 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,8 +49,6 @@ public class BigQueryDestination extends BaseConnector implements Destination { private final BigQuerySQLNameTransformer namingResolver; - private static final Pattern datasetIdPattern = Pattern.compile("^(([a-z]([a-z0-9\\-]*[a-z0-9])?):)?([a-zA-Z0-9_]+)$"); - public BigQueryDestination() { namingResolver = new BigQuerySQLNameTransformer(); } @@ -60,7 +56,7 @@ public BigQueryDestination() { @Override public AirbyteConnectionStatus check(final JsonNode config) { try { - final String datasetId = getDatasetId(config); + final String datasetId = BigQueryUtils.getDatasetId(config); final String datasetLocation = BigQueryUtils.getDatasetLocation(config); final BigQuery bigquery = getBigQuery(config); final UploadingMethod uploadingMethod = BigQueryUtils.getLoadingMethod(config); @@ -180,9 +176,10 @@ protected Map> getUp } /** - * BigQuery might have different structure of the Temporary table. - * If this method returns TRUE, temporary table will have only three common Airbyte attributes. - * In case of FALSE, temporary table structure will be in line with Airbyte message JsonSchema. + * BigQuery might have different structure of the Temporary table. If this method returns TRUE, + * temporary table will have only three common Airbyte attributes. In case of FALSE, temporary table + * structure will be in line with Airbyte message JsonSchema. + * * @return use default AirbyteSchema or build using JsonSchema */ protected boolean isDefaultAirbyteTmpTableSchema() { @@ -199,27 +196,6 @@ protected String getTargetTableName(final String streamName) { return namingResolver.getRawTableName(streamName); } - public static String getDatasetId(final JsonNode config) { - String datasetId = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); - Matcher matcher = datasetIdPattern.matcher(datasetId); - - if (matcher.matches()) { - if (!isNull(matcher.group(1))) { - final String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); - if (!(projectId.equals(matcher.group(2)))) { - throw new IllegalArgumentException(String.format( - "Project ID included in Dataset ID must match Project ID field's value: Project ID is %s, but you specified %s in Dataset ID", - projectId, - matcher.group(2))); - } - } - return matcher.group(4); - } - throw new IllegalArgumentException(String.format( - "BigQuery Dataset ID format must match '[project-id:]dataset_id': %s", - datasetId)); - } - protected AirbyteMessageConsumer getRecordConsumer(final Map> writeConfigs, final Consumer outputRecordCollector) { return new BigQueryRecordConsumer(writeConfigs, outputRecordCollector); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index c4dbde0a84e9..072c617a3343 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.bigquery; import static io.airbyte.integrations.destination.bigquery.helpers.LoggerHelper.getJobErrorMessage; +import static java.util.Objects.isNull; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -37,6 +38,8 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.ImmutablePair; import org.joda.time.DateTime; import org.slf4j.Logger; @@ -46,6 +49,7 @@ public class BigQueryUtils { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryUtils.class); private static final String BIG_QUERY_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSSSS"; + private static final Pattern datasetIdPattern = Pattern.compile("^(([a-z]([a-z0-9\\-]*[a-z0-9])?):)?([a-zA-Z0-9_]+)$"); public static ImmutablePair executeQuery(final BigQuery bigquery, final QueryJobConfiguration queryConfig) { final JobId jobId = JobId.of(UUID.randomUUID().toString()); @@ -160,6 +164,27 @@ public static JsonNode getGcsAvroJsonNodeConfig(final JsonNode config) { return gcsJsonNode; } + public static String getDatasetId(final JsonNode config) { + String datasetId = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + Matcher matcher = datasetIdPattern.matcher(datasetId); + + if (matcher.matches()) { + if (!isNull(matcher.group(1))) { + final String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + if (!(projectId.equals(matcher.group(2)))) { + throw new IllegalArgumentException(String.format( + "Project ID included in Dataset ID must match Project ID field's value: Project ID is %s, but you specified %s in Dataset ID", + projectId, + matcher.group(2))); + } + } + return matcher.group(4); + } + throw new IllegalArgumentException(String.format( + "BigQuery Dataset ID format must match '[project-id:]dataset_id': %s", + datasetId)); + } + public static String getDatasetLocation(final JsonNode config) { if (config.has(BigQueryConsts.CONFIG_DATASET_LOCATION)) { return config.get(BigQueryConsts.CONFIG_DATASET_LOCATION).asText(); @@ -212,7 +237,7 @@ public static void transformJsonDateTimeToBigDataFormat(List dateTimeFie } public static String getSchema(final JsonNode config, final ConfiguredAirbyteStream stream) { - final String defaultSchema = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + final String defaultSchema = getDatasetId(config); final String srcNamespace = stream.getStream().getNamespace(); if (srcNamespace == null) { return defaultSchema; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java similarity index 93% rename from airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java rename to airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 9ce5c3b8047f..33cb6d55d334 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -14,7 +14,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.*; -public class BigQueryDestinationTest { +public class BigQueryUtilsTest { private ImmutableMap.Builder configMapBuilder; @@ -39,7 +39,7 @@ public void testGetDatasetIdSuccess(String projectId, String datasetId, String e .put(BigQueryConsts.CONFIG_DATASET_ID, datasetId) .build()); - String actual = BigQueryDestination.getDatasetId(config); + String actual = BigQueryUtils.getDatasetId(config); assertEquals(expected, actual); } @@ -59,7 +59,7 @@ public void testGetDatasetIdFail(String projectId, String datasetId, String expe .put(BigQueryConsts.CONFIG_DATASET_ID, datasetId) .build()); - Exception exception = assertThrows(IllegalArgumentException.class, () -> BigQueryDestination.getDatasetId(config)); + Exception exception = assertThrows(IllegalArgumentException.class, () -> BigQueryUtils.getDatasetId(config)); assertEquals(expected, exception.getMessage()); } From f4ced6218481d20ce5080e78684aa2749117f6df Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 22 Dec 2021 14:29:57 +0000 Subject: [PATCH 05/13] update change log --- docs/integrations/destinations/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index abdc8216093d..dbc44fe23b1b 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -155,7 +155,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| 0.6.1 | 2021-12-02 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | +| 0.6.1 | 2021-12-22 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.6.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files | | 0.5.1 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | | 0.5.0 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | From a3838ebbbdc777a70f181fca4dabb5a6b8978bfd Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 29 Dec 2021 02:54:21 +0000 Subject: [PATCH 06/13] change var name to const name --- .../integrations/destination/bigquery/BigQueryUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 072c617a3343..a1b4c33c17ae 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -49,7 +49,7 @@ public class BigQueryUtils { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryUtils.class); private static final String BIG_QUERY_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSSSS"; - private static final Pattern datasetIdPattern = Pattern.compile("^(([a-z]([a-z0-9\\-]*[a-z0-9])?):)?([a-zA-Z0-9_]+)$"); + private static final Pattern DATASET_ID_PATTERN = Pattern.compile("^(([a-z]([a-z0-9\\-]*[a-z0-9])?):)?([a-zA-Z0-9_]+)$"); public static ImmutablePair executeQuery(final BigQuery bigquery, final QueryJobConfiguration queryConfig) { final JobId jobId = JobId.of(UUID.randomUUID().toString()); @@ -166,7 +166,7 @@ public static JsonNode getGcsAvroJsonNodeConfig(final JsonNode config) { public static String getDatasetId(final JsonNode config) { String datasetId = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); - Matcher matcher = datasetIdPattern.matcher(datasetId); + Matcher matcher = DATASET_ID_PATTERN.matcher(datasetId); if (matcher.matches()) { if (!isNull(matcher.group(1))) { From af8079848399d9d6bff5e4718b2bc11663f1e26e Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 29 Dec 2021 02:59:16 +0000 Subject: [PATCH 07/13] change public method to private --- .../bigquery/BigQueryUtilsTest.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 33cb6d55d334..5182986011aa 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -24,13 +24,7 @@ public void init() { .put(BigQueryConsts.CONFIG_CREDS, "test_secret") .put(BigQueryConsts.CONFIG_DATASET_LOCATION, "US"); } - - public static Stream validBigQueryIdProvider() { - return Stream.of( - Arguments.arguments("my-project", "my_dataset", "my_dataset"), - Arguments.arguments("my-project", "my-project:my_dataset", "my_dataset")); - } - + @ParameterizedTest @MethodSource("validBigQueryIdProvider") public void testGetDatasetIdSuccess(String projectId, String datasetId, String expected) throws Exception { @@ -44,13 +38,6 @@ public void testGetDatasetIdSuccess(String projectId, String datasetId, String e assertEquals(expected, actual); } - public static Stream invalidBigQueryIdProvider() { - return Stream.of( - Arguments.arguments("my-project", ":my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': :my_dataset"), - Arguments.arguments("my-project", "your-project:my_dataset", - "Project ID included in Dataset ID must match Project ID field's value: Project ID is my-project, but you specified your-project in Dataset ID")); - } - @ParameterizedTest @MethodSource("invalidBigQueryIdProvider") public void testGetDatasetIdFail(String projectId, String datasetId, String expected) throws Exception { @@ -64,4 +51,16 @@ public void testGetDatasetIdFail(String projectId, String datasetId, String expe assertEquals(expected, exception.getMessage()); } + private static Stream validBigQueryIdProvider() { + return Stream.of( + Arguments.arguments("my-project", "my_dataset", "my_dataset"), + Arguments.arguments("my-project", "my-project:my_dataset", "my_dataset")); + } + + private static Stream invalidBigQueryIdProvider() { + return Stream.of( + Arguments.arguments("my-project", ":my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': :my_dataset"), + Arguments.arguments("my-project", "your-project:my_dataset", + "Project ID included in Dataset ID must match Project ID field's value: Project ID is my-project, but you specified your-project in Dataset ID")); + } } From 4c78cb50839ea03330ab0941a48470a32071d2ba Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Wed, 29 Dec 2021 03:12:43 +0000 Subject: [PATCH 08/13] add test cases for testGetDatasetIdFail --- .../integrations/destination/bigquery/BigQueryUtilsTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 5182986011aa..d5c2e3aad867 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -60,6 +60,9 @@ private static Stream validBigQueryIdProvider() { private static Stream invalidBigQueryIdProvider() { return Stream.of( Arguments.arguments("my-project", ":my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': :my_dataset"), + Arguments.arguments("my-project", "my-project:my-project:my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': my-project:my-project:my_dataset"), + Arguments.arguments("my-project", "my-project-:my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': my-project-:my_dataset"), + Arguments.arguments("my-project", "my-project:", "BigQuery Dataset ID format must match '[project-id:]dataset_id': my-project:"), Arguments.arguments("my-project", "your-project:my_dataset", "Project ID included in Dataset ID must match Project ID field's value: Project ID is my-project, but you specified your-project in Dataset ID")); } From 71718710a686e388fdf6f24c3c5b5f53b4aac717 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Thu, 30 Dec 2021 11:04:46 +0000 Subject: [PATCH 09/13] add integration test for dataset-id prefixed with project-id --- .../bigquery/BigQueryDestinationTest.java | 60 +++++++++++++++---- .../bigquery/BigQueryUtilsTest.java | 6 +- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index d364c479715b..e0d06cb86c18 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -55,13 +55,18 @@ import java.time.Instant; import java.util.List; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -197,16 +202,20 @@ void testSpec() throws Exception { assertEquals(expected, actual); } - @Test - void testCheckSuccess() { + @ParameterizedTest + @MethodSource("datasetIdResetterProvider") + void testCheckSuccess(DatasetIdResetter resetDatasetId) { + resetDatasetId.accept(config); final AirbyteConnectionStatus actual = new BigQueryDestination().check(config); final AirbyteConnectionStatus expected = new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); assertEquals(expected, actual); } - @Test - void testCheckFailure() { + @ParameterizedTest + @MethodSource("datasetIdResetterProvider") + void testCheckFailure(DatasetIdResetter resetDatasetId) { ((ObjectNode) config).put(BigQueryConsts.CONFIG_PROJECT_ID, "fake"); + resetDatasetId.accept(config); final AirbyteConnectionStatus actual = new BigQueryDestination().check(config); final String actualMessage = actual.getMessage(); LOGGER.info("Checking expected failure message:" + actualMessage); @@ -215,8 +224,10 @@ void testCheckFailure() { assertEquals(expected, actual.withMessage("")); } - @Test - void testWriteSuccess() throws Exception { + @ParameterizedTest + @MethodSource("datasetIdResetterProvider") + void testWriteSuccess(DatasetIdResetter resetDatasetId) throws Exception { + resetDatasetId.accept(config); final BigQueryDestination destination = new BigQueryDestination(); final AirbyteMessageConsumer consumer = destination.getConsumer(config, catalog, Destination::defaultOutputRecordCollector); @@ -244,8 +255,10 @@ void testWriteSuccess() throws Exception { .collect(Collectors.toList())); } - @Test - void testWriteFailure() throws Exception { + @ParameterizedTest + @MethodSource("datasetIdResetterProvider") + void testWriteFailure(DatasetIdResetter resetDatasetId) throws Exception { + resetDatasetId.accept(config); // hack to force an exception to be thrown from within the consumer. final AirbyteMessage spiedMessage = spy(MESSAGE_USERS1); doThrow(new RuntimeException()).when(spiedMessage).getRecord(); @@ -305,8 +318,10 @@ private List retrieveRecords(final String tableName) throws Exception .collect(Collectors.toList()); } - @Test - void testWritePartitionOverUnpartitioned() throws Exception { + @ParameterizedTest + @MethodSource("datasetIdResetterProvider") + void testWritePartitionOverUnpartitioned(DatasetIdResetter resetDatasetId) throws Exception { + resetDatasetId.accept(config); final String raw_table_name = String.format("_airbyte_raw_%s", USERS_STREAM_NAME); createUnpartitionedTable(bigquery, dataset, raw_table_name); assertFalse(isTablePartitioned(bigquery, dataset, raw_table_name)); @@ -369,4 +384,29 @@ private boolean isTablePartitioned(final BigQuery bigquery, final Dataset datase return false; } + private static class DatasetIdResetter { + private Consumer consumer; + + DatasetIdResetter(Consumer consumer) { + this.consumer = consumer; + } + + public void accept(JsonNode config) { + consumer.accept(config); + } + } + + private static Stream datasetIdResetterProvider() { + return Stream.of( + Arguments.arguments(new DatasetIdResetter(config -> {})), + Arguments.arguments(new DatasetIdResetter( + config -> { + String projectId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + String datasetId = ((ObjectNode) config).get(BigQueryConsts.CONFIG_DATASET_ID).asText(); + ((ObjectNode) config).put(BigQueryConsts.CONFIG_DATASET_ID, + String.format("%s:%s", projectId, datasetId)); + } + )) + ); + } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index d5c2e3aad867..86b608b4a1ae 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -4,7 +4,8 @@ package io.airbyte.integrations.destination.bigquery; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; @@ -12,7 +13,8 @@ import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.*; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; public class BigQueryUtilsTest { From c505823144b4d09eb0ac9e9fe8b176317250a993 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Thu, 6 Jan 2022 14:12:31 +0000 Subject: [PATCH 10/13] fix getDatasetId --- .../destination/bigquery/BigQueryUtils.java | 31 ++++++++----------- .../bigquery/BigQueryUtilsTest.java | 8 ++--- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index a1b4c33c17ae..6f506a7cd36f 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -38,8 +38,6 @@ import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.ImmutablePair; import org.joda.time.DateTime; import org.slf4j.Logger; @@ -49,7 +47,6 @@ public class BigQueryUtils { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryUtils.class); private static final String BIG_QUERY_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSSSS"; - private static final Pattern DATASET_ID_PATTERN = Pattern.compile("^(([a-z]([a-z0-9\\-]*[a-z0-9])?):)?([a-zA-Z0-9_]+)$"); public static ImmutablePair executeQuery(final BigQuery bigquery, final QueryJobConfiguration queryConfig) { final JobId jobId = JobId.of(UUID.randomUUID().toString()); @@ -166,23 +163,21 @@ public static JsonNode getGcsAvroJsonNodeConfig(final JsonNode config) { public static String getDatasetId(final JsonNode config) { String datasetId = config.get(BigQueryConsts.CONFIG_DATASET_ID).asText(); - Matcher matcher = DATASET_ID_PATTERN.matcher(datasetId); - - if (matcher.matches()) { - if (!isNull(matcher.group(1))) { - final String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); - if (!(projectId.equals(matcher.group(2)))) { - throw new IllegalArgumentException(String.format( - "Project ID included in Dataset ID must match Project ID field's value: Project ID is %s, but you specified %s in Dataset ID", - projectId, - matcher.group(2))); - } + + int colonIndex = datasetId.indexOf(":"); + if (colonIndex != -1) { + String projectIdPart = datasetId.substring(0, colonIndex); + String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + if (!(projectId.equals(projectIdPart))) { + throw new IllegalArgumentException(String.format( + "Project ID included in Dataset ID must match Project ID field's value: Project ID is `%s`, but you specified `%s` in Dataset ID", + projectId, + projectIdPart)); } - return matcher.group(4); } - throw new IllegalArgumentException(String.format( - "BigQuery Dataset ID format must match '[project-id:]dataset_id': %s", - datasetId)); + // if colonIndex is -1, then this returns the entire string + // otherwise it returns everything after the colon + return datasetId.substring(colonIndex + 1); } public static String getDatasetLocation(final JsonNode config) { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 86b608b4a1ae..586e0cf7ce74 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -61,11 +61,9 @@ private static Stream validBigQueryIdProvider() { private static Stream invalidBigQueryIdProvider() { return Stream.of( - Arguments.arguments("my-project", ":my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': :my_dataset"), - Arguments.arguments("my-project", "my-project:my-project:my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': my-project:my-project:my_dataset"), - Arguments.arguments("my-project", "my-project-:my_dataset", "BigQuery Dataset ID format must match '[project-id:]dataset_id': my-project-:my_dataset"), - Arguments.arguments("my-project", "my-project:", "BigQuery Dataset ID format must match '[project-id:]dataset_id': my-project:"), + Arguments.arguments("my-project", ":my_dataset", + "Project ID included in Dataset ID must match Project ID field's value: Project ID is `my-project`, but you specified `` in Dataset ID"), Arguments.arguments("my-project", "your-project:my_dataset", - "Project ID included in Dataset ID must match Project ID field's value: Project ID is my-project, but you specified your-project in Dataset ID")); + "Project ID included in Dataset ID must match Project ID field's value: Project ID is `my-project`, but you specified `your-project` in Dataset ID")); } } From 686e2c71a60853ed55dd5b870e064a024f009a4a Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Thu, 6 Jan 2022 14:23:17 +0000 Subject: [PATCH 11/13] add comment to parameterized test provider --- .../destination/bigquery/BigQueryDestinationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index e0d06cb86c18..70c7e9dd1627 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -397,6 +397,7 @@ public void accept(JsonNode config) { } private static Stream datasetIdResetterProvider() { + // parameterized test with two dataset-id patterns: `dataset_id` and `project-id:dataset_id` return Stream.of( Arguments.arguments(new DatasetIdResetter(config -> {})), Arguments.arguments(new DatasetIdResetter( From 6c26a63bf7aacb2baac330647f9079482a6c7b30 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Sun, 16 Jan 2022 04:51:22 +0000 Subject: [PATCH 12/13] update docker image versions --- .../079d5540-f236-4294-ba7c-ade8fd918496.json | 2 +- .../22f6c74f-5699-40ff-833c-4a879ea40133.json | 2 +- .../init/src/main/resources/seed/destination_definitions.yaml | 4 ++-- .../init/src/main/resources/seed/destination_specs.yaml | 4 ++-- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- .../connectors/destination-bigquery/Dockerfile | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json index aac58b6e8ee7..aee3fcbfd47a 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "079d5540-f236-4294-ba7c-ade8fd918496", "name": "BigQuery (denormalized typed struct)", "dockerRepository": "airbyte/destination-bigquery-denormalized", - "dockerImageTag": "0.2.2", + "dockerImageTag": "0.2.3", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "icon": "bigquery.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json index 1f67b317fd09..d69347400d6f 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", "name": "BigQuery", "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "0.6.1", + "dockerImageTag": "0.6.2", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "icon": "bigquery.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index eaae8e1430c8..12f930456673 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -13,13 +13,13 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 0.6.1 + dockerImageTag: 0.6.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg - name: Cassandra diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 938177c9485a..992117865908 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -176,7 +176,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-bigquery:0.6.1" +- dockerImage: "airbyte/destination-bigquery:0.6.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: @@ -364,7 +364,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.2" +- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index be38d6f1cd9d..45801e0f0113 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-bigquery-denormalized COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index b174182a586f..e4bb7588c7ca 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.1 +LABEL io.airbyte.version=0.6.2 LABEL io.airbyte.name=airbyte/destination-bigquery From 25bf8174e6272d38cf3f6c8aae6b0bfcc24a9da7 Mon Sep 17 00:00:00 2001 From: Koji Matsumoto Date: Sun, 16 Jan 2022 05:06:41 +0000 Subject: [PATCH 13/13] update docker image versions again --- .../079d5540-f236-4294-ba7c-ade8fd918496.json | 2 +- .../22f6c74f-5699-40ff-833c-4a879ea40133.json | 2 +- .../init/src/main/resources/seed/destination_definitions.yaml | 4 ++-- .../init/src/main/resources/seed/destination_specs.yaml | 4 ++-- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- .../connectors/destination-bigquery/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 3 ++- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json index aee3fcbfd47a..d433add5e807 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/079d5540-f236-4294-ba7c-ade8fd918496.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "079d5540-f236-4294-ba7c-ade8fd918496", "name": "BigQuery (denormalized typed struct)", "dockerRepository": "airbyte/destination-bigquery-denormalized", - "dockerImageTag": "0.2.3", + "dockerImageTag": "0.2.4", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "icon": "bigquery.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json index d69347400d6f..d076c305f7c2 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133", "name": "BigQuery", "dockerRepository": "airbyte/destination-bigquery", - "dockerImageTag": "0.6.2", + "dockerImageTag": "0.6.4", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", "icon": "bigquery.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 12f930456673..b79a44373870 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -13,13 +13,13 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 0.6.2 + dockerImageTag: 0.6.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery icon: bigquery.svg - name: Cassandra diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 992117865908..f1911ba84ddb 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -176,7 +176,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-bigquery:0.6.2" +- dockerImage: "airbyte/destination-bigquery:0.6.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: @@ -364,7 +364,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.3" +- dockerImage: "airbyte/destination-bigquery-denormalized:0.2.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 45801e0f0113..11d11d4387ad 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-bigquery-denormalized COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index e4bb7588c7ca..ef038a965403 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.2 +LABEL io.airbyte.version=0.6.4 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index bf42c5043cbd..8da139e7d294 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -146,7 +146,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| 0.6.2 | 2021-12-22 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | +| 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.6.1 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | | 0.6.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files | | 0.5.1 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | @@ -164,6 +164,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.2.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | BigQuery/BiqQuery denorm Destinations : Support dataset-id prefixed by project-id | | 0.2.2 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | | 0.2.1 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | | 0.2.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/pull/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files |