From bbb86fd8488ad253f2e9cf3fb08360330bd860a3 Mon Sep 17 00:00:00 2001 From: Obada Alabbadi <76101898+obada-ab@users.noreply.github.com> Date: Wed, 13 Sep 2023 19:04:15 +0200 Subject: [PATCH] feat: add support for ExternalDatasetReference (#2871) API: https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#externaldatasetreference --- .../clirr-ignored-differences.xml | 5 ++ .../com/google/cloud/bigquery/Dataset.java | 6 ++ .../google/cloud/bigquery/DatasetInfo.java | 33 ++++++++ .../bigquery/ExternalDatasetReference.java | 79 +++++++++++++++++++ .../cloud/bigquery/DatasetInfoTest.java | 39 +++++++++ .../google/cloud/bigquery/DatasetTest.java | 31 ++++++++ .../ExternalDatasetReferenceTest.java | 70 ++++++++++++++++ pom.xml | 2 +- 8 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalDatasetReference.java create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalDatasetReferenceTest.java diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 41067bd21..eaeeded13 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -104,4 +104,9 @@ com/google/cloud/bigquery/IndexUnusedReason* *BaseTable(*) + + 7013 + com/google/cloud/bigquery/DatasetInfo* + *setExternalDatasetReference(*) + \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java index 5f9fea892..3ed4b8928 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java @@ -152,6 +152,12 @@ public Builder setDefaultCollation(String defaultCollation) { return this; } + @Override + public Builder setExternalDatasetReference(ExternalDatasetReference externalDatasetReference) { + infoBuilder.setExternalDatasetReference(externalDatasetReference); + return this; + } + @Override public Dataset build() { return new Dataset(bigquery, infoBuilder); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java index bd209952e..44583e0c8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java @@ -73,6 +73,7 @@ public Dataset apply(DatasetInfo datasetInfo) { private final EncryptionConfiguration defaultEncryptionConfiguration; private final Long defaultPartitionExpirationMs; private final String defaultCollation; + private final ExternalDatasetReference externalDatasetReference; /** A builder for {@code DatasetInfo} objects. */ public abstract static class Builder { @@ -127,6 +128,13 @@ public abstract static class Builder { public abstract Builder setLabels(Map labels); + /** + * Optional. Information about the external metadata storage where the dataset is defined. + * Filled out when the dataset type is EXTERNAL + */ + public abstract Builder setExternalDatasetReference( + ExternalDatasetReference externalDatasetReference); + /** * The default encryption key for all tables in the dataset. Once this property is set, all * newly-created partitioned tables in the dataset will have encryption key set to this value, @@ -183,6 +191,7 @@ static final class BuilderImpl extends Builder { private EncryptionConfiguration defaultEncryptionConfiguration; private Long defaultPartitionExpirationMs; private String defaultCollation; + private ExternalDatasetReference externalDatasetReference; BuilderImpl() {} @@ -202,6 +211,7 @@ static final class BuilderImpl extends Builder { this.defaultEncryptionConfiguration = datasetInfo.defaultEncryptionConfiguration; this.defaultPartitionExpirationMs = datasetInfo.defaultPartitionExpirationMs; this.defaultCollation = datasetInfo.defaultCollation; + this.externalDatasetReference = datasetInfo.externalDatasetReference; } BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) { @@ -236,6 +246,10 @@ public Acl apply(Dataset.Access accessPb) { } this.defaultPartitionExpirationMs = datasetPb.getDefaultPartitionExpirationMs(); this.defaultCollation = datasetPb.getDefaultCollation(); + if (datasetPb.getExternalDatasetReference() != null) { + this.externalDatasetReference = + ExternalDatasetReference.fromPb(datasetPb.getExternalDatasetReference()); + } } @Override @@ -336,6 +350,12 @@ public Builder setDefaultCollation(String defaultCollation) { return this; } + @Override + public Builder setExternalDatasetReference(ExternalDatasetReference externalDatasetReference) { + this.externalDatasetReference = externalDatasetReference; + return this; + } + @Override public DatasetInfo build() { return new DatasetInfo(this); @@ -358,6 +378,7 @@ public DatasetInfo build() { defaultEncryptionConfiguration = builder.defaultEncryptionConfiguration; defaultPartitionExpirationMs = builder.defaultPartitionExpirationMs; defaultCollation = builder.defaultCollation; + externalDatasetReference = builder.externalDatasetReference; } /** Returns the dataset identity. */ @@ -487,6 +508,14 @@ public String getDefaultCollation() { return defaultCollation; } + /** + * Returns information about the external metadata storage where the dataset is defined. Filled + * out when the dataset type is EXTERNAL. + */ + public ExternalDatasetReference getExternalDatasetReference() { + return externalDatasetReference; + } + /** Returns a builder for the dataset object. */ public Builder toBuilder() { return new BuilderImpl(this); @@ -510,6 +539,7 @@ public String toString() { .add("defaultEncryptionConfiguration", defaultEncryptionConfiguration) .add("defaultPartitionExpirationMs", defaultPartitionExpirationMs) .add("defaultCollation", defaultCollation) + .add("externalDatasetReference", externalDatasetReference) .toString(); } @@ -588,6 +618,9 @@ public Dataset.Access apply(Acl acl) { if (defaultCollation != null) { datasetPb.setDefaultCollation(defaultCollation); } + if (externalDatasetReference != null) { + datasetPb.setExternalDatasetReference(externalDatasetReference.toPb()); + } return datasetPb; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalDatasetReference.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalDatasetReference.java new file mode 100644 index 000000000..ecfe54c50 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalDatasetReference.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import java.io.Serializable; +import javax.annotation.Nullable; + +/** Configures the access a dataset defined in an external metadata storage. */ +@AutoValue +public abstract class ExternalDatasetReference implements Serializable { + + public static ExternalDatasetReference.Builder newBuilder() { + return new AutoValue_ExternalDatasetReference.Builder(); + } + + static ExternalDatasetReference fromPb( + com.google.api.services.bigquery.model.ExternalDatasetReference externalDatasetReference) { + ExternalDatasetReference.Builder builder = newBuilder(); + + if (externalDatasetReference.getConnection() != null) { + builder.setConnection(externalDatasetReference.getConnection()); + } + if (externalDatasetReference.getExternalSource() != null) { + builder.setExternalSource(externalDatasetReference.getExternalSource()); + } + + return builder.build(); + } + + public com.google.api.services.bigquery.model.ExternalDatasetReference toPb() { + com.google.api.services.bigquery.model.ExternalDatasetReference externalDatasetReference = + new com.google.api.services.bigquery.model.ExternalDatasetReference(); + + externalDatasetReference.setConnection(getConnection()); + externalDatasetReference.setExternalSource(getExternalSource()); + return externalDatasetReference; + } + + @Nullable + public abstract String getConnection(); + + @Nullable + public abstract String getExternalSource(); + + /** Returns a builder for an ExternalDatasetReference. */ + @VisibleForTesting + public abstract ExternalDatasetReference.Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + /** + * The connection id that is used to access the external_source. Format: + * projects/{project_id}/locations/{location_id}/connections/{connection_id} * + */ + public abstract ExternalDatasetReference.Builder setConnection(String connection); + + /** External source that backs this dataset * */ + public abstract ExternalDatasetReference.Builder setExternalSource(String externalSource); + + /** Creates a {@code ExternalDatasetReference} object. */ + public abstract ExternalDatasetReference build(); + } +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java index 453701e3a..c91cbc2f3 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java @@ -58,6 +58,12 @@ public class DatasetInfoTest { private static final DatasetId DATASET_ID_COMPLETE = DatasetId.of("project", "dataset"); private static final EncryptionConfiguration DATASET_ENCRYPTION_CONFIGURATION = EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); + + private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE = + ExternalDatasetReference.newBuilder() + .setExternalSource("source") + .setConnection("connection") + .build(); private static final DatasetInfo DATASET_INFO = DatasetInfo.newBuilder(DATASET_ID) .setAcl(ACCESS_RULES) @@ -82,6 +88,8 @@ public class DatasetInfoTest { .build(); private static final DatasetInfo DATASET_INFO_COMPLETE_WITH_IAM_MEMBER = DATASET_INFO.toBuilder().setAcl(ACCESS_RULES_IAM_MEMBER).build(); + private static final DatasetInfo DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE = + DATASET_INFO.toBuilder().setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE).build(); @Test public void testToBuilder() { @@ -108,6 +116,28 @@ public void testToBuilderIncomplete() { assertEquals(datasetInfo, datasetInfo.toBuilder().build()); } + @Test + public void testToBuilderWithExternalDatasetReference() { + compareDatasets( + DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE, + DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.toBuilder().build()); + + ExternalDatasetReference externalDatasetReference = + ExternalDatasetReference.newBuilder() + .setExternalSource("source2") + .setConnection("connection2") + .build(); + DatasetInfo datasetInfo = + DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE + .toBuilder() + .setExternalDatasetReference(externalDatasetReference) + .build(); + assertEquals(externalDatasetReference, datasetInfo.getExternalDatasetReference()); + datasetInfo = + datasetInfo.toBuilder().setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE).build(); + compareDatasets(DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE, datasetInfo); + } + @Test public void testBuilder() { assertNull(DATASET_INFO.getDatasetId().getProject()); @@ -137,6 +167,9 @@ public void testBuilder() { assertEquals(LOCATION, DATASET_INFO_COMPLETE.getLocation()); assertEquals(SELF_LINK, DATASET_INFO_COMPLETE.getSelfLink()); assertEquals(LABELS, DATASET_INFO_COMPLETE.getLabels()); + assertEquals( + EXTERNAL_DATASET_REFERENCE, + DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.getExternalDatasetReference()); } @Test @@ -156,6 +189,7 @@ public void testOf() { assertNull(datasetInfo.getDefaultEncryptionConfiguration()); assertNull(datasetInfo.getDefaultPartitionExpirationMs()); assertTrue(datasetInfo.getLabels().isEmpty()); + assertNull(datasetInfo.getExternalDatasetReference()); datasetInfo = DatasetInfo.of(DATASET_ID); assertEquals(DATASET_ID, datasetInfo.getDatasetId()); @@ -172,11 +206,15 @@ public void testOf() { assertNull(datasetInfo.getDefaultEncryptionConfiguration()); assertNull(datasetInfo.getDefaultPartitionExpirationMs()); assertTrue(datasetInfo.getLabels().isEmpty()); + assertNull(datasetInfo.getExternalDatasetReference()); } @Test public void testToPbAndFromPb() { compareDatasets(DATASET_INFO_COMPLETE, DatasetInfo.fromPb(DATASET_INFO_COMPLETE.toPb())); + compareDatasets( + DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE, + DatasetInfo.fromPb(DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.toPb())); DatasetInfo datasetInfo = DatasetInfo.newBuilder("project", "dataset").build(); compareDatasets(datasetInfo, DatasetInfo.fromPb(datasetInfo.toPb())); } @@ -204,5 +242,6 @@ private void compareDatasets(DatasetInfo expected, DatasetInfo value) { expected.getDefaultEncryptionConfiguration(), value.getDefaultEncryptionConfiguration()); assertEquals( expected.getDefaultPartitionExpirationMs(), value.getDefaultPartitionExpirationMs()); + assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java index d1c97a694..b244cf260 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java @@ -83,6 +83,11 @@ public class DatasetTest { TableInfo.newBuilder( TableId.of(NEW_PROJECT_ID, "dataset", "table3"), EXTERNAL_TABLE_DEFINITION) .build(); + private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE = + ExternalDatasetReference.newBuilder() + .setExternalSource("source") + .setConnection("connection") + .build(); @Rule public MockitoRule rule; @@ -319,6 +324,31 @@ public void testToAndFromPb() { compareDataset(expectedDataset, Dataset.fromPb(bigquery, expectedDataset.toPb())); } + @Test + public void testExternalDatasetReference() { + Dataset datasetWithExternalDatasetReference = + new Dataset.Builder(bigquery, DATASET_ID) + .setAcl(ACCESS_RULES) + .setCreationTime(CREATION_TIME) + .setDefaultTableLifetime(DEFAULT_TABLE_EXPIRATION) + .setDescription(DESCRIPTION) + .setEtag(ETAG) + .setFriendlyName(FRIENDLY_NAME) + .setGeneratedId(GENERATED_ID) + .setLastModified(LAST_MODIFIED) + .setLocation(LOCATION) + .setSelfLink(SELF_LINK) + .setLabels(LABELS) + .setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE) + .build(); + assertEquals( + EXTERNAL_DATASET_REFERENCE, + datasetWithExternalDatasetReference.getExternalDatasetReference()); + compareDataset( + datasetWithExternalDatasetReference, + datasetWithExternalDatasetReference.toBuilder().build()); + } + private void compareDataset(Dataset expected, Dataset value) { assertEquals(expected, value); compareDatasetInfo(expected, value); @@ -338,5 +368,6 @@ private void compareDatasetInfo(DatasetInfo expected, DatasetInfo value) { assertEquals(expected.getCreationTime(), value.getCreationTime()); assertEquals(expected.getDefaultTableLifetime(), value.getDefaultTableLifetime()); assertEquals(expected.getLastModified(), value.getLastModified()); + assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalDatasetReferenceTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalDatasetReferenceTest.java new file mode 100644 index 000000000..6d241948b --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalDatasetReferenceTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class ExternalDatasetReferenceTest { + private static final String EXTERNAL_SOURCE = "test_source"; + private static final String CONNECTION = "test_connection"; + private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE = + ExternalDatasetReference.newBuilder() + .setExternalSource(EXTERNAL_SOURCE) + .setConnection(CONNECTION) + .build(); + + @Test + public void testToBuilder() { + compareExternalDatasetReference( + EXTERNAL_DATASET_REFERENCE, EXTERNAL_DATASET_REFERENCE.toBuilder().build()); + ExternalDatasetReference externalDatasetReference = + EXTERNAL_DATASET_REFERENCE.toBuilder().setExternalSource("test_source2").build(); + assertEquals("test_source2", externalDatasetReference.getExternalSource()); + } + + @Test + public void testBuilder() { + assertEquals(EXTERNAL_SOURCE, EXTERNAL_DATASET_REFERENCE.getExternalSource()); + assertEquals(CONNECTION, EXTERNAL_DATASET_REFERENCE.getConnection()); + ExternalDatasetReference externalDatasetReference = + ExternalDatasetReference.newBuilder() + .setExternalSource(EXTERNAL_SOURCE) + .setConnection(CONNECTION) + .build(); + assertEquals(EXTERNAL_DATASET_REFERENCE, externalDatasetReference); + } + + @Test + public void testToAndFromPb() { + ExternalDatasetReference externalDatasetReference = + EXTERNAL_DATASET_REFERENCE.toBuilder().build(); + assertTrue( + ExternalDatasetReference.fromPb(externalDatasetReference.toPb()) + instanceof ExternalDatasetReference); + compareExternalDatasetReference( + externalDatasetReference, ExternalDatasetReference.fromPb(externalDatasetReference.toPb())); + } + + private void compareExternalDatasetReference( + ExternalDatasetReference expected, ExternalDatasetReference value) { + assertEquals(expected.getExternalSource(), value.getExternalSource()); + assertEquals(expected.getConnection(), value.getConnection()); + } +} diff --git a/pom.xml b/pom.xml index 508962ae5..be820053d 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ UTF-8 github google-cloud-bigquery-parent - v2-rev20230805-2.0.0 + v2-rev20230812-2.0.0 3.15.0 12.0.1