From 49f93c83c6feaf4e3f984fd798e03450bce3f5cf Mon Sep 17 00:00:00 2001 From: shollyman Date: Mon, 30 Jul 2018 13:47:11 -0700 Subject: [PATCH] BigQuery: Add Clustering support (#3415) * BigQuery: Add Clustering support to library. Initial changes for table. Next: plumb in configurations for load/query destination. * Plumb configuration options in. * add missing license header, remove unused import * Address reviewer comments: list immutability --- .../com/google/cloud/bigquery/Clustering.java | 66 +++++++++++++++++++ .../cloud/bigquery/LoadConfiguration.java | 20 ++++++ .../cloud/bigquery/LoadJobConfiguration.java | 42 +++++++++++- .../cloud/bigquery/QueryJobConfiguration.java | 52 ++++++++++++++- .../bigquery/StandardTableDefinition.java | 21 ++++++ .../bigquery/WriteChannelConfiguration.java | 44 ++++++++++++- .../bigquery/LoadJobConfigurationTest.java | 7 ++ .../bigquery/QueryJobConfigurationTest.java | 7 ++ .../bigquery/StandardTableDefinitionTest.java | 7 ++ .../WriteChannelConfigurationTest.java | 7 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 5 ++ 11 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Clustering.java diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Clustering.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Clustering.java new file mode 100644 index 000000000000..0d7a3faa6642 --- /dev/null +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Clustering.java @@ -0,0 +1,66 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery; + + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import java.io.Serializable; +import java.util.List; +import javax.annotation.Nullable; + +@AutoValue +public abstract class Clustering implements Serializable { + + private static final long serialVersionUID = 1L; + + + @Nullable + abstract ImmutableList getFieldsImmut(); + + public List getFields() {return getFieldsImmut();} + + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + + abstract Builder setFieldsImmut(ImmutableList fieldsImmut); + + public Builder setFields(List fields) { + return setFieldsImmut(ImmutableList.copyOf(fields)); + } + + public abstract Clustering build(); + } + + public static Builder newBuilder() { + return new AutoValue_Clustering.Builder(); + } + + com.google.api.services.bigquery.model.Clustering toPb() { + com.google.api.services.bigquery.model.Clustering clusterPb = + new com.google.api.services.bigquery.model.Clustering(); + clusterPb.setFields(getFields()); + return clusterPb; + } + + static Clustering fromPb(com.google.api.services.bigquery.model.Clustering clusterPb) { + return newBuilder().setFields(clusterPb.getFields()).build(); + } + +} diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java index 8d7204d61a29..9128e7afd662 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadConfiguration.java @@ -119,6 +119,16 @@ interface Builder { */ Builder setAutodetect(Boolean autodetect); + /** + * Sets the time partitioning specification for the destination table. + */ + Builder setTimePartitioning(TimePartitioning timePartitioning); + + /** + * Sets the clustering specification for the destination table. + */ + Builder setClustering(Clustering clustering); + LoadConfiguration build(); } @@ -211,6 +221,16 @@ interface Builder { */ Boolean getAutodetect(); + /** + * Returns the time partitioning specification defined for the destination table. + */ + TimePartitioning getTimePartitioning(); + + /** + * Returns the clustering specification for the definition table. + */ + Clustering getClustering(); + /** * Returns a builder for the load configuration object. */ diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index 3d0f41135c6f..90d1b137b94b 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -46,6 +46,8 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private final Boolean ignoreUnknownValues; private final List schemaUpdateOptions; private final Boolean autodetect; + private final TimePartitioning timePartitioning; + private final Clustering clustering; public static final class Builder extends JobConfiguration.Builder @@ -64,6 +66,8 @@ public static final class Builder private List projectionFields; private List schemaUpdateOptions; private Boolean autodetect; + private TimePartitioning timePartitioning; + private Clustering clustering; private Builder() { super(Type.LOAD); @@ -84,6 +88,8 @@ private Builder(LoadJobConfiguration loadConfiguration) { this.autodetect = loadConfiguration.autodetect; this.destinationEncryptionConfiguration = loadConfiguration.destinationEncryptionConfiguration; + this.timePartitioning = loadConfiguration.timePartitioning; + this.clustering = loadConfiguration.clustering; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -141,6 +147,12 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur } this.schemaUpdateOptions = schemaUpdateOptionsBuilder.build(); } + if (loadConfigurationPb.getTimePartitioning() != null) { + this.timePartitioning = TimePartitioning.fromPb(loadConfigurationPb.getTimePartitioning()); + } + if (loadConfigurationPb.getClustering() != null) { + this.clustering = Clustering.fromPb(loadConfigurationPb.getClustering()); + } this.autodetect = loadConfigurationPb.getAutodetect(); if (loadConfigurationPb.getDestinationEncryptionConfiguration() != null) { this.destinationEncryptionConfiguration = new EncryptionConfiguration.Builder( @@ -211,6 +223,18 @@ public Builder setIgnoreUnknownValues(Boolean ignoreUnknownValues) { return this; } + @Override + public Builder setTimePartitioning(TimePartitioning timePartitioning) { + this.timePartitioning = timePartitioning; + return this; + } + + @Override + public Builder setClustering(Clustering clustering) { + this.clustering = clustering; + return this; + } + /** * Sets the fully-qualified URIs that point to source data in Google Cloud Storage (e.g. * gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the @@ -253,6 +277,8 @@ private LoadJobConfiguration(Builder builder) { this.schemaUpdateOptions = builder.schemaUpdateOptions; this.autodetect = builder.autodetect; this.destinationEncryptionConfiguration = builder.destinationEncryptionConfiguration; + this.timePartitioning = builder.timePartitioning; + this.clustering = builder.clustering; } @@ -333,6 +359,12 @@ public Boolean getAutodetect() { return autodetect; } + @Override + public TimePartitioning getTimePartitioning() { return timePartitioning; } + + @Override + public Clustering getClustering() { return clustering; } + @Override public List getSchemaUpdateOptions() { return schemaUpdateOptions; @@ -357,7 +389,9 @@ ToStringHelper toStringHelper() { .add("ignoreUnknownValue", ignoreUnknownValues) .add("sourceUris", sourceUris) .add("schemaUpdateOptions", schemaUpdateOptions) - .add("autodetect", autodetect); + .add("autodetect", autodetect) + .add("timePartitioning", timePartitioning) + .add("clustering", clustering); } @Override @@ -429,6 +463,12 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { loadConfigurationPb.setDestinationEncryptionConfiguration( destinationEncryptionConfiguration.toPb()); } + if (timePartitioning != null) { + loadConfigurationPb.setTimePartitioning(timePartitioning.toPb()); + } + if (clustering != null) { + loadConfigurationPb.setClustering(clustering.toPb()); + } return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java index 724cc2b7403d..a955a4dd6c7c 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java @@ -62,6 +62,8 @@ public final class QueryJobConfiguration extends JobConfiguration { private final Integer maximumBillingTier; private final List schemaUpdateOptions; private final EncryptionConfiguration destinationEncryptionConfiguration; + private final TimePartitioning timePartitioning; + private final Clustering clustering; /** * Priority levels for a query. If not specified the priority is assumed to be @@ -104,6 +106,8 @@ public static final class Builder private Integer maximumBillingTier; private List schemaUpdateOptions; private EncryptionConfiguration destinationEncryptionConfiguration; + private TimePartitioning timePartitioning; + private Clustering clustering; private Builder() { super(Type.QUERY); @@ -129,6 +133,8 @@ private Builder(QueryJobConfiguration jobConfiguration) { this.maximumBillingTier = jobConfiguration.maximumBillingTier; this.schemaUpdateOptions = jobConfiguration.schemaUpdateOptions; this.destinationEncryptionConfiguration = jobConfiguration.destinationEncryptionConfiguration; + this.timePartitioning = jobConfiguration.timePartitioning; + this.clustering = jobConfiguration.clustering; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -195,6 +201,12 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.destinationEncryptionConfiguration = new EncryptionConfiguration.Builder( queryConfigurationPb.getDestinationEncryptionConfiguration()).build(); } + if (queryConfigurationPb.getTimePartitioning() != null) { + this.timePartitioning = TimePartitioning.fromPb(queryConfigurationPb.getTimePartitioning()); + } + if (queryConfigurationPb.getClustering() != null) { + this.clustering = Clustering.fromPb(queryConfigurationPb.getClustering()); + } } @@ -488,6 +500,22 @@ public Builder setSchemaUpdateOptions(List schemaUpdateOptio return this; } + /** + * Sets the time partitioning specification for the destination table. + */ + public Builder setTimePartitioning(TimePartitioning timePartitioning) { + this.timePartitioning = timePartitioning; + return this; + } + + /** + * Sets the clustering specification for the destination table. + */ + public Builder setClustering(Clustering clustering) { + this.clustering = clustering; + return this; + } + public QueryJobConfiguration build() { return new QueryJobConfiguration(this); } @@ -522,6 +550,8 @@ private QueryJobConfiguration(Builder builder) { this.maximumBillingTier = builder.maximumBillingTier; this.schemaUpdateOptions = builder.schemaUpdateOptions; this.destinationEncryptionConfiguration = builder.destinationEncryptionConfiguration; + this.timePartitioning = builder.timePartitioning; + this.clustering = builder.clustering; } /** @@ -693,6 +723,16 @@ public List getSchemaUpdateOptions() { return schemaUpdateOptions; } + /** + * Returns the time partitioning specification for the destination table. + */ + public TimePartitioning getTimePartitioning() { return timePartitioning; } + + /** + * Returns the clustering specification for the destination table. + */ + public Clustering getClustering() { return clustering; } + @Override public Builder toBuilder() { return new Builder(this); @@ -718,7 +758,9 @@ ToStringHelper toStringHelper() { .add("dryRun", dryRun) .add("useLegacySql", useLegacySql) .add("maximumBillingTier", maximumBillingTier) - .add("schemaUpdateOptions", schemaUpdateOptions); + .add("schemaUpdateOptions", schemaUpdateOptions) + .add("timePartitioning", timePartitioning) + .add("clustering", clustering); } @Override @@ -734,7 +776,7 @@ public int hashCode() { defaultDataset, flattenResults, priority, query, positionalParameters, namedParameters, tableDefinitions, useQueryCache, userDefinedFunctions, writeDisposition, dryRun, useLegacySql, maximumBillingTier, - schemaUpdateOptions); + schemaUpdateOptions, timePartitioning, clustering); } @Override @@ -813,6 +855,12 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (destinationEncryptionConfiguration != null) { queryConfigurationPb.setDestinationEncryptionConfiguration(destinationEncryptionConfiguration.toPb()); } + if (timePartitioning != null) { + queryConfigurationPb.setTimePartitioning(timePartitioning.toPb()); + } + if (clustering != null) { + queryConfigurationPb.setClustering(clustering.toPb()); + } return configurationPb.setQuery(queryConfigurationPb); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java index 40889fc036fd..b4329b19065b 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java @@ -142,6 +142,13 @@ public abstract static class Builder */ public abstract Builder setTimePartitioning(TimePartitioning timePartitioning); + /** + * Set the clustering configuration for the table. If not set, the table is not + * clustered. Clustering is only available for partitioned tables. + */ + public abstract Builder setClustering(Clustering clustering); + + /** Creates a {@code StandardTableDefinition} object. */ public abstract StandardTableDefinition build(); } @@ -179,6 +186,14 @@ public abstract static class Builder @Nullable public abstract TimePartitioning getTimePartitioning(); + + /** + * Returns the clustering configuration for this table. If {@code null}, the table is not + * clustered. + */ + @Nullable + public abstract Clustering getClustering(); + /** * Returns a builder for a BigQuery standard table definition. */ @@ -212,6 +227,9 @@ Table toPb() { if (getTimePartitioning() != null) { tablePb.setTimePartitioning(getTimePartitioning().toPb()); } + if (getClustering() != null) { + tablePb.setClustering(getClustering().toPb()); + } return tablePb; } @@ -227,6 +245,9 @@ static StandardTableDefinition fromPb(Table tablePb) { if (tablePb.getTimePartitioning() != null) { builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning())); } + if (tablePb.getClustering() != null) { + builder.setClustering(Clustering.fromPb(tablePb.getClustering())); + } return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build(); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java index 01f38c77acd0..00301997ea98 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java @@ -50,6 +50,8 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria private final List schemaUpdateOptions; private final Boolean autodetect; private final EncryptionConfiguration destinationEncryptionConfiguration; + private final TimePartitioning timePartitioning; + private final Clustering clustering; public static final class Builder implements LoadConfiguration.Builder { @@ -64,6 +66,8 @@ public static final class Builder implements LoadConfiguration.Builder { private List schemaUpdateOptions; private Boolean autodetect; private EncryptionConfiguration destinationEncryptionConfiguration; + private TimePartitioning timePartitioning; + private Clustering clustering; private Builder() {} @@ -79,6 +83,8 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) { this.schemaUpdateOptions = writeChannelConfiguration.schemaUpdateOptions; this.autodetect = writeChannelConfiguration.autodetect; this.destinationEncryptionConfiguration = writeChannelConfiguration.destinationEncryptionConfiguration; + this.timePartitioning = writeChannelConfiguration.timePartitioning; + this.clustering = writeChannelConfiguration.clustering; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -140,6 +146,12 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this.destinationEncryptionConfiguration = new EncryptionConfiguration.Builder( configurationPb.getLoad().getDestinationEncryptionConfiguration()).build(); } + if (loadConfigurationPb.getTimePartitioning() != null) { + this.timePartitioning = TimePartitioning.fromPb(loadConfigurationPb.getTimePartitioning()); + } + if (loadConfigurationPb.getClustering() != null) { + this.clustering = Clustering.fromPb(loadConfigurationPb.getClustering()); + } } @@ -218,6 +230,18 @@ public Builder setAutodetect(Boolean autodetect) { return this; } + @Override + public Builder setTimePartitioning(TimePartitioning timePartitioning) { + this.timePartitioning = timePartitioning; + return this; + } + + @Override + public Builder setClustering(Clustering clustering) { + this.clustering = clustering; + return this; + } + @Override public WriteChannelConfiguration build() { return new WriteChannelConfiguration(this); @@ -236,6 +260,8 @@ protected WriteChannelConfiguration(Builder builder) { this.schemaUpdateOptions = builder.schemaUpdateOptions; this.autodetect = builder.autodetect; this.destinationEncryptionConfiguration = builder.destinationEncryptionConfiguration; + this.timePartitioning = builder.timePartitioning; + this.clustering = builder.clustering; } @@ -313,6 +339,12 @@ public Boolean getAutodetect() { return autodetect; } + @Override + public TimePartitioning getTimePartitioning() { return timePartitioning; } + + @Override + public Clustering getClustering() { return clustering; } + @Override public Builder toBuilder() { return new Builder(this); @@ -330,7 +362,9 @@ MoreObjects.ToStringHelper toStringHelper() { .add("schema", schema) .add("ignoreUnknownValue", ignoreUnknownValues) .add("schemaUpdateOptions", schemaUpdateOptions) - .add("autodetect", autodetect); + .add("autodetect", autodetect) + .add("timePartitioning", timePartitioning) + .add("clustering", clustering); } @Override @@ -348,7 +382,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions, - nullMarker, maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions, autodetect); + nullMarker, maxBadRecords, schema, ignoreUnknownValues, schemaUpdateOptions, autodetect, timePartitioning, clustering); } WriteChannelConfiguration setProjectId(String projectId) { @@ -403,6 +437,12 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { loadConfigurationPb.setDestinationEncryptionConfiguration( destinationEncryptionConfiguration.toPb()); } + if (timePartitioning != null) { + loadConfigurationPb.setTimePartitioning(timePartitioning.toPb()); + } + if (clustering != null) { + loadConfigurationPb.setClustering(clustering.toPb()); + } return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index 0552cc881150..754e5e648803 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -21,6 +21,7 @@ import com.google.cloud.bigquery.JobInfo.CreateDisposition; import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption; +import com.google.cloud.bigquery.TimePartitioning.Type; import com.google.common.collect.ImmutableList; import org.junit.Test; @@ -53,6 +54,8 @@ public class LoadJobConfigurationTest { private static final Boolean AUTODETECT = true; private static final EncryptionConfiguration JOB_ENCRYPTION_CONFIGURATION = EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); + private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY); + private static final Clustering CLUSTERING = Clustering.newBuilder().setFields(ImmutableList.of("Foo", "Bar")).build(); private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV = LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS) .setCreateDisposition(CREATE_DISPOSITION) @@ -64,6 +67,8 @@ public class LoadJobConfigurationTest { .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) .setAutodetect(AUTODETECT) .setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION) + .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) .build(); private static final DatastoreBackupOptions BACKUP_OPTIONS = DatastoreBackupOptions.newBuilder() .setProjectionFields(ImmutableList.of("field_1", "field_2")) @@ -163,5 +168,7 @@ private void compareLoadJobConfiguration(LoadJobConfiguration expected, assertEquals(expected.getAutodetect(), value.getAutodetect()); assertEquals(expected.getSchemaUpdateOptions(), value.getSchemaUpdateOptions()); assertEquals(expected.getDestinationEncryptionConfiguration(), value.getDestinationEncryptionConfiguration()); + assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); + assertEquals(expected.getClustering(), value.getClustering()); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java index 7fff324aafb8..787622408075 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java @@ -24,6 +24,7 @@ import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.JobInfo.SchemaUpdateOption; import com.google.cloud.bigquery.QueryJobConfiguration.Priority; +import com.google.cloud.bigquery.TimePartitioning.Type; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.List; @@ -78,6 +79,8 @@ public class QueryJobConfigurationTest { UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); private static final EncryptionConfiguration JOB_ENCRYPTION_CONFIGURATION = EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); + private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY); + private static final Clustering CLUSTERING = Clustering.newBuilder().setFields(ImmutableList.of("Foo","Bar")).build(); private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = QueryJobConfiguration.newBuilder(QUERY) .setUseQueryCache(USE_QUERY_CACHE) @@ -95,6 +98,8 @@ public class QueryJobConfigurationTest { .setMaximumBillingTier(MAX_BILLING_TIER) .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) .setDestinationEncryptionConfiguration(JOB_ENCRYPTION_CONFIGURATION) + .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) .build(); @Test @@ -166,5 +171,7 @@ private void compareQueryJobConfiguration(QueryJobConfiguration expected, assertEquals(expected.getMaximumBillingTier(), value.getMaximumBillingTier()); assertEquals(expected.getSchemaUpdateOptions(), value.getSchemaUpdateOptions()); assertEquals(expected.getDestinationEncryptionConfiguration(), value.getDestinationEncryptionConfiguration()); + assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); + assertEquals(expected.getClustering(), value.getClustering()); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java index 038fcfd1da5c..85649eebc7c9 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java @@ -22,6 +22,7 @@ import com.google.cloud.bigquery.StandardTableDefinition.StreamingBuffer; +import com.google.common.collect.ImmutableList; import org.junit.Test; public class StandardTableDefinitionTest { @@ -48,6 +49,8 @@ public class StandardTableDefinitionTest { private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L); private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(TimePartitioning.Type.DAY, 42); + private static final Clustering CLUSTERING = + Clustering.newBuilder().setFields(ImmutableList.of("Foo","Bar")).build(); private static final StandardTableDefinition TABLE_DEFINITION = StandardTableDefinition.newBuilder() .setLocation(LOCATION) @@ -56,6 +59,7 @@ public class StandardTableDefinitionTest { .setStreamingBuffer(STREAMING_BUFFER) .setSchema(TABLE_SCHEMA) .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) .build(); @Test @@ -85,6 +89,7 @@ public void testBuilder() { assertEquals(NUM_ROWS, TABLE_DEFINITION.getNumRows()); assertEquals(STREAMING_BUFFER, TABLE_DEFINITION.getStreamingBuffer()); assertEquals(TIME_PARTITIONING, TABLE_DEFINITION.getTimePartitioning()); + assertEquals(CLUSTERING, TABLE_DEFINITION.getClustering()); } @@ -98,6 +103,7 @@ public void testOf() { assertNull(definition.getNumRows()); assertNull(definition.getStreamingBuffer()); assertNull(definition.getTimePartitioning()); + assertNull(definition.getClustering()); } @Test @@ -122,6 +128,7 @@ private void compareStandardTableDefinition(StandardTableDefinition expected, assertEquals(expected.getStreamingBuffer(), value.getStreamingBuffer()); assertEquals(expected.getType(), value.getType()); assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); + assertEquals(expected.getClustering(), value.getClustering()); assertEquals(expected.hashCode(), value.hashCode()); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java index fb0048850ccd..8128f099c0c4 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java @@ -21,6 +21,7 @@ import com.google.cloud.bigquery.JobInfo.CreateDisposition; import com.google.cloud.bigquery.JobInfo.WriteDisposition; +import com.google.cloud.bigquery.TimePartitioning.Type; import com.google.common.collect.ImmutableList; import org.junit.Test; @@ -51,6 +52,8 @@ public class WriteChannelConfigurationTest { private static final Boolean AUTODETECT = true; private static final List SCHEMA_UPDATE_OPTIONS = ImmutableList.of(JobInfo.SchemaUpdateOption.ALLOW_FIELD_ADDITION); + private static final TimePartitioning TIME_PARTITIONING = TimePartitioning.of(Type.DAY); + private static final Clustering CLUSTERING = Clustering.newBuilder().setFields(ImmutableList.of("Foo","Bar")).build(); private static final WriteChannelConfiguration LOAD_CONFIGURATION_CSV = WriteChannelConfiguration.newBuilder(TABLE_ID) .setCreateDisposition(CREATE_DISPOSITION) @@ -62,6 +65,8 @@ public class WriteChannelConfigurationTest { .setSchema(TABLE_SCHEMA) .setSchemaUpdateOptions(SCHEMA_UPDATE_OPTIONS) .setAutodetect(AUTODETECT) + .setTimePartitioning(TIME_PARTITIONING) + .setClustering(CLUSTERING) .build(); private static final DatastoreBackupOptions BACKUP_OPTIONS = DatastoreBackupOptions.newBuilder() @@ -173,5 +178,7 @@ private void compareLoadConfiguration(WriteChannelConfiguration expected, assertEquals(expected.getSchema(), value.getSchema()); assertEquals(expected.getSchemaUpdateOptions(), value.getSchemaUpdateOptions()); assertEquals(expected.getAutodetect(), value.getAutodetect()); + assertEquals(expected.getTimePartitioning(), value.getTimePartitioning()); + assertEquals(expected.getClustering(), value.getClustering()); } } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 572c9506c67b..3a07a4166ef7 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -39,6 +39,7 @@ import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.Clustering; import com.google.cloud.bigquery.CopyJobConfiguration; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.DatasetId; @@ -388,9 +389,11 @@ public void testCreateAndGetTable() { String tableName = "test_create_and_get_table"; TableId tableId = TableId.of(DATASET, tableName); TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); StandardTableDefinition tableDefinition = StandardTableDefinition.newBuilder() .setSchema(TABLE_SCHEMA) .setTimePartitioning(partitioning) + .setClustering(clustering) .build(); Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); assertNotNull(createdTable); @@ -408,6 +411,7 @@ public void testCreateAndGetTable() { assertNotNull(remoteTable.getDefinition().getNumRows()); assertEquals(partitioning, remoteTable.getDefinition().getTimePartitioning()); + assertEquals(clustering, remoteTable.getDefinition().getClustering()); assertTrue(remoteTable.delete()); } @@ -432,6 +436,7 @@ public void testCreateAndGetTableWithSelectedField() { assertNull(remoteTable.getDefinition().getNumBytes()); assertNull(remoteTable.getDefinition().getNumRows()); assertNull(remoteTable.getDefinition().getTimePartitioning()); + assertNull(remoteTable.getDefinition().getClustering()); assertTrue(remoteTable.delete()); }