From d170550ff2f0da484ae2a9b110c22a1db439ba9c Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 27 Jan 2016 13:05:49 +0100 Subject: [PATCH] Fix comments. Make JobConfiguration an abstract class --- .../com/google/gcloud/bigquery/BigQuery.java | 4 +- .../google/gcloud/bigquery/BigQueryImpl.java | 11 +- .../gcloud/bigquery/CopyJobConfiguration.java | 55 +-- .../bigquery/ExtractJobConfiguration.java | 40 +-- .../gcloud/bigquery/JobConfiguration.java | 107 +++++- .../com/google/gcloud/bigquery/JobInfo.java | 32 +- .../gcloud/bigquery/LoadConfiguration.java | 283 ++-------------- .../gcloud/bigquery/LoadJobConfiguration.java | 238 +++++++++++-- .../bigquery/QueryJobConfiguration.java | 42 +-- .../bigquery/TableDataWriteChannel.java | 23 +- .../bigquery/WriteChannelConfiguration.java | 316 ++++++++++++++++++ .../google/gcloud/spi/DefaultBigQueryRpc.java | 2 - .../gcloud/bigquery/BigQueryImplTest.java | 9 +- .../bigquery/CopyJobConfigurationTest.java | 2 +- .../bigquery/ExtractJobConfigurationTest.java | 2 +- .../gcloud/bigquery/ITBigQueryTest.java | 2 +- .../google/gcloud/bigquery/JobInfoTest.java | 2 +- .../bigquery/LoadJobConfigurationTest.java | 2 +- .../bigquery/QueryJobConfigurationTest.java | 2 +- .../gcloud/bigquery/SerializationTest.java | 17 +- .../bigquery/TableDataWriteChannelTest.java | 15 +- ...ava => WriteChannelConfigurationTest.java} | 20 +- .../gcloud/examples/BigQueryExample.java | 17 +- 23 files changed, 806 insertions(+), 437 deletions(-) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java rename gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/{LoadConfigurationTest.java => WriteChannelConfigurationTest.java} (84%) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java index f5009c83eb41..6bc6a2ebabb5 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java @@ -666,9 +666,9 @@ Page> listTableData(TableId tableId, TableDataListOption... opt /** * Returns a channel to write data to be inserted into a BigQuery table. Data format and other - * options can be configured using the {@link LoadConfiguration} parameter. + * options can be configured using the {@link WriteChannelConfiguration} parameter. * * @throws BigQueryException upon failure */ - TableDataWriteChannel writer(LoadConfiguration loadConfiguration); + TableDataWriteChannel writer(WriteChannelConfiguration writeChannelConfiguration); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java index a3f708bb4c96..3186a0ac196b 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java @@ -596,8 +596,8 @@ private static QueryResult.Builder transformQueryResults(JobId jobId, List optionMap(Option... options) { @@ -681,7 +681,7 @@ public TableId apply(TableId tableId) { break; case LOAD: LoadJobConfiguration loadConfiguration = (LoadJobConfiguration) configuration; - jobBuilder.configuration((LoadJobConfiguration) setProjectId(loadConfiguration)); + jobBuilder.configuration(setProjectId(loadConfiguration)); break; default: // never reached @@ -698,9 +698,10 @@ private QueryRequest setProjectId(QueryRequest request) { return builder.build(); } - private LoadConfiguration setProjectId(LoadConfiguration configuration) { + @SuppressWarnings("unchecked") + private T setProjectId(T configuration) { LoadConfiguration.Builder builder = configuration.toBuilder(); builder.destinationTable(setProjectId(configuration.destinationTable())); - return builder.build(); + return (T) builder.build(); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java index 88d420b38054..0877977f7243 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java @@ -1,21 +1,36 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.gcloud.bigquery; import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.JobConfigurationTableCopy; -import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import java.io.Serializable; import java.util.List; import java.util.Objects; /** - * Google BigQuery Copy Job configuration. A Copy Job copies an existing table to another new or - * existing table. + * Google BigQuery copy job configuration. A copy job copies an existing table to another new or + * existing table. Copy job configurations have {@link JobConfiguration.Type#COPY} type. */ -public final class CopyJobConfiguration implements JobConfiguration, Serializable { +public final class CopyJobConfiguration extends JobConfiguration { private static final long serialVersionUID = 1140509641399762967L; @@ -24,16 +39,20 @@ public final class CopyJobConfiguration implements JobConfiguration, Serializabl private final JobInfo.CreateDisposition createDisposition; private final JobInfo.WriteDisposition writeDisposition; - public static final class Builder { + public static final class Builder + extends JobConfiguration.Builder { private List sourceTables; private TableId destinationTable; private JobInfo.CreateDisposition createDisposition; private JobInfo.WriteDisposition writeDisposition; - private Builder() {} + private Builder() { + type(Type.COPY); + } private Builder(CopyJobConfiguration jobConfiguration) { + type(Type.COPY); this.sourceTables = jobConfiguration.sourceTables; this.destinationTable = jobConfiguration.destinationTable; this.createDisposition = jobConfiguration.createDisposition; @@ -41,6 +60,7 @@ private Builder(CopyJobConfiguration jobConfiguration) { } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + type(Type.COPY); JobConfigurationTableCopy copyConfigurationPb = configurationPb.getCopy(); this.destinationTable = TableId.fromPb(copyConfigurationPb.getDestinationTable()); if (copyConfigurationPb.getSourceTables() != null) { @@ -103,17 +123,13 @@ public CopyJobConfiguration build() { } private CopyJobConfiguration(Builder builder) { + super(builder); this.sourceTables = checkNotNull(builder.sourceTables); this.destinationTable = checkNotNull(builder.destinationTable); this.createDisposition = builder.createDisposition; this.writeDisposition = builder.writeDisposition; } - @Override - public Type type() { - return Type.COPY; - } - /** * Returns the source tables to copy. */ @@ -148,29 +164,29 @@ public JobInfo.WriteDisposition writeDisposition() { return writeDisposition; } + @Override public Builder toBuilder() { return new Builder(this); } @Override - public String toString() { - return MoreObjects.toStringHelper(this) + protected ToStringHelper toStringHelper() { + return super.toStringHelper() .add("sourceTables", sourceTables) .add("destinationTable", destinationTable) .add("createDisposition", createDisposition) - .add("writeDisposition", writeDisposition) - .toString(); + .add("writeDisposition", writeDisposition); } @Override public boolean equals(Object obj) { - return obj instanceof CopyJobConfiguration - && Objects.equals(toPb(), ((CopyJobConfiguration) obj).toPb()); + return obj instanceof CopyJobConfiguration && baseEquals((CopyJobConfiguration) obj); } @Override public int hashCode() { - return Objects.hash(sourceTables, destinationTable, createDisposition, writeDisposition); + return Objects.hash(baseHashCode(), sourceTables, destinationTable, createDisposition, + writeDisposition); } com.google.api.services.bigquery.model.JobConfiguration toPb() { @@ -218,6 +234,7 @@ public static CopyJobConfiguration of(TableId destinationTable, List so return builder(destinationTable, sourceTables).build(); } + @SuppressWarnings("unchecked") static CopyJobConfiguration fromPb( com.google.api.services.bigquery.model.JobConfiguration jobPb) { return new Builder(jobPb).build(); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java index 04f493d4af20..ef676d9b8841 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,19 +19,18 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.JobConfigurationExtract; -import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; -import java.io.Serializable; import java.util.List; import java.util.Objects; /** - * Google BigQuery Extract Job configuration. An Extract Job exports a BigQuery table to Google + * Google BigQuery extract job configuration. An extract job exports a BigQuery table to Google * Cloud Storage. The extract destination provided as URIs that point to objects in Google Cloud - * Storage. + * Storage. Extract job configurations have {@link JobConfiguration.Type#EXTRACT} type. */ -public final class ExtractJobConfiguration implements JobConfiguration, Serializable { +public final class ExtractJobConfiguration extends JobConfiguration { private static final long serialVersionUID = 4147749733166593761L; @@ -42,7 +41,8 @@ public final class ExtractJobConfiguration implements JobConfiguration, Seriali private final String format; private final String compression; - public static final class Builder { + public static final class Builder + extends JobConfiguration.Builder { private TableId sourceTable; private List destinationUris; @@ -51,9 +51,12 @@ public static final class Builder { private String format; private String compression; - private Builder() {} + private Builder() { + type(Type.EXTRACT); + } private Builder(ExtractJobConfiguration jobInfo) { + type(Type.EXTRACT); this.sourceTable = jobInfo.sourceTable; this.destinationUris = jobInfo.destinationUris; this.printHeader = jobInfo.printHeader; @@ -63,6 +66,7 @@ private Builder(ExtractJobConfiguration jobInfo) { } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + type(Type.EXTRACT); JobConfigurationExtract extractConfigurationPb = configurationPb.getExtract(); this.sourceTable = TableId.fromPb(extractConfigurationPb.getSourceTable()); this.destinationUris = extractConfigurationPb.getDestinationUris(); @@ -134,6 +138,7 @@ public ExtractJobConfiguration build() { } private ExtractJobConfiguration(Builder builder) { + super(builder); this.sourceTable = checkNotNull(builder.sourceTable); this.destinationUris = checkNotNull(builder.destinationUris); this.printHeader = builder.printHeader; @@ -142,11 +147,6 @@ private ExtractJobConfiguration(Builder builder) { this.compression = builder.compression; } - @Override - public Type type() { - return Type.EXTRACT; - } - /** * Returns the table to export. */ @@ -193,31 +193,30 @@ public String compression() { return compression; } + @Override public Builder toBuilder() { return new Builder(this); } @Override - public String toString() { - return MoreObjects.toStringHelper(this) + protected ToStringHelper toStringHelper() { + return super.toStringHelper() .add("sourceTable", sourceTable) .add("destinationUris", destinationUris) .add("format", format) .add("printHeader", printHeader) .add("fieldDelimiter", fieldDelimiter) - .add("compression", compression) - .toString(); + .add("compression", compression); } @Override public boolean equals(Object obj) { - return obj instanceof ExtractJobConfiguration - && Objects.equals(toPb(), ((ExtractJobConfiguration) obj).toPb()); + return obj instanceof ExtractJobConfiguration && baseEquals((ExtractJobConfiguration) obj); } @Override public int hashCode() { - return Objects.hash(sourceTable, destinationUris, printHeader, fieldDelimiter, + return Objects.hash(baseHashCode(), sourceTable, destinationUris, printHeader, fieldDelimiter, format, compression); } @@ -281,6 +280,7 @@ public static ExtractJobConfiguration of(TableId sourceTable, List desti return builder(sourceTable, destinationUris).format(format).build(); } + @SuppressWarnings("unchecked") static ExtractJobConfiguration fromPb( com.google.api.services.bigquery.model.JobConfiguration confPb) { return new Builder(confPb).build(); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java index ea19a6d9cb67..9e4e22942502 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,35 +16,126 @@ package com.google.gcloud.bigquery; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; + +import java.io.Serializable; +import java.util.Objects; + /** - * Common interface for a BigQuery Job configuration. + * Base class for a BigQuery job configuration. */ -public interface JobConfiguration { +public abstract class JobConfiguration implements Serializable { + + private static final long serialVersionUID = -548132177415406526L; + + private final Type type; /** * Type of a BigQuery Job. */ enum Type { /** - * A Copy Job copies an existing table to another new or existing table. + * A Copy Job copies an existing table to another new or existing table. Instances of + * {@code JobConfiguration} for this type are implemented by {@link CopyJobConfiguration}. */ COPY, /** - * An Extract Job exports a BigQuery table to Google Cloud Storage. + * An Extract Job exports a BigQuery table to Google Cloud Storage. Instances of + * {@code JobConfiguration} for this type are implemented by {@link ExtractJobConfiguration}. */ EXTRACT, /** - * A Load Job loads data from one of several formats into a table. + * A Load Job loads data from one of several formats into a table. Instances of + * {@code JobConfiguration} for this type are implemented by {@link LoadJobConfiguration}. */ LOAD, /** - * A Query Job runs a query against BigQuery data. + * A Query Job runs a query against BigQuery data. Instances of + * {@code JobConfiguration} for this type are implemented by {@link QueryJobConfiguration}. */ QUERY } + /** + * Base builder for job configurations. + * + * @param the job configuration type + * @param the job configuration builder + */ + public abstract static class Builder> { + + private Type type; + + protected Builder() {} + + @SuppressWarnings("unchecked") + protected B self() { + return (B) this; + } + + B type(Type type) { + this.type = checkNotNull(type); + return self(); + } + + /** + * Creates an object. + */ + public abstract T build(); + } + + protected JobConfiguration(Builder builder) { + this.type = builder.type; + } + /** * Returns the type of the job configuration. */ - Type type(); + public Type type() { + return type; + } + + /** + * Returns a builder for the object. + */ + public abstract Builder toBuilder(); + + protected ToStringHelper toStringHelper() { + return MoreObjects.toStringHelper(this).add("type", type); + } + + @Override + public String toString() { + return toStringHelper().toString(); + } + + protected final int baseHashCode() { + return Objects.hash(type); + } + + protected final boolean baseEquals(JobConfiguration jobConfiguration) { + return Objects.equals(toPb(), jobConfiguration.toPb()); + } + + abstract com.google.api.services.bigquery.model.JobConfiguration toPb(); + + @SuppressWarnings("unchecked") + static T fromPb( + com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + if (configurationPb.getCopy() != null) { + return (T) CopyJobConfiguration.fromPb(configurationPb); + } else if (configurationPb.getExtract() != null) { + return (T) ExtractJobConfiguration.fromPb(configurationPb); + } else if (configurationPb.getLoad() != null) { + return (T) LoadJobConfiguration.fromPb(configurationPb); + } else if (configurationPb.getQuery() != null) { + return (T) QueryJobConfiguration.fromPb(configurationPb); + } else { + // never reached + throw new IllegalArgumentException("Job configuration is not supported"); + } + } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java index 519f3bf6163b..355f0c52fb25 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java @@ -125,19 +125,7 @@ protected Builder(Job jobPb) { this.statistics = JobStatistics.fromPb(jobPb.getStatistics()); } this.userEmail = jobPb.getUserEmail(); - com.google.api.services.bigquery.model.JobConfiguration confPb = jobPb.getConfiguration(); - if (confPb.getCopy() != null) { - this.configuration = CopyJobConfiguration.fromPb(confPb); - } else if (confPb.getExtract() != null) { - this.configuration = ExtractJobConfiguration.fromPb(confPb); - } else if (confPb.getLoad() != null) { - this.configuration = LoadJobConfiguration.fromPb(confPb); - } else if (confPb.getQuery() != null) { - this.configuration = QueryJobConfiguration.fromPb(confPb); - } else { - // never reached - throw new IllegalArgumentException("Job configuration is not supported"); - } + this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); } Builder etag(String etag) { @@ -305,23 +293,7 @@ Job toPb() { if (statistics != null) { jobPb.setStatistics(statistics.toPb()); } - switch (configuration.type()) { - case COPY: - jobPb.setConfiguration(this.configuration().toPb()); - break; - case EXTRACT: - jobPb.setConfiguration(this.configuration().toPb()); - break; - case LOAD: - jobPb.setConfiguration(this.configuration().toPb()); - break; - case QUERY: - jobPb.setConfiguration(this.configuration().toPb()); - break; - default: - // never reached - throw new IllegalArgumentException("Job configuration is not supported"); - } + jobPb.setConfiguration(configuration.toPb()); return jobPb; } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java index 115701a5932f..223a25a478e0 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java @@ -16,108 +16,26 @@ package com.google.gcloud.bigquery; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; import com.google.gcloud.bigquery.JobInfo.CreateDisposition; import com.google.gcloud.bigquery.JobInfo.WriteDisposition; -import java.io.Serializable; import java.util.List; -import java.util.Objects; /** - * Google BigQuery Configuration for a load operation. A load configuration can be used to load data - * into a table with a {@link com.google.gcloud.WriteChannel} - * ({@link BigQuery#writer(LoadConfiguration)}). + * Common interface for a load configuration. A load configuration + * ({@link WriteChannelConfiguration}) can be used to load data into a table with a + * {@link com.google.gcloud.WriteChannel} ({@link BigQuery#writer(WriteChannelConfiguration)}). + * A load configuration ({@link LoadJobConfiguration}) can also be used to create a load job + * ({@link JobInfo#of(JobConfiguration)}). */ -public class LoadConfiguration implements Serializable { - - private static final long serialVersionUID = 470267591917413578L; - - protected final TableId destinationTable; - protected final CreateDisposition createDisposition; - protected final WriteDisposition writeDisposition; - protected final FormatOptions formatOptions; - protected final Integer maxBadRecords; - protected final Schema schema; - protected final Boolean ignoreUnknownValues; - protected final List projectionFields; - - public static class Builder { - - private TableId destinationTable; - private CreateDisposition createDisposition; - private WriteDisposition writeDisposition; - private FormatOptions formatOptions; - private Integer maxBadRecords; - private Schema schema; - private Boolean ignoreUnknownValues; - private List projectionFields; - - protected Builder() {} +public interface LoadConfiguration { - protected Builder(LoadConfiguration loadConfiguration) { - this.destinationTable = loadConfiguration.destinationTable; - this.createDisposition = loadConfiguration.createDisposition; - this.writeDisposition = loadConfiguration.writeDisposition; - this.formatOptions = loadConfiguration.formatOptions; - this.maxBadRecords = loadConfiguration.maxBadRecords; - this.schema = loadConfiguration.schema; - this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues; - this.projectionFields = loadConfiguration.projectionFields; - } - - protected Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { - JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); - this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); - if (loadConfigurationPb.getCreateDisposition() != null) { - this.createDisposition = - CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); - } - if (loadConfigurationPb.getWriteDisposition() != null) { - this.writeDisposition = WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); - } - if (loadConfigurationPb.getSourceFormat() != null) { - this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); - } - if (loadConfigurationPb.getAllowJaggedRows() != null - || loadConfigurationPb.getAllowQuotedNewlines() != null - || loadConfigurationPb.getEncoding() != null - || loadConfigurationPb.getFieldDelimiter() != null - || loadConfigurationPb.getQuote() != null - || loadConfigurationPb.getSkipLeadingRows() != null) { - CsvOptions.Builder builder = CsvOptions.builder() - .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) - .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) - .encoding(loadConfigurationPb.getEncoding()) - .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) - .quote(loadConfigurationPb.getQuote()) - .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); - this.formatOptions = builder.build(); - } - this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); - if (loadConfigurationPb.getSchema() != null) { - this.schema = Schema.fromPb(loadConfigurationPb.getSchema()); - } - this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues(); - this.projectionFields = loadConfigurationPb.getProjectionFields(); - } - - @SuppressWarnings("unchecked") - B self() { - return (B) this; - } + interface Builder { /** * Sets the destination table to load the data into. */ - public B destinationTable(TableId destinationTable) { - this.destinationTable = destinationTable; - return self(); - } + Builder destinationTable(TableId destinationTable); /** * Sets whether the job is allowed to create new tables. @@ -125,10 +43,7 @@ public B destinationTable(TableId destinationTable) { * @see * Create Disposition */ - public B createDisposition(CreateDisposition createDisposition) { - this.createDisposition = createDisposition; - return self(); - } + Builder createDisposition(CreateDisposition createDisposition); /** * Sets the action that should occur if the destination table already exists. @@ -136,10 +51,7 @@ public B createDisposition(CreateDisposition createDisposition) { * @see * Write Disposition */ - public B writeDisposition(WriteDisposition writeDisposition) { - this.writeDisposition = writeDisposition; - return self(); - } + Builder writeDisposition(WriteDisposition writeDisposition); /** * Sets the source format, and possibly some parsing options, of the external data. Supported @@ -149,30 +61,21 @@ public B writeDisposition(WriteDisposition writeDisposition) { * * Source Format */ - public B formatOptions(FormatOptions formatOptions) { - this.formatOptions = formatOptions; - return self(); - } + Builder formatOptions(FormatOptions formatOptions); /** * Sets the maximum number of bad records that BigQuery can ignore when running the job. If the * number of bad records exceeds this value, an invalid error is returned in the job result. * By default no bad record is ignored. */ - public B maxBadRecords(Integer maxBadRecords) { - this.maxBadRecords = maxBadRecords; - return self(); - } + Builder maxBadRecords(Integer maxBadRecords); /** * Sets the schema for the destination table. The schema can be omitted if the destination table * already exists, or if you're loading data from a Google Cloud Datastore backup (i.e. * {@code DATASTORE_BACKUP} format option). */ - public B schema(Schema schema) { - this.schema = schema; - return self(); - } + Builder schema(Schema schema); /** * Sets whether BigQuery should allow extra values that are not represented in the table schema. @@ -180,10 +83,7 @@ public B schema(Schema schema) { * are treated as bad records, and if there are too many bad records, an invalid error is * returned in the job result. By default unknown values are not allowed. */ - public B ignoreUnknownValues(Boolean ignoreUnknownValues) { - this.ignoreUnknownValues = ignoreUnknownValues; - return self(); - } + Builder ignoreUnknownValues(Boolean ignoreUnknownValues); /** * Sets which entity properties to load into BigQuery from a Cloud Datastore backup. This field @@ -192,36 +92,15 @@ public B ignoreUnknownValues(Boolean ignoreUnknownValues) { * all properties. If any named property isn't found in the Cloud Datastore backup, an invalid * error is returned in the job result. */ - public B projectionFields(List projectionFields) { - this.projectionFields = - projectionFields != null ? ImmutableList.copyOf(projectionFields) : null; - return self(); - } - - @SuppressWarnings("unchecked") - public T build() { - return (T) new LoadConfiguration(this); - } - } + Builder projectionFields(List projectionFields); - @SuppressWarnings("unchecked") - protected LoadConfiguration(Builder builder) { - this.destinationTable = checkNotNull(builder.destinationTable); - this.createDisposition = builder.createDisposition; - this.writeDisposition = builder.writeDisposition; - this.formatOptions = builder.formatOptions; - this.maxBadRecords = builder.maxBadRecords; - this.schema = builder.schema; - this.ignoreUnknownValues = builder.ignoreUnknownValues; - this.projectionFields = (List) builder.projectionFields; + LoadConfiguration build(); } /** * Returns the destination table to load the data into. */ - public TableId destinationTable() { - return destinationTable; - } + TableId destinationTable(); /** * Returns whether the job is allowed to create new tables. @@ -229,9 +108,7 @@ public TableId destinationTable() { * @see * Create Disposition */ - public CreateDisposition createDisposition() { - return this.createDisposition; - } + CreateDisposition createDisposition(); /** * Returns the action that should occur if the destination table already exists. @@ -239,40 +116,30 @@ public CreateDisposition createDisposition() { * @see * Write Disposition */ - public WriteDisposition writeDisposition() { - return writeDisposition; - } + WriteDisposition writeDisposition(); /** * Returns additional properties used to parse CSV data (used when {@link #format()} is set * to CSV). Returns {@code null} if not set. */ - public CsvOptions csvOptions() { - return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; - } + CsvOptions csvOptions(); /** * Returns the maximum number of bad records that BigQuery can ignore when running the job. If the * number of bad records exceeds this value, an invalid error is returned in the job result. * By default no bad record is ignored. */ - public Integer maxBadRecords() { - return maxBadRecords; - } + Integer maxBadRecords(); /** * Returns the schema for the destination table, if set. Returns {@code null} otherwise. */ - public Schema schema() { - return schema; - } + Schema schema(); /** * Returns the format of the data files. */ - public String format() { - return formatOptions != null ? formatOptions.type() : null; - } + String format(); /** * Returns whether BigQuery should allow extra values that are not represented in the table @@ -280,9 +147,7 @@ public String format() { * columns are treated as bad records, and if there are too many bad records, an invalid error is * returned in the job result. By default unknown values are not allowed. */ - public Boolean ignoreUnknownValues() { - return ignoreUnknownValues; - } + Boolean ignoreUnknownValues(); /** * Returns which entity properties to load into BigQuery from a Cloud Datastore backup. This field @@ -291,104 +156,10 @@ public Boolean ignoreUnknownValues() { * all properties. If any named property isn't found in the Cloud Datastore backup, an invalid * error is returned in the job result. */ - public List projectionFields() { - return projectionFields; - } - - public Builder toBuilder() { - return new Builder(this); - } - - MoreObjects.ToStringHelper toStringHelper() { - return MoreObjects.toStringHelper(this) - .add("destinationTable", destinationTable) - .add("createDisposition", createDisposition) - .add("writeDisposition", writeDisposition) - .add("formatOptions", formatOptions) - .add("maxBadRecords", maxBadRecords) - .add("schema", schema) - .add("ignoreUnknownValue", ignoreUnknownValues) - .add("projectionFields", projectionFields); - } - - @Override - public String toString() { - return toStringHelper().toString(); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof LoadConfiguration - && Objects.equals(toPb(), ((LoadConfiguration) obj).toPb()); - } - - @Override - public int hashCode() { - return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions, - maxBadRecords, schema, ignoreUnknownValues, projectionFields); - } - - com.google.api.services.bigquery.model.JobConfiguration toPb() { - JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); - loadConfigurationPb.setDestinationTable(destinationTable.toPb()); - if (createDisposition != null) { - loadConfigurationPb.setCreateDisposition(createDisposition.toString()); - } - if (writeDisposition != null) { - loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); - } - if (csvOptions() != null) { - CsvOptions csvOptions = csvOptions(); - loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) - .setAllowJaggedRows(csvOptions.allowJaggedRows()) - .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) - .setEncoding(csvOptions.encoding()) - .setQuote(csvOptions.quote()) - .setSkipLeadingRows(csvOptions.skipLeadingRows()); - } - if (schema != null) { - loadConfigurationPb.setSchema(schema.toPb()); - } - if (formatOptions != null) { - loadConfigurationPb.setSourceFormat(formatOptions.type()); - } - loadConfigurationPb.setMaxBadRecords(maxBadRecords); - loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); - loadConfigurationPb.setProjectionFields(projectionFields); - return new com.google.api.services.bigquery.model.JobConfiguration() - .setLoad(loadConfigurationPb); - } - - static LoadConfiguration fromPb( - com.google.api.services.bigquery.model.JobConfiguration configurationPb) { - return new Builder<>(configurationPb).build(); - } - - /** - * Creates a builder for a BigQuery Load Configuration given the destination table. - */ - public static Builder builder(TableId destinationTable) { - return new Builder<>().destinationTable(destinationTable); - } - - /** - * Creates a builder for a BigQuery Load Configuration given the destination table and format. - */ - public static Builder builder(TableId destinationTable, FormatOptions format) { - return new Builder<>().destinationTable(destinationTable).formatOptions(format); - } + List projectionFields(); /** - * Returns a BigQuery Load Configuration for the given destination table. + * Returns a builder for the load configuration object. */ - public static LoadConfiguration of(TableId destinationTable) { - return builder(destinationTable).build(); - } - - /** - * Returns a BigQuery Load Configuration for the given destination table and format. - */ - public static LoadConfiguration of(TableId destinationTable, FormatOptions format) { - return builder(destinationTable).formatOptions(format).build(); - } + Builder toBuilder(); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java index 93ac10d4385b..a7c1c9a4a289 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,52 +16,162 @@ package com.google.gcloud.bigquery; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.JobConfigurationLoad; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; -import java.io.Serializable; import java.util.List; import java.util.Objects; /** - * Google BigQuery Load Job configuration. A Load Job loads data from one of several formats into a - * table. Data is provided as URIs that point to objects in Google Cloud Storage. + * Google BigQuery load job configuration. A load job loads data from one of several formats into a + * table. Data is provided as URIs that point to objects in Google Cloud Storage. Load job + * configurations have {@link JobConfiguration.Type#LOAD} type. */ -public class LoadJobConfiguration extends LoadConfiguration - implements JobConfiguration, Serializable { +public class LoadJobConfiguration extends JobConfiguration implements LoadConfiguration { private static final long serialVersionUID = -2673554846792429829L; private final List sourceUris; + protected final TableId destinationTable; + protected final JobInfo.CreateDisposition createDisposition; + protected final JobInfo.WriteDisposition writeDisposition; + protected final FormatOptions formatOptions; + protected final Integer maxBadRecords; + protected final Schema schema; + protected final Boolean ignoreUnknownValues; + protected final List projectionFields; public static final class Builder - extends LoadConfiguration.Builder { + extends JobConfiguration.Builder + implements LoadConfiguration.Builder { private List sourceUris; + private TableId destinationTable; + private JobInfo.CreateDisposition createDisposition; + private JobInfo.WriteDisposition writeDisposition; + private FormatOptions formatOptions; + private Integer maxBadRecords; + private Schema schema; + private Boolean ignoreUnknownValues; + private List projectionFields; private Builder() { - super(); + type(Type.LOAD); } private Builder(LoadJobConfiguration loadConfiguration) { - super(loadConfiguration); + type(Type.LOAD); + this.destinationTable = loadConfiguration.destinationTable; + this.createDisposition = loadConfiguration.createDisposition; + this.writeDisposition = loadConfiguration.writeDisposition; + this.formatOptions = loadConfiguration.formatOptions; + this.maxBadRecords = loadConfiguration.maxBadRecords; + this.schema = loadConfiguration.schema; + this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues; + this.projectionFields = loadConfiguration.projectionFields; this.sourceUris = loadConfiguration.sourceUris; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { - super(configurationPb); - if (configurationPb.getLoad().getSourceUris() != null) { + type(Type.LOAD); + JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); + this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); + if (loadConfigurationPb.getCreateDisposition() != null) { + this.createDisposition = + JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); + } + if (loadConfigurationPb.getWriteDisposition() != null) { + this.writeDisposition = + JobInfo.WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); + } + if (loadConfigurationPb.getSourceFormat() != null) { + this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); + } + if (loadConfigurationPb.getAllowJaggedRows() != null + || loadConfigurationPb.getAllowQuotedNewlines() != null + || loadConfigurationPb.getEncoding() != null + || loadConfigurationPb.getFieldDelimiter() != null + || loadConfigurationPb.getQuote() != null + || loadConfigurationPb.getSkipLeadingRows() != null) { + CsvOptions.Builder builder = CsvOptions.builder() + .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) + .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) + .encoding(loadConfigurationPb.getEncoding()) + .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) + .quote(loadConfigurationPb.getQuote()) + .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + this.formatOptions = builder.build(); + } + this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); + if (loadConfigurationPb.getSchema() != null) { + this.schema = Schema.fromPb(loadConfigurationPb.getSchema()); + } + this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues(); + this.projectionFields = loadConfigurationPb.getProjectionFields(); + if (loadConfigurationPb.getSourceUris() != null) { this.sourceUris = ImmutableList.copyOf(configurationPb.getLoad().getSourceUris()); } } + @Override + public Builder destinationTable(TableId destinationTable) { + this.destinationTable = destinationTable; + return this; + } + + @Override + public Builder createDisposition(JobInfo.CreateDisposition createDisposition) { + this.createDisposition = createDisposition; + return this; + } + + @Override + public Builder writeDisposition(JobInfo.WriteDisposition writeDisposition) { + this.writeDisposition = writeDisposition; + return this; + } + + @Override + public Builder formatOptions(FormatOptions formatOptions) { + this.formatOptions = formatOptions; + return this; + } + + @Override + public Builder maxBadRecords(Integer maxBadRecords) { + this.maxBadRecords = maxBadRecords; + return this; + } + + @Override + public Builder schema(Schema schema) { + this.schema = schema; + return this; + } + + @Override + public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { + this.ignoreUnknownValues = ignoreUnknownValues; + return this; + } + + @Override + public Builder projectionFields(List projectionFields) { + this.projectionFields = + projectionFields != null ? ImmutableList.copyOf(projectionFields) : null; + return this; + } + /** * Sets the fully-qualified URIs that point to source data in Google Cloud Storage (e.g. * gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the * 'bucket' name. */ public Builder sourceUris(List sourceUris) { - this.sourceUris = sourceUris != null ? ImmutableList.copyOf(sourceUris) : null; + this.sourceUris = ImmutableList.copyOf(checkNotNull(sourceUris)); return this; } @@ -74,11 +184,59 @@ public LoadJobConfiguration build() { private LoadJobConfiguration(Builder builder) { super(builder); this.sourceUris = builder.sourceUris; + this.destinationTable = builder.destinationTable; + this.createDisposition = builder.createDisposition; + this.writeDisposition = builder.writeDisposition; + this.formatOptions = builder.formatOptions; + this.maxBadRecords = builder.maxBadRecords; + this.schema = builder.schema; + this.ignoreUnknownValues = builder.ignoreUnknownValues; + this.projectionFields = builder.projectionFields; + } + + @Override + public TableId destinationTable() { + return destinationTable; + } + + @Override + public JobInfo.CreateDisposition createDisposition() { + return this.createDisposition; } @Override - public Type type() { - return Type.LOAD; + public JobInfo.WriteDisposition writeDisposition() { + return writeDisposition; + } + + @Override + public CsvOptions csvOptions() { + return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; + } + + @Override + public Integer maxBadRecords() { + return maxBadRecords; + } + + @Override + public Schema schema() { + return schema; + } + + @Override + public String format() { + return formatOptions != null ? formatOptions.type() : null; + } + + @Override + public Boolean ignoreUnknownValues() { + return ignoreUnknownValues; + } + + @Override + public List projectionFields() { + return projectionFields; } /** @@ -90,32 +248,67 @@ public List sourceUris() { return sourceUris; } + @Override public Builder toBuilder() { return new Builder(this); } @Override - ToStringHelper toStringHelper() { - return super.toStringHelper().add("sourceUris", sourceUris); + protected ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("destinationTable", destinationTable) + .add("createDisposition", createDisposition) + .add("writeDisposition", writeDisposition) + .add("formatOptions", formatOptions) + .add("maxBadRecords", maxBadRecords) + .add("schema", schema) + .add("ignoreUnknownValue", ignoreUnknownValues) + .add("projectionFields", projectionFields) + .add("sourceUris", sourceUris); } @Override public boolean equals(Object obj) { - return obj instanceof LoadJobConfiguration - && Objects.equals(toPb(), ((LoadJobConfiguration) obj).toPb()); + return obj instanceof LoadJobConfiguration && baseEquals((LoadJobConfiguration) obj); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), sourceUris); + return Objects.hash(baseHashCode(), sourceUris); } com.google.api.services.bigquery.model.JobConfiguration toPb() { - com.google.api.services.bigquery.model.JobConfiguration configurationPb = super.toPb(); + JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); + loadConfigurationPb.setDestinationTable(destinationTable.toPb()); + if (createDisposition != null) { + loadConfigurationPb.setCreateDisposition(createDisposition.toString()); + } + if (writeDisposition != null) { + loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); + } + if (csvOptions() != null) { + CsvOptions csvOptions = csvOptions(); + loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) + .setAllowJaggedRows(csvOptions.allowJaggedRows()) + .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) + .setEncoding(csvOptions.encoding()) + .setQuote(csvOptions.quote()) + .setSkipLeadingRows(csvOptions.skipLeadingRows()); + } + if (schema != null) { + loadConfigurationPb.setSchema(schema.toPb()); + } + if (formatOptions != null) { + loadConfigurationPb.setSourceFormat(formatOptions.type()); + } + loadConfigurationPb.setMaxBadRecords(maxBadRecords); + loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); + loadConfigurationPb.setProjectionFields(projectionFields); if (sourceUris != null) { - configurationPb.getLoad().setSourceUris(ImmutableList.copyOf(sourceUris)); + loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris)); } - return configurationPb; + return new com.google.api.services.bigquery.model.JobConfiguration() + .setLoad(loadConfigurationPb); } /** @@ -183,6 +376,7 @@ public static LoadJobConfiguration of(TableId destinationTable, String sourceUri return of(destinationTable, ImmutableList.of(sourceUri), format); } + @SuppressWarnings("unchecked") static LoadJobConfiguration fromPb( com.google.api.services.bigquery.model.JobConfiguration confPb) { return new Builder(confPb).build(); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java index 0cf12f546258..44837bca4126 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.bigquery.model.JobConfigurationQuery; -import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -27,15 +27,15 @@ import com.google.gcloud.bigquery.JobInfo.CreateDisposition; import com.google.gcloud.bigquery.JobInfo.WriteDisposition; -import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; /** - * Google BigQuery Query Job configuration. A Query Job runs a query against BigQuery data. + * Google BigQuery Query Job configuration. A Query Job runs a query against BigQuery data. Query + * job configurations have {@link JobConfiguration.Type#QUERY} type. */ -public final class QueryJobConfiguration implements JobConfiguration, Serializable { +public final class QueryJobConfiguration extends JobConfiguration { private static final long serialVersionUID = -1108948249081804890L; @@ -72,7 +72,7 @@ public enum Priority { private final Boolean flattenResults; private final Boolean dryRun; - public static final class Builder { + public static final class Builder extends JobConfiguration.Builder { private String query; private TableId destinationTable; @@ -87,9 +87,12 @@ public static final class Builder { private Boolean flattenResults; private Boolean dryRun; - private Builder() {} + private Builder() { + type(Type.QUERY); + } private Builder(QueryJobConfiguration jobConfiguration) { + type(Type.QUERY); this.query = jobConfiguration.query; this.destinationTable = jobConfiguration.destinationTable; this.tableDefinitions = jobConfiguration.tableDefinitions; @@ -105,6 +108,7 @@ private Builder(QueryJobConfiguration jobConfiguration) { } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + type(Type.QUERY); JobConfigurationQuery queryConfigurationPb = configurationPb.getQuery(); this.query = queryConfigurationPb.getQuery(); allowLargeResults = queryConfigurationPb.getAllowLargeResults(); @@ -294,6 +298,7 @@ public QueryJobConfiguration build() { } private QueryJobConfiguration(Builder builder) { + super(builder); this.query = checkNotNull(builder.query); this.allowLargeResults = builder.allowLargeResults; this.createDisposition = builder.createDisposition; @@ -309,11 +314,6 @@ private QueryJobConfiguration(Builder builder) { this.dryRun = builder.dryRun; } - @Override - public Type type() { - return Type.QUERY; - } - /** * Returns whether the job is enabled to create arbitrarily large results. If {@code true} * the query is allowed to create large results at a slight cost in performance. @@ -425,13 +425,14 @@ public Boolean dryRun() { return dryRun; } + @Override public Builder toBuilder() { return new Builder(this); } @Override - public String toString() { - return MoreObjects.toStringHelper(this) + protected ToStringHelper toStringHelper() { + return super.toStringHelper() .add("query", query) .add("destinationTable", destinationTable) .add("defaultDataset", defaultDataset) @@ -443,21 +444,19 @@ public String toString() { .add("userDefinedFunctions", userDefinedFunctions) .add("createDisposition", createDisposition) .add("writeDisposition", writeDisposition) - .add("dryRun", dryRun) - .toString(); + .add("dryRun", dryRun); } @Override public boolean equals(Object obj) { - return obj instanceof QueryJobConfiguration - && Objects.equals(toPb(), ((QueryJobConfiguration) obj).toPb()); + return obj instanceof QueryJobConfiguration && baseEquals((QueryJobConfiguration) obj); } @Override public int hashCode() { - return Objects.hash(allowLargeResults, createDisposition, destinationTable, defaultDataset, - flattenResults, priority, query, tableDefinitions, useQueryCache, userDefinedFunctions, - writeDisposition, dryRun); + return Objects.hash(baseHashCode(), allowLargeResults, createDisposition, destinationTable, + defaultDataset, flattenResults, priority, query, tableDefinitions, useQueryCache, + userDefinedFunctions, writeDisposition, dryRun); } com.google.api.services.bigquery.model.JobConfiguration toPb() { @@ -516,6 +515,7 @@ public static QueryJobConfiguration of(String query) { return builder(query).build(); } + @SuppressWarnings("unchecked") static QueryJobConfiguration fromPb( com.google.api.services.bigquery.model.JobConfiguration jobPb) { return new Builder(jobPb).build(); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java index c4cee5a9a303..bee0340a29a8 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java @@ -24,18 +24,18 @@ import com.google.gcloud.RetryHelper; import com.google.gcloud.WriteChannel; -import java.util.Arrays; - /** * WriteChannel implementation to stream data into a BigQuery table. */ -class TableDataWriteChannel extends BaseWriteChannel { +class TableDataWriteChannel extends BaseWriteChannel { - TableDataWriteChannel(BigQueryOptions options, LoadConfiguration loadConfiguration) { - this(options, loadConfiguration, options.rpc().open(loadConfiguration.toPb())); + TableDataWriteChannel(BigQueryOptions options, + WriteChannelConfiguration writeChannelConfiguration) { + this(options, writeChannelConfiguration, options.rpc().open(writeChannelConfiguration.toPb())); } - TableDataWriteChannel(BigQueryOptions options, LoadConfiguration config, String uploadId) { + TableDataWriteChannel(BigQueryOptions options, WriteChannelConfiguration config, + String uploadId) { super(options, config, uploadId); } @@ -57,7 +57,8 @@ protected StateImpl.Builder stateBuilder() { return StateImpl.builder(options(), entity(), uploadId()); } - static class StateImpl extends BaseWriteChannel.BaseState { + static class StateImpl + extends BaseWriteChannel.BaseState { private static final long serialVersionUID = -787362105981823738L; @@ -66,9 +67,10 @@ static class StateImpl extends BaseWriteChannel.BaseState { + extends BaseWriteChannel.BaseState.Builder { - private Builder(BigQueryOptions options, LoadConfiguration configuration, String uploadId) { + private Builder(BigQueryOptions options, WriteChannelConfiguration configuration, + String uploadId) { super(options, configuration, uploadId); } @@ -77,7 +79,8 @@ public RestorableState build() { } } - static Builder builder(BigQueryOptions options, LoadConfiguration config, String uploadId) { + static Builder builder(BigQueryOptions options, WriteChannelConfiguration config, + String uploadId) { return new Builder(options, config, uploadId); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java new file mode 100644 index 000000000000..7c026c100929 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java @@ -0,0 +1,316 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.JobConfigurationLoad; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.gcloud.bigquery.JobInfo.CreateDisposition; +import com.google.gcloud.bigquery.JobInfo.WriteDisposition; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery Configuration for a load operation. A load configuration can be used to load data + * into a table with a {@link com.google.gcloud.WriteChannel} + * ({@link BigQuery#writer(WriteChannelConfiguration)}). + */ +public class WriteChannelConfiguration implements LoadConfiguration, Serializable { + + private static final long serialVersionUID = 470267591917413578L; + + protected final TableId destinationTable; + protected final CreateDisposition createDisposition; + protected final WriteDisposition writeDisposition; + protected final FormatOptions formatOptions; + protected final Integer maxBadRecords; + protected final Schema schema; + protected final Boolean ignoreUnknownValues; + protected final List projectionFields; + + public static final class Builder implements LoadConfiguration.Builder { + + private TableId destinationTable; + private CreateDisposition createDisposition; + private WriteDisposition writeDisposition; + private FormatOptions formatOptions; + private Integer maxBadRecords; + private Schema schema; + private Boolean ignoreUnknownValues; + private List projectionFields; + + protected Builder() {} + + protected Builder(WriteChannelConfiguration writeChannelConfiguration) { + this.destinationTable = writeChannelConfiguration.destinationTable; + this.createDisposition = writeChannelConfiguration.createDisposition; + this.writeDisposition = writeChannelConfiguration.writeDisposition; + this.formatOptions = writeChannelConfiguration.formatOptions; + this.maxBadRecords = writeChannelConfiguration.maxBadRecords; + this.schema = writeChannelConfiguration.schema; + this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues; + this.projectionFields = writeChannelConfiguration.projectionFields; + } + + protected Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); + this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); + if (loadConfigurationPb.getCreateDisposition() != null) { + this.createDisposition = + CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); + } + if (loadConfigurationPb.getWriteDisposition() != null) { + this.writeDisposition = WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); + } + if (loadConfigurationPb.getSourceFormat() != null) { + this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); + } + if (loadConfigurationPb.getAllowJaggedRows() != null + || loadConfigurationPb.getAllowQuotedNewlines() != null + || loadConfigurationPb.getEncoding() != null + || loadConfigurationPb.getFieldDelimiter() != null + || loadConfigurationPb.getQuote() != null + || loadConfigurationPb.getSkipLeadingRows() != null) { + CsvOptions.Builder builder = CsvOptions.builder() + .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) + .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) + .encoding(loadConfigurationPb.getEncoding()) + .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) + .quote(loadConfigurationPb.getQuote()) + .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + this.formatOptions = builder.build(); + } + this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); + if (loadConfigurationPb.getSchema() != null) { + this.schema = Schema.fromPb(loadConfigurationPb.getSchema()); + } + this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues(); + this.projectionFields = loadConfigurationPb.getProjectionFields(); + } + + @Override + public Builder destinationTable(TableId destinationTable) { + this.destinationTable = destinationTable; + return this; + } + + @Override + public Builder createDisposition(CreateDisposition createDisposition) { + this.createDisposition = createDisposition; + return this; + } + + @Override + public Builder writeDisposition(WriteDisposition writeDisposition) { + this.writeDisposition = writeDisposition; + return this; + } + + @Override + public Builder formatOptions(FormatOptions formatOptions) { + this.formatOptions = formatOptions; + return this; + } + + @Override + public Builder maxBadRecords(Integer maxBadRecords) { + this.maxBadRecords = maxBadRecords; + return this; + } + + @Override + public Builder schema(Schema schema) { + this.schema = schema; + return this; + } + + @Override + public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { + this.ignoreUnknownValues = ignoreUnknownValues; + return this; + } + + @Override + public Builder projectionFields(List projectionFields) { + this.projectionFields = + projectionFields != null ? ImmutableList.copyOf(projectionFields) : null; + return this; + } + + public WriteChannelConfiguration build() { + return new WriteChannelConfiguration(this); + } + } + + protected WriteChannelConfiguration(Builder builder) { + this.destinationTable = checkNotNull(builder.destinationTable); + this.createDisposition = builder.createDisposition; + this.writeDisposition = builder.writeDisposition; + this.formatOptions = builder.formatOptions; + this.maxBadRecords = builder.maxBadRecords; + this.schema = builder.schema; + this.ignoreUnknownValues = builder.ignoreUnknownValues; + this.projectionFields = builder.projectionFields; + } + + @Override + public TableId destinationTable() { + return destinationTable; + } + + @Override + public CreateDisposition createDisposition() { + return this.createDisposition; + } + + @Override + public WriteDisposition writeDisposition() { + return writeDisposition; + } + + @Override + public CsvOptions csvOptions() { + return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; + } + + @Override + public Integer maxBadRecords() { + return maxBadRecords; + } + + @Override + public Schema schema() { + return schema; + } + + @Override + public String format() { + return formatOptions != null ? formatOptions.type() : null; + } + + @Override + public Boolean ignoreUnknownValues() { + return ignoreUnknownValues; + } + + @Override + public List projectionFields() { + return projectionFields; + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + MoreObjects.ToStringHelper toStringHelper() { + return MoreObjects.toStringHelper(this) + .add("destinationTable", destinationTable) + .add("createDisposition", createDisposition) + .add("writeDisposition", writeDisposition) + .add("formatOptions", formatOptions) + .add("maxBadRecords", maxBadRecords) + .add("schema", schema) + .add("ignoreUnknownValue", ignoreUnknownValues) + .add("projectionFields", projectionFields); + } + + @Override + public String toString() { + return toStringHelper().toString(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof WriteChannelConfiguration + && Objects.equals(toPb(), ((WriteChannelConfiguration) obj).toPb()); + } + + @Override + public int hashCode() { + return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions, + maxBadRecords, schema, ignoreUnknownValues, projectionFields); + } + + com.google.api.services.bigquery.model.JobConfiguration toPb() { + JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); + loadConfigurationPb.setDestinationTable(destinationTable.toPb()); + if (createDisposition != null) { + loadConfigurationPb.setCreateDisposition(createDisposition.toString()); + } + if (writeDisposition != null) { + loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); + } + if (csvOptions() != null) { + CsvOptions csvOptions = csvOptions(); + loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) + .setAllowJaggedRows(csvOptions.allowJaggedRows()) + .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) + .setEncoding(csvOptions.encoding()) + .setQuote(csvOptions.quote()) + .setSkipLeadingRows(csvOptions.skipLeadingRows()); + } + if (schema != null) { + loadConfigurationPb.setSchema(schema.toPb()); + } + if (formatOptions != null) { + loadConfigurationPb.setSourceFormat(formatOptions.type()); + } + loadConfigurationPb.setMaxBadRecords(maxBadRecords); + loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); + loadConfigurationPb.setProjectionFields(projectionFields); + return new com.google.api.services.bigquery.model.JobConfiguration() + .setLoad(loadConfigurationPb); + } + + static WriteChannelConfiguration fromPb( + com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + return new Builder(configurationPb).build(); + } + + /** + * Creates a builder for a BigQuery Load Configuration given the destination table. + */ + public static Builder builder(TableId destinationTable) { + return new Builder().destinationTable(destinationTable); + } + + /** + * Creates a builder for a BigQuery Load Configuration given the destination table and format. + */ + public static Builder builder(TableId destinationTable, FormatOptions format) { + return builder(destinationTable).formatOptions(format); + } + + /** + * Returns a BigQuery Load Configuration for the given destination table. + */ + public static WriteChannelConfiguration of(TableId destinationTable) { + return builder(destinationTable).build(); + } + + /** + * Returns a BigQuery Load Configuration for the given destination table and format. + */ + public static WriteChannelConfiguration of(TableId destinationTable, FormatOptions format) { + return builder(destinationTable).formatOptions(format).build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java index 592ae1b93d6d..b57f1dc8a128 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java @@ -24,7 +24,6 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; -import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; @@ -43,7 +42,6 @@ import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.Job; import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationLoad; import com.google.api.services.bigquery.model.JobList; import com.google.api.services.bigquery.model.JobStatus; import com.google.api.services.bigquery.model.QueryRequest; diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java index 862dd5b930fa..8af8c700cd8c 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java @@ -1010,12 +1010,13 @@ public void testGetQueryResultsWithOptions() { @Test public void testWriter() { - LoadConfiguration loadConfiguration = LoadConfiguration.of(TABLE_ID); - EasyMock.expect(bigqueryRpcMock.open(LoadConfiguration.of(TABLE_ID_WITH_PROJECT).toPb())) - .andReturn("upload-id"); + WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.of(TABLE_ID); + EasyMock.expect( + bigqueryRpcMock.open(WriteChannelConfiguration.of(TABLE_ID_WITH_PROJECT).toPb())) + .andReturn("upload-id"); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - WriteChannel channel = bigquery.writer(loadConfiguration); + WriteChannel channel = bigquery.writer(writeChannelConfiguration); assertNotNull(channel); assertTrue(channel.isOpen()); } diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java index 0a94b4fbded2..ff038cf0a37d 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java index a81c912dbb58..d4d7a50f47f2 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java index be30b5eb5050..e083d3682d8c 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java @@ -878,7 +878,7 @@ public void testCancelNonExistingJob() throws InterruptedException { public void testInsertFromFile() throws InterruptedException, FileNotFoundException { String destinationTableName = "test_insert_from_file_table"; TableId tableId = TableId.of(DATASET, destinationTableName); - LoadConfiguration configuration = LoadConfiguration.builder(tableId) + WriteChannelConfiguration configuration = WriteChannelConfiguration.builder(tableId) .formatOptions(FormatOptions.json()) .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .schema(TABLE_SCHEMA) diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java index 5c45215633c4..5bc3643c288d 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java index f58df04cfcad..495b5e4d64f6 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java index 6febe38b73f3..c59deff5020f 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index 1d0673a0183b..19b281f073b3 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -172,14 +172,15 @@ public class SerializationTest { CopyJobConfiguration.of(TABLE_ID, TABLE_ID); private static final ExtractJobConfiguration EXTRACT_JOB_CONFIGURATION = ExtractJobConfiguration.of(TABLE_ID, SOURCE_URIS); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) - .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) - .formatOptions(CSV_OPTIONS) - .ignoreUnknownValues(true) - .maxBadRecords(10) - .schema(TABLE_SCHEMA) - .build(); + private static final WriteChannelConfiguration LOAD_CONFIGURATION = + WriteChannelConfiguration.builder(TABLE_ID) + .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) + .formatOptions(CSV_OPTIONS) + .ignoreUnknownValues(true) + .maxBadRecords(10) + .schema(TABLE_SCHEMA) + .build(); private static final LoadJobConfiguration LOAD_JOB_CONFIGURATION = LoadJobConfiguration.of(TABLE_ID, SOURCE_URIS); private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java index 67933407e377..6b7edcd76db1 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java @@ -50,13 +50,14 @@ public class TableDataWriteChannelTest { private static final String UPLOAD_ID = "uploadid"; private static final TableId TABLE_ID = TableId.of("dataset", "table"); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) - .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) - .formatOptions(FormatOptions.json()) - .ignoreUnknownValues(true) - .maxBadRecords(10) - .build(); + private static final WriteChannelConfiguration LOAD_CONFIGURATION = + WriteChannelConfiguration.builder(TABLE_ID) + .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) + .formatOptions(FormatOptions.json()) + .ignoreUnknownValues(true) + .maxBadRecords(10) + .build(); private static final int MIN_CHUNK_SIZE = 256 * 1024; private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE; private static final int CUSTOM_CHUNK_SIZE = 4 * MIN_CHUNK_SIZE; diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/WriteChannelConfigurationTest.java similarity index 84% rename from gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadConfigurationTest.java rename to gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/WriteChannelConfigurationTest.java index a245026c8854..17fa8446d097 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/WriteChannelConfigurationTest.java @@ -28,7 +28,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; -public class LoadConfigurationTest { +public class WriteChannelConfigurationTest { private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() .allowJaggedRows(true) @@ -47,7 +47,7 @@ public class LoadConfigurationTest { .description("FieldDescription") .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) + private static final WriteChannelConfiguration LOAD_CONFIGURATION = WriteChannelConfiguration.builder(TABLE_ID) .createDisposition(CREATE_DISPOSITION) .writeDisposition(WRITE_DISPOSITION) .formatOptions(CSV_OPTIONS) @@ -60,7 +60,7 @@ public class LoadConfigurationTest { @Test public void testToBuilder() { compareLoadConfiguration(LOAD_CONFIGURATION, LOAD_CONFIGURATION.toBuilder().build()); - LoadConfiguration configuration = LOAD_CONFIGURATION.toBuilder() + WriteChannelConfiguration configuration = LOAD_CONFIGURATION.toBuilder() .destinationTable(TableId.of("dataset", "newTable")) .build(); assertEquals("newTable", configuration.destinationTable().table()); @@ -70,9 +70,9 @@ public void testToBuilder() { @Test public void testOf() { - LoadConfiguration configuration = LoadConfiguration.of(TABLE_ID); + WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); assertEquals(TABLE_ID, configuration.destinationTable()); - configuration = LoadConfiguration.of(TABLE_ID, CSV_OPTIONS); + configuration = WriteChannelConfiguration.of(TABLE_ID, CSV_OPTIONS); assertEquals(TABLE_ID, configuration.destinationTable()); assertEquals(FORMAT, configuration.format()); assertEquals(CSV_OPTIONS, configuration.csvOptions()); @@ -80,7 +80,7 @@ public void testOf() { @Test public void testToBuilderIncomplete() { - LoadConfiguration configuration = LoadConfiguration.of(TABLE_ID); + WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); compareLoadConfiguration(configuration, configuration.toBuilder().build()); } @@ -101,12 +101,12 @@ public void testBuilder() { public void testToPbAndFromPb() { assertNull(LOAD_CONFIGURATION.toPb().getLoad().getSourceUris()); compareLoadConfiguration(LOAD_CONFIGURATION, - LoadConfiguration.fromPb(LOAD_CONFIGURATION.toPb())); - LoadConfiguration configuration = LoadConfiguration.of(TABLE_ID); - compareLoadConfiguration(configuration, LoadConfiguration.fromPb(configuration.toPb())); + WriteChannelConfiguration.fromPb(LOAD_CONFIGURATION.toPb())); + WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); + compareLoadConfiguration(configuration, WriteChannelConfiguration.fromPb(configuration.toPb())); } - private void compareLoadConfiguration(LoadConfiguration expected, LoadConfiguration value) { + private void compareLoadConfiguration(WriteChannelConfiguration expected, WriteChannelConfiguration value) { assertEquals(expected, value); assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java index 2143be188338..8fe78cbd50ad 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java @@ -34,7 +34,7 @@ import com.google.gcloud.bigquery.JobId; import com.google.gcloud.bigquery.JobInfo; import com.google.gcloud.bigquery.JobStatus; -import com.google.gcloud.bigquery.LoadConfiguration; +import com.google.gcloud.bigquery.WriteChannelConfiguration; import com.google.gcloud.bigquery.LoadJobConfiguration; import com.google.gcloud.bigquery.QueryRequest; import com.google.gcloud.bigquery.QueryResponse; @@ -576,8 +576,8 @@ JobInfo parse(String... args) throws Exception { String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); - ExtractJobConfiguration configuration = - ExtractJobConfiguration.of(tableId, Arrays.asList(args).subList(3, args.length)); + ExtractJobConfiguration configuration = ExtractJobConfiguration.of( + tableId, Arrays.asList(args).subList(3, args.length), format); return JobInfo.of(configuration); } throw new IllegalArgumentException("Missing required arguments."); @@ -671,9 +671,11 @@ protected String params() { * @see Resumable * Upload */ - private static class LoadFileAction extends BigQueryAction> { + private static class LoadFileAction + extends BigQueryAction> { @Override - void run(BigQuery bigquery, Tuple configuration) throws Exception { + void run(BigQuery bigquery, Tuple configuration) + throws Exception { System.out.println("Running insert"); try (FileChannel fileChannel = FileChannel.open(Paths.get(configuration.y()))) { WriteChannel writeChannel = bigquery.writer(configuration.x()); @@ -688,13 +690,14 @@ void run(BigQuery bigquery, Tuple configuration) thro } @Override - Tuple parse(String... args) throws Exception { + Tuple parse(String... args) throws Exception { if (args.length == 4) { String dataset = args[0]; String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); - LoadConfiguration configuration = LoadConfiguration.of(tableId, FormatOptions.of(format)); + WriteChannelConfiguration configuration = + WriteChannelConfiguration.of(tableId, FormatOptions.of(format)); return Tuple.of(configuration, args[3]); } throw new IllegalArgumentException("Missing required arguments.");