From 8d2249e332d6701d02a283e30e64577d4a1e35fa Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 30 Nov 2015 16:31:51 +0100 Subject: [PATCH 01/10] Add TableInfo, CsvOptions, ExternalDataConfiguration and UserDefinedFunction models and tests --- .../google/gcloud/bigquery/CsvOptions.java | 267 +++++++ .../bigquery/ExternalDataConfiguration.java | 419 ++++++++++ .../google/gcloud/bigquery/FieldSchema.java | 287 +++++++ .../com/google/gcloud/bigquery/TableInfo.java | 743 ++++++++++++++++++ .../google/gcloud/bigquery/TableSchema.java | 151 ++++ .../gcloud/bigquery/UserDefinedFunction.java | 158 ++++ .../gcloud/bigquery/CsvOptionsTest.java | 83 ++ .../ExternalDataConfigurationTest.java | 109 +++ .../gcloud/bigquery/FieldSchemaTest.java | 105 +++ .../gcloud/bigquery/SerializationTest.java | 51 +- .../google/gcloud/bigquery/TableInfoTest.java | 229 ++++++ .../gcloud/bigquery/TableSchemaTest.java | 77 ++ .../bigquery/UserDefinedFunctionTest.java | 56 ++ 13 files changed, 2734 insertions(+), 1 deletion(-) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java new file mode 100644 index 000000000000..dc9e88e97a15 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java @@ -0,0 +1,267 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import com.google.common.base.MoreObjects; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Google BigQuery CSV options. This class wraps some properties of CSV files used by BigQuery to + * parse external data. + */ +public class CsvOptions implements Serializable { + + private static final long serialVersionUID = 2193570529308612708L; + + private final Boolean allowJaggedRows; + private final Boolean allowQuotedNewLines; + private final String encoding; + private final String fieldDelimiter; + private final String quote; + private final Integer skipLeadingRows; + + public static final class Builder { + + private Boolean allowJaggedRows; + private Boolean allowQuotedNewLines; + private String encoding; + private String fieldDelimiter; + private String quote; + private Integer skipLeadingRows; + + private Builder() {} + + /** + * Set whether BigQuery should accept rows that are missing trailing optional columns. If + * {@code true}, BigQuery treats missing trailing columns as null values. If {@code false}, + * records with missing trailing columns are treated as bad records, and if there are too many + * bad records, an invalid error is returned in the job result. By default, rows with missing + * trailing columns are considered bad records. + */ + public Builder allowJaggedRows(Boolean allowJaggedRows) { + this.allowJaggedRows = allowJaggedRows; + return this; + } + + /** + * Sets whether BigQuery should allow quoted data sections that contain newline characters in a + * CSV file. By default quoted newline are not allowed. + */ + public Builder allowQuotedNewLines(Boolean allowQuotedNewLines) { + this.allowQuotedNewLines = allowQuotedNewLines; + return this; + } + + /** + * Sets the character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The + * default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split + * using the values set in {@link #quote(String)} and {@link #fieldDelimiter(String)}. + */ + public Builder encoding(String encoding) { + this.encoding = encoding; + return this; + } + + /** + * Sets the separator for fields in a CSV file. BigQuery converts the string to ISO-8859-1 + * encoding, and then uses the first byte of the encoded string to split the data in its raw, + * binary state. BigQuery also supports the escape sequence "\t" to specify a tab separator. + * The default value is a comma (','). + */ + public Builder fieldDelimiter(String fieldDelimiter) { + this.fieldDelimiter = fieldDelimiter; + return this; + } + + /** + * Sets the value that is used to quote data sections in a CSV file. BigQuery converts the + * string to ISO-8859-1 encoding, and then uses the first byte of the encoded string to split + * the data in its raw, binary state. The default value is a double-quote ('"'). If your data + * does not contain quoted sections, set the property value to an empty string. If your data + * contains quoted newline characters, you must also set {@link #allowQuotedNewLines(Boolean)} + * property to {@code true}. + */ + public Builder quote(String quote) { + this.quote = quote; + return this; + } + + /** + * Sets the number of rows at the top of a CSV file that BigQuery will skip when reading the + * data. The default value is 0. This property is useful if you have header rows in the file + * that should be skipped. + */ + public Builder skipLeadingRows(Integer skipLeadingRows) { + this.skipLeadingRows = skipLeadingRows; + return this; + } + + /** + * Creates an {@code ExternalDataConfiguration} object. + */ + public CsvOptions build() { + return new CsvOptions(this); + } + } + + private CsvOptions(Builder builder) { + this.allowJaggedRows = builder.allowJaggedRows; + this.allowQuotedNewLines = builder.allowQuotedNewLines; + this.encoding = builder.encoding; + this.fieldDelimiter = builder.fieldDelimiter; + this.quote = builder.quote; + this.skipLeadingRows = builder.skipLeadingRows; + } + + /** + * Returns whether BigQuery should accept rows that are missing trailing optional columns. If + * {@code true}, BigQuery treats missing trailing columns as null values. If {@code false}, + * records with missing trailing columns are treated as bad records, and if there are too many + * bad records, an invalid error is returned in the job result. + */ + public Boolean allowJaggedRows() { + return allowJaggedRows; + } + + /** + * Returns whether BigQuery should allow quoted data sections that contain newline characters in a + * CSV file. + */ + public Boolean allowQuotedNewLines() { + return allowQuotedNewLines; + } + + /** + * Returns the character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The + * default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split + * using the values set in {@link #quote()} and {@link #fieldDelimiter()}. + */ + public String encoding() { + return encoding; + } + + /** + * Returns the separator for fields in a CSV file. + */ + public String fieldDelimiter() { + return fieldDelimiter; + } + + /** + * Returns the value that is used to quote data sections in a CSV file. + */ + public String quote() { + return quote; + } + + /** + * Returns the number of rows at the top of a CSV file that BigQuery will skip when reading the + * data. + */ + public Integer skipLeadingRows() { + return skipLeadingRows; + } + + public Builder toBuilder() { + return new Builder() + .allowJaggedRows(allowJaggedRows) + .allowQuotedNewLines(allowQuotedNewLines) + .encoding(encoding) + .fieldDelimiter(fieldDelimiter) + .quote(quote) + .skipLeadingRows(skipLeadingRows); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("allowJaggedRows", allowJaggedRows) + .add("allowQuotedNewLines", allowQuotedNewLines) + .add("encoding", encoding) + .add("fieldDelimiter", fieldDelimiter) + .add("quote", quote) + .add("skipLeadingRows", skipLeadingRows) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(allowJaggedRows, allowQuotedNewLines, encoding, fieldDelimiter, quote, + skipLeadingRows); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof CsvOptions && Objects.equals(toPb(), ((CsvOptions) obj).toPb()); + } + + com.google.api.services.bigquery.model.CsvOptions toPb() { + com.google.api.services.bigquery.model.CsvOptions csvOptions = + new com.google.api.services.bigquery.model.CsvOptions(); + if (allowJaggedRows != null) { + csvOptions.setAllowJaggedRows(allowJaggedRows); + } + if (allowQuotedNewLines != null) { + csvOptions.setAllowQuotedNewlines(allowQuotedNewLines); + } + if (encoding != null) { + csvOptions.setEncoding(encoding); + } + if (fieldDelimiter != null) { + csvOptions.setFieldDelimiter(fieldDelimiter); + } + if (quote != null) { + csvOptions.setQuote(quote); + } + if (skipLeadingRows != null) { + csvOptions.setSkipLeadingRows(skipLeadingRows); + } + return csvOptions; + } + + /** + * Returns a builder for a CsvOptions object. + */ + public static Builder builder() { + return new Builder(); + } + + static CsvOptions fromPb(com.google.api.services.bigquery.model.CsvOptions csvOptions) { + Builder builder = builder(); + if (csvOptions.getAllowJaggedRows() != null) { + builder.allowJaggedRows(csvOptions.getAllowJaggedRows()); + } + if (csvOptions.getAllowQuotedNewlines() != null) { + builder.allowQuotedNewLines(csvOptions.getAllowQuotedNewlines()); + } + if (csvOptions.getEncoding() != null) { + builder.encoding(csvOptions.getEncoding()); + } + if (csvOptions.getFieldDelimiter() != null) { + builder.fieldDelimiter(csvOptions.getFieldDelimiter()); + } + if (csvOptions.getQuote() != null) { + builder.quote(csvOptions.getQuote()); + } + if (csvOptions.getSkipLeadingRows() != null) { + builder.skipLeadingRows(csvOptions.getSkipLeadingRows()); + } + return builder.build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java new file mode 100644 index 000000000000..f7eb792c675c --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java @@ -0,0 +1,419 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery configuration for tables backed by external data. Objects of this class describe + * the data format, location, and other properties of a table stored outside of BigQuery. + * By defining these properties, the data source can then be queried as if it were a standard + * BigQuery table. Support for external tables is experimental and might be subject to changes or + * removed. + * + * @see Federated Data Sources + * + */ +public class ExternalDataConfiguration implements Serializable { + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public ExternalDataConfiguration apply( + com.google.api.services.bigquery.model.ExternalDataConfiguration configurationPb) { + return ExternalDataConfiguration.fromPb(configurationPb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public com.google.api.services.bigquery.model.ExternalDataConfiguration apply( + ExternalDataConfiguration configuration) { + return configuration.toPb(); + } + }; + + private static final long serialVersionUID = -8004288831035566549L; + + private final List sourceUris; + private final TableSchema schema; + private final String sourceFormat; + private final Integer maxBadRecords; + private final Boolean ignoreUnknownValues; + private final String compression; + private final CsvOptions csvOptions; + + public static final class Builder { + + private List sourceUris; + private TableSchema schema; + private String sourceFormat; + private Integer maxBadRecords; + private Boolean ignoreUnknownValues; + private String compression; + private CsvOptions csvOptions; + + private Builder() {} + + /** + * Sets the fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can + * contain one '*' wildcard character that must come after the bucket's name. Size limits + * related to load jobs apply to external data sources, plus an additional limit of 10 GB + * maximum size across all URIs. + * + * @see Quota + */ + public Builder sourceUris(List sourceUris) { + this.sourceUris = ImmutableList.copyOf(checkNotNull(sourceUris)); + return this; + } + + /** + * Sets the schema for the external data. + */ + public Builder schema(TableSchema schema) { + this.schema = checkNotNull(schema); + return this; + } + + /** + * Sets the source format of the external data. + * + * + * Source Format + */ + public Builder sourceFormat(String sourceFormat) { + this.sourceFormat = checkNotNull(sourceFormat); + return this; + } + + /** + * Sets the maximum number of bad records that BigQuery can ignore when reading data. If the + * number of bad records exceeds this value, an invalid error is returned in the job result. + * The default value is 0, which requires that all records are valid. + */ + public Builder maxBadRecords(Integer maxBadRecords) { + this.maxBadRecords = maxBadRecords; + return this; + } + + /** + * Sets whether BigQuery should allow extra values that are not represented in the table schema. + * If true, the extra values are ignored. If false, records with extra columns are treated as + * bad records, and if there are too many bad records, an invalid error is returned in the job + * result. The default value is false. The value set with + * {@link #sourceFormat(String)} property determines what + * BigQuery treats as an extra value. + * + * @see + * Ignore Unknown Values + */ + public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { + this.ignoreUnknownValues = ignoreUnknownValues; + return this; + } + + /** + * Sets compression type of the data source. By default no compression is assumed. + * + * @see + * Compression + */ + public Builder compression(String compression) { + this.compression = compression; + return this; + } + + /** + * Sets additional properties to be used to parse CSV data (used when + * {@link #sourceFormat(String)} is set to CSV). + */ + public Builder csvOptions(CsvOptions csvOptions) { + this.csvOptions = csvOptions; + return this; + } + + /** + * Creates an {@code ExternalDataConfiguration} object. + */ + public ExternalDataConfiguration build() { + return new ExternalDataConfiguration(this); + } + } + + ExternalDataConfiguration(Builder builder) { + this.compression = builder.compression; + this.ignoreUnknownValues = builder.ignoreUnknownValues; + this.maxBadRecords = builder.maxBadRecords; + this.schema = builder.schema; + this.sourceFormat = builder.sourceFormat; + this.sourceUris = builder.sourceUris; + this.csvOptions = builder.csvOptions; + } + + /** + * Returns the compression type of the data source. + * + * @see + * Compression + */ + public String compression() { + return compression; + } + + /** + * Returns whether BigQuery should allow extra values that are not represented in the table + * schema. If true, the extra values are ignored. If false, records with extra columns are treated + * as bad records, and if there are too many bad records, an invalid error is returned in the job + * result. The default value is false. The value of {@link #sourceFormat()} determines what + * BigQuery treats as an extra value. + * + * @see + * Ignore Unknown Values + */ + public Boolean ignoreUnknownValues() { + return ignoreUnknownValues; + } + + /** + * Returns the maximum number of bad records that BigQuery can ignore when reading data. If the + * number of bad records exceeds this value, an invalid error is returned in the job result. + */ + public Integer maxBadRecords() { + return maxBadRecords; + } + + /** + * Returns the schema for the external data. + */ + public TableSchema schema() { + return schema; + } + + /** + * Sets the source format of the external data. + * + * + * Source Format + */ + public String sourceFormat() { + return sourceFormat; + } + + /** + * Returns the fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can + * contain one '*' wildcard character that must come after the bucket's name. Size limits + * related to load jobs apply to external data sources, plus an additional limit of 10 GB + * maximum size across all URIs. + * + * @see Quota + */ + public List sourceUris() { + return sourceUris; + } + + /** + * Returns additional properties used to parse CSV data (used when {@link #sourceFormat()} is set + * to CSV). + */ + public CsvOptions csvOptions() { + return csvOptions; + } + + /** + * Returns a builder for the {@code ExternalDataConfiguration} object. + */ + public Builder toBuilder() { + return new Builder() + .compression(compression) + .ignoreUnknownValues(ignoreUnknownValues) + .maxBadRecords(maxBadRecords) + .schema(schema) + .sourceFormat(sourceFormat) + .sourceUris(sourceUris) + .csvOptions(csvOptions); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("sourceUris", sourceUris) + .add("sourceFormat", sourceFormat) + .add("schema", schema) + .add("compression", compression) + .add("ignoreUnknownValues", ignoreUnknownValues) + .add("maxBadRecords", maxBadRecords) + .add("csvOptions", csvOptions) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(compression, ignoreUnknownValues, maxBadRecords, schema, sourceFormat, + sourceUris, csvOptions); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ExternalDataConfiguration + && Objects.equals(toPb(), ((ExternalDataConfiguration) obj).toPb()); + } + + com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() { + com.google.api.services.bigquery.model.ExternalDataConfiguration externalConfigurationPb = + new com.google.api.services.bigquery.model.ExternalDataConfiguration(); + if (compression != null) { + externalConfigurationPb.setCompression(compression); + } + if (ignoreUnknownValues != null) { + externalConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); + } + if (maxBadRecords != null) { + externalConfigurationPb.setMaxBadRecords(maxBadRecords); + } + if (schema != null) { + externalConfigurationPb.setSchema(schema.toPb()); + } + if (sourceFormat != null) { + externalConfigurationPb.setSourceFormat(sourceFormat); + } + if (sourceUris != null) { + externalConfigurationPb.setSourceUris(sourceUris); + } + if (csvOptions != null) { + externalConfigurationPb.setCsvOptions(csvOptions.toPb()); + } + return externalConfigurationPb; + } + + /** + * Creates a builder for an ExternalDataConfiguration object. + * + * @param sourceUris the fully-qualified URIs that point to your data in Google Cloud Storage. + * Each URI can contain one '*' wildcard character that must come after the bucket's name. + * Size limits related to load jobs apply to external data sources, plus an additional limit + * of 10 GB maximum size across all URIs. + * @param schema the schema for the external data + * @param format the source format of the external data + * @return a builder for an ExternalDataConfiguration object given source URIs, schema and format + * + * @see Quota + * @see + * Source Format + */ + public static Builder builder(List sourceUris, TableSchema schema, String format) { + return new Builder().sourceUris(sourceUris).schema(schema).sourceFormat(format); + } + + /** + * Creates a builder for an ExternalDataConfiguration object. + * + * @param sourceUri a fully-qualified URI that points to your data in Google Cloud Storage. The + * URI can contain one '*' wildcard character that must come after the bucket's name. Size + * limits related to load jobs apply to external data sources. + * @param schema the schema for the external data + * @param format the source format of the external data + * @return a builder for an ExternalDataConfiguration object given source URI, schema and format + * + * @see Quota + * @see + * Source Format + */ + public static Builder builder(String sourceUri, TableSchema schema, String format) { + return new Builder() + .sourceUris(ImmutableList.of(sourceUri)) + .schema(schema) + .sourceFormat(format); + } + + /** + * Creates an ExternalDataConfiguration object. + * + * @param sourceUris the fully-qualified URIs that point to your data in Google Cloud Storage. + * Each URI can contain one '*' wildcard character that must come after the bucket's name. + * Size limits related to load jobs apply to external data sources, plus an additional limit + * of 10 GB maximum size across all URIs. + * @param schema the schema for the external data + * @param format the source format of the external data + * @return an ExternalDataConfiguration object given source URIs, schema and format + * + * @see Quota + * @see + * Source Format + */ + public static ExternalDataConfiguration of( + List sourceUris, TableSchema schema, String format) { + return builder(sourceUris, schema, format).build(); + } + + /** + * Creates an ExternalDataConfiguration object. + * + * @param sourceUri a fully-qualified URI that points to your data in Google Cloud Storage. The + * URI can contain one '*' wildcard character that must come after the bucket's name. Size + * limits related to load jobs apply to external data sources. + * @param schema the schema for the external data + * @param format the source format of the external data + * @return an ExternalDataConfiguration object given source URIs, schema and format + * + * @see Quota + * @see + * Source Format + */ + public static ExternalDataConfiguration of(String sourceUri, TableSchema schema, String format) { + return builder(sourceUri, schema, format).build(); + } + + static ExternalDataConfiguration fromPb( + com.google.api.services.bigquery.model.ExternalDataConfiguration externalDataConfiguration) { + Builder builder = new Builder(); + if (externalDataConfiguration.getSourceUris() != null) { + builder.sourceUris(externalDataConfiguration.getSourceUris()); + } + if (externalDataConfiguration.getSchema() != null) { + builder.schema(TableSchema.fromPb(externalDataConfiguration.getSchema())); + } + if (externalDataConfiguration.getSourceFormat() != null) { + builder.sourceFormat(externalDataConfiguration.getSourceFormat()); + } + if (externalDataConfiguration.getCompression() != null) { + builder.compression(externalDataConfiguration.getCompression()); + } + if (externalDataConfiguration.getIgnoreUnknownValues() != null) { + builder.ignoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues()); + } + if (externalDataConfiguration.getCsvOptions() != null) { + builder.csvOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); + } + if (externalDataConfiguration.getMaxBadRecords() != null) { + builder.maxBadRecords(externalDataConfiguration.getMaxBadRecords()); + } + return builder.build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java new file mode 100644 index 000000000000..fa622bf28976 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java @@ -0,0 +1,287 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * Google Bigquery schema for a Table field. Several data types and modes are supported. + */ +public class FieldSchema implements Serializable { + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public FieldSchema apply(TableFieldSchema pb) { + return FieldSchema.fromPb(pb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public TableFieldSchema apply(FieldSchema fieldSchema) { + return fieldSchema.toPb(); + } + }; + + private static final long serialVersionUID = -8154262932305199256L; + + /** + * Data Types for a BigQuery Table field. + * + * @see + * Data Types + */ + public enum Type { + STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, RECORD + } + + /** + * Mode for a BigQuery Table field. {@link Mode#NULLABLE} fields can be set to {@code null}, + * {@link Mode#REQUIRED} fields must be provided. {@link Mode#REPEATED} fields can contain more + * than one value. + */ + public enum Mode { + NULLABLE, REQUIRED, REPEATED + } + + private final String name; + private final Type type; + private final Mode mode; + private final String description; + private final List fields; + + public static final class Builder { + + private String name; + private Type type; + private Mode mode; + private String description; + private List fields; + + private Builder() {} + + /** + * Sets the field name. The name must contain only letters (a-z, A-Z), numbers (0-9), or + * underscores (_), and must start with a letter or underscore. The maximum length is 128 + * characters. + */ + public Builder name(String name) { + this.name = checkNotNull(name); + return this; + } + + /** + * Sets the type of the field. + * + * @see + * Data Types + */ + public Builder type(Type type) { + this.type = checkNotNull(type); + return this; + } + + /** + * Sets the mode of the field. By default {@link Mode#NULLABLE} is used. + */ + public Builder mode(Mode mode) { + this.mode = mode; + return this; + } + + /** + * Sets the field description. The maximum length is 16K characters. + */ + public Builder description(String description) { + this.description = description; + return this; + } + + /** + * Sets the nested schema fields if {@link #type(Type)} is set to {@link Type#RECORD}. + */ + Builder fields(Iterable fields) { + this.fields = fields != null ? ImmutableList.copyOf(fields) : null; + return this; + } + + /** + * Creates an {@code FieldSchema} object. + */ + public FieldSchema build() { + FieldSchema fieldSchema = new FieldSchema(this); + checkNotNull(fieldSchema.name); + checkNotNull(fieldSchema.type); + return fieldSchema; + } + } + + private FieldSchema(Builder builder) { + this.name = builder.name; + this.type = builder.type; + this.mode = builder.mode; + this.description = builder.description; + this.fields = builder.fields; + } + + /** + * Returns the field name. + */ + public String name() { + return name; + } + + /** + * Returns the field type. + * + * @see + * Data Types + */ + public Type type() { + return type; + } + + /** + * Returns the field mode. By default {@link Mode#NULLABLE} is used. + */ + public Mode mode() { + return mode; + } + + /** + * Returns the field description. + */ + public String description() { + return description; + } + + /** + * Returns the nested schema fields if {@link #type()} is set to {@link Type#RECORD}. Returns + * {@code null} otherwise. + */ + public List fields() { + return fields; + } + + /** + * Returns a builder for the {@code FieldSchema} object. + */ + public Builder toBuilder() { + return new Builder() + .name(this.name) + .type(this.type) + .mode(this.mode) + .description(this.description) + .fields(this.fields); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("type", type) + .add("mode", mode) + .add("description", description) + .add("fields", fields) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(name, type, mode, description, fields); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof FieldSchema && Objects.equals(toPb(), ((FieldSchema) obj).toPb()); + } + + TableFieldSchema toPb() { + TableFieldSchema fieldSchemaPb = new TableFieldSchema(); + fieldSchemaPb.setName(name); + fieldSchemaPb.setType(type.name()); + if (mode != null) { + fieldSchemaPb.setMode(mode.name()); + } + if (description != null) { + fieldSchemaPb.setDescription(description); + } + if (fields != null) { + List fieldsPb = Lists.transform(fields, TO_PB_FUNCTION); + fieldSchemaPb.setFields(fieldsPb); + } + return fieldSchemaPb; + } + + /** + * Returns a FieldSchema object with given name and type. This method should only be used to + * create fields with primitive types (i.e. {@link Type#FLOAT}, {@link Type#BOOLEAN}, + * {@link Type#INTEGER}, {@link Type#STRING} and {@link Type#TIMESTAMP}). + */ + public static FieldSchema of(String name, Type type) { + return builder(name, type).build(); + } + + /** + * Returns a FieldSchema object of type {@link Type#RECORD} for the provided name and subfields. + */ + public static FieldSchema of(String name, List fields) { + return builder(name, fields).type(Type.RECORD).build(); + } + + /** + * Returns a builder for a FieldSchema object with given name and type. + */ + public static Builder builder(String name, Type type) { + return new Builder().name(name).type(type); + } + + /** + * Returns a builder for a FieldSchema object with given name and list of subfields. Type is set + * to {@link Type#RECORD}. + */ + public static Builder builder(String name, List fields) { + return new Builder().name(name).type(Type.RECORD).fields(fields); + } + + static FieldSchema fromPb(TableFieldSchema fieldSchemaPb) { + Builder fieldBuilder = new Builder(); + fieldBuilder.name(fieldSchemaPb.getName()); + fieldBuilder.type(Type.valueOf(fieldSchemaPb.getType())); + if (fieldSchemaPb.getMode() != null) { + fieldBuilder.mode(Mode.valueOf(fieldSchemaPb.getMode())); + } + if (fieldSchemaPb.getDescription() != null) { + fieldBuilder.description(fieldSchemaPb.getDescription()); + } + if (fieldSchemaPb.getFields() != null) { + fieldBuilder.fields(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)); + } + return fieldBuilder.build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java new file mode 100644 index 000000000000..df08025517f6 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java @@ -0,0 +1,743 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.client.util.Data; +import com.google.api.services.bigquery.model.Streamingbuffer; +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.ViewDefinition; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Lists; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery Table information. A BigQuery table is a standard, two-dimensional table with + * individual records organized in rows, and a data type assigned to each column + * (also called a field). Individual fields within a record may contain nested and repeated children + * fields. Every table is described by a schema that describes field names, types, and other + * information. + * + * @see Managing Tables + */ +public class TableInfo implements Serializable { + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public TableInfo apply(Table pb) { + return TableInfo.fromPb(pb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public Table apply(TableInfo tableInfo) { + return tableInfo.toPb(); + } + }; + + private static final long serialVersionUID = -7679032506430816205L; + + /** + * The table type. + */ + public enum Type { + /** + * A normal BigQuery table. + */ + TABLE, + /** + * A virtual table defined by a SQL query. + * + * @see Views + */ + VIEW, + /** + * A BigQuery table backed by external data. + * + * @see Federated Data + * Sources + */ + EXTERNAL + } + + /** + * Google BigQuery Table's Streaming Buffer information. This class contains information on a + * table's streaming buffer as the estimated size in number of rows/bytes. + */ + public static class StreamingBuffer implements Serializable { + + private static final long serialVersionUID = -6713971364725267597L; + private final long estimatedRows; + private final long estimatedBytes; + private final long oldestEntryTime; + + StreamingBuffer(long estimatedRows, long estimatedBytes, long oldestEntryTime) { + this.estimatedRows = estimatedRows; + this.estimatedBytes = estimatedBytes; + this.oldestEntryTime = oldestEntryTime; + } + + /** + * Returns a lower-bound estimate of the number of rows currently in the streaming buffer. + */ + public long estimatedRows() { + return estimatedRows; + } + + /** + * Returns a lower-bound estimate of the number of bytes currently in the streaming buffer. + */ + public long estimatedBytes() { + return estimatedBytes; + } + + /** + * Returns the timestamp of the oldest entry in the streaming buffer, in milliseconds since + * epoch. + */ + public long oldestEntryTime() { + return oldestEntryTime; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("estimatedRows", estimatedRows) + .add("estimatedBytes", estimatedBytes) + .add("oldestEntryTime", oldestEntryTime) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(estimatedRows, estimatedBytes, oldestEntryTime); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof StreamingBuffer + && Objects.equals(toPb(), ((StreamingBuffer) obj).toPb()); + } + + public Streamingbuffer toPb() { + return new Streamingbuffer() + .setEstimatedBytes(BigInteger.valueOf(estimatedBytes)) + .setEstimatedRows(BigInteger.valueOf(estimatedRows)) + .setOldestEntryTime(BigInteger.valueOf(oldestEntryTime)); + } + + static StreamingBuffer fromPb(Streamingbuffer streamingBufferPb) { + return new StreamingBuffer(streamingBufferPb.getEstimatedRows().longValue(), + streamingBufferPb.getEstimatedBytes().longValue(), + streamingBufferPb.getOldestEntryTime().longValue()); + } + } + + private final String etag; + private final String id; + private final String selfLink; + private final TableId tableId; + private final String friendlyName; + private final String description; + private final TableSchema schema; + private final Long numBytes; + private final Long numRows; + private final Long creationTime; + private final Long expirationTime; + private final Long lastModifiedTime; + private final String viewQuery; + private final List userDefinedFunctions; + private final Type type; + private final ExternalDataConfiguration externalConfiguration; + private final String location; + private final StreamingBuffer streamingBuffer; + + public static final class Builder { + + private String etag; + private String id; + private String selfLink; + private TableId tableId; + private String friendlyName; + private String description; + private TableSchema schema; + private Long numBytes; + private Long numRows; + private Long creationTime; + private Long expirationTime; + private Long lastModifiedTime; + private String viewQuery; + private List userDefinedFunctions; + private Type type; + private ExternalDataConfiguration externalConfiguration; + private String location; + private StreamingBuffer streamingBuffer; + + private Builder() {} + + private Builder(TableInfo tableInfo) { + this.etag = tableInfo.etag; + this.id = tableInfo.id; + this.selfLink = tableInfo.selfLink; + this.tableId = tableInfo.tableId; + this.friendlyName = tableInfo.friendlyName; + this.description = tableInfo.description; + this.schema = tableInfo.schema; + this.numBytes = tableInfo.numBytes; + this.numRows = tableInfo.numRows; + this.creationTime = tableInfo.creationTime; + this.expirationTime = tableInfo.expirationTime; + this.lastModifiedTime = tableInfo.lastModifiedTime; + this.viewQuery = tableInfo.viewQuery; + this.userDefinedFunctions = tableInfo.userDefinedFunctions; + this.type = tableInfo.type; + this.externalConfiguration = tableInfo.externalConfiguration; + this.location = tableInfo.location; + this.streamingBuffer = tableInfo.streamingBuffer; + } + + Builder creationTime(Long creationTime) { + this.creationTime = creationTime; + return this; + } + + /** + * Sets a user-friendly description for the table. + */ + public Builder description(String description) { + this.description = firstNonNull(description, Data.nullOf(String.class)); + return this; + } + + Builder etag(String etag) { + this.etag = etag; + return this; + } + + /** + * Sets the time when this table expires, in milliseconds since the epoch. If not present, the + * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. + */ + public Builder expirationTime(Long expirationTime) { + this.expirationTime = firstNonNull(expirationTime, Data.nullOf(Long.class)); + return this; + } + + /** + * Sets the data format, location, and other properties of a table stored outside of BigQuery. + * This property is experimental and might be subject to change or removed. + * + * @see Federated Data + * Sources + */ + public Builder externalConfiguration(ExternalDataConfiguration externalConfiguration) { + this.externalConfiguration = externalConfiguration; + return this; + } + + /** + * Sets a user-friendly name for the dataset. + */ + public Builder friendlyName(String friendlyName) { + this.friendlyName = firstNonNull(friendlyName, Data.nullOf(String.class)); + return this; + } + + Builder id(String id) { + this.id = id; + return this; + } + + Builder lastModifiedTime(Long lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + return this; + } + + Builder location(String location) { + this.location = location; + return this; + } + + Builder numBytes(Long numBytes) { + this.numBytes = numBytes; + return this; + } + + Builder numRows(Long numRows) { + this.numRows = numRows; + return this; + } + + /** + * Sets the table's schema. Providing a schema is not necessary when {@link #viewQuery} is + * provided. + */ + public Builder schema(TableSchema schema) { + this.schema = schema; + return this; + } + + Builder selfLink(String selfLink) { + this.selfLink = selfLink; + return this; + } + + Builder streamingBuffer(StreamingBuffer streamingBuffer) { + this.streamingBuffer = streamingBuffer; + return this; + } + + /** + * Sets the table identity. + */ + public Builder tableId(TableId tableId) { + this.tableId = tableId; + return this; + } + + Builder type(Type type) { + this.type = type; + return this; + } + + /** + * Sets the query used to create a table of {@link Type#VIEW} type. + */ + public Builder viewQuery(String viewQuery) { + this.viewQuery = viewQuery; + return this; + } + + /** + * Sets user-defined functions to be used by the view's query. User defined functions are only + * used when {@link #type(TableInfo.Type)} is set to {@link Type#VIEW} and a query is provided via + * {@link #viewQuery(String)}. + * + * @see User-Defined + * Functions + */ + public Builder userDefinedFunctions(List userDefinedFunctions) { + this.userDefinedFunctions = userDefinedFunctions; + return this; + } + + /** + * Creates a {@code TableInfo} object. + */ + public TableInfo build() { + checkNotNull(tableId); + return new TableInfo(this); + } + } + + private TableInfo(Builder builder) { + this.etag = builder.etag; + this.id = builder.id; + this.selfLink = builder.selfLink; + this.tableId = builder.tableId; + this.friendlyName = builder.friendlyName; + this.description = builder.description; + this.schema = builder.schema; + this.numBytes = builder.numBytes; + this.numRows = builder.numRows; + this.creationTime = builder.creationTime; + this.expirationTime = builder.expirationTime; + this.lastModifiedTime = builder.lastModifiedTime; + this.viewQuery = builder.viewQuery; + this.userDefinedFunctions = builder.userDefinedFunctions; + this.type = builder.type; + this.externalConfiguration = builder.externalConfiguration; + this.location = builder.location; + this.streamingBuffer = builder.streamingBuffer; + } + + /** + * Returns the hash of the table resource. + */ + public String etag() { + return etag; + } + + /** + * Returns an opaque id for the dataset. + */ + public String id() { + return id; + } + + /** + * Returns an URL that can be used to access the resource again. The returned URL can be used for + * get or update requests. + */ + public String selfLink() { + return selfLink; + } + + /** + * Returns the table identity. + */ + public TableId tableId() { + return tableId; + } + + /** + * Returns a user-friendly name for the table. + */ + public String friendlyName() { + return friendlyName; + } + + /** + * Returns a user-friendly description for the table. + */ + public String description() { + return description; + } + + /** + * Returns the table's schema. + */ + public TableSchema schema() { + return schema; + } + + /** + * Returns the size of this table in bytes, excluding any data in the streaming buffer. + */ + public Long numBytes() { + return numBytes; + } + + /** + * Returns the number of rows in this table, excluding any data in the streaming buffer. + */ + public Long numRows() { + return numRows; + } + + /** + * Returns the time when this table was created, in milliseconds since the epoch. + */ + public Long creationTime() { + return creationTime; + } + + /** + * Returns the time when this table expires, in milliseconds since the epoch. If not present, the + * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. + */ + public Long expirationTime() { + return expirationTime; + } + + /** + * Returns the time when this table was last modified, in milliseconds since the epoch. + */ + public Long lastModifiedTime() { + return lastModifiedTime; + } + + /** + * If this table is a view ({@link #type()} returns {@link Type#VIEW}) this method returns the + * query used to create the view. Returns {@code null} otherwise. + */ + public String viewQuery() { + return viewQuery; + } + + /** + * If this table is a view ({@link #type()} returns {@link Type#VIEW} this method returns user + * defined functions that can be used by {@link #viewQuery}. + * + * @see User-Defined Functions + * + */ + public List userDefinedFunctions() { + return userDefinedFunctions; + } + + /** + * Returns the type of the table. + */ + public Type type() { + return type; + } + + /** + * If this table is external ({@link #type()} returns {@link Type#EXTERNAL}) this method returns + * the data format, location, and other properties of a table stored outside of BigQuery. + * If the table is not {@link Type#EXTERNAL} this method returns {@code null}. This property + * is experimental and might be subject to change or removed. + * + * @see Federated Data Sources + * + */ + public ExternalDataConfiguration externalConfiguration() { + return externalConfiguration; + } + + /** + * Returns the geographic location where the table should reside. This value is inherited from the + * dataset. + * + * @see + * Dataset Location + */ + public String location() { + return location; + } + + /** + * Returns information on the table's streaming buffer if any exists. Returns {@code null} if no + * streaming buffer exists. + */ + public StreamingBuffer streamingBuffer() { + return streamingBuffer; + } + + /** + * Returns a builder for the {@code TableInfo} object. + */ + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("tableId", tableId) + .add("schema", schema) + .add("externalDataConfiguration", externalConfiguration) + .add("query", viewQuery) + .add("userDefinedFunctions", userDefinedFunctions) + .add("type", type) + .add("location", location) + .add("streamingBuffer", streamingBuffer) + .add("etag", etag) + .add("id", id) + .add("selfLink", selfLink) + .add("friendlyName", friendlyName) + .add("description", description) + .add("numBytes", numBytes) + .add("numRows", numRows) + .add("expirationTime", expirationTime) + .add("creationTime", creationTime) + .add("lastModifiedTime", lastModifiedTime) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(tableId); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof TableInfo && Objects.equals(toPb(), ((TableInfo) obj).toPb()); + } + + Table toPb() { + Table tablePb = new Table(); + tablePb.setTableReference(tableId.toPb()); + if (externalConfiguration != null) { + tablePb.setExternalDataConfiguration(externalConfiguration.toPb()); + } + if (lastModifiedTime != null) { + tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); + } + if (numRows != null) { + tablePb.setNumRows(BigInteger.valueOf(numRows)); + } + if (schema != null) { + tablePb.setSchema(schema.toPb()); + } + if (streamingBuffer != null) { + tablePb.setStreamingBuffer(streamingBuffer.toPb()); + } + if (viewQuery != null) { + ViewDefinition viewDefinition = new ViewDefinition().setQuery(viewQuery); + if (userDefinedFunctions != null) { + viewDefinition.setUserDefinedFunctionResources( + Lists.transform(userDefinedFunctions, UserDefinedFunction.TO_PB_FUNCTION)); + } + tablePb.setView(new ViewDefinition().setQuery(viewQuery)); + } + if (type != null) { + tablePb.setType(type.name()); + } + tablePb.setCreationTime(creationTime); + tablePb.setDescription(description); + tablePb.setEtag(etag); + tablePb.setExpirationTime(expirationTime); + tablePb.setFriendlyName(friendlyName); + tablePb.setId(id); + tablePb.setLocation(location); + tablePb.setNumBytes(numBytes); + tablePb.setSelfLink(selfLink); + return tablePb; + } + + /** + * Returns a builder for a BigQuery table backed by external data. External tables are + * experimental and might be subject to change or removed. + * + * @param tableId table id + * @param configuration configuration for external data source + */ + public static Builder builder(TableId tableId, ExternalDataConfiguration configuration) { + return new Builder().tableId(tableId).externalConfiguration(configuration); + } + + /** + * Returns a builder for a BigQuery view. + * + * @param tableId table id + * @param query query used to create the view + */ + public static Builder builder(TableId tableId, String query) { + return new Builder().tableId(tableId).viewQuery(query); + } + + /** + * Returns a builder for a BigQuery view. + * + * @param tableId table id + * @param query query used to create the view + */ + public static Builder builder(TableId tableId, String query, List functions) { + return new Builder().tableId(tableId).viewQuery(query).userDefinedFunctions(functions); + } + + /** + * Returns a builder for a BigQuery table. + * + * @param tableId table id + * @param schema table's schema definition + */ + public static Builder builder(TableId tableId, TableSchema schema) { + return new Builder().tableId(tableId).schema(schema); + } + + /** + * Creates BigQuery table backed by external data. + * + * @param tableId table id + * @param configuration configuration for external data source + */ + public static TableInfo of(TableId tableId, ExternalDataConfiguration configuration) { + return builder(tableId, configuration).build(); + } + + /** + * Creates a BigQuery view. + * + * @param tableId table id + * @param query query used to create the view + */ + public static TableInfo of(TableId tableId, String query) { + return builder(tableId, query).build(); + } + + /** + * Creates a BigQuery view. + * + * @param tableId table id + * @param query query used to create the view + * @param functions user defined funtions that can be used in {@code query} + */ + public static TableInfo of(TableId tableId, String query, List functions) { + return builder(tableId, query, functions).build(); + } + + /** + * Creates a BigQuery table. + * + * @param tableId table id + * @param schema table's schema definition + */ + public static TableInfo of(TableId tableId, TableSchema schema) { + return builder(tableId, schema).build(); + } + + static TableInfo fromPb(Table tablePb) { + Builder builder = new Builder().tableId(TableId.fromPb(tablePb.getTableReference())); + if (tablePb.getDescription() != null) { + builder.description(tablePb.getDescription()); + } + if (tablePb.getExpirationTime() != null) { + builder.expirationTime(tablePb.getExpirationTime()); + } + if (tablePb.getExternalDataConfiguration() != null) { + builder.externalConfiguration( + ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration())); + } + if (tablePb.getFriendlyName() != null) { + builder.friendlyName(tablePb.getFriendlyName()); + } + if (tablePb.getLastModifiedTime() != null) { + builder.lastModifiedTime(tablePb.getLastModifiedTime().longValue()); + } + if (tablePb.getLocation() != null) { + builder.location(tablePb.getLocation()); + } + if (tablePb.getNumRows() != null) { + builder.numRows(tablePb.getNumRows().longValue()); + } + if (tablePb.getSchema() != null) { + builder.schema(TableSchema.fromPb(tablePb.getSchema())); + } + if (tablePb.getStreamingBuffer() != null) { + builder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); + } + if (tablePb.getType() != null) { + builder.type(Type.valueOf(tablePb.getType())); + } + if (tablePb.getView() != null) { + builder.viewQuery(tablePb.getView().getQuery()); + if (tablePb.getView().getUserDefinedFunctionResources() != null) { + builder.userDefinedFunctions( + Lists.transform(tablePb.getView().getUserDefinedFunctionResources(), + UserDefinedFunction.FROM_PB_FUNCTION)); + } + } + if (tablePb.getCreationTime() != null) { + builder.creationTime(tablePb.getCreationTime()); + } + if (tablePb.getEtag() != null) { + builder.etag(tablePb.getEtag()); + } + if (tablePb.getId() != null) { + builder.id(tablePb.getId()); + } + if (tablePb.getNumBytes() != null) { + builder.numBytes(tablePb.getNumBytes()); + } + if (tablePb.getSelfLink() != null) { + builder.selfLink(tablePb.getSelfLink()); + } + return builder.build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java new file mode 100644 index 000000000000..987d838db36c --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java @@ -0,0 +1,151 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * This class represents the schema for a Google BigQuery Table or data source. + */ +public class TableSchema implements Serializable { + + static final Function + FROM_PB_FUNCTION = new Function() { + @Override + public TableSchema apply(com.google.api.services.bigquery.model.TableSchema pb) { + return TableSchema.fromPb(pb); + } + }; + static final Function + TO_PB_FUNCTION = new Function() { + @Override + public com.google.api.services.bigquery.model.TableSchema apply(TableSchema schema) { + return schema.toPb(); + } + }; + + private static final long serialVersionUID = 2007400596384553696L; + + private final List fields; + + public static final class Builder { + + private List fields; + + private Builder() {} + + /** + * Adds a field's schema to the table's schema. + */ + public Builder addField(FieldSchema field) { + if (fields == null) { + fields = Lists.newArrayList(); + } + fields.add(checkNotNull(field)); + return this; + } + + /** + * Sets table's schema fields. + */ + public Builder fields(Iterable fields) { + this.fields = Lists.newArrayList(checkNotNull(fields)); + return this; + } + + /** + * Creates an {@code TableSchema} object. + */ + public TableSchema build() { + return new TableSchema(this); + } + } + + private TableSchema(Builder builder) { + this.fields = builder.fields != null ? ImmutableList.copyOf(builder.fields) : + ImmutableList.of(); + } + + /** + * Returns the fields in the current table schema. + */ + public List fields() { + return fields; + } + + /** + * Returns a builder for the {@code TableSchema} object. + */ + public Builder toBuilder() { + Builder builder = new Builder(); + builder.fields(fields); + return builder; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("fields", fields) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(fields); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof TableSchema && Objects.equals(toPb(), ((TableSchema) obj).toPb()); + } + + com.google.api.services.bigquery.model.TableSchema toPb() { + com.google.api.services.bigquery.model.TableSchema tableSchemaPb = + new com.google.api.services.bigquery.model.TableSchema(); + if (fields != null) { + List fieldsPb = Lists.transform(fields, FieldSchema.TO_PB_FUNCTION); + tableSchemaPb.setFields(fieldsPb); + } + return tableSchemaPb; + } + + public static Builder builder() { + return new Builder(); + } + + public static TableSchema of(Iterable fields) { + checkNotNull(fields); + return builder().fields(fields).build(); + } + + static TableSchema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { + Builder schemaBuilder = new Builder(); + return TableSchema.of(Lists.transform(tableSchemaPb.getFields(), FieldSchema.FROM_PB_FUNCTION)); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java new file mode 100644 index 000000000000..a7874a738126 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java @@ -0,0 +1,158 @@ +package com.google.gcloud.bigquery; + +import com.google.api.services.bigquery.model.UserDefinedFunctionResource; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Google BigQuery User Defined Function. BigQuery supports user-defined functions (UDFs) written in + * JavaScript. A UDF is similar to the "Map" function in a MapReduce: it takes a single row as input + * and produces zero or more rows as output. The output can potentially have a different schema than + * the input. + * + * @see User-Defined Functions + */ +public abstract class UserDefinedFunction implements Serializable { + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public UserDefinedFunction apply(UserDefinedFunctionResource userDefinedFunctionPb) { + return UserDefinedFunction.fromPb(userDefinedFunctionPb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public UserDefinedFunctionResource apply(UserDefinedFunction userDefinedFunction) { + return userDefinedFunction.toPb(); + } + }; + + private static final long serialVersionUID = 8704260561787440287L; + + /** + * Type of user-defined function. User defined functions can be provided inline as code blobs + * ({@link #INLINE}) or as a Google Cloud Storage URI ({@link #FROM_URI}). + */ + public enum Type { + INLINE, + FROM_URI + } + + private final Type type; + private final String functionDefinition; + + UserDefinedFunction(Type type, String functionDefinition) { + this.type = type; + this.functionDefinition = functionDefinition; + } + + public Type type() { + return type; + } + + /** + * Returns function's definition. If {@link #type()} is {@link Type#INLINE} this method returns + * a code blob. If {@link #type()} is {@link Type#FROM_URI} this method returns a Google Cloud + * Storage URI (e.g. gs://bucket/path). + */ + public String functionDefinition() { + return functionDefinition; + } + + /** + * A Google Cloud BigQuery user-defined function, as a code blob. + */ + public static final class InlineFunction extends UserDefinedFunction { + + private static final long serialVersionUID = 1083672109192091686L; + + /** + * Creates a Google Cloud BigQuery user-defined function given a code blob. + */ + public InlineFunction(String inlineCode) { + super(Type.INLINE, inlineCode); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("inlineCode", functionDefinition()).toString(); + } + + @Override + public com.google.api.services.bigquery.model.UserDefinedFunctionResource toPb() { + return new com.google.api.services.bigquery.model.UserDefinedFunctionResource() + .setInlineCode(functionDefinition()); + } + } + + /** + * A Google Cloud BigQuery user-defined function, as an URI to Google Cloud Storage. + */ + public static final class UriFunction extends UserDefinedFunction { + + private static final long serialVersionUID = 4660331691852223839L; + + /** + * Creates a Google Cloud BigQuery user-defined function given a Google Cloud Storage URI (e.g. + * gs://bucket/path). + */ + public UriFunction(String functionUri) { + super(Type.FROM_URI, functionUri); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("functionUri", functionDefinition()).toString(); + } + + @Override + public com.google.api.services.bigquery.model.UserDefinedFunctionResource toPb() { + return new com.google.api.services.bigquery.model.UserDefinedFunctionResource() + .setResourceUri(functionDefinition()); + } + } + + @Override + public int hashCode() { + return Objects.hash(type, functionDefinition); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof UserDefinedFunction + && Objects.equals(toPb(), ((UserDefinedFunction) obj).toPb()); + } + + public abstract com.google.api.services.bigquery.model.UserDefinedFunctionResource toPb(); + + /** + * Creates a Google Cloud BigQuery user-defined function given a code blob. + */ + public static UserDefinedFunction inline(String functionDefinition) { + return new InlineFunction(functionDefinition); + } + + /** + * Creates a Google Cloud BigQuery user-defined function given a Google Cloud Storage URI (e.g. + * gs://bucket/path). + */ + public static UserDefinedFunction fromUri(String functionDefinition) { + return new UriFunction(functionDefinition); + } + + public static UserDefinedFunction fromPb( + com.google.api.services.bigquery.model.UserDefinedFunctionResource pb) { + if (pb.getInlineCode() != null) { + return new InlineFunction(pb.getInlineCode()); + } + if (pb.getResourceUri() != null) { + return new UriFunction(pb.getResourceUri()); + } + throw new IllegalArgumentException("Invalid user-defined function"); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java new file mode 100644 index 000000000000..ac797ae88171 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class CsvOptionsTest { + + private static final Boolean ALLOW_JAGGED_ROWS = true; + private static final Boolean ALLOW_QUOTED_NEWLINE = true; + private static final String ENCODING = "UTF_8"; + private static final String FIELD_DELIMITER = ","; + private static final String QUOTE = "\""; + private static final Integer SKIP_LEADING_ROWS = 42; + private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() + .allowJaggedRows(ALLOW_JAGGED_ROWS) + .allowQuotedNewLines(ALLOW_QUOTED_NEWLINE) + .encoding(ENCODING) + .fieldDelimiter(FIELD_DELIMITER) + .quote(QUOTE) + .skipLeadingRows(SKIP_LEADING_ROWS) + .build(); + + @Test + public void testToBuilder() { + compareCsvOptions(CSV_OPTIONS, CSV_OPTIONS.toBuilder().build()); + CsvOptions csvOptions = CSV_OPTIONS.toBuilder() + .fieldDelimiter(";") + .build(); + assertEquals(";", csvOptions.fieldDelimiter()); + csvOptions = csvOptions.toBuilder().fieldDelimiter(",").build(); + compareCsvOptions(CSV_OPTIONS, csvOptions); + } + + @Test + public void testToBuilderIncomplete() { + CsvOptions csvOptions = CsvOptions.builder().fieldDelimiter("|").build(); + assertEquals(csvOptions, csvOptions.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(ALLOW_JAGGED_ROWS, CSV_OPTIONS.allowJaggedRows()); + assertEquals(ALLOW_QUOTED_NEWLINE, CSV_OPTIONS.allowQuotedNewLines()); + assertEquals(ENCODING, CSV_OPTIONS.encoding()); + assertEquals(FIELD_DELIMITER, CSV_OPTIONS.fieldDelimiter()); + assertEquals(QUOTE, CSV_OPTIONS.quote()); + assertEquals(SKIP_LEADING_ROWS, CSV_OPTIONS.skipLeadingRows()); + } + + @Test + public void testToAndFromPb() { + compareCsvOptions(CSV_OPTIONS, CsvOptions.fromPb(CSV_OPTIONS.toPb())); + CsvOptions csvOptions = CsvOptions.builder().allowJaggedRows(ALLOW_JAGGED_ROWS).build(); + compareCsvOptions(csvOptions, CsvOptions.fromPb(csvOptions.toPb())); + } + + private void compareCsvOptions(CsvOptions expected, CsvOptions value) { + assertEquals(expected, value); + assertEquals(expected.allowJaggedRows(), value.allowJaggedRows()); + assertEquals(expected.allowQuotedNewLines(), value.allowQuotedNewLines()); + assertEquals(expected.encoding(), value.encoding()); + assertEquals(expected.fieldDelimiter(), value.fieldDelimiter()); + assertEquals(expected.quote(), value.quote()); + assertEquals(expected.skipLeadingRows(), value.skipLeadingRows()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java new file mode 100644 index 000000000000..7f23d3b5501a --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +import java.util.List; + +public class ExternalDataConfigurationTest { + + private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final FieldSchema FIELD_SCHEMA1 = + FieldSchema.builder("StringField", FieldSchema.Type.STRING) + .mode(FieldSchema.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final FieldSchema FIELD_SCHEMA2 = + FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) + .mode(FieldSchema.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final FieldSchema FIELD_SCHEMA3 = + FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(FieldSchema.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, + FIELD_SCHEMA3); + private static final TableSchema TABLE_SCHEMA = TableSchema.of(FIELDS); + private static final String SOURCE_FORMAT = "CSV"; + private static final Integer MAX_BAD_RECORDS = 42; + private static final Boolean IGNORE_UNKNOWN_VALUES = true; + private static final String COMPRESSION = "GZIP"; + private static final CsvOptions CSV_OPTIONS = CsvOptions.builder().build(); + private static final ExternalDataConfiguration CONFIGURATION = ExternalDataConfiguration + .builder(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT) + .compression(COMPRESSION) + .csvOptions(CSV_OPTIONS) + .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .maxBadRecords(MAX_BAD_RECORDS) + .build(); + + @Test + public void testToBuilder() { + compareConfiguration(CONFIGURATION, CONFIGURATION.toBuilder().build()); + ExternalDataConfiguration configuration = CONFIGURATION.toBuilder().compression("NONE").build(); + assertEquals("NONE", configuration.compression()); + configuration = configuration.toBuilder() + .compression(COMPRESSION) + .build(); + compareConfiguration(CONFIGURATION, configuration); + } + + @Test + public void testToBuilderIncomplete() { + ExternalDataConfiguration configuration = + ExternalDataConfiguration.of(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT); + assertEquals(configuration, configuration.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(COMPRESSION, CONFIGURATION.compression()); + assertEquals(CSV_OPTIONS, CONFIGURATION.csvOptions()); + assertEquals(IGNORE_UNKNOWN_VALUES, CONFIGURATION.ignoreUnknownValues()); + assertEquals(MAX_BAD_RECORDS, CONFIGURATION.maxBadRecords()); + assertEquals(TABLE_SCHEMA, CONFIGURATION.schema()); + assertEquals(SOURCE_FORMAT, CONFIGURATION.sourceFormat()); + assertEquals(SOURCE_URIS, CONFIGURATION.sourceUris()); + } + + @Test + public void testToAndFromPb() { + compareConfiguration(CONFIGURATION, ExternalDataConfiguration.fromPb(CONFIGURATION.toPb())); + ExternalDataConfiguration configuration = + ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT).build(); + compareConfiguration(configuration, ExternalDataConfiguration.fromPb(configuration.toPb())); + } + + private void compareConfiguration(ExternalDataConfiguration expected, + ExternalDataConfiguration value) { + assertEquals(expected, value); + assertEquals(expected.compression(), value.compression()); + assertEquals(expected.csvOptions(), value.csvOptions()); + assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues()); + assertEquals(expected.maxBadRecords(), value.maxBadRecords()); + assertEquals(expected.schema(), value.schema()); + assertEquals(expected.sourceFormat(), value.sourceFormat()); + assertEquals(expected.sourceUris(), value.sourceUris()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java new file mode 100644 index 000000000000..fa26419abbba --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +public class FieldSchemaTest { + + private static final String FIELD_NAME1 = "StringField"; + private static final String FIELD_NAME2 = "IntegerField"; + private static final String FIELD_NAME3 = "RecordField"; + private static final FieldSchema.Type FIELD_TYPE1 = FieldSchema.Type.STRING; + private static final FieldSchema.Type FIELD_TYPE2 = FieldSchema.Type.INTEGER; + private static final FieldSchema.Type FIELD_TYPE3 = FieldSchema.Type.RECORD; + private static final FieldSchema.Mode FIELD_MODE1 = FieldSchema.Mode.NULLABLE; + private static final FieldSchema.Mode FIELD_MODE2 = FieldSchema.Mode.REPEATED; + private static final FieldSchema.Mode FIELD_MODE3 = FieldSchema.Mode.REQUIRED; + private static final String FIELD_DESCRIPTION1 = "FieldDescription1"; + private static final String FIELD_DESCRIPTION2 = "FieldDescription2"; + private static final String FIELD_DESCRIPTION3 = "FieldDescription3"; + private static final FieldSchema FIELD_SCHEMA1 = FieldSchema.builder(FIELD_NAME1, FIELD_TYPE1) + .mode(FIELD_MODE1) + .description(FIELD_DESCRIPTION1) + .build(); + private static final FieldSchema FIELD_SCHEMA2 = FieldSchema.builder(FIELD_NAME2, FIELD_TYPE2) + .mode(FIELD_MODE2) + .description(FIELD_DESCRIPTION2) + .build(); + private static final FieldSchema FIELD_SCHEMA3 = FieldSchema + .builder(FIELD_NAME3, ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(FIELD_MODE3) + .description(FIELD_DESCRIPTION3) + .build(); + + @Test + public void testToBuilder() { + compareFieldSchemas(FIELD_SCHEMA1, FIELD_SCHEMA1.toBuilder().build()); + compareFieldSchemas(FIELD_SCHEMA2, FIELD_SCHEMA2.toBuilder().build()); + compareFieldSchemas(FIELD_SCHEMA3, FIELD_SCHEMA3.toBuilder().build()); + FieldSchema fieldSchema = FIELD_SCHEMA1.toBuilder() + .description("New Description") + .build(); + assertEquals("New Description", fieldSchema.description()); + fieldSchema = fieldSchema.toBuilder().description(FIELD_DESCRIPTION1).build(); + compareFieldSchemas(FIELD_SCHEMA1, fieldSchema); + } + + @Test + public void testToBuilderIncomplete() { + FieldSchema fieldSchema = FieldSchema.of(FIELD_NAME1, FIELD_TYPE1); + compareFieldSchemas(fieldSchema, fieldSchema.toBuilder().build()); + fieldSchema = FieldSchema.of(FIELD_NAME2, ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)); + compareFieldSchemas(fieldSchema, fieldSchema.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(FIELD_NAME1, FIELD_SCHEMA1.name()); + assertEquals(FIELD_TYPE1, FIELD_SCHEMA1.type()); + assertEquals(FIELD_MODE1, FIELD_SCHEMA1.mode()); + assertEquals(FIELD_DESCRIPTION1, FIELD_SCHEMA1.description()); + assertEquals(null, FIELD_SCHEMA1.fields()); + assertEquals(FIELD_NAME3, FIELD_SCHEMA3.name()); + assertEquals(FIELD_TYPE3, FIELD_SCHEMA3.type()); + assertEquals(FIELD_MODE3, FIELD_SCHEMA3.mode()); + assertEquals(FIELD_DESCRIPTION3, FIELD_SCHEMA3.description()); + assertEquals(ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2), FIELD_SCHEMA3.fields()); + } + + @Test + public void testToAndFromPb() { + compareFieldSchemas(FIELD_SCHEMA1, FieldSchema.fromPb(FIELD_SCHEMA1.toPb())); + compareFieldSchemas(FIELD_SCHEMA2, FieldSchema.fromPb(FIELD_SCHEMA2.toPb())); + compareFieldSchemas(FIELD_SCHEMA3, FieldSchema.fromPb(FIELD_SCHEMA3.toPb())); + FieldSchema fieldSchema = FieldSchema.builder(FIELD_NAME1, FIELD_TYPE1).build(); + compareFieldSchemas(fieldSchema, FieldSchema.fromPb(fieldSchema.toPb())); + } + + private void compareFieldSchemas(FieldSchema expected, FieldSchema value) { + assertEquals(expected, value); + assertEquals(expected.name(), value.name()); + assertEquals(expected.type(), value.type()); + assertEquals(expected.mode(), value.mode()); + assertEquals(expected.description(), value.description()); + assertEquals(expected.fields(), value.fields()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index 93165724e11f..8001dffb1cf9 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -67,6 +67,54 @@ public class SerializationTest { .selfLink(SELF_LINK) .build(); private static final TableId TABLE_ID = TableId.of("project", "dataset", "table"); + private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() + .allowJaggedRows(true) + .allowQuotedNewLines(false) + .encoding("CSV") + .fieldDelimiter(",") + .quote("\"") + .skipLeadingRows(42) + .build(); + private static final FieldSchema FIELD_SCHEMA1 = + FieldSchema.builder("StringField", FieldSchema.Type.STRING) + .mode(FieldSchema.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final FieldSchema FIELD_SCHEMA2 = + FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) + .mode(FieldSchema.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final FieldSchema FIELD_SCHEMA3 = + FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(FieldSchema.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, + FIELD_SCHEMA3); + private static final TableSchema TABLE_SCHEMA = TableSchema.of(FIELDS); + private static final TableInfo.StreamingBuffer STREAMING_BUFFER = + new TableInfo.StreamingBuffer(1L, 2L, 3L); + private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final ExternalDataConfiguration EXTERNAL_DATA_CONFIGURATION = + ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, "CSV") + .csvOptions(CSV_OPTIONS) + .ignoreUnknownValues(true) + .maxBadRecords(42) + .build(); + private static final UserDefinedFunction INLINE_FUNCTION = + new UserDefinedFunction.InlineFunction("inline"); + private static final UserDefinedFunction URI_FUNCTION = + new UserDefinedFunction.UriFunction("URI"); + private static final TableInfo TABLE_INFO = TableInfo.builder(TABLE_ID, TABLE_SCHEMA) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .id(ID) + .location(LOCATION) + .type(TableInfo.Type.TABLE) + .streamingBuffer(STREAMING_BUFFER) + .build(); @Test public void testServiceOptions() throws Exception { @@ -89,7 +137,8 @@ public void testServiceOptions() throws Exception { @Test public void testModelAndRequests() throws Exception { Serializable[] objects = {DOMAIN_ACCESS, GROUP_ACCESS, USER_ACCESS, VIEW_ACCESS, DATASET_ID, - DATASET_INFO, TABLE_ID}; + DATASET_INFO, TABLE_ID, CSV_OPTIONS, STREAMING_BUFFER, EXTERNAL_DATA_CONFIGURATION, + TABLE_SCHEMA, TABLE_INFO, INLINE_FUNCTION, URI_FUNCTION}; for (Serializable obj : objects) { Object copy = serializeAndDeserialize(obj); assertEquals(obj, obj); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java new file mode 100644 index 000000000000..5e4d7c220197 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.google.gcloud.bigquery.TableInfo.StreamingBuffer; + +import org.junit.Test; + +import java.util.List; + +public class TableInfoTest { + + private static final FieldSchema FIELD_SCHEMA1 = + FieldSchema.builder("StringField", FieldSchema.Type.STRING) + .mode(FieldSchema.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final FieldSchema FIELD_SCHEMA2 = + FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) + .mode(FieldSchema.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final FieldSchema FIELD_SCHEMA3 = + FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(FieldSchema.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, + FIELD_SCHEMA3); + private static final TableSchema TABLE_SCHEMA = TableSchema.of(FIELDS); + private static final String VIEW_QUERY = "VIEW QUERY"; + private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final String SOURCE_FORMAT = "CSV"; + private static final Integer MAX_BAD_RECORDS = 42; + private static final Boolean IGNORE_UNKNOWN_VALUES = true; + private static final String COMPRESSION = "GZIP"; + private static final CsvOptions CSV_OPTIONS = CsvOptions.builder().build(); + private static final ExternalDataConfiguration CONFIGURATION = ExternalDataConfiguration + .builder(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT) + .compression(COMPRESSION) + .csvOptions(CSV_OPTIONS) + .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .maxBadRecords(MAX_BAD_RECORDS) + .build(); + private static final String ETAG = "etag"; + private static final String ID = "project:dataset:table"; + private static final String SELF_LINK = "selfLink"; + private static final TableId TABLE_ID = TableId.of("dataset", "table"); + private static final String FRIENDLY_NAME = "friendlyName"; + private static final String DESCRIPTION = "description"; + private static final Long NUM_BYTES = 42L; + private static final Long NUM_ROWS = 43L; + private static final Long CREATION_TIME = 10L; + private static final Long EXPIRATION_TIME = 100L; + private static final Long LAST_MODIFIED_TIME = 20L; + private static final String LOCATION = "US"; + private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L); + private static List USER_DEFINED_FUNCTIONS = ImmutableList.of( + UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); + private static final TableInfo TABLE_INFO = TableInfo.builder(TABLE_ID, TABLE_SCHEMA) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .expirationTime(EXPIRATION_TIME) + .friendlyName(FRIENDLY_NAME) + .id(ID) + .lastModifiedTime(LAST_MODIFIED_TIME) + .location(LOCATION) + .numBytes(NUM_BYTES) + .numRows(NUM_ROWS) + .selfLink(SELF_LINK) + .streamingBuffer(STREAMING_BUFFER) + .type(TableInfo.Type.TABLE) + .build(); + private static final TableInfo EXTERNAL_TABLE_INFO = TableInfo.builder(TABLE_ID, CONFIGURATION) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .expirationTime(EXPIRATION_TIME) + .friendlyName(FRIENDLY_NAME) + .id(ID) + .lastModifiedTime(LAST_MODIFIED_TIME) + .location(LOCATION) + .numBytes(NUM_BYTES) + .numRows(NUM_ROWS) + .selfLink(SELF_LINK) + .streamingBuffer(STREAMING_BUFFER) + .type(TableInfo.Type.TABLE) + .build(); + private static final TableInfo VIEW_INFO = + TableInfo.builder(TABLE_ID, VIEW_QUERY, USER_DEFINED_FUNCTIONS) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .expirationTime(EXPIRATION_TIME) + .friendlyName(FRIENDLY_NAME) + .id(ID) + .lastModifiedTime(LAST_MODIFIED_TIME) + .location(LOCATION) + .numBytes(NUM_BYTES) + .numRows(NUM_ROWS) + .selfLink(SELF_LINK) + .streamingBuffer(STREAMING_BUFFER) + .type(TableInfo.Type.VIEW) + .build(); + + @Test + public void testToBuilder() { + compareTableInfo(TABLE_INFO, TABLE_INFO.toBuilder().build()); + compareTableInfo(VIEW_INFO, VIEW_INFO.toBuilder().build()); + compareTableInfo(EXTERNAL_TABLE_INFO, EXTERNAL_TABLE_INFO.toBuilder().build()); + TableInfo tableInfo = TABLE_INFO.toBuilder() + .type(TableInfo.Type.VIEW) + .description("newDescription") + .build(); + assertEquals(TableInfo.Type.VIEW, tableInfo.type()); + assertEquals("newDescription", tableInfo.description()); + tableInfo = tableInfo.toBuilder() + .type(TableInfo.Type.TABLE) + .description("description") + .build(); + compareTableInfo(TABLE_INFO, tableInfo); + } + + @Test + public void testToBuilderIncomplete() { + TableInfo tableInfo = TableInfo.of(TABLE_ID, VIEW_QUERY); + assertEquals(tableInfo, tableInfo.toBuilder().build()); + } + @Test + public void testBuilder() { + assertEquals(TABLE_ID, TABLE_INFO.tableId()); + assertEquals(TABLE_SCHEMA, TABLE_INFO.schema()); + assertEquals(null, TABLE_INFO.viewQuery()); + assertEquals(null, TABLE_INFO.externalConfiguration()); + assertEquals(CREATION_TIME, TABLE_INFO.creationTime()); + assertEquals(DESCRIPTION, TABLE_INFO.description()); + assertEquals(ETAG, TABLE_INFO.etag()); + assertEquals(EXPIRATION_TIME, TABLE_INFO.expirationTime()); + assertEquals(FRIENDLY_NAME, TABLE_INFO.friendlyName()); + assertEquals(ID, TABLE_INFO.id()); + assertEquals(LAST_MODIFIED_TIME, TABLE_INFO.lastModifiedTime()); + assertEquals(LOCATION, TABLE_INFO.location()); + assertEquals(NUM_BYTES, TABLE_INFO.numBytes()); + assertEquals(NUM_ROWS, TABLE_INFO.numRows()); + assertEquals(SELF_LINK, TABLE_INFO.selfLink()); + assertEquals(STREAMING_BUFFER, TABLE_INFO.streamingBuffer()); + assertEquals(TableInfo.Type.TABLE, TABLE_INFO.type()); + assertEquals(TABLE_ID, VIEW_INFO.tableId()); + assertEquals(null, VIEW_INFO.schema()); + assertEquals(VIEW_QUERY, VIEW_INFO.viewQuery()); + assertEquals(null, VIEW_INFO.externalConfiguration()); + assertEquals(CREATION_TIME, VIEW_INFO.creationTime()); + assertEquals(DESCRIPTION, VIEW_INFO.description()); + assertEquals(ETAG, VIEW_INFO.etag()); + assertEquals(EXPIRATION_TIME, VIEW_INFO.expirationTime()); + assertEquals(FRIENDLY_NAME, VIEW_INFO.friendlyName()); + assertEquals(ID, VIEW_INFO.id()); + assertEquals(LAST_MODIFIED_TIME, VIEW_INFO.lastModifiedTime()); + assertEquals(LOCATION, VIEW_INFO.location()); + assertEquals(NUM_BYTES, VIEW_INFO.numBytes()); + assertEquals(NUM_ROWS, VIEW_INFO.numRows()); + assertEquals(SELF_LINK, VIEW_INFO.selfLink()); + assertEquals(STREAMING_BUFFER, VIEW_INFO.streamingBuffer()); + assertEquals(TableInfo.Type.VIEW, VIEW_INFO.type()); + assertEquals(TABLE_ID, EXTERNAL_TABLE_INFO.tableId()); + assertEquals(null, EXTERNAL_TABLE_INFO.schema()); + assertEquals(null, EXTERNAL_TABLE_INFO.viewQuery()); + assertEquals(CONFIGURATION, EXTERNAL_TABLE_INFO.externalConfiguration()); + assertEquals(CREATION_TIME, EXTERNAL_TABLE_INFO.creationTime()); + assertEquals(DESCRIPTION, EXTERNAL_TABLE_INFO.description()); + assertEquals(ETAG, EXTERNAL_TABLE_INFO.etag()); + assertEquals(EXPIRATION_TIME, EXTERNAL_TABLE_INFO.expirationTime()); + assertEquals(FRIENDLY_NAME, EXTERNAL_TABLE_INFO.friendlyName()); + assertEquals(ID, EXTERNAL_TABLE_INFO.id()); + assertEquals(LAST_MODIFIED_TIME, EXTERNAL_TABLE_INFO.lastModifiedTime()); + assertEquals(LOCATION, EXTERNAL_TABLE_INFO.location()); + assertEquals(NUM_BYTES, EXTERNAL_TABLE_INFO.numBytes()); + assertEquals(NUM_ROWS, EXTERNAL_TABLE_INFO.numRows()); + assertEquals(SELF_LINK, EXTERNAL_TABLE_INFO.selfLink()); + assertEquals(STREAMING_BUFFER, EXTERNAL_TABLE_INFO.streamingBuffer()); + assertEquals(TableInfo.Type.TABLE, EXTERNAL_TABLE_INFO.type()); + } + + @Test + public void testToAndFromPb() { + compareTableInfo(TABLE_INFO, TableInfo.fromPb(TABLE_INFO.toPb())); + compareTableInfo(VIEW_INFO, TableInfo.fromPb(VIEW_INFO.toPb())); + compareTableInfo(EXTERNAL_TABLE_INFO, TableInfo.fromPb(EXTERNAL_TABLE_INFO.toPb())); + } + + private void compareTableInfo(TableInfo expected, TableInfo value) { + assertEquals(expected, value); + assertEquals(expected.tableId(), value.tableId()); + assertEquals(expected.schema(), value.schema()); + assertEquals(expected.viewQuery(), value.viewQuery()); + assertEquals(expected.externalConfiguration(), value.externalConfiguration()); + assertEquals(expected.creationTime(), value.creationTime()); + assertEquals(expected.description(), value.description()); + assertEquals(expected.etag(), value.etag()); + assertEquals(expected.expirationTime(), value.expirationTime()); + assertEquals(expected.friendlyName(), value.friendlyName()); + assertEquals(expected.id(), value.id()); + assertEquals(expected.lastModifiedTime(), value.lastModifiedTime()); + assertEquals(expected.location(), value.location()); + assertEquals(expected.numBytes(), value.numBytes()); + assertEquals(expected.numRows(), value.numRows()); + assertEquals(expected.selfLink(), value.selfLink()); + assertEquals(expected.streamingBuffer(), value.streamingBuffer()); + assertEquals(expected.type(), value.type()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java new file mode 100644 index 000000000000..8237d94cf7a8 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +import java.util.List; + +public class TableSchemaTest { + + private static final FieldSchema FIELD_SCHEMA1 = + FieldSchema.builder("StringField", FieldSchema.Type.STRING) + .mode(FieldSchema.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final FieldSchema FIELD_SCHEMA2 = + FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) + .mode(FieldSchema.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final FieldSchema FIELD_SCHEMA3 = + FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(FieldSchema.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, + FIELD_SCHEMA3); + private static final TableSchema TABLE_SCHEMA = TableSchema.builder().fields(FIELDS).build(); + + @Test + public void testToBuilder() { + compareTableSchema(TABLE_SCHEMA, TABLE_SCHEMA.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(FIELDS, TABLE_SCHEMA.fields()); + TableSchema tableSchema = TABLE_SCHEMA.toBuilder() + .fields(ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .addField(FIELD_SCHEMA3) + .build(); + compareTableSchema(TABLE_SCHEMA, tableSchema); + } + + @Test + public void testOf() { + compareTableSchema(TABLE_SCHEMA, TableSchema.of(FIELDS)); + } + + @Test + public void testToAndFromPb() { + compareTableSchema(TABLE_SCHEMA, TableSchema.fromPb(TABLE_SCHEMA.toPb())); + } + + private void compareTableSchema(TableSchema expected, TableSchema value) { + assertEquals(expected, value); + assertEquals(expected.fields(), value.fields()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java new file mode 100644 index 000000000000..f2640f240a62 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class UserDefinedFunctionTest { + + private static final String INLINE = "inline"; + private static final String URI = "uri"; + private static final UserDefinedFunction INLINE_FUNCTION = + new UserDefinedFunction.InlineFunction(INLINE); + private static final UserDefinedFunction URI_FUNCTION = new UserDefinedFunction.UriFunction(URI); + + @Test + public void testConstructor() { + assertEquals(INLINE, INLINE_FUNCTION.functionDefinition()); + assertEquals(UserDefinedFunction.Type.INLINE, INLINE_FUNCTION.type()); + assertEquals(URI, URI_FUNCTION.functionDefinition()); + assertEquals(UserDefinedFunction.Type.FROM_URI, URI_FUNCTION.type()); + } + + @Test + public void testFactoryMethod() { + compareUserDefinedFunction(INLINE_FUNCTION, UserDefinedFunction.inline(INLINE)); + compareUserDefinedFunction(URI_FUNCTION, UserDefinedFunction.fromUri(URI)); + } + + @Test + public void testToAndFromPb() { + compareUserDefinedFunction(INLINE_FUNCTION, UserDefinedFunction.fromPb(INLINE_FUNCTION.toPb())); + compareUserDefinedFunction(URI_FUNCTION, UserDefinedFunction.fromPb(URI_FUNCTION.toPb())); + } + + private void compareUserDefinedFunction(UserDefinedFunction expected, UserDefinedFunction value) { + assertEquals(expected, value); + assertEquals(expected.type(), value.type()); + assertEquals(expected.functionDefinition(), value.functionDefinition()); + } +} From b18666b4abd2dfdab3d772ebf7c731632733c990 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 1 Dec 2015 11:28:05 +0100 Subject: [PATCH 02/10] Refactor TableInfo and related classes - TableSchema renamed to Schema - FieldSchema renamed to Field - Add class Type to Field to represent the field's type (and wrap record subfields) - Add factory method and setter from varargs for Schema - Better javadoc for CsvOptions and Field - Update tests according to changes --- .../google/gcloud/bigquery/CsvOptions.java | 22 +- .../bigquery/ExternalDataConfiguration.java | 22 +- .../com/google/gcloud/bigquery/Field.java | 370 ++++++++++++++++++ .../google/gcloud/bigquery/FieldSchema.java | 287 -------------- .../{TableSchema.java => Schema.java} | 65 +-- .../com/google/gcloud/bigquery/TableInfo.java | 24 +- .../gcloud/bigquery/UserDefinedFunction.java | 11 +- .../gcloud/bigquery/CsvOptionsTest.java | 7 +- .../ExternalDataConfigurationTest.java | 22 +- .../{FieldSchemaTest.java => FieldTest.java} | 51 +-- .../google/gcloud/bigquery/SchemaTest.java | 77 ++++ .../gcloud/bigquery/SerializationTest.java | 25 +- .../google/gcloud/bigquery/TableInfoTest.java | 26 +- .../gcloud/bigquery/TableSchemaTest.java | 77 ---- 14 files changed, 591 insertions(+), 495 deletions(-) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java delete mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java rename gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/{TableSchema.java => Schema.java} (68%) rename gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/{FieldSchemaTest.java => FieldTest.java} (59%) create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SchemaTest.java delete mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java index dc9e88e97a15..48fe50b7c540 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java @@ -19,6 +19,7 @@ import com.google.common.base.MoreObjects; import java.io.Serializable; +import java.nio.charset.Charset; import java.util.Objects; /** @@ -78,6 +79,16 @@ public Builder encoding(String encoding) { return this; } + /** + * Sets the character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The + * default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split + * using the values set in {@link #quote(String)} and {@link #fieldDelimiter(String)}. + */ + public Builder encoding(Charset encoding) { + this.encoding = encoding.name(); + return this; + } + /** * Sets the separator for fields in a CSV file. BigQuery converts the string to ISO-8859-1 * encoding, and then uses the first byte of the encoded string to split the data in its raw, @@ -113,7 +124,7 @@ public Builder skipLeadingRows(Integer skipLeadingRows) { } /** - * Creates an {@code ExternalDataConfiguration} object. + * Creates a {@code CsvOptions} object. */ public CsvOptions build() { return new CsvOptions(this); @@ -132,8 +143,9 @@ private CsvOptions(Builder builder) { /** * Returns whether BigQuery should accept rows that are missing trailing optional columns. If * {@code true}, BigQuery treats missing trailing columns as null values. If {@code false}, - * records with missing trailing columns are treated as bad records, and if there are too many - * bad records, an invalid error is returned in the job result. + * records with missing trailing columns are treated as bad records, and if the number of bad + * records exceeds {@link ExternalDataConfiguration#maxBadRecords()}, an invalid error is returned + * in the job result. */ public Boolean allowJaggedRows() { return allowJaggedRows; @@ -148,8 +160,8 @@ public Boolean allowQuotedNewLines() { } /** - * Returns the character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The - * default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split + * Returns the character encoding of the data. The supported values are UTF-8 or ISO-8859-1. If + * not set, UTF-8 is used. BigQuery decodes the data after the raw, binary data has been split * using the values set in {@link #quote()} and {@link #fieldDelimiter()}. */ public String encoding() { diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java index f7eb792c675c..9425ca980ebb 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java @@ -62,7 +62,7 @@ public com.google.api.services.bigquery.model.ExternalDataConfiguration apply( private static final long serialVersionUID = -8004288831035566549L; private final List sourceUris; - private final TableSchema schema; + private final Schema schema; private final String sourceFormat; private final Integer maxBadRecords; private final Boolean ignoreUnknownValues; @@ -72,7 +72,7 @@ public com.google.api.services.bigquery.model.ExternalDataConfiguration apply( public static final class Builder { private List sourceUris; - private TableSchema schema; + private Schema schema; private String sourceFormat; private Integer maxBadRecords; private Boolean ignoreUnknownValues; @@ -97,13 +97,15 @@ public Builder sourceUris(List sourceUris) { /** * Sets the schema for the external data. */ - public Builder schema(TableSchema schema) { + public Builder schema(Schema schema) { this.schema = checkNotNull(schema); return this; } /** - * Sets the source format of the external data. + * Sets the source format of the external data. Supported values are {@code CSV} for CSV files, + * and {@code NEWLINE_DELIMITED_JSON} for newline-delimited JSON. If not set, files are assumed + * to be in CSV format. * * * Source Format @@ -212,7 +214,7 @@ public Integer maxBadRecords() { /** * Returns the schema for the external data. */ - public TableSchema schema() { + public Schema schema() { return schema; } @@ -327,7 +329,7 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() { * @see * Source Format */ - public static Builder builder(List sourceUris, TableSchema schema, String format) { + public static Builder builder(List sourceUris, Schema schema, String format) { return new Builder().sourceUris(sourceUris).schema(schema).sourceFormat(format); } @@ -345,7 +347,7 @@ public static Builder builder(List sourceUris, TableSchema schema, Strin * @see * Source Format */ - public static Builder builder(String sourceUri, TableSchema schema, String format) { + public static Builder builder(String sourceUri, Schema schema, String format) { return new Builder() .sourceUris(ImmutableList.of(sourceUri)) .schema(schema) @@ -368,7 +370,7 @@ public static Builder builder(String sourceUri, TableSchema schema, String forma * Source Format */ public static ExternalDataConfiguration of( - List sourceUris, TableSchema schema, String format) { + List sourceUris, Schema schema, String format) { return builder(sourceUris, schema, format).build(); } @@ -386,7 +388,7 @@ public static ExternalDataConfiguration of( * @see * Source Format */ - public static ExternalDataConfiguration of(String sourceUri, TableSchema schema, String format) { + public static ExternalDataConfiguration of(String sourceUri, Schema schema, String format) { return builder(sourceUri, schema, format).build(); } @@ -397,7 +399,7 @@ static ExternalDataConfiguration fromPb( builder.sourceUris(externalDataConfiguration.getSourceUris()); } if (externalDataConfiguration.getSchema() != null) { - builder.schema(TableSchema.fromPb(externalDataConfiguration.getSchema())); + builder.schema(Schema.fromPb(externalDataConfiguration.getSchema())); } if (externalDataConfiguration.getSourceFormat() != null) { builder.sourceFormat(externalDataConfiguration.getSourceFormat()); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java new file mode 100644 index 000000000000..3139431b3545 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java @@ -0,0 +1,370 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.TableFieldSchema; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery Table field. A table field has a name, a value, a mode and possibly a description. + * Supported types are: {@link Type#integer()}, {@link Type#bool()}, {@link Type#string()}, + * {@link Type#floatingPoint()}, {@link Type#timestamp()} and {@link Type#record(Field...)}. One or + * more fields form a table's schema. + */ +public class Field implements Serializable { + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public Field apply(TableFieldSchema pb) { + return Field.fromPb(pb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public TableFieldSchema apply(Field field) { + return field.toPb(); + } + }; + + private static final long serialVersionUID = -8154262932305199256L; + + /** + * Data Types for a BigQuery Table field. This class provides factory methods for all BigQuery + * field types. To instantiate a RECORD value the list of sub-fields must be provided. + * + * @see + * Data Types + */ + public static class Type implements Serializable { + + private static final long serialVersionUID = 2841484762609576959L; + + public enum Value { + STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, RECORD + } + + private final Value value; + private final List fields; + + private Type(Value value) { + this.value = checkNotNull(value); + this.fields = null; + } + + private Type(Value value, List fields) { + checkArgument(fields.size() > 0, "Record must have at least one field"); + this.value = value; + this.fields = fields; + } + + /** + * Returns the value identifier. + * + * @see + * Data Types + */ + public Value value() { + return value; + } + + /** + * Returns the list of sub-fields if {@link #value()} is set to {@link Value#RECORD}. Returns + * {@code null} otherwise. + */ + public List fields() { + return fields; + } + + /** + * Returns a {@link Value#STRING} field value. + */ + public static Type string() { + return new Type(Value.STRING); + } + + /** + * Returns an {@link Value#INTEGER} field value. + */ + public static Type integer() { + return new Type(Value.INTEGER); + } + + /** + * Returns a {@link Value#FLOAT} field value. + */ + public static Type floatingPoint() { + return new Type(Value.FLOAT); + } + + /** + * Returns a {@link Value#BOOLEAN} field value. + */ + public static Type bool() { + return new Type(Value.BOOLEAN); + } + + /** + * Returns a {@link Value#TIMESTAMP} field value. + */ + public static Type timestamp() { + return new Type(Value.TIMESTAMP); + } + + /** + * Returns a {@link Value#RECORD} field value with associated list of sub-fields. + */ + public static Type record(Field... fields) { + return new Type(Value.RECORD, ImmutableList.copyOf(fields)); + } + + /** + * Returns a {@link Value#RECORD} field value with associated list of sub-fields. + */ + public static Type record(List fields) { + return new Type(Value.RECORD, ImmutableList.copyOf(checkNotNull(fields))); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("value", value) + .add("fields", fields) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(value, fields); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Type)) { + return false; + } + Type other = (Type) obj; + return Objects.equals(value, other.value) + && Objects.equals(fields, other.fields); + } + } + + /** + * Mode for a BigQuery Table field. {@link Mode#NULLABLE} fields can be set to {@code null}, + * {@link Mode#REQUIRED} fields must be provided. {@link Mode#REPEATED} fields can contain more + * than one value. + */ + public enum Mode { + NULLABLE, REQUIRED, REPEATED + } + + private final String name; + private final Type type; + private final Mode mode; + private final String description; + + public static final class Builder { + + private String name; + private Type type; + private Mode mode; + private String description; + + private Builder() {} + + /** + * Sets the field name. The name must contain only letters (a-z, A-Z), numbers (0-9), or + * underscores (_), and must start with a letter or underscore. The maximum length is 128 + * characters. + */ + public Builder name(String name) { + this.name = checkNotNull(name); + return this; + } + + /** + * Sets the value of the field. + * + * @see + * Data Types + */ + public Builder type(Type type) { + this.type = checkNotNull(type); + return this; + } + + /** + * Sets the mode of the field. By default {@link Mode#NULLABLE} is used. + */ + public Builder mode(Mode mode) { + this.mode = mode; + return this; + } + + /** + * Sets the field description. The maximum length is 16K characters. + */ + public Builder description(String description) { + this.description = description; + return this; + } + + /** + * Creates an {@code Field} object. + */ + public Field build() { + return new Field(this); + } + } + + private Field(Builder builder) { + this.name = checkNotNull(builder.name); + this.type = checkNotNull(builder.type); + this.mode = builder.mode; + this.description = builder.description; + } + + /** + * Returns the field name. + */ + public String name() { + return name; + } + + /** + * Returns the field value. + * + * @see + * Data Types + */ + public Type type() { + return type; + } + + /** + * Returns the field mode. By default {@link Mode#NULLABLE} is used. + */ + public Mode mode() { + return mode; + } + + /** + * Returns the field description. + */ + public String description() { + return description; + } + + /** + * Returns the list of sub-fields if {@link #type()} is a {@link Type.Value#RECORD}. Returns + * {@code null} otherwise. + */ + public List fields() { + return type.fields(); + } + + /** + * Returns a builder for the {@code Field} object. + */ + public Builder toBuilder() { + return new Builder() + .name(this.name) + .type(this.type) + .mode(this.mode) + .description(this.description); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("value", type) + .add("mode", mode) + .add("description", description) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(name, type, mode, description); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof Field && Objects.equals(toPb(), ((Field) obj).toPb()); + } + + TableFieldSchema toPb() { + TableFieldSchema fieldSchemaPb = new TableFieldSchema(); + fieldSchemaPb.setName(name); + fieldSchemaPb.setType(type.value().name()); + if (mode != null) { + fieldSchemaPb.setMode(mode.name()); + } + if (description != null) { + fieldSchemaPb.setDescription(description); + } + if (fields() != null) { + List fieldsPb = Lists.transform(fields(), TO_PB_FUNCTION); + fieldSchemaPb.setFields(fieldsPb); + } + return fieldSchemaPb; + } + + /** + * Returns a Field object with given name and value. + */ + public static Field of(String name, Type type) { + return builder(name, type).build(); + } + + /** + * Returns a builder for a Field object with given name and value. + */ + public static Builder builder(String name, Type type) { + return new Builder().name(name).type(type); + } + + static Field fromPb(TableFieldSchema fieldSchemaPb) { + Builder fieldBuilder = new Builder(); + fieldBuilder.name(fieldSchemaPb.getName()); + Type.Value enumValue = Type.Value.valueOf(fieldSchemaPb.getType()); + if (fieldSchemaPb.getMode() != null) { + fieldBuilder.mode(Mode.valueOf(fieldSchemaPb.getMode())); + } + if (fieldSchemaPb.getDescription() != null) { + fieldBuilder.description(fieldSchemaPb.getDescription()); + } + if (fieldSchemaPb.getFields() != null) { + fieldBuilder.type(Type.record(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION))); + } else { + fieldBuilder.type(new Type(enumValue)); + } + return fieldBuilder.build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java deleted file mode 100644 index fa622bf28976..000000000000 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FieldSchema.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.common.base.Function; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - -import java.io.Serializable; -import java.util.List; -import java.util.Objects; - -/** - * Google Bigquery schema for a Table field. Several data types and modes are supported. - */ -public class FieldSchema implements Serializable { - - static final Function FROM_PB_FUNCTION = - new Function() { - @Override - public FieldSchema apply(TableFieldSchema pb) { - return FieldSchema.fromPb(pb); - } - }; - static final Function TO_PB_FUNCTION = - new Function() { - @Override - public TableFieldSchema apply(FieldSchema fieldSchema) { - return fieldSchema.toPb(); - } - }; - - private static final long serialVersionUID = -8154262932305199256L; - - /** - * Data Types for a BigQuery Table field. - * - * @see - * Data Types - */ - public enum Type { - STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, RECORD - } - - /** - * Mode for a BigQuery Table field. {@link Mode#NULLABLE} fields can be set to {@code null}, - * {@link Mode#REQUIRED} fields must be provided. {@link Mode#REPEATED} fields can contain more - * than one value. - */ - public enum Mode { - NULLABLE, REQUIRED, REPEATED - } - - private final String name; - private final Type type; - private final Mode mode; - private final String description; - private final List fields; - - public static final class Builder { - - private String name; - private Type type; - private Mode mode; - private String description; - private List fields; - - private Builder() {} - - /** - * Sets the field name. The name must contain only letters (a-z, A-Z), numbers (0-9), or - * underscores (_), and must start with a letter or underscore. The maximum length is 128 - * characters. - */ - public Builder name(String name) { - this.name = checkNotNull(name); - return this; - } - - /** - * Sets the type of the field. - * - * @see - * Data Types - */ - public Builder type(Type type) { - this.type = checkNotNull(type); - return this; - } - - /** - * Sets the mode of the field. By default {@link Mode#NULLABLE} is used. - */ - public Builder mode(Mode mode) { - this.mode = mode; - return this; - } - - /** - * Sets the field description. The maximum length is 16K characters. - */ - public Builder description(String description) { - this.description = description; - return this; - } - - /** - * Sets the nested schema fields if {@link #type(Type)} is set to {@link Type#RECORD}. - */ - Builder fields(Iterable fields) { - this.fields = fields != null ? ImmutableList.copyOf(fields) : null; - return this; - } - - /** - * Creates an {@code FieldSchema} object. - */ - public FieldSchema build() { - FieldSchema fieldSchema = new FieldSchema(this); - checkNotNull(fieldSchema.name); - checkNotNull(fieldSchema.type); - return fieldSchema; - } - } - - private FieldSchema(Builder builder) { - this.name = builder.name; - this.type = builder.type; - this.mode = builder.mode; - this.description = builder.description; - this.fields = builder.fields; - } - - /** - * Returns the field name. - */ - public String name() { - return name; - } - - /** - * Returns the field type. - * - * @see - * Data Types - */ - public Type type() { - return type; - } - - /** - * Returns the field mode. By default {@link Mode#NULLABLE} is used. - */ - public Mode mode() { - return mode; - } - - /** - * Returns the field description. - */ - public String description() { - return description; - } - - /** - * Returns the nested schema fields if {@link #type()} is set to {@link Type#RECORD}. Returns - * {@code null} otherwise. - */ - public List fields() { - return fields; - } - - /** - * Returns a builder for the {@code FieldSchema} object. - */ - public Builder toBuilder() { - return new Builder() - .name(this.name) - .type(this.type) - .mode(this.mode) - .description(this.description) - .fields(this.fields); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("name", name) - .add("type", type) - .add("mode", mode) - .add("description", description) - .add("fields", fields) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash(name, type, mode, description, fields); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof FieldSchema && Objects.equals(toPb(), ((FieldSchema) obj).toPb()); - } - - TableFieldSchema toPb() { - TableFieldSchema fieldSchemaPb = new TableFieldSchema(); - fieldSchemaPb.setName(name); - fieldSchemaPb.setType(type.name()); - if (mode != null) { - fieldSchemaPb.setMode(mode.name()); - } - if (description != null) { - fieldSchemaPb.setDescription(description); - } - if (fields != null) { - List fieldsPb = Lists.transform(fields, TO_PB_FUNCTION); - fieldSchemaPb.setFields(fieldsPb); - } - return fieldSchemaPb; - } - - /** - * Returns a FieldSchema object with given name and type. This method should only be used to - * create fields with primitive types (i.e. {@link Type#FLOAT}, {@link Type#BOOLEAN}, - * {@link Type#INTEGER}, {@link Type#STRING} and {@link Type#TIMESTAMP}). - */ - public static FieldSchema of(String name, Type type) { - return builder(name, type).build(); - } - - /** - * Returns a FieldSchema object of type {@link Type#RECORD} for the provided name and subfields. - */ - public static FieldSchema of(String name, List fields) { - return builder(name, fields).type(Type.RECORD).build(); - } - - /** - * Returns a builder for a FieldSchema object with given name and type. - */ - public static Builder builder(String name, Type type) { - return new Builder().name(name).type(type); - } - - /** - * Returns a builder for a FieldSchema object with given name and list of subfields. Type is set - * to {@link Type#RECORD}. - */ - public static Builder builder(String name, List fields) { - return new Builder().name(name).type(Type.RECORD).fields(fields); - } - - static FieldSchema fromPb(TableFieldSchema fieldSchemaPb) { - Builder fieldBuilder = new Builder(); - fieldBuilder.name(fieldSchemaPb.getName()); - fieldBuilder.type(Type.valueOf(fieldSchemaPb.getType())); - if (fieldSchemaPb.getMode() != null) { - fieldBuilder.mode(Mode.valueOf(fieldSchemaPb.getMode())); - } - if (fieldSchemaPb.getDescription() != null) { - fieldBuilder.description(fieldSchemaPb.getDescription()); - } - if (fieldSchemaPb.getFields() != null) { - fieldBuilder.fields(Lists.transform(fieldSchemaPb.getFields(), FROM_PB_FUNCTION)); - } - return fieldBuilder.build(); - } -} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java similarity index 68% rename from gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java rename to gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java index 987d838db36c..68959928984d 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableSchema.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java @@ -31,39 +31,39 @@ /** * This class represents the schema for a Google BigQuery Table or data source. */ -public class TableSchema implements Serializable { +public class Schema implements Serializable { - static final Function + static final Function FROM_PB_FUNCTION = new Function() { + Schema>() { @Override - public TableSchema apply(com.google.api.services.bigquery.model.TableSchema pb) { - return TableSchema.fromPb(pb); + public Schema apply(com.google.api.services.bigquery.model.TableSchema pb) { + return Schema.fromPb(pb); } }; - static final Function - TO_PB_FUNCTION = new Function + TO_PB_FUNCTION = new Function() { @Override - public com.google.api.services.bigquery.model.TableSchema apply(TableSchema schema) { + public com.google.api.services.bigquery.model.TableSchema apply(Schema schema) { return schema.toPb(); } }; private static final long serialVersionUID = 2007400596384553696L; - private final List fields; + private final List fields; public static final class Builder { - private List fields; + private List fields; private Builder() {} /** * Adds a field's schema to the table's schema. */ - public Builder addField(FieldSchema field) { + public Builder addField(Field field) { if (fields == null) { fields = Lists.newArrayList(); } @@ -74,38 +74,44 @@ public Builder addField(FieldSchema field) { /** * Sets table's schema fields. */ - public Builder fields(Iterable fields) { + public Builder fields(Iterable fields) { this.fields = Lists.newArrayList(checkNotNull(fields)); return this; } /** - * Creates an {@code TableSchema} object. + * Sets table's schema fields. */ - public TableSchema build() { - return new TableSchema(this); + public Builder fields(Field... fields) { + this.fields = Lists.newArrayList(fields); + return this; + } + + /** + * Creates an {@code Schema} object. + */ + public Schema build() { + return new Schema(this); } } - private TableSchema(Builder builder) { + private Schema(Builder builder) { this.fields = builder.fields != null ? ImmutableList.copyOf(builder.fields) : - ImmutableList.of(); + ImmutableList.of(); } /** * Returns the fields in the current table schema. */ - public List fields() { + public List fields() { return fields; } /** - * Returns a builder for the {@code TableSchema} object. + * Returns a builder for the {@code Schema} object. */ public Builder toBuilder() { - Builder builder = new Builder(); - builder.fields(fields); - return builder; + return builder().fields(fields); } @Override @@ -122,14 +128,14 @@ public int hashCode() { @Override public boolean equals(Object obj) { - return obj instanceof TableSchema && Objects.equals(toPb(), ((TableSchema) obj).toPb()); + return obj instanceof Schema && Objects.equals(toPb(), ((Schema) obj).toPb()); } com.google.api.services.bigquery.model.TableSchema toPb() { com.google.api.services.bigquery.model.TableSchema tableSchemaPb = new com.google.api.services.bigquery.model.TableSchema(); if (fields != null) { - List fieldsPb = Lists.transform(fields, FieldSchema.TO_PB_FUNCTION); + List fieldsPb = Lists.transform(fields, Field.TO_PB_FUNCTION); tableSchemaPb.setFields(fieldsPb); } return tableSchemaPb; @@ -139,13 +145,16 @@ public static Builder builder() { return new Builder(); } - public static TableSchema of(Iterable fields) { + public static Schema of(Iterable fields) { checkNotNull(fields); return builder().fields(fields).build(); } - static TableSchema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { - Builder schemaBuilder = new Builder(); - return TableSchema.of(Lists.transform(tableSchemaPb.getFields(), FieldSchema.FROM_PB_FUNCTION)); + public static Schema of(Field... fields) { + return builder().fields(fields).build(); + } + + static Schema fromPb(com.google.api.services.bigquery.model.TableSchema tableSchemaPb) { + return Schema.of(Lists.transform(tableSchemaPb.getFields(), Field.FROM_PB_FUNCTION)); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java index df08025517f6..756b94f6d9ac 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java @@ -34,10 +34,9 @@ /** * Google BigQuery Table information. A BigQuery table is a standard, two-dimensional table with - * individual records organized in rows, and a data type assigned to each column - * (also called a field). Individual fields within a record may contain nested and repeated children - * fields. Every table is described by a schema that describes field names, types, and other - * information. + * individual records organized in rows, and a data type assigned to each column (also called a + * field). Individual fields within a record may contain nested and repeated children fields. Every + * table is described by a schema that describes field names, types, and other information. * * @see Managing Tables */ @@ -162,7 +161,7 @@ static StreamingBuffer fromPb(Streamingbuffer streamingBufferPb) { private final TableId tableId; private final String friendlyName; private final String description; - private final TableSchema schema; + private final Schema schema; private final Long numBytes; private final Long numRows; private final Long creationTime; @@ -183,7 +182,7 @@ public static final class Builder { private TableId tableId; private String friendlyName; private String description; - private TableSchema schema; + private Schema schema; private Long numBytes; private Long numRows; private Long creationTime; @@ -295,7 +294,7 @@ Builder numRows(Long numRows) { * Sets the table's schema. Providing a schema is not necessary when {@link #viewQuery} is * provided. */ - public Builder schema(TableSchema schema) { + public Builder schema(Schema schema) { this.schema = schema; return this; } @@ -348,16 +347,15 @@ public Builder userDefinedFunctions(List userDefinedFunctio * Creates a {@code TableInfo} object. */ public TableInfo build() { - checkNotNull(tableId); return new TableInfo(this); } } private TableInfo(Builder builder) { + this.tableId = checkNotNull(builder.tableId); this.etag = builder.etag; this.id = builder.id; this.selfLink = builder.selfLink; - this.tableId = builder.tableId; this.friendlyName = builder.friendlyName; this.description = builder.description; this.schema = builder.schema; @@ -420,7 +418,7 @@ public String description() { /** * Returns the table's schema. */ - public TableSchema schema() { + public Schema schema() { return schema; } @@ -637,7 +635,7 @@ public static Builder builder(TableId tableId, String query, List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); - private static final FieldSchema FIELD_SCHEMA1 = - FieldSchema.builder("StringField", FieldSchema.Type.STRING) - .mode(FieldSchema.Mode.NULLABLE) + private static final Field FIELD_SCHEMA1 = + Field.builder("StringField", Field.Type.string()) + .mode(Field.Mode.NULLABLE) .description("FieldDescription1") .build(); - private static final FieldSchema FIELD_SCHEMA2 = - FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) - .mode(FieldSchema.Mode.REPEATED) + private static final Field FIELD_SCHEMA2 = + Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REPEATED) .description("FieldDescription2") .build(); - private static final FieldSchema FIELD_SCHEMA3 = - FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) - .mode(FieldSchema.Mode.REQUIRED) + private static final Field FIELD_SCHEMA3 = + Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(Field.Mode.REQUIRED) .description("FieldDescription3") .build(); - private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, - FIELD_SCHEMA3); - private static final TableSchema TABLE_SCHEMA = TableSchema.of(FIELDS); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); private static final String SOURCE_FORMAT = "CSV"; private static final Integer MAX_BAD_RECORDS = 42; private static final Boolean IGNORE_UNKNOWN_VALUES = true; diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldTest.java similarity index 59% rename from gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java rename to gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldTest.java index fa26419abbba..5f039eaed206 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldSchemaTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FieldTest.java @@ -22,30 +22,31 @@ import org.junit.Test; -public class FieldSchemaTest { +public class FieldTest { private static final String FIELD_NAME1 = "StringField"; private static final String FIELD_NAME2 = "IntegerField"; private static final String FIELD_NAME3 = "RecordField"; - private static final FieldSchema.Type FIELD_TYPE1 = FieldSchema.Type.STRING; - private static final FieldSchema.Type FIELD_TYPE2 = FieldSchema.Type.INTEGER; - private static final FieldSchema.Type FIELD_TYPE3 = FieldSchema.Type.RECORD; - private static final FieldSchema.Mode FIELD_MODE1 = FieldSchema.Mode.NULLABLE; - private static final FieldSchema.Mode FIELD_MODE2 = FieldSchema.Mode.REPEATED; - private static final FieldSchema.Mode FIELD_MODE3 = FieldSchema.Mode.REQUIRED; + private static final Field.Type FIELD_TYPE1 = Field.Type.string(); + private static final Field.Type FIELD_TYPE2 = Field.Type.integer(); + private static final Field.Mode FIELD_MODE1 = Field.Mode.NULLABLE; + private static final Field.Mode FIELD_MODE2 = Field.Mode.REPEATED; + private static final Field.Mode FIELD_MODE3 = Field.Mode.REQUIRED; private static final String FIELD_DESCRIPTION1 = "FieldDescription1"; private static final String FIELD_DESCRIPTION2 = "FieldDescription2"; private static final String FIELD_DESCRIPTION3 = "FieldDescription3"; - private static final FieldSchema FIELD_SCHEMA1 = FieldSchema.builder(FIELD_NAME1, FIELD_TYPE1) + private static final Field FIELD_SCHEMA1 = Field.builder(FIELD_NAME1, FIELD_TYPE1) .mode(FIELD_MODE1) .description(FIELD_DESCRIPTION1) .build(); - private static final FieldSchema FIELD_SCHEMA2 = FieldSchema.builder(FIELD_NAME2, FIELD_TYPE2) + private static final Field FIELD_SCHEMA2 = Field.builder(FIELD_NAME2, FIELD_TYPE2) .mode(FIELD_MODE2) .description(FIELD_DESCRIPTION2) .build(); - private static final FieldSchema FIELD_SCHEMA3 = FieldSchema - .builder(FIELD_NAME3, ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) + private static final Field.Type FIELD_TYPE3 = + Field.Type.record(ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)); + private static final Field FIELD_SCHEMA3 = Field + .builder(FIELD_NAME3, FIELD_TYPE3) .mode(FIELD_MODE3) .description(FIELD_DESCRIPTION3) .build(); @@ -55,20 +56,20 @@ public void testToBuilder() { compareFieldSchemas(FIELD_SCHEMA1, FIELD_SCHEMA1.toBuilder().build()); compareFieldSchemas(FIELD_SCHEMA2, FIELD_SCHEMA2.toBuilder().build()); compareFieldSchemas(FIELD_SCHEMA3, FIELD_SCHEMA3.toBuilder().build()); - FieldSchema fieldSchema = FIELD_SCHEMA1.toBuilder() + Field field = FIELD_SCHEMA1.toBuilder() .description("New Description") .build(); - assertEquals("New Description", fieldSchema.description()); - fieldSchema = fieldSchema.toBuilder().description(FIELD_DESCRIPTION1).build(); - compareFieldSchemas(FIELD_SCHEMA1, fieldSchema); + assertEquals("New Description", field.description()); + field = field.toBuilder().description(FIELD_DESCRIPTION1).build(); + compareFieldSchemas(FIELD_SCHEMA1, field); } @Test public void testToBuilderIncomplete() { - FieldSchema fieldSchema = FieldSchema.of(FIELD_NAME1, FIELD_TYPE1); - compareFieldSchemas(fieldSchema, fieldSchema.toBuilder().build()); - fieldSchema = FieldSchema.of(FIELD_NAME2, ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)); - compareFieldSchemas(fieldSchema, fieldSchema.toBuilder().build()); + Field field = Field.of(FIELD_NAME1, FIELD_TYPE1); + compareFieldSchemas(field, field.toBuilder().build()); + field = Field.of(FIELD_NAME2, FIELD_TYPE3); + compareFieldSchemas(field, field.toBuilder().build()); } @Test @@ -87,14 +88,14 @@ public void testBuilder() { @Test public void testToAndFromPb() { - compareFieldSchemas(FIELD_SCHEMA1, FieldSchema.fromPb(FIELD_SCHEMA1.toPb())); - compareFieldSchemas(FIELD_SCHEMA2, FieldSchema.fromPb(FIELD_SCHEMA2.toPb())); - compareFieldSchemas(FIELD_SCHEMA3, FieldSchema.fromPb(FIELD_SCHEMA3.toPb())); - FieldSchema fieldSchema = FieldSchema.builder(FIELD_NAME1, FIELD_TYPE1).build(); - compareFieldSchemas(fieldSchema, FieldSchema.fromPb(fieldSchema.toPb())); + compareFieldSchemas(FIELD_SCHEMA1, Field.fromPb(FIELD_SCHEMA1.toPb())); + compareFieldSchemas(FIELD_SCHEMA2, Field.fromPb(FIELD_SCHEMA2.toPb())); + compareFieldSchemas(FIELD_SCHEMA3, Field.fromPb(FIELD_SCHEMA3.toPb())); + Field field = Field.builder(FIELD_NAME1, FIELD_TYPE1).build(); + compareFieldSchemas(field, Field.fromPb(field.toPb())); } - private void compareFieldSchemas(FieldSchema expected, FieldSchema value) { + private void compareFieldSchemas(Field expected, Field value) { assertEquals(expected, value); assertEquals(expected.name(), value.name()); assertEquals(expected.type(), value.type()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SchemaTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SchemaTest.java new file mode 100644 index 000000000000..d24268d2e7cd --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SchemaTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +import java.util.List; + +public class SchemaTest { + + private static final Field FIELD_SCHEMA1 = + Field.builder("StringField", Field.Type.string()) + .mode(Field.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final Field FIELD_SCHEMA2 = + Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final Field FIELD_SCHEMA3 = + Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(Field.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, + FIELD_SCHEMA3); + private static final Schema TABLE_SCHEMA = Schema.builder().fields(FIELDS).build(); + + @Test + public void testToBuilder() { + compareTableSchema(TABLE_SCHEMA, TABLE_SCHEMA.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(FIELDS, TABLE_SCHEMA.fields()); + Schema schema = TABLE_SCHEMA.toBuilder() + .fields(FIELD_SCHEMA1, FIELD_SCHEMA2) + .addField(FIELD_SCHEMA3) + .build(); + compareTableSchema(TABLE_SCHEMA, schema); + } + + @Test + public void testOf() { + compareTableSchema(TABLE_SCHEMA, Schema.of(FIELDS)); + } + + @Test + public void testToAndFromPb() { + compareTableSchema(TABLE_SCHEMA, Schema.fromPb(TABLE_SCHEMA.toPb())); + } + + private void compareTableSchema(Schema expected, Schema value) { + assertEquals(expected, value); + assertEquals(expected.fields(), value.fields()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index 8001dffb1cf9..f80dbf1d5225 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -31,6 +31,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.List; public class SerializationTest { @@ -70,29 +71,27 @@ public class SerializationTest { private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() .allowJaggedRows(true) .allowQuotedNewLines(false) - .encoding("CSV") + .encoding(StandardCharsets.ISO_8859_1) .fieldDelimiter(",") .quote("\"") .skipLeadingRows(42) .build(); - private static final FieldSchema FIELD_SCHEMA1 = - FieldSchema.builder("StringField", FieldSchema.Type.STRING) - .mode(FieldSchema.Mode.NULLABLE) + private static final Field FIELD_SCHEMA1 = + Field.builder("StringField", Field.Type.string()) + .mode(Field.Mode.NULLABLE) .description("FieldDescription1") .build(); - private static final FieldSchema FIELD_SCHEMA2 = - FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) - .mode(FieldSchema.Mode.REPEATED) + private static final Field FIELD_SCHEMA2 = + Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REPEATED) .description("FieldDescription2") .build(); - private static final FieldSchema FIELD_SCHEMA3 = - FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) - .mode(FieldSchema.Mode.REQUIRED) + private static final Field FIELD_SCHEMA3 = + Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(Field.Mode.REQUIRED) .description("FieldDescription3") .build(); - private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, - FIELD_SCHEMA3); - private static final TableSchema TABLE_SCHEMA = TableSchema.of(FIELDS); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); private static final TableInfo.StreamingBuffer STREAMING_BUFFER = new TableInfo.StreamingBuffer(1L, 2L, 3L); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java index 5e4d7c220197..b29ddef665d5 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java @@ -27,24 +27,22 @@ public class TableInfoTest { - private static final FieldSchema FIELD_SCHEMA1 = - FieldSchema.builder("StringField", FieldSchema.Type.STRING) - .mode(FieldSchema.Mode.NULLABLE) + private static final Field FIELD_SCHEMA1 = + Field.builder("StringField", Field.Type.string()) + .mode(Field.Mode.NULLABLE) .description("FieldDescription1") .build(); - private static final FieldSchema FIELD_SCHEMA2 = - FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) - .mode(FieldSchema.Mode.REPEATED) + private static final Field FIELD_SCHEMA2 = + Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REPEATED) .description("FieldDescription2") .build(); - private static final FieldSchema FIELD_SCHEMA3 = - FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) - .mode(FieldSchema.Mode.REQUIRED) + private static final Field FIELD_SCHEMA3 = + Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(Field.Mode.REQUIRED) .description("FieldDescription3") .build(); - private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, - FIELD_SCHEMA3); - private static final TableSchema TABLE_SCHEMA = TableSchema.of(FIELDS); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); private static final String VIEW_QUERY = "VIEW QUERY"; private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); private static final String SOURCE_FORMAT = "CSV"; @@ -72,8 +70,6 @@ public class TableInfoTest { private static final Long LAST_MODIFIED_TIME = 20L; private static final String LOCATION = "US"; private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L); - private static List USER_DEFINED_FUNCTIONS = ImmutableList.of( - UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); private static final TableInfo TABLE_INFO = TableInfo.builder(TABLE_ID, TABLE_SCHEMA) .creationTime(CREATION_TIME) .description(DESCRIPTION) @@ -104,6 +100,8 @@ public class TableInfoTest { .streamingBuffer(STREAMING_BUFFER) .type(TableInfo.Type.TABLE) .build(); + private static List USER_DEFINED_FUNCTIONS = ImmutableList.of( + UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); private static final TableInfo VIEW_INFO = TableInfo.builder(TABLE_ID, VIEW_QUERY, USER_DEFINED_FUNCTIONS) .creationTime(CREATION_TIME) diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java deleted file mode 100644 index 8237d94cf7a8..000000000000 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableSchemaTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.ImmutableList; - -import org.junit.Test; - -import java.util.List; - -public class TableSchemaTest { - - private static final FieldSchema FIELD_SCHEMA1 = - FieldSchema.builder("StringField", FieldSchema.Type.STRING) - .mode(FieldSchema.Mode.NULLABLE) - .description("FieldDescription1") - .build(); - private static final FieldSchema FIELD_SCHEMA2 = - FieldSchema.builder("IntegerField", FieldSchema.Type.INTEGER) - .mode(FieldSchema.Mode.REPEATED) - .description("FieldDescription2") - .build(); - private static final FieldSchema FIELD_SCHEMA3 = - FieldSchema.builder("RecordField", ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) - .mode(FieldSchema.Mode.REQUIRED) - .description("FieldDescription3") - .build(); - private static final List FIELDS = ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2, - FIELD_SCHEMA3); - private static final TableSchema TABLE_SCHEMA = TableSchema.builder().fields(FIELDS).build(); - - @Test - public void testToBuilder() { - compareTableSchema(TABLE_SCHEMA, TABLE_SCHEMA.toBuilder().build()); - } - - @Test - public void testBuilder() { - assertEquals(FIELDS, TABLE_SCHEMA.fields()); - TableSchema tableSchema = TABLE_SCHEMA.toBuilder() - .fields(ImmutableList.of(FIELD_SCHEMA1, FIELD_SCHEMA2)) - .addField(FIELD_SCHEMA3) - .build(); - compareTableSchema(TABLE_SCHEMA, tableSchema); - } - - @Test - public void testOf() { - compareTableSchema(TABLE_SCHEMA, TableSchema.of(FIELDS)); - } - - @Test - public void testToAndFromPb() { - compareTableSchema(TABLE_SCHEMA, TableSchema.fromPb(TABLE_SCHEMA.toPb())); - } - - private void compareTableSchema(TableSchema expected, TableSchema value) { - assertEquals(expected, value); - assertEquals(expected.fields(), value.fields()); - } -} From 5f6357eaf4c35342f43add62f11c4cc384465ee6 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 2 Dec 2015 11:43:40 +0100 Subject: [PATCH 03/10] Minor javadoc updates, rename functionDefinition to content --- .../com/google/gcloud/bigquery/Field.java | 4 +-- .../com/google/gcloud/bigquery/Schema.java | 1 - .../com/google/gcloud/bigquery/TableInfo.java | 8 ++--- .../gcloud/bigquery/UserDefinedFunction.java | 29 +++++++++---------- .../bigquery/UserDefinedFunctionTest.java | 6 ++-- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java index 3139431b3545..73ca87640ff8 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java @@ -219,7 +219,7 @@ public Builder type(Type type) { } /** - * Sets the mode of the field. By default {@link Mode#NULLABLE} is used. + * Sets the mode of the field. When not specified {@link Mode#NULLABLE} is used. */ public Builder mode(Mode mode) { this.mode = mode; @@ -235,7 +235,7 @@ public Builder description(String description) { } /** - * Creates an {@code Field} object. + * Creates a {@code Field} object. */ public Field build() { return new Field(this); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java index 68959928984d..0ac6e1b84ade 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Schema.java @@ -146,7 +146,6 @@ public static Builder builder() { } public static Schema of(Iterable fields) { - checkNotNull(fields); return builder().fields(fields).build(); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java index 756b94f6d9ac..9ca74eb3c1f7 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java @@ -258,7 +258,7 @@ public Builder externalConfiguration(ExternalDataConfiguration externalConfigura } /** - * Sets a user-friendly name for the dataset. + * Sets a user-friendly name for the table. */ public Builder friendlyName(String friendlyName) { this.friendlyName = firstNonNull(friendlyName, Data.nullOf(String.class)); @@ -291,8 +291,8 @@ Builder numRows(Long numRows) { } /** - * Sets the table's schema. Providing a schema is not necessary when {@link #viewQuery} is - * provided. + * Sets the table's schema. Providing a schema is not necessary when {@link #viewQuery(String)} + * or {@link #externalConfiguration(ExternalDataConfiguration)} are provided. */ public Builder schema(Schema schema) { this.schema = schema; @@ -380,7 +380,7 @@ public String etag() { } /** - * Returns an opaque id for the dataset. + * Returns an opaque id for the table. */ public String id() { return id; diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java index c9ef65f10d98..931c1eaf024a 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/UserDefinedFunction.java @@ -44,11 +44,11 @@ public enum Type { } private final Type type; - private final String functionDefinition; + private final String content; - UserDefinedFunction(Type type, String functionDefinition) { + UserDefinedFunction(Type type, String content) { this.type = type; - this.functionDefinition = functionDefinition; + this.content = content; } public Type type() { @@ -56,18 +56,17 @@ public Type type() { } /** - * Returns function's definition. If {@link #type()} is {@link Type#INLINE} this method returns - * a code blob. If {@link #type()} is {@link Type#FROM_URI} this method returns a Google Cloud - * Storage URI (e.g. gs://bucket/path). + * If {@link #type()} is {@link Type#INLINE} this method returns a code blob. If {@link #type()} + * is {@link Type#FROM_URI} the method returns a Google Cloud Storage URI (e.g. gs://bucket/path). */ - public String functionDefinition() { - return functionDefinition; + public String content() { + return content; } /** * A Google Cloud BigQuery user-defined function, as a code blob. */ - public static final class InlineFunction extends UserDefinedFunction { + static final class InlineFunction extends UserDefinedFunction { private static final long serialVersionUID = 1083672109192091686L; @@ -77,20 +76,20 @@ public static final class InlineFunction extends UserDefinedFunction { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("inlineCode", functionDefinition()).toString(); + return MoreObjects.toStringHelper(this).add("inlineCode", content()).toString(); } @Override public com.google.api.services.bigquery.model.UserDefinedFunctionResource toPb() { return new com.google.api.services.bigquery.model.UserDefinedFunctionResource() - .setInlineCode(functionDefinition()); + .setInlineCode(content()); } } /** * A Google Cloud BigQuery user-defined function, as an URI to Google Cloud Storage. */ - public static final class UriFunction extends UserDefinedFunction { + static final class UriFunction extends UserDefinedFunction { private static final long serialVersionUID = 4660331691852223839L; @@ -100,19 +99,19 @@ public static final class UriFunction extends UserDefinedFunction { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("functionUri", functionDefinition()).toString(); + return MoreObjects.toStringHelper(this).add("functionUri", content()).toString(); } @Override public com.google.api.services.bigquery.model.UserDefinedFunctionResource toPb() { return new com.google.api.services.bigquery.model.UserDefinedFunctionResource() - .setResourceUri(functionDefinition()); + .setResourceUri(content()); } } @Override public int hashCode() { - return Objects.hash(type, functionDefinition); + return Objects.hash(type, content); } @Override diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java index f2640f240a62..2741aaed89a5 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/UserDefinedFunctionTest.java @@ -30,9 +30,9 @@ public class UserDefinedFunctionTest { @Test public void testConstructor() { - assertEquals(INLINE, INLINE_FUNCTION.functionDefinition()); + assertEquals(INLINE, INLINE_FUNCTION.content()); assertEquals(UserDefinedFunction.Type.INLINE, INLINE_FUNCTION.type()); - assertEquals(URI, URI_FUNCTION.functionDefinition()); + assertEquals(URI, URI_FUNCTION.content()); assertEquals(UserDefinedFunction.Type.FROM_URI, URI_FUNCTION.type()); } @@ -51,6 +51,6 @@ public void testToAndFromPb() { private void compareUserDefinedFunction(UserDefinedFunction expected, UserDefinedFunction value) { assertEquals(expected, value); assertEquals(expected.type(), value.type()); - assertEquals(expected.functionDefinition(), value.functionDefinition()); + assertEquals(expected.content(), value.content()); } } From 79001cf046ca1f33427231655dd457cd41015e09 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 2 Dec 2015 11:48:17 +0100 Subject: [PATCH 04/10] Replace format with FormatOptions in ExternalDataConfiguration --- .../google/gcloud/bigquery/CsvOptions.java | 32 ++----- .../bigquery/ExternalDataConfiguration.java | 95 ++++++++----------- .../google/gcloud/bigquery/FormatOptions.java | 90 ++++++++++++++++++ .../gcloud/bigquery/CsvOptionsTest.java | 1 + .../ExternalDataConfigurationTest.java | 11 +-- .../gcloud/bigquery/FormatOptionsTest.java | 52 ++++++++++ .../gcloud/bigquery/SerializationTest.java | 3 +- .../google/gcloud/bigquery/TableInfoTest.java | 4 +- 8 files changed, 200 insertions(+), 88 deletions(-) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FormatOptionsTest.java diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java index 48fe50b7c540..5c52cd78cc8f 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java @@ -18,15 +18,14 @@ import com.google.common.base.MoreObjects; -import java.io.Serializable; import java.nio.charset.Charset; import java.util.Objects; /** - * Google BigQuery CSV options. This class wraps some properties of CSV files used by BigQuery to - * parse external data. + * Google BigQuery options for CSV format. This class wraps some properties of CSV files used by + * BigQuery to parse external data. */ -public class CsvOptions implements Serializable { +public class CsvOptions extends FormatOptions { private static final long serialVersionUID = 2193570529308612708L; @@ -132,6 +131,7 @@ public CsvOptions build() { } private CsvOptions(Builder builder) { + super(FormatOptions.CSV); this.allowJaggedRows = builder.allowJaggedRows; this.allowQuotedNewLines = builder.allowQuotedNewLines; this.encoding = builder.encoding; @@ -226,24 +226,12 @@ public boolean equals(Object obj) { com.google.api.services.bigquery.model.CsvOptions toPb() { com.google.api.services.bigquery.model.CsvOptions csvOptions = new com.google.api.services.bigquery.model.CsvOptions(); - if (allowJaggedRows != null) { - csvOptions.setAllowJaggedRows(allowJaggedRows); - } - if (allowQuotedNewLines != null) { - csvOptions.setAllowQuotedNewlines(allowQuotedNewLines); - } - if (encoding != null) { - csvOptions.setEncoding(encoding); - } - if (fieldDelimiter != null) { - csvOptions.setFieldDelimiter(fieldDelimiter); - } - if (quote != null) { - csvOptions.setQuote(quote); - } - if (skipLeadingRows != null) { - csvOptions.setSkipLeadingRows(skipLeadingRows); - } + csvOptions.setAllowJaggedRows(allowJaggedRows); + csvOptions.setAllowQuotedNewlines(allowQuotedNewLines); + csvOptions.setEncoding(encoding); + csvOptions.setFieldDelimiter(fieldDelimiter); + csvOptions.setQuote(quote); + csvOptions.setSkipLeadingRows(skipLeadingRows); return csvOptions; } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java index 9425ca980ebb..5d55080a88d5 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java @@ -63,29 +63,27 @@ public com.google.api.services.bigquery.model.ExternalDataConfiguration apply( private final List sourceUris; private final Schema schema; - private final String sourceFormat; + private final FormatOptions formatOptions; private final Integer maxBadRecords; private final Boolean ignoreUnknownValues; private final String compression; - private final CsvOptions csvOptions; public static final class Builder { private List sourceUris; private Schema schema; - private String sourceFormat; + private FormatOptions formatOptions; private Integer maxBadRecords; private Boolean ignoreUnknownValues; private String compression; - private CsvOptions csvOptions; private Builder() {} /** - * Sets the fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can - * contain one '*' wildcard character that must come after the bucket's name. Size limits - * related to load jobs apply to external data sources, plus an additional limit of 10 GB - * maximum size across all URIs. + * Sets the fully-qualified URIs that point to your data in Google Cloud Storage (e.g. + * gs://bucket/path). Each URI can contain one '*' wildcard character that must come after the + * bucket's name. Size limits related to load jobs apply to external data sources, plus an + * additional limit of 10 GB maximum size across all URIs. * * @see Quota */ @@ -103,15 +101,14 @@ public Builder schema(Schema schema) { } /** - * Sets the source format of the external data. Supported values are {@code CSV} for CSV files, - * and {@code NEWLINE_DELIMITED_JSON} for newline-delimited JSON. If not set, files are assumed - * to be in CSV format. + * Sets the source format, and possibly some parsing options, of the external data. Supported + * formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}. * * * Source Format */ - public Builder sourceFormat(String sourceFormat) { - this.sourceFormat = checkNotNull(sourceFormat); + public Builder formatOptions(FormatOptions formatOptions) { + this.formatOptions = checkNotNull(formatOptions); return this; } @@ -129,9 +126,8 @@ public Builder maxBadRecords(Integer maxBadRecords) { * Sets whether BigQuery should allow extra values that are not represented in the table schema. * If true, the extra values are ignored. If false, records with extra columns are treated as * bad records, and if there are too many bad records, an invalid error is returned in the job - * result. The default value is false. The value set with - * {@link #sourceFormat(String)} property determines what - * BigQuery treats as an extra value. + * result. The default value is false. The value set with {@link #formatOptions(FormatOptions)} + * property determines what BigQuery treats as an extra value. * * @see * Ignore Unknown Values @@ -152,15 +148,6 @@ public Builder compression(String compression) { return this; } - /** - * Sets additional properties to be used to parse CSV data (used when - * {@link #sourceFormat(String)} is set to CSV). - */ - public Builder csvOptions(CsvOptions csvOptions) { - this.csvOptions = csvOptions; - return this; - } - /** * Creates an {@code ExternalDataConfiguration} object. */ @@ -174,9 +161,8 @@ public ExternalDataConfiguration build() { this.ignoreUnknownValues = builder.ignoreUnknownValues; this.maxBadRecords = builder.maxBadRecords; this.schema = builder.schema; - this.sourceFormat = builder.sourceFormat; + this.formatOptions = builder.formatOptions; this.sourceUris = builder.sourceUris; - this.csvOptions = builder.csvOptions; } /** @@ -193,8 +179,8 @@ public String compression() { * Returns whether BigQuery should allow extra values that are not represented in the table * schema. If true, the extra values are ignored. If false, records with extra columns are treated * as bad records, and if there are too many bad records, an invalid error is returned in the job - * result. The default value is false. The value of {@link #sourceFormat()} determines what - * BigQuery treats as an extra value. + * result. The default value is false. The value of {@link #format()} determines what BigQuery + * treats as an extra value. * * @see * Ignore Unknown Values @@ -219,13 +205,13 @@ public Schema schema() { } /** - * Sets the source format of the external data. + * Returns the source format of the external data. * * * Source Format */ - public String sourceFormat() { - return sourceFormat; + public String format() { + return formatOptions.type(); } /** @@ -241,11 +227,11 @@ public List sourceUris() { } /** - * Returns additional properties used to parse CSV data (used when {@link #sourceFormat()} is set - * to CSV). + * Returns additional properties used to parse CSV data (used when {@link #format()} is set to + * CSV). Returns {@code null} if not set. */ public CsvOptions csvOptions() { - return csvOptions; + return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; } /** @@ -257,28 +243,26 @@ public Builder toBuilder() { .ignoreUnknownValues(ignoreUnknownValues) .maxBadRecords(maxBadRecords) .schema(schema) - .sourceFormat(sourceFormat) - .sourceUris(sourceUris) - .csvOptions(csvOptions); + .formatOptions(formatOptions) + .sourceUris(sourceUris); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("sourceUris", sourceUris) - .add("sourceFormat", sourceFormat) + .add("formatOptions", formatOptions) .add("schema", schema) .add("compression", compression) .add("ignoreUnknownValues", ignoreUnknownValues) .add("maxBadRecords", maxBadRecords) - .add("csvOptions", csvOptions) .toString(); } @Override public int hashCode() { - return Objects.hash(compression, ignoreUnknownValues, maxBadRecords, schema, sourceFormat, - sourceUris, csvOptions); + return Objects.hash(compression, ignoreUnknownValues, maxBadRecords, schema, formatOptions, + sourceUris); } @Override @@ -302,14 +286,14 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() { if (schema != null) { externalConfigurationPb.setSchema(schema.toPb()); } - if (sourceFormat != null) { - externalConfigurationPb.setSourceFormat(sourceFormat); + if (formatOptions != null) { + externalConfigurationPb.setSourceFormat(formatOptions.type()); } if (sourceUris != null) { externalConfigurationPb.setSourceUris(sourceUris); } - if (csvOptions != null) { - externalConfigurationPb.setCsvOptions(csvOptions.toPb()); + if (csvOptions() != null) { + externalConfigurationPb.setCsvOptions(csvOptions().toPb()); } return externalConfigurationPb; } @@ -329,8 +313,8 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() { * @see * Source Format */ - public static Builder builder(List sourceUris, Schema schema, String format) { - return new Builder().sourceUris(sourceUris).schema(schema).sourceFormat(format); + public static Builder builder(List sourceUris, Schema schema, FormatOptions format) { + return new Builder().sourceUris(sourceUris).schema(schema).formatOptions(format); } /** @@ -347,11 +331,11 @@ public static Builder builder(List sourceUris, Schema schema, String for * @see * Source Format */ - public static Builder builder(String sourceUri, Schema schema, String format) { + public static Builder builder(String sourceUri, Schema schema, FormatOptions format) { return new Builder() .sourceUris(ImmutableList.of(sourceUri)) .schema(schema) - .sourceFormat(format); + .formatOptions(format); } /** @@ -369,8 +353,8 @@ public static Builder builder(String sourceUri, Schema schema, String format) { * @see * Source Format */ - public static ExternalDataConfiguration of( - List sourceUris, Schema schema, String format) { + public static ExternalDataConfiguration of(List sourceUris, Schema schema, + FormatOptions format) { return builder(sourceUris, schema, format).build(); } @@ -388,7 +372,8 @@ public static ExternalDataConfiguration of( * @see * Source Format */ - public static ExternalDataConfiguration of(String sourceUri, Schema schema, String format) { + public static ExternalDataConfiguration of(String sourceUri, Schema schema, + FormatOptions format) { return builder(sourceUri, schema, format).build(); } @@ -402,7 +387,7 @@ static ExternalDataConfiguration fromPb( builder.schema(Schema.fromPb(externalDataConfiguration.getSchema())); } if (externalDataConfiguration.getSourceFormat() != null) { - builder.sourceFormat(externalDataConfiguration.getSourceFormat()); + builder.formatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat())); } if (externalDataConfiguration.getCompression() != null) { builder.compression(externalDataConfiguration.getCompression()); @@ -411,7 +396,7 @@ static ExternalDataConfiguration fromPb( builder.ignoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues()); } if (externalDataConfiguration.getCsvOptions() != null) { - builder.csvOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); + builder.formatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions())); } if (externalDataConfiguration.getMaxBadRecords() != null) { builder.maxBadRecords(externalDataConfiguration.getMaxBadRecords()); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java new file mode 100644 index 000000000000..8a6884daf65d --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import com.google.common.base.MoreObjects; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Base class for Google BigQuery format options. These class define the format of external data + * used by BigQuery, for either federated tables or load jobs. + */ +public class FormatOptions implements Serializable { + + static final String CSV = "CSV"; + static final String JSON = "NEWLINE_DELIMITED_JSON"; + static final String DATASTORE_BACKUP = "DATASTORE_BACKUP"; + private static final long serialVersionUID = -443376052020423691L; + + private final String type; + + FormatOptions(String type) { + this.type = type; + } + + /** + * Returns the external data format, as a string. + */ + public String type() { + return type; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("format", type).toString(); + } + + @Override + public int hashCode() { + return Objects.hash(type); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof FormatOptions && Objects.equals(type, ((FormatOptions) obj).type()); + } + + /** + * Default options for CSV format. + */ + public static FormatOptions csv() { + return new FormatOptions(CSV); + } + + /** + * Default options for NEWLINE_DELIMITED_JSON format. + */ + public static FormatOptions json() { + return new FormatOptions(JSON); + } + + /** + * Default options for DATASTORE_BACKUP format. + */ + public static FormatOptions datastoreBackup() { + return new FormatOptions(DATASTORE_BACKUP); + } + + /** + * Default options for the provided format. + */ + public static FormatOptions of(String format) { + return new FormatOptions(format); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java index bfa99957cdbd..371202174431 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CsvOptionsTest.java @@ -59,6 +59,7 @@ public void testToBuilderIncomplete() { @Test public void testBuilder() { + assertEquals(FormatOptions.CSV, CSV_OPTIONS.type()); assertEquals(ALLOW_JAGGED_ROWS, CSV_OPTIONS.allowJaggedRows()); assertEquals(ALLOW_QUOTED_NEWLINE, CSV_OPTIONS.allowQuotedNewLines()); assertEquals(ENCODING.name(), CSV_OPTIONS.encoding()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java index 4c7f5d37dda8..cba03317ccb9 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java @@ -49,9 +49,8 @@ public class ExternalDataConfigurationTest { private static final String COMPRESSION = "GZIP"; private static final CsvOptions CSV_OPTIONS = CsvOptions.builder().build(); private static final ExternalDataConfiguration CONFIGURATION = ExternalDataConfiguration - .builder(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT) + .builder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) .compression(COMPRESSION) - .csvOptions(CSV_OPTIONS) .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) .maxBadRecords(MAX_BAD_RECORDS) .build(); @@ -70,7 +69,7 @@ public void testToBuilder() { @Test public void testToBuilderIncomplete() { ExternalDataConfiguration configuration = - ExternalDataConfiguration.of(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT); + ExternalDataConfiguration.of(SOURCE_URIS, TABLE_SCHEMA, FormatOptions.json()); assertEquals(configuration, configuration.toBuilder().build()); } @@ -81,7 +80,7 @@ public void testBuilder() { assertEquals(IGNORE_UNKNOWN_VALUES, CONFIGURATION.ignoreUnknownValues()); assertEquals(MAX_BAD_RECORDS, CONFIGURATION.maxBadRecords()); assertEquals(TABLE_SCHEMA, CONFIGURATION.schema()); - assertEquals(SOURCE_FORMAT, CONFIGURATION.sourceFormat()); + assertEquals(SOURCE_FORMAT, CONFIGURATION.format()); assertEquals(SOURCE_URIS, CONFIGURATION.sourceUris()); } @@ -89,7 +88,7 @@ public void testBuilder() { public void testToAndFromPb() { compareConfiguration(CONFIGURATION, ExternalDataConfiguration.fromPb(CONFIGURATION.toPb())); ExternalDataConfiguration configuration = - ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT).build(); + ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS).build(); compareConfiguration(configuration, ExternalDataConfiguration.fromPb(configuration.toPb())); } @@ -101,7 +100,7 @@ private void compareConfiguration(ExternalDataConfiguration expected, assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues()); assertEquals(expected.maxBadRecords(), value.maxBadRecords()); assertEquals(expected.schema(), value.schema()); - assertEquals(expected.sourceFormat(), value.sourceFormat()); + assertEquals(expected.format(), value.format()); assertEquals(expected.sourceUris(), value.sourceUris()); } } diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FormatOptionsTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FormatOptionsTest.java new file mode 100644 index 000000000000..df939143156b --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/FormatOptionsTest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class FormatOptionsTest { + + @Test + public void testConstructor() { + FormatOptions options = new FormatOptions(FormatOptions.CSV); + assertEquals(FormatOptions.CSV, options.type()); + options = new FormatOptions(FormatOptions.JSON); + assertEquals(FormatOptions.JSON, options.type()); + options = new FormatOptions(FormatOptions.DATASTORE_BACKUP); + assertEquals(FormatOptions.DATASTORE_BACKUP, options.type()); + } + + @Test + public void testFactoryMethods() { + assertEquals(FormatOptions.CSV, FormatOptions.csv().type()); + assertEquals(FormatOptions.JSON, FormatOptions.json().type()); + assertEquals(FormatOptions.DATASTORE_BACKUP, FormatOptions.datastoreBackup().type()); + } + + @Test + public void testEquals() { + assertEquals(FormatOptions.csv(), FormatOptions.csv()); + assertEquals(FormatOptions.csv().hashCode(), FormatOptions.csv().hashCode()); + assertEquals(FormatOptions.json(), FormatOptions.json()); + assertEquals(FormatOptions.json().hashCode(), FormatOptions.json().hashCode()); + assertEquals(FormatOptions.datastoreBackup(), FormatOptions.datastoreBackup()); + assertEquals(FormatOptions.datastoreBackup().hashCode(), + FormatOptions.datastoreBackup().hashCode()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index f80dbf1d5225..b3bdbd3eda57 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -96,8 +96,7 @@ public class SerializationTest { new TableInfo.StreamingBuffer(1L, 2L, 3L); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); private static final ExternalDataConfiguration EXTERNAL_DATA_CONFIGURATION = - ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, "CSV") - .csvOptions(CSV_OPTIONS) + ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) .ignoreUnknownValues(true) .maxBadRecords(42) .build(); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java index b29ddef665d5..8fa84f766d17 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java @@ -49,11 +49,9 @@ public class TableInfoTest { private static final Integer MAX_BAD_RECORDS = 42; private static final Boolean IGNORE_UNKNOWN_VALUES = true; private static final String COMPRESSION = "GZIP"; - private static final CsvOptions CSV_OPTIONS = CsvOptions.builder().build(); private static final ExternalDataConfiguration CONFIGURATION = ExternalDataConfiguration - .builder(SOURCE_URIS, TABLE_SCHEMA, SOURCE_FORMAT) + .builder(SOURCE_URIS, TABLE_SCHEMA, FormatOptions.datastoreBackup()) .compression(COMPRESSION) - .csvOptions(CSV_OPTIONS) .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) .maxBadRecords(MAX_BAD_RECORDS) .build(); From f84a3b115196ca702d3c4a420870edcbb3f86596 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 3 Dec 2015 22:31:58 +0100 Subject: [PATCH 05/10] Handle unset of Field's mode and description --- .../com/google/gcloud/bigquery/Field.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java index 73ca87640ff8..d28e5eb9fed7 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java @@ -16,9 +16,11 @@ package com.google.gcloud.bigquery; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.client.util.Data; import com.google.api.services.bigquery.model.TableFieldSchema; import com.google.common.base.Function; import com.google.common.base.MoreObjects; @@ -180,7 +182,7 @@ public boolean equals(Object obj) { * than one value. */ public enum Mode { - NULLABLE, REQUIRED, REPEATED + NULLABLE, REQUIRED, REPEATED, NOT_SET } private final String name; @@ -197,6 +199,13 @@ public static final class Builder { private Builder() {} + private Builder(Field field) { + this.name = field.name; + this.type = field.type; + this.mode = field.mode; + this.description = field.description; + } + /** * Sets the field name. The name must contain only letters (a-z, A-Z), numbers (0-9), or * underscores (_), and must start with a letter or underscore. The maximum length is 128 @@ -222,7 +231,7 @@ public Builder type(Type type) { * Sets the mode of the field. When not specified {@link Mode#NULLABLE} is used. */ public Builder mode(Mode mode) { - this.mode = mode; + this.mode = firstNonNull(mode, Mode.NOT_SET); return this; } @@ -230,7 +239,7 @@ public Builder mode(Mode mode) { * Sets the field description. The maximum length is 16K characters. */ public Builder description(String description) { - this.description = description; + this.description = firstNonNull(description, Data.nullOf(String.class)); return this; } @@ -270,14 +279,14 @@ public Type type() { * Returns the field mode. By default {@link Mode#NULLABLE} is used. */ public Mode mode() { - return mode; + return mode == Mode.NOT_SET ? null : mode; } /** * Returns the field description. */ public String description() { - return description; + return Data.isNull(description) ? null : description; } /** @@ -292,11 +301,7 @@ public List fields() { * Returns a builder for the {@code Field} object. */ public Builder toBuilder() { - return new Builder() - .name(this.name) - .type(this.type) - .mode(this.mode) - .description(this.description); + return new Builder(this); } @Override @@ -324,7 +329,7 @@ TableFieldSchema toPb() { fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { - fieldSchemaPb.setMode(mode.name()); + fieldSchemaPb.setMode(mode == Mode.NOT_SET ? Data.nullOf(String.class) : mode.name()); } if (description != null) { fieldSchemaPb.setDescription(description); From bd58ee509d5663189d74485e92d70d70d44da43b Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 3 Dec 2015 22:35:36 +0100 Subject: [PATCH 06/10] Make FormatOptions.csv return CsvOptions, remove getters from ExternalDataConfiguration --- .../google/gcloud/bigquery/CsvOptions.java | 5 ++-- .../bigquery/ExternalDataConfiguration.java | 27 +++++++------------ .../google/gcloud/bigquery/FormatOptions.java | 4 +-- .../ExternalDataConfigurationTest.java | 7 ++--- 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java index 5c52cd78cc8f..40655e2c0c36 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java @@ -203,6 +203,7 @@ public Builder toBuilder() { @Override public String toString() { return MoreObjects.toStringHelper(this) + .add("type", type()) .add("allowJaggedRows", allowJaggedRows) .add("allowQuotedNewLines", allowQuotedNewLines) .add("encoding", encoding) @@ -214,8 +215,8 @@ public String toString() { @Override public int hashCode() { - return Objects.hash(allowJaggedRows, allowQuotedNewLines, encoding, fieldDelimiter, quote, - skipLeadingRows); + return Objects.hash(type(), allowJaggedRows, allowQuotedNewLines, encoding, fieldDelimiter, + quote, skipLeadingRows); } @Override diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java index 5d55080a88d5..9e049c4f13d6 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java @@ -179,8 +179,8 @@ public String compression() { * Returns whether BigQuery should allow extra values that are not represented in the table * schema. If true, the extra values are ignored. If false, records with extra columns are treated * as bad records, and if there are too many bad records, an invalid error is returned in the job - * result. The default value is false. The value of {@link #format()} determines what BigQuery - * treats as an extra value. + * result. The default value is false. The value of {@link #formatOptions()} determines what + * BigQuery treats as an extra value. * * @see * Ignore Unknown Values @@ -204,16 +204,6 @@ public Schema schema() { return schema; } - /** - * Returns the source format of the external data. - * - * - * Source Format - */ - public String format() { - return formatOptions.type(); - } - /** * Returns the fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can * contain one '*' wildcard character that must come after the bucket's name. Size limits @@ -227,11 +217,12 @@ public List sourceUris() { } /** - * Returns additional properties used to parse CSV data (used when {@link #format()} is set to - * CSV). Returns {@code null} if not set. + * Returns the source format, and possibly some parsing options, of the external data. Supported + * formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}. */ - public CsvOptions csvOptions() { - return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; + @SuppressWarnings("unchecked") + public F formatOptions() { + return (F) formatOptions; } /** @@ -292,8 +283,8 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() { if (sourceUris != null) { externalConfigurationPb.setSourceUris(sourceUris); } - if (csvOptions() != null) { - externalConfigurationPb.setCsvOptions(csvOptions().toPb()); + if (formatOptions instanceof CsvOptions) { + externalConfigurationPb.setCsvOptions(((CsvOptions) formatOptions).toPb()); } return externalConfigurationPb; } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java index 8a6884daf65d..e1f9d5aeb545 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/FormatOptions.java @@ -63,8 +63,8 @@ public boolean equals(Object obj) { /** * Default options for CSV format. */ - public static FormatOptions csv() { - return new FormatOptions(CSV); + public static CsvOptions csv() { + return CsvOptions.builder().build(); } /** diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java index cba03317ccb9..f9b7c31e1071 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExternalDataConfigurationTest.java @@ -43,7 +43,6 @@ public class ExternalDataConfigurationTest { .description("FieldDescription3") .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); - private static final String SOURCE_FORMAT = "CSV"; private static final Integer MAX_BAD_RECORDS = 42; private static final Boolean IGNORE_UNKNOWN_VALUES = true; private static final String COMPRESSION = "GZIP"; @@ -76,11 +75,10 @@ public void testToBuilderIncomplete() { @Test public void testBuilder() { assertEquals(COMPRESSION, CONFIGURATION.compression()); - assertEquals(CSV_OPTIONS, CONFIGURATION.csvOptions()); + assertEquals(CSV_OPTIONS, CONFIGURATION.formatOptions()); assertEquals(IGNORE_UNKNOWN_VALUES, CONFIGURATION.ignoreUnknownValues()); assertEquals(MAX_BAD_RECORDS, CONFIGURATION.maxBadRecords()); assertEquals(TABLE_SCHEMA, CONFIGURATION.schema()); - assertEquals(SOURCE_FORMAT, CONFIGURATION.format()); assertEquals(SOURCE_URIS, CONFIGURATION.sourceUris()); } @@ -96,11 +94,10 @@ private void compareConfiguration(ExternalDataConfiguration expected, ExternalDataConfiguration value) { assertEquals(expected, value); assertEquals(expected.compression(), value.compression()); - assertEquals(expected.csvOptions(), value.csvOptions()); + assertEquals(expected.formatOptions(), value.formatOptions()); assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues()); assertEquals(expected.maxBadRecords(), value.maxBadRecords()); assertEquals(expected.schema(), value.schema()); - assertEquals(expected.format(), value.format()); assertEquals(expected.sourceUris(), value.sourceUris()); } } From 66972df7c2eb649a3d6da8df5903607d4028a973 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 3 Dec 2015 22:38:12 +0100 Subject: [PATCH 07/10] Add TableType hierarcy to wrap different types of tables - Move schema from TableInfo to TableType - Add TableType.View class with attributes: view, userDefinedFunctions - Add TableType.ExternalTable class with attributes: ExternalDataConfiguration - Fix return null in TableInfo.description/friendlyName/expirationTime --- .../com/google/gcloud/bigquery/TableInfo.java | 284 ++++-------------- .../com/google/gcloud/bigquery/TableType.java | 260 ++++++++++++++++ .../gcloud/bigquery/SerializationTest.java | 18 +- .../google/gcloud/bigquery/TableInfoTest.java | 90 +++--- .../google/gcloud/bigquery/TableTypeTest.java | 126 ++++++++ 5 files changed, 494 insertions(+), 284 deletions(-) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java create mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java index 9ca74eb3c1f7..c899ac53da94 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java @@ -59,29 +59,6 @@ public Table apply(TableInfo tableInfo) { private static final long serialVersionUID = -7679032506430816205L; - /** - * The table type. - */ - public enum Type { - /** - * A normal BigQuery table. - */ - TABLE, - /** - * A virtual table defined by a SQL query. - * - * @see Views - */ - VIEW, - /** - * A BigQuery table backed by external data. - * - * @see Federated Data - * Sources - */ - EXTERNAL - } - /** * Google BigQuery Table's Streaming Buffer information. This class contains information on a * table's streaming buffer as the estimated size in number of rows/bytes. @@ -159,18 +136,14 @@ static StreamingBuffer fromPb(Streamingbuffer streamingBufferPb) { private final String id; private final String selfLink; private final TableId tableId; + private final TableType type; private final String friendlyName; private final String description; - private final Schema schema; private final Long numBytes; private final Long numRows; private final Long creationTime; private final Long expirationTime; private final Long lastModifiedTime; - private final String viewQuery; - private final List userDefinedFunctions; - private final Type type; - private final ExternalDataConfiguration externalConfiguration; private final String location; private final StreamingBuffer streamingBuffer; @@ -180,18 +153,14 @@ public static final class Builder { private String id; private String selfLink; private TableId tableId; + private TableType type; private String friendlyName; private String description; - private Schema schema; private Long numBytes; private Long numRows; private Long creationTime; private Long expirationTime; private Long lastModifiedTime; - private String viewQuery; - private List userDefinedFunctions; - private Type type; - private ExternalDataConfiguration externalConfiguration; private String location; private StreamingBuffer streamingBuffer; @@ -202,18 +171,14 @@ private Builder(TableInfo tableInfo) { this.id = tableInfo.id; this.selfLink = tableInfo.selfLink; this.tableId = tableInfo.tableId; + this.type = tableInfo.type; this.friendlyName = tableInfo.friendlyName; this.description = tableInfo.description; - this.schema = tableInfo.schema; this.numBytes = tableInfo.numBytes; this.numRows = tableInfo.numRows; this.creationTime = tableInfo.creationTime; this.expirationTime = tableInfo.expirationTime; this.lastModifiedTime = tableInfo.lastModifiedTime; - this.viewQuery = tableInfo.viewQuery; - this.userDefinedFunctions = tableInfo.userDefinedFunctions; - this.type = tableInfo.type; - this.externalConfiguration = tableInfo.externalConfiguration; this.location = tableInfo.location; this.streamingBuffer = tableInfo.streamingBuffer; } @@ -245,18 +210,6 @@ public Builder expirationTime(Long expirationTime) { return this; } - /** - * Sets the data format, location, and other properties of a table stored outside of BigQuery. - * This property is experimental and might be subject to change or removed. - * - * @see Federated Data - * Sources - */ - public Builder externalConfiguration(ExternalDataConfiguration externalConfiguration) { - this.externalConfiguration = externalConfiguration; - return this; - } - /** * Sets a user-friendly name for the table. */ @@ -290,15 +243,6 @@ Builder numRows(Long numRows) { return this; } - /** - * Sets the table's schema. Providing a schema is not necessary when {@link #viewQuery(String)} - * or {@link #externalConfiguration(ExternalDataConfiguration)} are provided. - */ - public Builder schema(Schema schema) { - this.schema = schema; - return this; - } - Builder selfLink(String selfLink) { this.selfLink = selfLink; return this; @@ -317,29 +261,11 @@ public Builder tableId(TableId tableId) { return this; } - Builder type(Type type) { - this.type = type; - return this; - } - /** - * Sets the query used to create a table of {@link Type#VIEW} type. + * Sets the table type. */ - public Builder viewQuery(String viewQuery) { - this.viewQuery = viewQuery; - return this; - } - - /** - * Sets user-defined functions to be used by the view's query. User defined functions are only - * used when {@link #type(TableInfo.Type)} is set to {@link Type#VIEW} and a query is provided via - * {@link #viewQuery(String)}. - * - * @see User-Defined - * Functions - */ - public Builder userDefinedFunctions(List userDefinedFunctions) { - this.userDefinedFunctions = userDefinedFunctions; + public Builder type(TableType type) { + this.type = type; return this; } @@ -358,16 +284,12 @@ private TableInfo(Builder builder) { this.selfLink = builder.selfLink; this.friendlyName = builder.friendlyName; this.description = builder.description; - this.schema = builder.schema; + this.type = builder.type; this.numBytes = builder.numBytes; this.numRows = builder.numRows; this.creationTime = builder.creationTime; this.expirationTime = builder.expirationTime; this.lastModifiedTime = builder.lastModifiedTime; - this.viewQuery = builder.viewQuery; - this.userDefinedFunctions = builder.userDefinedFunctions; - this.type = builder.type; - this.externalConfiguration = builder.externalConfiguration; this.location = builder.location; this.streamingBuffer = builder.streamingBuffer; } @@ -405,21 +327,14 @@ public TableId tableId() { * Returns a user-friendly name for the table. */ public String friendlyName() { - return friendlyName; + return Data.isNull(friendlyName) ? null : friendlyName; } /** * Returns a user-friendly description for the table. */ public String description() { - return description; - } - - /** - * Returns the table's schema. - */ - public Schema schema() { - return schema; + return Data.isNull(description) ? null : description; } /** @@ -448,7 +363,7 @@ public Long creationTime() { * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. */ public Long expirationTime() { - return expirationTime; + return Data.isNull(expirationTime) ? null : expirationTime; } /** @@ -458,43 +373,22 @@ public Long lastModifiedTime() { return lastModifiedTime; } - /** - * If this table is a view ({@link #type()} returns {@link Type#VIEW}) this method returns the - * query used to create the view. Returns {@code null} otherwise. - */ - public String viewQuery() { - return viewQuery; - } - - /** - * If this table is a view ({@link #type()} returns {@link Type#VIEW} this method returns user - * defined functions that can be used by {@link #viewQuery}. - * - * @see User-Defined Functions - * - */ - public List userDefinedFunctions() { - return userDefinedFunctions; - } - /** * Returns the type of the table. */ - public Type type() { - return type; + public Schema schema() { + return type.schema(); } /** - * If this table is external ({@link #type()} returns {@link Type#EXTERNAL}) this method returns - * the data format, location, and other properties of a table stored outside of BigQuery. - * If the table is not {@link Type#EXTERNAL} this method returns {@code null}. This property - * is experimental and might be subject to change or removed. - * - * @see Federated Data Sources - * + * Returns the table's type. If this table is simple table the method returns an instance of + * {@link TableType}. If this table is an external table this method returns an instance of + * {@link com.google.gcloud.bigquery.TableType.ExternalTable}. If this table is a view table this + * method returns an instance of {@link com.google.gcloud.bigquery.TableType.View}. */ - public ExternalDataConfiguration externalConfiguration() { - return externalConfiguration; + @SuppressWarnings("unchecked") + public T type() { + return (T) type; } /** @@ -527,10 +421,6 @@ public Builder toBuilder() { public String toString() { return MoreObjects.toStringHelper(this) .add("tableId", tableId) - .add("schema", schema) - .add("externalDataConfiguration", externalConfiguration) - .add("query", viewQuery) - .add("userDefinedFunctions", userDefinedFunctions) .add("type", type) .add("location", location) .add("streamingBuffer", streamingBuffer) @@ -560,31 +450,30 @@ public boolean equals(Object obj) { Table toPb() { Table tablePb = new Table(); tablePb.setTableReference(tableId.toPb()); - if (externalConfiguration != null) { - tablePb.setExternalDataConfiguration(externalConfiguration.toPb()); - } if (lastModifiedTime != null) { tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); } if (numRows != null) { tablePb.setNumRows(BigInteger.valueOf(numRows)); } - if (schema != null) { - tablePb.setSchema(schema.toPb()); + if (schema() != null) { + tablePb.setSchema(schema().toPb()); } - if (streamingBuffer != null) { - tablePb.setStreamingBuffer(streamingBuffer.toPb()); - } - if (viewQuery != null) { - ViewDefinition viewDefinition = new ViewDefinition().setQuery(viewQuery); - if (userDefinedFunctions != null) { + if (type instanceof TableType.View) { + TableType.View viewType = (TableType.View) type; + ViewDefinition viewDefinition = new ViewDefinition().setQuery(viewType.query()); + if (viewType.userDefinedFunctions() != null) { viewDefinition.setUserDefinedFunctionResources( - Lists.transform(userDefinedFunctions, UserDefinedFunction.TO_PB_FUNCTION)); + Lists.transform(viewType.userDefinedFunctions(), UserDefinedFunction.TO_PB_FUNCTION)); } - tablePb.setView(new ViewDefinition().setQuery(viewQuery)); + tablePb.setView(viewDefinition); } - if (type != null) { - tablePb.setType(type.name()); + if (type instanceof TableType.ExternalTable) { + tablePb.setExternalDataConfiguration( + ((TableType.ExternalTable) type).configuration().toPb()); + } + if (streamingBuffer != null) { + tablePb.setStreamingBuffer(streamingBuffer.toPb()); } tablePb.setCreationTime(creationTime); tablePb.setDescription(description); @@ -599,99 +488,38 @@ Table toPb() { } /** - * Returns a builder for a BigQuery table backed by external data. External tables are - * experimental and might be subject to change or removed. - * - * @param tableId table id - * @param configuration configuration for external data source - */ - public static Builder builder(TableId tableId, ExternalDataConfiguration configuration) { - return new Builder().tableId(tableId).externalConfiguration(configuration); - } - - /** - * Returns a builder for a BigQuery view. - * - * @param tableId table id - * @param query query used to create the view - */ - public static Builder builder(TableId tableId, String query) { - return new Builder().tableId(tableId).viewQuery(query); - } - - /** - * Returns a builder for a BigQuery view. + * Returns a builder for a BigQuery table given its type. * * @param tableId table id - * @param query query used to create the view + * @param type the type of the table */ - public static Builder builder(TableId tableId, String query, List functions) { - return new Builder().tableId(tableId).viewQuery(query).userDefinedFunctions(functions); + public static Builder builder(TableId tableId, TableType type) { + return new Builder().tableId(tableId).type(type); } /** - * Returns a builder for a BigQuery table. + * Creates BigQuery table given its type * * @param tableId table id - * @param schema table's schema definition + * @param type the type of the table */ - public static Builder builder(TableId tableId, Schema schema) { - return new Builder().tableId(tableId).schema(schema); - } - - /** - * Creates BigQuery table backed by external data. - * - * @param tableId table id - * @param configuration configuration for external data source - */ - public static TableInfo of(TableId tableId, ExternalDataConfiguration configuration) { - return builder(tableId, configuration).build(); - } - - /** - * Creates a BigQuery view. - * - * @param tableId table id - * @param query query used to create the view - */ - public static TableInfo of(TableId tableId, String query) { - return builder(tableId, query).build(); - } - - /** - * Creates a BigQuery view. - * - * @param tableId table id - * @param query query used to create the view - * @param functions user defined funtions that can be used in {@code query} - */ - public static TableInfo of(TableId tableId, String query, List functions) { - return builder(tableId, query, functions).build(); - } - - /** - * Creates a BigQuery table. - * - * @param tableId table id - * @param schema table's schema definition - */ - public static TableInfo of(TableId tableId, Schema schema) { - return builder(tableId, schema).build(); + public static TableInfo of(TableId tableId, TableType type) { + return builder(tableId, type).build(); } static TableInfo fromPb(Table tablePb) { Builder builder = new Builder().tableId(TableId.fromPb(tablePb.getTableReference())); + Schema schema = null; + if (tablePb.getSchema() != null) { + schema = Schema.fromPb(tablePb.getSchema()); + builder.type(TableType.table(schema)); + } if (tablePb.getDescription() != null) { builder.description(tablePb.getDescription()); } if (tablePb.getExpirationTime() != null) { builder.expirationTime(tablePb.getExpirationTime()); } - if (tablePb.getExternalDataConfiguration() != null) { - builder.externalConfiguration( - ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration())); - } if (tablePb.getFriendlyName() != null) { builder.friendlyName(tablePb.getFriendlyName()); } @@ -704,22 +532,22 @@ static TableInfo fromPb(Table tablePb) { if (tablePb.getNumRows() != null) { builder.numRows(tablePb.getNumRows().longValue()); } - if (tablePb.getSchema() != null) { - builder.schema(Schema.fromPb(tablePb.getSchema())); - } if (tablePb.getStreamingBuffer() != null) { builder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); } - if (tablePb.getType() != null) { - builder.type(Type.valueOf(tablePb.getType())); - } if (tablePb.getView() != null) { - builder.viewQuery(tablePb.getView().getQuery()); + String query = tablePb.getView().getQuery(); + List functions = null; if (tablePb.getView().getUserDefinedFunctionResources() != null) { - builder.userDefinedFunctions( - Lists.transform(tablePb.getView().getUserDefinedFunctionResources(), - UserDefinedFunction.FROM_PB_FUNCTION)); + functions = Lists.transform(tablePb.getView().getUserDefinedFunctionResources(), + UserDefinedFunction.FROM_PB_FUNCTION); } + builder.type(new TableType.View(schema, query, functions)); + } + if (tablePb.getExternalDataConfiguration() != null) { + ExternalDataConfiguration configuration = + ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration()); + builder.type(new TableType.ExternalTable(schema, configuration)); } if (tablePb.getCreationTime() != null) { builder.creationTime(tablePb.getCreationTime()); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java new file mode 100644 index 000000000000..9a74d55e8bbb --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java @@ -0,0 +1,260 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery Table type. A BigQuery table can be of three types. {@link Type#TABLE} is a + * normal BigQuery table. {@link Type#VIEW} is a view table, created from a query. + * {@link Type#EXTERNAL} is an external table, backed by Google Cloud Storage data. + */ +public class TableType implements Serializable { + + /** + * The table type. + */ + public enum Type { + /** + * A normal BigQuery table. + */ + TABLE, + /** + * A virtual table defined by a SQL query. + * + * @see Views + */ + VIEW, + /** + * A BigQuery table backed by external data. + * + * @see Federated Data + * Sources + */ + EXTERNAL + } + + private final Type type; + private final Schema schema; + + TableType(Type type, Schema schema) { + this.type = type; + this.schema = schema; + } + + /** + * Returns the type of the table. + */ + public Type type() { + return type; + } + + /** + * Returns the table's schema. + */ + public Schema schema() { + return schema; + } + + /** + * Google BigQuery View type. + */ + public static class View extends TableType { + + private static final long serialVersionUID = 8275064752050652381L; + + private final String query; + private final List userDefinedFunctions; + + View(String query) { + this(null, query, null); + } + + View(String query, List userDefinedFunctions) { + this(null, query, userDefinedFunctions); + } + + View(Schema schema, String query, List userDefinedFunctions) { + super(Type.VIEW, schema); + this.query = query; + this.userDefinedFunctions = userDefinedFunctions; + } + + /** + * Returns the query used to create the view. + */ + public String query() { + return query; + }; + + /** + * Returns user defined functions that can be used by {@link #query()}. Returns {@code null} if + * not set. + * + * @see User-Defined Functions + * + */ + public List userDefinedFunctions() { + return userDefinedFunctions; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("type", type()) + .add("query", query) + .add("userDefinedFunctions", userDefinedFunctions) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(schema(), query, userDefinedFunctions); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof View)) { + return false; + } + View other = (View) obj; + return Objects.equals(type(), other.type()) + && Objects.equals(query, other.query) + && Objects.equals(userDefinedFunctions, other.userDefinedFunctions); + } + } + + /** + * Google BigQuery External Table type. + */ + public static class ExternalTable extends TableType { + + private static final long serialVersionUID = -6912214693318638552L; + + private final ExternalDataConfiguration configuration; + + ExternalTable(ExternalDataConfiguration configuration) { + this(null, configuration); + } + + ExternalTable(Schema schema, ExternalDataConfiguration configuration) { + super(Type.EXTERNAL, schema); + this.configuration = configuration; + } + + /** + * Returns the data format, location and other properties of a table stored outside of BigQuery. + * This property is experimental and might be subject to change or removed. + * + * @see Federated Data + * Sources + */ + public ExternalDataConfiguration configuration() { + return configuration; + }; + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("type", type()) + .add("configuration", configuration) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(schema(), configuration); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExternalTable)) { + return false; + } + ExternalTable other = (ExternalTable) obj; + return Objects.equals(type(), other.type()) + && Objects.equals(configuration, other.configuration); + } + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("type", type) + .add("schema", schema) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(type, schema); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof TableType)) { + return false; + } + TableType other = (TableType) obj; + return Objects.equals(type, other.type) + && Objects.equals(schema, other.schema); + } + + /** + * Returns a simple BigQuery Table type. + */ + public static TableType table(Schema schema) { + return new TableType(Type.TABLE, checkNotNull(schema)); + } + + /** + * Returns a BigQuery View type given the query needed to create the view. + */ + public static View view(String query) { + return new View(checkNotNull(query)); + } + + /** + * Returns a BigQuery View type given the query needed to create the view and a list of user + * defined functions that can be used by the query. + */ + public static View view(String query, List userDefinedFunctions) { + return new View(checkNotNull(query), checkNotNull(userDefinedFunctions)); + } + + /** + * Returns a BigQuery View type given the query needed to create the view and a list of user + * defined functions that can be used by the query. + */ + public static View view(String query, UserDefinedFunction... userDefinedFunctions) { + return new View(checkNotNull(query), ImmutableList.copyOf(userDefinedFunctions)); + } + + /** + * Returns a BigQuery External Table type given the external data configuration. + */ + public static ExternalTable externalTable(ExternalDataConfiguration configuration) { + return new ExternalTable(checkNotNull(configuration)); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index b3bdbd3eda57..86e18ac74c97 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -104,15 +104,15 @@ public class SerializationTest { new UserDefinedFunction.InlineFunction("inline"); private static final UserDefinedFunction URI_FUNCTION = new UserDefinedFunction.UriFunction("URI"); - private static final TableInfo TABLE_INFO = TableInfo.builder(TABLE_ID, TABLE_SCHEMA) - .creationTime(CREATION_TIME) - .description(DESCRIPTION) - .etag(ETAG) - .id(ID) - .location(LOCATION) - .type(TableInfo.Type.TABLE) - .streamingBuffer(STREAMING_BUFFER) - .build(); + private static final TableInfo TABLE_INFO = + TableInfo.builder(TABLE_ID, TableType.table(TABLE_SCHEMA)) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .id(ID) + .location(LOCATION) + .streamingBuffer(STREAMING_BUFFER) + .build(); @Test public void testServiceOptions() throws Exception { diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java index 8fa84f766d17..ecde5f2c907c 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java @@ -17,6 +17,7 @@ package com.google.gcloud.bigquery; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.google.common.collect.ImmutableList; import com.google.gcloud.bigquery.TableInfo.StreamingBuffer; @@ -68,40 +69,40 @@ public class TableInfoTest { private static final Long LAST_MODIFIED_TIME = 20L; private static final String LOCATION = "US"; private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L); - private static final TableInfo TABLE_INFO = TableInfo.builder(TABLE_ID, TABLE_SCHEMA) - .creationTime(CREATION_TIME) - .description(DESCRIPTION) - .etag(ETAG) - .expirationTime(EXPIRATION_TIME) - .friendlyName(FRIENDLY_NAME) - .id(ID) - .lastModifiedTime(LAST_MODIFIED_TIME) - .location(LOCATION) - .numBytes(NUM_BYTES) - .numRows(NUM_ROWS) - .selfLink(SELF_LINK) - .streamingBuffer(STREAMING_BUFFER) - .type(TableInfo.Type.TABLE) - .build(); - private static final TableInfo EXTERNAL_TABLE_INFO = TableInfo.builder(TABLE_ID, CONFIGURATION) - .creationTime(CREATION_TIME) - .description(DESCRIPTION) - .etag(ETAG) - .expirationTime(EXPIRATION_TIME) - .friendlyName(FRIENDLY_NAME) - .id(ID) - .lastModifiedTime(LAST_MODIFIED_TIME) - .location(LOCATION) - .numBytes(NUM_BYTES) - .numRows(NUM_ROWS) - .selfLink(SELF_LINK) - .streamingBuffer(STREAMING_BUFFER) - .type(TableInfo.Type.TABLE) - .build(); + private static final TableInfo TABLE_INFO = + TableInfo.builder(TABLE_ID, TableType.table(TABLE_SCHEMA)) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .expirationTime(EXPIRATION_TIME) + .friendlyName(FRIENDLY_NAME) + .id(ID) + .lastModifiedTime(LAST_MODIFIED_TIME) + .location(LOCATION) + .numBytes(NUM_BYTES) + .numRows(NUM_ROWS) + .selfLink(SELF_LINK) + .streamingBuffer(STREAMING_BUFFER) + .build(); + private static final TableInfo EXTERNAL_TABLE_INFO = + TableInfo.builder(TABLE_ID, TableType.externalTable(CONFIGURATION)) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .expirationTime(EXPIRATION_TIME) + .friendlyName(FRIENDLY_NAME) + .id(ID) + .lastModifiedTime(LAST_MODIFIED_TIME) + .location(LOCATION) + .numBytes(NUM_BYTES) + .numRows(NUM_ROWS) + .selfLink(SELF_LINK) + .streamingBuffer(STREAMING_BUFFER) + .build(); private static List USER_DEFINED_FUNCTIONS = ImmutableList.of( UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); private static final TableInfo VIEW_INFO = - TableInfo.builder(TABLE_ID, VIEW_QUERY, USER_DEFINED_FUNCTIONS) + TableInfo.builder(TABLE_ID, TableType.view(VIEW_QUERY, USER_DEFINED_FUNCTIONS)) .creationTime(CREATION_TIME) .description(DESCRIPTION) .etag(ETAG) @@ -114,7 +115,6 @@ public class TableInfoTest { .numRows(NUM_ROWS) .selfLink(SELF_LINK) .streamingBuffer(STREAMING_BUFFER) - .type(TableInfo.Type.VIEW) .build(); @Test @@ -123,13 +123,10 @@ public void testToBuilder() { compareTableInfo(VIEW_INFO, VIEW_INFO.toBuilder().build()); compareTableInfo(EXTERNAL_TABLE_INFO, EXTERNAL_TABLE_INFO.toBuilder().build()); TableInfo tableInfo = TABLE_INFO.toBuilder() - .type(TableInfo.Type.VIEW) .description("newDescription") .build(); - assertEquals(TableInfo.Type.VIEW, tableInfo.type()); assertEquals("newDescription", tableInfo.description()); tableInfo = tableInfo.toBuilder() - .type(TableInfo.Type.TABLE) .description("description") .build(); compareTableInfo(TABLE_INFO, tableInfo); @@ -137,15 +134,14 @@ public void testToBuilder() { @Test public void testToBuilderIncomplete() { - TableInfo tableInfo = TableInfo.of(TABLE_ID, VIEW_QUERY); + TableInfo tableInfo = TableInfo.of(TABLE_ID, TableType.view(VIEW_QUERY)); assertEquals(tableInfo, tableInfo.toBuilder().build()); } @Test public void testBuilder() { assertEquals(TABLE_ID, TABLE_INFO.tableId()); + assertTrue(VIEW_INFO.type() instanceof TableType); assertEquals(TABLE_SCHEMA, TABLE_INFO.schema()); - assertEquals(null, TABLE_INFO.viewQuery()); - assertEquals(null, TABLE_INFO.externalConfiguration()); assertEquals(CREATION_TIME, TABLE_INFO.creationTime()); assertEquals(DESCRIPTION, TABLE_INFO.description()); assertEquals(ETAG, TABLE_INFO.etag()); @@ -158,11 +154,11 @@ public void testBuilder() { assertEquals(NUM_ROWS, TABLE_INFO.numRows()); assertEquals(SELF_LINK, TABLE_INFO.selfLink()); assertEquals(STREAMING_BUFFER, TABLE_INFO.streamingBuffer()); - assertEquals(TableInfo.Type.TABLE, TABLE_INFO.type()); + assertEquals(TableType.Type.TABLE, TABLE_INFO.type().type()); assertEquals(TABLE_ID, VIEW_INFO.tableId()); assertEquals(null, VIEW_INFO.schema()); - assertEquals(VIEW_QUERY, VIEW_INFO.viewQuery()); - assertEquals(null, VIEW_INFO.externalConfiguration()); + assertTrue(VIEW_INFO.type() instanceof TableType.View); + assertEquals(VIEW_QUERY, ((TableType.View) VIEW_INFO.type()).query()); assertEquals(CREATION_TIME, VIEW_INFO.creationTime()); assertEquals(DESCRIPTION, VIEW_INFO.description()); assertEquals(ETAG, VIEW_INFO.etag()); @@ -175,11 +171,12 @@ public void testBuilder() { assertEquals(NUM_ROWS, VIEW_INFO.numRows()); assertEquals(SELF_LINK, VIEW_INFO.selfLink()); assertEquals(STREAMING_BUFFER, VIEW_INFO.streamingBuffer()); - assertEquals(TableInfo.Type.VIEW, VIEW_INFO.type()); + assertEquals(TableType.Type.VIEW, VIEW_INFO.type().type()); assertEquals(TABLE_ID, EXTERNAL_TABLE_INFO.tableId()); assertEquals(null, EXTERNAL_TABLE_INFO.schema()); - assertEquals(null, EXTERNAL_TABLE_INFO.viewQuery()); - assertEquals(CONFIGURATION, EXTERNAL_TABLE_INFO.externalConfiguration()); + assertTrue(EXTERNAL_TABLE_INFO.type() instanceof TableType.ExternalTable); + assertEquals(CONFIGURATION, + ((TableType.ExternalTable) EXTERNAL_TABLE_INFO.type()).configuration()); assertEquals(CREATION_TIME, EXTERNAL_TABLE_INFO.creationTime()); assertEquals(DESCRIPTION, EXTERNAL_TABLE_INFO.description()); assertEquals(ETAG, EXTERNAL_TABLE_INFO.etag()); @@ -192,7 +189,7 @@ public void testBuilder() { assertEquals(NUM_ROWS, EXTERNAL_TABLE_INFO.numRows()); assertEquals(SELF_LINK, EXTERNAL_TABLE_INFO.selfLink()); assertEquals(STREAMING_BUFFER, EXTERNAL_TABLE_INFO.streamingBuffer()); - assertEquals(TableInfo.Type.TABLE, EXTERNAL_TABLE_INFO.type()); + assertEquals(TableType.Type.EXTERNAL, EXTERNAL_TABLE_INFO.type().type()); } @Test @@ -206,8 +203,7 @@ private void compareTableInfo(TableInfo expected, TableInfo value) { assertEquals(expected, value); assertEquals(expected.tableId(), value.tableId()); assertEquals(expected.schema(), value.schema()); - assertEquals(expected.viewQuery(), value.viewQuery()); - assertEquals(expected.externalConfiguration(), value.externalConfiguration()); + assertEquals(expected.type(), value.type()); assertEquals(expected.creationTime(), value.creationTime()); assertEquals(expected.description(), value.description()); assertEquals(expected.etag(), value.etag()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java new file mode 100644 index 000000000000..6b67e8968e03 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TableTypeTest { + + private static final Field FIELD_SCHEMA1 = + Field.builder("StringField", Field.Type.string()) + .mode(Field.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final Field FIELD_SCHEMA2 = + Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final Field FIELD_SCHEMA3 = + Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(Field.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); + private static final String VIEW_QUERY = "VIEW QUERY"; + private static final ExternalDataConfiguration CONFIGURATION = ExternalDataConfiguration + .builder(ImmutableList.of("URI"), TABLE_SCHEMA, FormatOptions.datastoreBackup()) + .compression("ZIP") + .ignoreUnknownValues(true) + .maxBadRecords(42) + .build(); + private static List FUNCTIONS = ImmutableList.of( + UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); + private static final TableType TABLE = TableType.table(TABLE_SCHEMA); + private static final TableType.View VIEW = TableType.view(VIEW_QUERY); + private static final TableType.View VIEW_WITH_FUNCTIONS = TableType.view(VIEW_QUERY, FUNCTIONS); + private static final TableType.ExternalTable EXTERNAL = TableType.externalTable(CONFIGURATION); + + @Test + public void testConstructor() { + TableType table = new TableType(TableType.Type.TABLE, TABLE_SCHEMA); + assertEquals(TableType.Type.TABLE, table.type()); + assertEquals(TABLE_SCHEMA, table.schema()); + TableType.View view = new TableType.View(VIEW_QUERY); + assertEquals(TableType.Type.VIEW, view.type()); + assertEquals(null, view.schema()); + assertEquals(VIEW_QUERY, view.query()); + assertEquals(null, view.userDefinedFunctions()); + view = new TableType.View(VIEW_QUERY, FUNCTIONS); + assertEquals(TableType.Type.VIEW, view.type()); + assertEquals(null, view.schema()); + assertEquals(VIEW_QUERY, view.query()); + assertEquals(FUNCTIONS, view.userDefinedFunctions()); + view = new TableType.View(TABLE_SCHEMA, VIEW_QUERY, FUNCTIONS); + assertEquals(TableType.Type.VIEW, view.type()); + assertEquals(TABLE_SCHEMA, view.schema()); + assertEquals(VIEW_QUERY, view.query()); + assertEquals(FUNCTIONS, view.userDefinedFunctions()); + TableType.ExternalTable extern = new TableType.ExternalTable(CONFIGURATION); + assertEquals(TableType.Type.EXTERNAL, extern.type()); + assertEquals(null, extern.schema()); + assertEquals(CONFIGURATION, extern.configuration()); + extern = new TableType.ExternalTable(TABLE_SCHEMA, CONFIGURATION); + assertEquals(TableType.Type.EXTERNAL, extern.type()); + assertEquals(TABLE_SCHEMA, extern.schema()); + assertEquals(CONFIGURATION, extern.configuration()); + } + + @Test + public void testFactoryMethods() { + TableType table = TableType.table(TABLE_SCHEMA); + assertEquals(TableType.Type.TABLE, table.type()); + assertEquals(TABLE_SCHEMA, table.schema()); + TableType.View view = TableType.view(VIEW_QUERY); + assertEquals(TableType.Type.VIEW, view.type()); + assertEquals(null, view.schema()); + assertEquals(VIEW_QUERY, view.query()); + assertEquals(null, view.userDefinedFunctions()); + view = TableType.view(VIEW_QUERY, FUNCTIONS); + assertEquals(TableType.Type.VIEW, view.type()); + assertEquals(null, view.schema()); + assertEquals(VIEW_QUERY, view.query()); + assertEquals(FUNCTIONS, view.userDefinedFunctions()); + TableType.ExternalTable extern = TableType.externalTable(CONFIGURATION); + assertEquals(TableType.Type.EXTERNAL, extern.type()); + assertEquals(null, extern.schema()); + assertEquals(CONFIGURATION, extern.configuration()); + } + + @Test + public void testEquals() { + assertEquals(new TableType(TableType.Type.TABLE, TABLE_SCHEMA), TableType.table(TABLE_SCHEMA)); + assertEquals(new TableType.View(VIEW_QUERY), TableType.view(VIEW_QUERY)); + assertEquals(new TableType.View(VIEW_QUERY, FUNCTIONS), TableType.view(VIEW_QUERY, FUNCTIONS)); + assertEquals(new TableType.ExternalTable(CONFIGURATION), + TableType.externalTable(CONFIGURATION)); + assertEquals(new TableType(TableType.Type.TABLE, TABLE_SCHEMA).hashCode(), + TableType.table(TABLE_SCHEMA).hashCode()); + assertEquals(new TableType.View(VIEW_QUERY).hashCode(), + TableType.view(VIEW_QUERY).hashCode()); + assertEquals(new TableType.View(VIEW_QUERY, FUNCTIONS).hashCode(), + TableType.view(VIEW_QUERY, FUNCTIONS).hashCode()); + assertEquals(new TableType.ExternalTable(CONFIGURATION).hashCode(), + TableType.externalTable(CONFIGURATION).hashCode()); + } +} From ae2a484006337782084ee62096a6b7114051d836 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 4 Dec 2015 14:54:22 +0100 Subject: [PATCH 08/10] Remove TableType, add TableInfo, ViewInfo ExternalTableInfo hierarcy --- .../google/gcloud/bigquery/BaseTableInfo.java | 541 ++++++++++++++++++ .../gcloud/bigquery/ExternalTableInfo.java | 159 +++++ .../com/google/gcloud/bigquery/TableInfo.java | 500 ++-------------- .../com/google/gcloud/bigquery/TableType.java | 260 --------- .../com/google/gcloud/bigquery/ViewInfo.java | 229 ++++++++ .../gcloud/bigquery/SerializationTest.java | 25 +- .../google/gcloud/bigquery/TableInfoTest.java | 83 +-- .../google/gcloud/bigquery/TableTypeTest.java | 126 ---- 8 files changed, 1041 insertions(+), 882 deletions(-) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java delete mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java delete mode 100644 gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java new file mode 100644 index 000000000000..d409e440aabd --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java @@ -0,0 +1,541 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.client.util.Data; +import com.google.api.services.bigquery.model.Streamingbuffer; +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.ViewDefinition; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.Lists; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Objects; + +/** + * Base class for Google BigQuery table information. Use {@link TableInfo} for a simple BigQuery + * Table. Use {@link ViewInfo} for a BigQuery View Table. Use {@link ExternalTableInfo} for a + * BigQuery Table backed by external data. + * + * @see Managing Tables + */ +public class BaseTableInfo implements Serializable { + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public BaseTableInfo apply(Table pb) { + return BaseTableInfo.fromPb(pb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public Table apply(BaseTableInfo tableInfo) { + return tableInfo.toPb(); + } + }; + + private static final long serialVersionUID = -7679032506430816205L; + + /** + * The table type. + */ + public enum Type { + /** + * A normal BigQuery table. + */ + TABLE, + /** + * A virtual table defined by a SQL query. + * + * @see Views + */ + VIEW, + /** + * A BigQuery table backed by external data. + * + * @see Federated Data + * Sources + */ + EXTERNAL + } + + /** + * Google BigQuery Table's Streaming Buffer information. This class contains information on a + * table's streaming buffer as the estimated size in number of rows/bytes. + */ + public static class StreamingBuffer implements Serializable { + + private static final long serialVersionUID = -6713971364725267597L; + private final long estimatedRows; + private final long estimatedBytes; + private final long oldestEntryTime; + + StreamingBuffer(long estimatedRows, long estimatedBytes, long oldestEntryTime) { + this.estimatedRows = estimatedRows; + this.estimatedBytes = estimatedBytes; + this.oldestEntryTime = oldestEntryTime; + } + + /** + * Returns a lower-bound estimate of the number of rows currently in the streaming buffer. + */ + public long estimatedRows() { + return estimatedRows; + } + + /** + * Returns a lower-bound estimate of the number of bytes currently in the streaming buffer. + */ + public long estimatedBytes() { + return estimatedBytes; + } + + /** + * Returns the timestamp of the oldest entry in the streaming buffer, in milliseconds since + * epoch. + */ + public long oldestEntryTime() { + return oldestEntryTime; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("estimatedRows", estimatedRows) + .add("estimatedBytes", estimatedBytes) + .add("oldestEntryTime", oldestEntryTime) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hash(estimatedRows, estimatedBytes, oldestEntryTime); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof StreamingBuffer + && Objects.equals(toPb(), ((StreamingBuffer) obj).toPb()); + } + + public Streamingbuffer toPb() { + return new Streamingbuffer() + .setEstimatedBytes(BigInteger.valueOf(estimatedBytes)) + .setEstimatedRows(BigInteger.valueOf(estimatedRows)) + .setOldestEntryTime(BigInteger.valueOf(oldestEntryTime)); + } + + static StreamingBuffer fromPb(Streamingbuffer streamingBufferPb) { + return new StreamingBuffer(streamingBufferPb.getEstimatedRows().longValue(), + streamingBufferPb.getEstimatedBytes().longValue(), + streamingBufferPb.getOldestEntryTime().longValue()); + } + } + + private final String etag; + private final String id; + private final String selfLink; + private final TableId tableId; + private final Type type; + private final Schema schema; + private final String friendlyName; + private final String description; + private final Long numBytes; + private final Long numRows; + private final Long creationTime; + private final Long expirationTime; + private final Long lastModifiedTime; + + public static class Builder> { + + private String etag; + private String id; + private String selfLink; + private TableId tableId; + private Type type; + private Schema schema; + private String friendlyName; + private String description; + private Long numBytes; + private Long numRows; + private Long creationTime; + private Long expirationTime; + private Long lastModifiedTime; + + protected Builder() {} + + protected Builder(BaseTableInfo tableInfo) { + this.etag = tableInfo.etag; + this.id = tableInfo.id; + this.selfLink = tableInfo.selfLink; + this.tableId = tableInfo.tableId; + this.type = tableInfo.type; + this.schema = tableInfo.schema; + this.friendlyName = tableInfo.friendlyName; + this.description = tableInfo.description; + this.numBytes = tableInfo.numBytes; + this.numRows = tableInfo.numRows; + this.creationTime = tableInfo.creationTime; + this.expirationTime = tableInfo.expirationTime; + this.lastModifiedTime = tableInfo.lastModifiedTime; + } + + @SuppressWarnings("unchecked") + protected B self() { + return (B) this; + } + + B creationTime(Long creationTime) { + this.creationTime = creationTime; + return self(); + } + + /** + * Sets a user-friendly description for the table. + */ + public B description(String description) { + this.description = firstNonNull(description, Data.nullOf(String.class)); + return self(); + } + + B etag(String etag) { + this.etag = etag; + return self(); + } + + /** + * Sets the time when this table expires, in milliseconds since the epoch. If not present, the + * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. + */ + public B expirationTime(Long expirationTime) { + this.expirationTime = firstNonNull(expirationTime, Data.nullOf(Long.class)); + return self(); + } + + /** + * Sets a user-friendly name for the table. + */ + public B friendlyName(String friendlyName) { + this.friendlyName = firstNonNull(friendlyName, Data.nullOf(String.class)); + return self(); + } + + B id(String id) { + this.id = id; + return self(); + } + + B lastModifiedTime(Long lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + return self(); + } + + B numBytes(Long numBytes) { + this.numBytes = numBytes; + return self(); + } + + B numRows(Long numRows) { + this.numRows = numRows; + return self(); + } + + B selfLink(String selfLink) { + this.selfLink = selfLink; + return self(); + } + + /** + * Sets the table identity. + */ + public B tableId(TableId tableId) { + this.tableId = checkNotNull(tableId); + return self(); + } + + B type(Type type) { + this.type = type; + return self(); + } + + /** + * Sets the table schema. + */ + public B schema(Schema schema) { + this.schema = checkNotNull(schema); + return self(); + } + + /** + * Creates a {@code BaseTableInfo} object. + */ + @SuppressWarnings("unchecked") + public T build() { + return (T) new BaseTableInfo(this); + } + } + + protected BaseTableInfo(Builder builder) { + this.tableId = checkNotNull(builder.tableId); + this.etag = builder.etag; + this.id = builder.id; + this.selfLink = builder.selfLink; + this.friendlyName = builder.friendlyName; + this.description = builder.description; + this.type = builder.type; + this.schema = builder.schema; + this.numBytes = builder.numBytes; + this.numRows = builder.numRows; + this.creationTime = builder.creationTime; + this.expirationTime = builder.expirationTime; + this.lastModifiedTime = builder.lastModifiedTime; + } + + /** + * Returns the hash of the table resource. + */ + public String etag() { + return etag; + } + + /** + * Returns an opaque id for the table. + */ + public String id() { + return id; + } + + /** + * Returns the table's type. If this table is simple table the method returns {@link Type#TABLE}. + * If this table is an external table this method returns {@link Type#EXTERNAL}. If this table is + * a view table this method returns {@link Type#VIEW}. + */ + public Type type() { + return type; + } + + /** + * Returns the table's schema. + */ + public Schema schema() { + return schema; + } + + /** + * Returns an URL that can be used to access the resource again. The returned URL can be used for + * get or update requests. + */ + public String selfLink() { + return selfLink; + } + + /** + * Returns the table identity. + */ + public TableId tableId() { + return tableId; + } + + /** + * Returns a user-friendly name for the table. + */ + public String friendlyName() { + return Data.isNull(friendlyName) ? null : friendlyName; + } + + /** + * Returns a user-friendly description for the table. + */ + public String description() { + return Data.isNull(description) ? null : description; + } + + /** + * Returns the size of this table in bytes, excluding any data in the streaming buffer. + */ + public Long numBytes() { + return numBytes; + } + + /** + * Returns the number of rows in this table, excluding any data in the streaming buffer. + */ + public Long numRows() { + return numRows; + } + + /** + * Returns the time when this table was created, in milliseconds since the epoch. + */ + public Long creationTime() { + return creationTime; + } + + /** + * Returns the time when this table expires, in milliseconds since the epoch. If not present, the + * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. + */ + public Long expirationTime() { + return Data.isNull(expirationTime) ? null : expirationTime; + } + + /** + * Returns the time when this table was last modified, in milliseconds since the epoch. + */ + public Long lastModifiedTime() { + return lastModifiedTime; + } + + /** + * Returns a builder for the {@code BaseTableInfo} object. + */ + public Builder toBuilder() { + return new Builder(this); + } + + protected MoreObjects.ToStringHelper toStringHelper() { + return MoreObjects.toStringHelper(this) + .add("tableId", tableId) + .add("type", type) + .add("schema", schema) + .add("etag", etag) + .add("id", id) + .add("selfLink", selfLink) + .add("friendlyName", friendlyName) + .add("description", description) + .add("numBytes", numBytes) + .add("numRows", numRows) + .add("expirationTime", expirationTime) + .add("creationTime", creationTime) + .add("lastModifiedTime", lastModifiedTime); + } + + @Override + public String toString() { + return toStringHelper().toString(); + } + + @Override + public int hashCode() { + return Objects.hash(tableId); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof BaseTableInfo && Objects.equals(toPb(), ((BaseTableInfo) obj).toPb()); + } + + Table toPb() { + Table tablePb = new Table(); + tablePb.setTableReference(tableId.toPb()); + if (lastModifiedTime != null) { + tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); + } + if (numRows != null) { + tablePb.setNumRows(BigInteger.valueOf(numRows)); + } + if (schema != null) { + tablePb.setSchema(schema.toPb()); + } + tablePb.setCreationTime(creationTime); + tablePb.setDescription(description); + tablePb.setEtag(etag); + tablePb.setExpirationTime(expirationTime); + tablePb.setFriendlyName(friendlyName); + tablePb.setId(id); + tablePb.setNumBytes(numBytes); + tablePb.setSelfLink(selfLink); + return tablePb; + } + + @SuppressWarnings("unchecked") + static T fromPb(Table tablePb) { + Builder builder; + TableId tableId = TableId.fromPb(tablePb.getTableReference()); + Schema schema = tablePb.getSchema() != null ? Schema.fromPb(tablePb.getSchema()) : null; + if (Objects.equals(tablePb.getType(), Type.VIEW.name()) || tablePb.getView() != null) { + ViewDefinition viewPb = tablePb.getView(); + ViewInfo.Builder viewBuilder = ViewInfo.builder(tableId, viewPb.getQuery()); + if (tablePb.getView().getUserDefinedFunctionResources() != null) { + viewBuilder.userDefinedFunctions(Lists.transform(viewPb.getUserDefinedFunctionResources(), + UserDefinedFunction.FROM_PB_FUNCTION)); + } + builder = viewBuilder; + } else if (Objects.equals(tablePb.getType(), Type.EXTERNAL.name()) + || tablePb.getExternalDataConfiguration() != null) { + ExternalTableInfo.Builder externalBuilder = ExternalTableInfo.builder(tableId, + ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration())); + if (tablePb.getStreamingBuffer() != null) { + externalBuilder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); + } + builder = externalBuilder; + } else if (Objects.equals(tablePb.getType(), Type.TABLE.name()) || schema != null) { + TableInfo.Builder tableBuilder = TableInfo.builder(tableId, schema); + if (tablePb.getLocation() != null) { + tableBuilder.location(tablePb.getLocation()); + } + if (tablePb.getStreamingBuffer() != null) { + tableBuilder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); + } + builder = tableBuilder; + } else { + // This is for incomplete tables returned by bigquery.listTables + builder = new Builder().tableId(tableId); + } + if (schema != null) { + builder.schema(schema); + } + if (tablePb.getDescription() != null) { + builder.description(tablePb.getDescription()); + } + if (tablePb.getExpirationTime() != null) { + builder.expirationTime(tablePb.getExpirationTime()); + } + if (tablePb.getFriendlyName() != null) { + builder.friendlyName(tablePb.getFriendlyName()); + } + if (tablePb.getLastModifiedTime() != null) { + builder.lastModifiedTime(tablePb.getLastModifiedTime().longValue()); + } + if (tablePb.getNumRows() != null) { + builder.numRows(tablePb.getNumRows().longValue()); + } + if (tablePb.getCreationTime() != null) { + builder.creationTime(tablePb.getCreationTime()); + } + if (tablePb.getEtag() != null) { + builder.etag(tablePb.getEtag()); + } + if (tablePb.getId() != null) { + builder.id(tablePb.getId()); + } + if (tablePb.getNumBytes() != null) { + builder.numBytes(tablePb.getNumBytes()); + } + if (tablePb.getSelfLink() != null) { + builder.selfLink(tablePb.getSelfLink()); + } + return (T) builder.build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java new file mode 100644 index 000000000000..a885eeb77d1b --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java @@ -0,0 +1,159 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.Table; +import com.google.common.base.MoreObjects; + +import java.util.Objects; + +/** + * Google BigQuery External Table information. BigQuery's external tables are tables whose data + * reside outside of BigQuery but can be queried as normal BigQuery tables. External tables are + * experimental and might be subject to change or removed. + * + * @see Federated Data + * Sources + */ +public class ExternalTableInfo extends BaseTableInfo { + + private static final long serialVersionUID = -5893406738246214865L; + + private final ExternalDataConfiguration configuration; + private final StreamingBuffer streamingBuffer; + + public static final class Builder extends BaseTableInfo.Builder { + + private ExternalDataConfiguration configuration; + private StreamingBuffer streamingBuffer; + + private Builder() {} + + private Builder(ExternalTableInfo tableInfo) { + super(tableInfo); + this.configuration = tableInfo.configuration; + this.streamingBuffer = tableInfo.streamingBuffer; + } + + @Override + protected Builder self() { + return this; + } + + /** + * Sets the data format, location and other properties of a table stored outside of BigQuery. + * + * @see Federated Data + * Sources + */ + public Builder configuration(ExternalDataConfiguration configuration) { + this.configuration = checkNotNull(configuration); + return self(); + } + + Builder streamingBuffer(StreamingBuffer streamingBuffer) { + this.streamingBuffer = streamingBuffer; + return self(); + } + + /** + * Creates a {@code ExternalTableInfo} object. + */ + @Override + public ExternalTableInfo build() { + return new ExternalTableInfo(this); + } + } + + private ExternalTableInfo(Builder builder) { + super(builder); + this.configuration = checkNotNull(builder.configuration); + this.streamingBuffer = builder.streamingBuffer; + } + + /** + * Returns the data format, location and other properties of a table stored outside of BigQuery. + * This property is experimental and might be subject to change or removed. + * + * @see Federated Data + * Sources + */ + public ExternalDataConfiguration configuration() { + return configuration; + }; + + /** + * Returns information on the table's streaming buffer if any exists. Returns {@code null} if no + * streaming buffer exists. + */ + public StreamingBuffer streamingBuffer() { + return streamingBuffer; + } + + /** + * Returns a builder for the {@code ExternalTableInfo} object. + */ + @Override + public Builder toBuilder() { + return new Builder(this); + } + + @Override + protected MoreObjects.ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("configuration", configuration) + .add("streamingBuffer", streamingBuffer); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ExternalTableInfo + && Objects.equals(toPb(), ((ExternalTableInfo) obj).toPb()); + } + + @Override + Table toPb() { + Table tablePb = super.toPb(); + tablePb.setExternalDataConfiguration(configuration.toPb()); + if (streamingBuffer != null) { + tablePb.setStreamingBuffer(streamingBuffer.toPb()); + } + return tablePb; + } + + /** + * Returns a builder for a BigQuery External Table. + * + * @param tableId table id + * @param configuration data format, location and other properties of an External Table + */ + public static Builder builder(TableId tableId, ExternalDataConfiguration configuration) { + return new Builder().tableId(tableId).type(Type.EXTERNAL).configuration(configuration); + } + + /** + * Returns a BigQuery External Table. + * + * @param table table id + * @param configuration data format, location and other properties of an External Table + */ + public static ExternalTableInfo of(TableId table, ExternalDataConfiguration configuration) { + return builder(table, configuration).build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java index c899ac53da94..732444ebdf26 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java @@ -16,381 +16,69 @@ package com.google.gcloud.bigquery; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.api.client.util.Data; -import com.google.api.services.bigquery.model.Streamingbuffer; import com.google.api.services.bigquery.model.Table; -import com.google.api.services.bigquery.model.ViewDefinition; -import com.google.common.base.Function; import com.google.common.base.MoreObjects; -import com.google.common.collect.Lists; -import java.io.Serializable; -import java.math.BigInteger; -import java.util.List; import java.util.Objects; /** - * Google BigQuery Table information. A BigQuery table is a standard, two-dimensional table with + * A Google BigQuery Table information. A BigQuery table is a standard, two-dimensional table with * individual records organized in rows, and a data type assigned to each column (also called a * field). Individual fields within a record may contain nested and repeated children fields. Every * table is described by a schema that describes field names, types, and other information. * * @see Managing Tables */ -public class TableInfo implements Serializable { - - static final Function FROM_PB_FUNCTION = - new Function() { - @Override - public TableInfo apply(Table pb) { - return TableInfo.fromPb(pb); - } - }; - static final Function TO_PB_FUNCTION = - new Function() { - @Override - public Table apply(TableInfo tableInfo) { - return tableInfo.toPb(); - } - }; - - private static final long serialVersionUID = -7679032506430816205L; - - /** - * Google BigQuery Table's Streaming Buffer information. This class contains information on a - * table's streaming buffer as the estimated size in number of rows/bytes. - */ - public static class StreamingBuffer implements Serializable { - - private static final long serialVersionUID = -6713971364725267597L; - private final long estimatedRows; - private final long estimatedBytes; - private final long oldestEntryTime; - - StreamingBuffer(long estimatedRows, long estimatedBytes, long oldestEntryTime) { - this.estimatedRows = estimatedRows; - this.estimatedBytes = estimatedBytes; - this.oldestEntryTime = oldestEntryTime; - } - - /** - * Returns a lower-bound estimate of the number of rows currently in the streaming buffer. - */ - public long estimatedRows() { - return estimatedRows; - } - - /** - * Returns a lower-bound estimate of the number of bytes currently in the streaming buffer. - */ - public long estimatedBytes() { - return estimatedBytes; - } - - /** - * Returns the timestamp of the oldest entry in the streaming buffer, in milliseconds since - * epoch. - */ - public long oldestEntryTime() { - return oldestEntryTime; - } +public class TableInfo extends BaseTableInfo { - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("estimatedRows", estimatedRows) - .add("estimatedBytes", estimatedBytes) - .add("oldestEntryTime", oldestEntryTime) - .toString(); - } + private static final long serialVersionUID = -5910575573063546949L; - @Override - public int hashCode() { - return Objects.hash(estimatedRows, estimatedBytes, oldestEntryTime); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof StreamingBuffer - && Objects.equals(toPb(), ((StreamingBuffer) obj).toPb()); - } - - public Streamingbuffer toPb() { - return new Streamingbuffer() - .setEstimatedBytes(BigInteger.valueOf(estimatedBytes)) - .setEstimatedRows(BigInteger.valueOf(estimatedRows)) - .setOldestEntryTime(BigInteger.valueOf(oldestEntryTime)); - } - - static StreamingBuffer fromPb(Streamingbuffer streamingBufferPb) { - return new StreamingBuffer(streamingBufferPb.getEstimatedRows().longValue(), - streamingBufferPb.getEstimatedBytes().longValue(), - streamingBufferPb.getOldestEntryTime().longValue()); - } - } - - private final String etag; - private final String id; - private final String selfLink; - private final TableId tableId; - private final TableType type; - private final String friendlyName; - private final String description; - private final Long numBytes; - private final Long numRows; - private final Long creationTime; - private final Long expirationTime; - private final Long lastModifiedTime; private final String location; private final StreamingBuffer streamingBuffer; - public static final class Builder { + public static final class Builder extends BaseTableInfo.Builder { - private String etag; - private String id; - private String selfLink; - private TableId tableId; - private TableType type; - private String friendlyName; - private String description; - private Long numBytes; - private Long numRows; - private Long creationTime; - private Long expirationTime; - private Long lastModifiedTime; private String location; private StreamingBuffer streamingBuffer; private Builder() {} private Builder(TableInfo tableInfo) { - this.etag = tableInfo.etag; - this.id = tableInfo.id; - this.selfLink = tableInfo.selfLink; - this.tableId = tableInfo.tableId; - this.type = tableInfo.type; - this.friendlyName = tableInfo.friendlyName; - this.description = tableInfo.description; - this.numBytes = tableInfo.numBytes; - this.numRows = tableInfo.numRows; - this.creationTime = tableInfo.creationTime; - this.expirationTime = tableInfo.expirationTime; - this.lastModifiedTime = tableInfo.lastModifiedTime; + super(tableInfo); this.location = tableInfo.location; this.streamingBuffer = tableInfo.streamingBuffer; } - Builder creationTime(Long creationTime) { - this.creationTime = creationTime; - return this; - } - - /** - * Sets a user-friendly description for the table. - */ - public Builder description(String description) { - this.description = firstNonNull(description, Data.nullOf(String.class)); - return this; - } - - Builder etag(String etag) { - this.etag = etag; - return this; - } - - /** - * Sets the time when this table expires, in milliseconds since the epoch. If not present, the - * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. - */ - public Builder expirationTime(Long expirationTime) { - this.expirationTime = firstNonNull(expirationTime, Data.nullOf(Long.class)); - return this; - } - - /** - * Sets a user-friendly name for the table. - */ - public Builder friendlyName(String friendlyName) { - this.friendlyName = firstNonNull(friendlyName, Data.nullOf(String.class)); - return this; - } - - Builder id(String id) { - this.id = id; - return this; - } - - Builder lastModifiedTime(Long lastModifiedTime) { - this.lastModifiedTime = lastModifiedTime; + @Override + protected Builder self() { return this; } Builder location(String location) { this.location = location; - return this; - } - - Builder numBytes(Long numBytes) { - this.numBytes = numBytes; - return this; - } - - Builder numRows(Long numRows) { - this.numRows = numRows; - return this; - } - - Builder selfLink(String selfLink) { - this.selfLink = selfLink; - return this; + return self(); } Builder streamingBuffer(StreamingBuffer streamingBuffer) { this.streamingBuffer = streamingBuffer; - return this; - } - - /** - * Sets the table identity. - */ - public Builder tableId(TableId tableId) { - this.tableId = tableId; - return this; - } - - /** - * Sets the table type. - */ - public Builder type(TableType type) { - this.type = type; - return this; + return self(); } /** * Creates a {@code TableInfo} object. */ + @Override public TableInfo build() { return new TableInfo(this); } } private TableInfo(Builder builder) { - this.tableId = checkNotNull(builder.tableId); - this.etag = builder.etag; - this.id = builder.id; - this.selfLink = builder.selfLink; - this.friendlyName = builder.friendlyName; - this.description = builder.description; - this.type = builder.type; - this.numBytes = builder.numBytes; - this.numRows = builder.numRows; - this.creationTime = builder.creationTime; - this.expirationTime = builder.expirationTime; - this.lastModifiedTime = builder.lastModifiedTime; + super(builder); this.location = builder.location; this.streamingBuffer = builder.streamingBuffer; } - /** - * Returns the hash of the table resource. - */ - public String etag() { - return etag; - } - - /** - * Returns an opaque id for the table. - */ - public String id() { - return id; - } - - /** - * Returns an URL that can be used to access the resource again. The returned URL can be used for - * get or update requests. - */ - public String selfLink() { - return selfLink; - } - - /** - * Returns the table identity. - */ - public TableId tableId() { - return tableId; - } - - /** - * Returns a user-friendly name for the table. - */ - public String friendlyName() { - return Data.isNull(friendlyName) ? null : friendlyName; - } - - /** - * Returns a user-friendly description for the table. - */ - public String description() { - return Data.isNull(description) ? null : description; - } - - /** - * Returns the size of this table in bytes, excluding any data in the streaming buffer. - */ - public Long numBytes() { - return numBytes; - } - - /** - * Returns the number of rows in this table, excluding any data in the streaming buffer. - */ - public Long numRows() { - return numRows; - } - - /** - * Returns the time when this table was created, in milliseconds since the epoch. - */ - public Long creationTime() { - return creationTime; - } - - /** - * Returns the time when this table expires, in milliseconds since the epoch. If not present, the - * table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. - */ - public Long expirationTime() { - return Data.isNull(expirationTime) ? null : expirationTime; - } - - /** - * Returns the time when this table was last modified, in milliseconds since the epoch. - */ - public Long lastModifiedTime() { - return lastModifiedTime; - } - - /** - * Returns the type of the table. - */ - public Schema schema() { - return type.schema(); - } - - /** - * Returns the table's type. If this table is simple table the method returns an instance of - * {@link TableType}. If this table is an external table this method returns an instance of - * {@link com.google.gcloud.bigquery.TableType.ExternalTable}. If this table is a view table this - * method returns an instance of {@link com.google.gcloud.bigquery.TableType.View}. - */ - @SuppressWarnings("unchecked") - public T type() { - return (T) type; - } - /** * Returns the geographic location where the table should reside. This value is inherited from the * dataset. @@ -411,159 +99,53 @@ public StreamingBuffer streamingBuffer() { } /** - * Returns a builder for the {@code TableInfo} object. + * Returns a builder for a BigQuery Table. + * + * @param tableId table id + * @param schema the schema of the table */ - public Builder toBuilder() { - return new Builder(this); + public static Builder builder(TableId tableId, Schema schema) { + return new Builder().tableId(tableId).type(Type.TABLE).schema(schema); + } + + /** + * Creates BigQuery table given its type + * + * @param tableId table id + * @param schema the schema of the table + */ + public static BaseTableInfo of(TableId tableId, Schema schema) { + return builder(tableId, schema).build(); } + /** + * Returns a builder for the {@code ExternalTableInfo} object. + */ @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("tableId", tableId) - .add("type", type) - .add("location", location) - .add("streamingBuffer", streamingBuffer) - .add("etag", etag) - .add("id", id) - .add("selfLink", selfLink) - .add("friendlyName", friendlyName) - .add("description", description) - .add("numBytes", numBytes) - .add("numRows", numRows) - .add("expirationTime", expirationTime) - .add("creationTime", creationTime) - .add("lastModifiedTime", lastModifiedTime) - .toString(); + public Builder toBuilder() { + return new Builder(this); } @Override - public int hashCode() { - return Objects.hash(tableId); + protected MoreObjects.ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("location", location) + .add("streamingBuffer", streamingBuffer); } @Override public boolean equals(Object obj) { - return obj instanceof TableInfo && Objects.equals(toPb(), ((TableInfo) obj).toPb()); + return obj instanceof TableInfo + && Objects.equals(toPb(), ((TableInfo) obj).toPb()); } + @Override Table toPb() { - Table tablePb = new Table(); - tablePb.setTableReference(tableId.toPb()); - if (lastModifiedTime != null) { - tablePb.setLastModifiedTime(BigInteger.valueOf(lastModifiedTime)); - } - if (numRows != null) { - tablePb.setNumRows(BigInteger.valueOf(numRows)); - } - if (schema() != null) { - tablePb.setSchema(schema().toPb()); - } - if (type instanceof TableType.View) { - TableType.View viewType = (TableType.View) type; - ViewDefinition viewDefinition = new ViewDefinition().setQuery(viewType.query()); - if (viewType.userDefinedFunctions() != null) { - viewDefinition.setUserDefinedFunctionResources( - Lists.transform(viewType.userDefinedFunctions(), UserDefinedFunction.TO_PB_FUNCTION)); - } - tablePb.setView(viewDefinition); - } - if (type instanceof TableType.ExternalTable) { - tablePb.setExternalDataConfiguration( - ((TableType.ExternalTable) type).configuration().toPb()); - } + Table tablePb = super.toPb(); + tablePb.setLocation(location); if (streamingBuffer != null) { tablePb.setStreamingBuffer(streamingBuffer.toPb()); } - tablePb.setCreationTime(creationTime); - tablePb.setDescription(description); - tablePb.setEtag(etag); - tablePb.setExpirationTime(expirationTime); - tablePb.setFriendlyName(friendlyName); - tablePb.setId(id); - tablePb.setLocation(location); - tablePb.setNumBytes(numBytes); - tablePb.setSelfLink(selfLink); return tablePb; } - - /** - * Returns a builder for a BigQuery table given its type. - * - * @param tableId table id - * @param type the type of the table - */ - public static Builder builder(TableId tableId, TableType type) { - return new Builder().tableId(tableId).type(type); - } - - /** - * Creates BigQuery table given its type - * - * @param tableId table id - * @param type the type of the table - */ - public static TableInfo of(TableId tableId, TableType type) { - return builder(tableId, type).build(); - } - - static TableInfo fromPb(Table tablePb) { - Builder builder = new Builder().tableId(TableId.fromPb(tablePb.getTableReference())); - Schema schema = null; - if (tablePb.getSchema() != null) { - schema = Schema.fromPb(tablePb.getSchema()); - builder.type(TableType.table(schema)); - } - if (tablePb.getDescription() != null) { - builder.description(tablePb.getDescription()); - } - if (tablePb.getExpirationTime() != null) { - builder.expirationTime(tablePb.getExpirationTime()); - } - if (tablePb.getFriendlyName() != null) { - builder.friendlyName(tablePb.getFriendlyName()); - } - if (tablePb.getLastModifiedTime() != null) { - builder.lastModifiedTime(tablePb.getLastModifiedTime().longValue()); - } - if (tablePb.getLocation() != null) { - builder.location(tablePb.getLocation()); - } - if (tablePb.getNumRows() != null) { - builder.numRows(tablePb.getNumRows().longValue()); - } - if (tablePb.getStreamingBuffer() != null) { - builder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); - } - if (tablePb.getView() != null) { - String query = tablePb.getView().getQuery(); - List functions = null; - if (tablePb.getView().getUserDefinedFunctionResources() != null) { - functions = Lists.transform(tablePb.getView().getUserDefinedFunctionResources(), - UserDefinedFunction.FROM_PB_FUNCTION); - } - builder.type(new TableType.View(schema, query, functions)); - } - if (tablePb.getExternalDataConfiguration() != null) { - ExternalDataConfiguration configuration = - ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration()); - builder.type(new TableType.ExternalTable(schema, configuration)); - } - if (tablePb.getCreationTime() != null) { - builder.creationTime(tablePb.getCreationTime()); - } - if (tablePb.getEtag() != null) { - builder.etag(tablePb.getEtag()); - } - if (tablePb.getId() != null) { - builder.id(tablePb.getId()); - } - if (tablePb.getNumBytes() != null) { - builder.numBytes(tablePb.getNumBytes()); - } - if (tablePb.getSelfLink() != null) { - builder.selfLink(tablePb.getSelfLink()); - } - return builder.build(); - } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java deleted file mode 100644 index 9a74d55e8bbb..000000000000 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableType.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; - -import java.io.Serializable; -import java.util.List; -import java.util.Objects; - -/** - * Google BigQuery Table type. A BigQuery table can be of three types. {@link Type#TABLE} is a - * normal BigQuery table. {@link Type#VIEW} is a view table, created from a query. - * {@link Type#EXTERNAL} is an external table, backed by Google Cloud Storage data. - */ -public class TableType implements Serializable { - - /** - * The table type. - */ - public enum Type { - /** - * A normal BigQuery table. - */ - TABLE, - /** - * A virtual table defined by a SQL query. - * - * @see Views - */ - VIEW, - /** - * A BigQuery table backed by external data. - * - * @see Federated Data - * Sources - */ - EXTERNAL - } - - private final Type type; - private final Schema schema; - - TableType(Type type, Schema schema) { - this.type = type; - this.schema = schema; - } - - /** - * Returns the type of the table. - */ - public Type type() { - return type; - } - - /** - * Returns the table's schema. - */ - public Schema schema() { - return schema; - } - - /** - * Google BigQuery View type. - */ - public static class View extends TableType { - - private static final long serialVersionUID = 8275064752050652381L; - - private final String query; - private final List userDefinedFunctions; - - View(String query) { - this(null, query, null); - } - - View(String query, List userDefinedFunctions) { - this(null, query, userDefinedFunctions); - } - - View(Schema schema, String query, List userDefinedFunctions) { - super(Type.VIEW, schema); - this.query = query; - this.userDefinedFunctions = userDefinedFunctions; - } - - /** - * Returns the query used to create the view. - */ - public String query() { - return query; - }; - - /** - * Returns user defined functions that can be used by {@link #query()}. Returns {@code null} if - * not set. - * - * @see User-Defined Functions - * - */ - public List userDefinedFunctions() { - return userDefinedFunctions; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("type", type()) - .add("query", query) - .add("userDefinedFunctions", userDefinedFunctions) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash(schema(), query, userDefinedFunctions); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof View)) { - return false; - } - View other = (View) obj; - return Objects.equals(type(), other.type()) - && Objects.equals(query, other.query) - && Objects.equals(userDefinedFunctions, other.userDefinedFunctions); - } - } - - /** - * Google BigQuery External Table type. - */ - public static class ExternalTable extends TableType { - - private static final long serialVersionUID = -6912214693318638552L; - - private final ExternalDataConfiguration configuration; - - ExternalTable(ExternalDataConfiguration configuration) { - this(null, configuration); - } - - ExternalTable(Schema schema, ExternalDataConfiguration configuration) { - super(Type.EXTERNAL, schema); - this.configuration = configuration; - } - - /** - * Returns the data format, location and other properties of a table stored outside of BigQuery. - * This property is experimental and might be subject to change or removed. - * - * @see Federated Data - * Sources - */ - public ExternalDataConfiguration configuration() { - return configuration; - }; - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("type", type()) - .add("configuration", configuration) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash(schema(), configuration); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof ExternalTable)) { - return false; - } - ExternalTable other = (ExternalTable) obj; - return Objects.equals(type(), other.type()) - && Objects.equals(configuration, other.configuration); - } - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("type", type) - .add("schema", schema) - .toString(); - } - - @Override - public int hashCode() { - return Objects.hash(type, schema); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof TableType)) { - return false; - } - TableType other = (TableType) obj; - return Objects.equals(type, other.type) - && Objects.equals(schema, other.schema); - } - - /** - * Returns a simple BigQuery Table type. - */ - public static TableType table(Schema schema) { - return new TableType(Type.TABLE, checkNotNull(schema)); - } - - /** - * Returns a BigQuery View type given the query needed to create the view. - */ - public static View view(String query) { - return new View(checkNotNull(query)); - } - - /** - * Returns a BigQuery View type given the query needed to create the view and a list of user - * defined functions that can be used by the query. - */ - public static View view(String query, List userDefinedFunctions) { - return new View(checkNotNull(query), checkNotNull(userDefinedFunctions)); - } - - /** - * Returns a BigQuery View type given the query needed to create the view and a list of user - * defined functions that can be used by the query. - */ - public static View view(String query, UserDefinedFunction... userDefinedFunctions) { - return new View(checkNotNull(query), ImmutableList.copyOf(userDefinedFunctions)); - } - - /** - * Returns a BigQuery External Table type given the external data configuration. - */ - public static ExternalTable externalTable(ExternalDataConfiguration configuration) { - return new ExternalTable(checkNotNull(configuration)); - } -} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java new file mode 100644 index 000000000000..6a98a446e294 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java @@ -0,0 +1,229 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.ViewDefinition; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery View Table information. BigQuery's views are logical views, not materialized + * views, which means that the query that defines the view is re-executed every time the view is + * queried. + * + * @see Views + */ +public class ViewInfo extends BaseTableInfo { + + private static final long serialVersionUID = 7567772157817454901L; + + private final String query; + private final List userDefinedFunctions; + + public static final class Builder extends BaseTableInfo.Builder { + + private String query; + private List userDefinedFunctions; + + private Builder() {} + + private Builder(ViewInfo viewInfo) { + super(viewInfo); + this.query = viewInfo.query; + this.userDefinedFunctions = viewInfo.userDefinedFunctions; + } + + @Override + protected Builder self() { + return this; + } + + /** + * Sets the query used to create the view. + */ + public Builder query(String query) { + this.query = checkNotNull(query); + return self(); + } + + /** + * Sets user defined functions that can be used by {@link #query()}. + * + * @see User-Defined Functions + * + */ + public Builder userDefinedFunctions(List userDefinedFunctions) { + this.userDefinedFunctions = ImmutableList.copyOf(checkNotNull(userDefinedFunctions)); + return self(); + } + + /** + * Sets user defined functions that can be used by {@link #query()}. + * + * @see User-Defined Functions + * + */ + public Builder userDefinedFunctions(UserDefinedFunction... userDefinedFunctions) { + this.userDefinedFunctions = ImmutableList.copyOf(userDefinedFunctions); + return self(); + } + + /** + * Creates a {@code ViewInfo} object. + */ + @Override + public ViewInfo build() { + return new ViewInfo(this); + } + } + + private ViewInfo(Builder builder) { + super(builder); + this.query = checkNotNull(builder.query); + this.userDefinedFunctions = builder.userDefinedFunctions; + } + + /** + * Returns the query used to create the view. + */ + public String query() { + return query; + }; + + /** + * Returns user defined functions that can be used by {@link #query()}. Returns {@code null} if + * not set. + * + * @see User-Defined Functions + * + */ + public List userDefinedFunctions() { + return userDefinedFunctions; + } + + /** + * Returns a builder for the {@code ViewInfo} object. + */ + @Override + public Builder toBuilder() { + return new Builder(this); + } + + @Override + protected MoreObjects.ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("query", query) + .add("userDefinedFunctions", userDefinedFunctions); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ViewInfo && Objects.equals(toPb(), ((ViewInfo) obj).toPb()); + } + + @Override + Table toPb() { + Table tablePb = super.toPb(); + ViewDefinition viewDefinition = new ViewDefinition() + .setQuery(query); + if (userDefinedFunctions != null) { + viewDefinition.setUserDefinedFunctionResources(Lists.transform(userDefinedFunctions, + UserDefinedFunction.TO_PB_FUNCTION)); + } + tablePb.setView(viewDefinition); + return tablePb; + } + + /** + * Returns a builder for a BigQuery View Table. + * + * @param tableId table id + * @param query the query used to generate the table + */ + public static Builder builder(TableId tableId, String query) { + return new Builder().tableId(tableId).type(Type.VIEW).query(query); + } + + /** + * Returns a builder for a BigQuery View Table. + * + * @param table table id + * @param query the query used to generate the table + * @param functions user-defined functions that can be used by the query + */ + public static Builder builder(TableId table, String query, List functions) { + return new Builder() + .tableId(table) + .type(Type.VIEW) + .userDefinedFunctions(functions) + .query(query); + } + + /** + * Returns a builder for a BigQuery View Table. + * + * @param table table id + * @param query the query used to generate the table + * @param functions user-defined functions that can be used by the query + */ + public static Builder builder(TableId table, String query, UserDefinedFunction... functions) { + return new Builder() + .tableId(table) + .type(Type.VIEW) + .userDefinedFunctions(functions) + .query(query); + } + + /** + * Creates a BigQuery View given table identity and query. + * + * @param tableId table id + * @param query the query used to generate the table + */ + public static ViewInfo of(TableId tableId, String query) { + return builder(tableId, query).build(); + } + + /** + * Creates a BigQuery View given table identity, a query and some user-defined functions. + * + * @param table table id + * @param query the query used to generate the table + * @param functions user-defined functions that can be used by the query + */ + public static ViewInfo of(TableId table, String query, List functions) { + return builder(table, query, functions).build(); + } + + /** + * Creates a BigQuery View given table identity, a query and some user-defined functions. + * + * @param table table id + * @param query the query used to generate the table + * @param functions user-defined functions that can be used by the query + */ + public static ViewInfo of(TableId table, String query, UserDefinedFunction... functions) { + return builder(table, query, functions).build(); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index 86e18ac74c97..c75337431580 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -92,8 +92,8 @@ public class SerializationTest { .description("FieldDescription3") .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); - private static final TableInfo.StreamingBuffer STREAMING_BUFFER = - new TableInfo.StreamingBuffer(1L, 2L, 3L); + private static final BaseTableInfo.StreamingBuffer STREAMING_BUFFER = + new BaseTableInfo.StreamingBuffer(1L, 2L, 3L); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); private static final ExternalDataConfiguration EXTERNAL_DATA_CONFIGURATION = ExternalDataConfiguration.builder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) @@ -104,8 +104,8 @@ public class SerializationTest { new UserDefinedFunction.InlineFunction("inline"); private static final UserDefinedFunction URI_FUNCTION = new UserDefinedFunction.UriFunction("URI"); - private static final TableInfo TABLE_INFO = - TableInfo.builder(TABLE_ID, TableType.table(TABLE_SCHEMA)) + private static final BaseTableInfo TABLE_INFO = + TableInfo.builder(TABLE_ID, TABLE_SCHEMA) .creationTime(CREATION_TIME) .description(DESCRIPTION) .etag(ETAG) @@ -113,6 +113,21 @@ public class SerializationTest { .location(LOCATION) .streamingBuffer(STREAMING_BUFFER) .build(); + private static final ViewInfo VIEW_INFO = + ViewInfo.builder(TABLE_ID, "QUERY") + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .id(ID) + .build(); + private static final ExternalTableInfo EXTERNAL_TABLE_INFO = + ExternalTableInfo.builder(TABLE_ID, EXTERNAL_DATA_CONFIGURATION) + .creationTime(CREATION_TIME) + .description(DESCRIPTION) + .etag(ETAG) + .id(ID) + .streamingBuffer(STREAMING_BUFFER) + .build(); @Test public void testServiceOptions() throws Exception { @@ -136,7 +151,7 @@ public void testServiceOptions() throws Exception { public void testModelAndRequests() throws Exception { Serializable[] objects = {DOMAIN_ACCESS, GROUP_ACCESS, USER_ACCESS, VIEW_ACCESS, DATASET_ID, DATASET_INFO, TABLE_ID, CSV_OPTIONS, STREAMING_BUFFER, EXTERNAL_DATA_CONFIGURATION, - TABLE_SCHEMA, TABLE_INFO, INLINE_FUNCTION, URI_FUNCTION}; + TABLE_SCHEMA, TABLE_INFO, VIEW_INFO, EXTERNAL_TABLE_INFO, INLINE_FUNCTION, URI_FUNCTION}; for (Serializable obj : objects) { Object copy = serializeAndDeserialize(obj); assertEquals(obj, obj); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java index ecde5f2c907c..f9e073906578 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java @@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue; import com.google.common.collect.ImmutableList; -import com.google.gcloud.bigquery.TableInfo.StreamingBuffer; +import com.google.gcloud.bigquery.BaseTableInfo.StreamingBuffer; import org.junit.Test; @@ -70,7 +70,7 @@ public class TableInfoTest { private static final String LOCATION = "US"; private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L); private static final TableInfo TABLE_INFO = - TableInfo.builder(TABLE_ID, TableType.table(TABLE_SCHEMA)) + TableInfo.builder(TABLE_ID, TABLE_SCHEMA) .creationTime(CREATION_TIME) .description(DESCRIPTION) .etag(ETAG) @@ -84,8 +84,8 @@ public class TableInfoTest { .selfLink(SELF_LINK) .streamingBuffer(STREAMING_BUFFER) .build(); - private static final TableInfo EXTERNAL_TABLE_INFO = - TableInfo.builder(TABLE_ID, TableType.externalTable(CONFIGURATION)) + private static final ExternalTableInfo EXTERNAL_TABLE_INFO = + ExternalTableInfo.builder(TABLE_ID, CONFIGURATION) .creationTime(CREATION_TIME) .description(DESCRIPTION) .etag(ETAG) @@ -93,7 +93,6 @@ public class TableInfoTest { .friendlyName(FRIENDLY_NAME) .id(ID) .lastModifiedTime(LAST_MODIFIED_TIME) - .location(LOCATION) .numBytes(NUM_BYTES) .numRows(NUM_ROWS) .selfLink(SELF_LINK) @@ -101,8 +100,8 @@ public class TableInfoTest { .build(); private static List USER_DEFINED_FUNCTIONS = ImmutableList.of( UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); - private static final TableInfo VIEW_INFO = - TableInfo.builder(TABLE_ID, TableType.view(VIEW_QUERY, USER_DEFINED_FUNCTIONS)) + private static final ViewInfo VIEW_INFO = + ViewInfo.builder(TABLE_ID, VIEW_QUERY, USER_DEFINED_FUNCTIONS) .creationTime(CREATION_TIME) .description(DESCRIPTION) .etag(ETAG) @@ -110,37 +109,39 @@ public class TableInfoTest { .friendlyName(FRIENDLY_NAME) .id(ID) .lastModifiedTime(LAST_MODIFIED_TIME) - .location(LOCATION) .numBytes(NUM_BYTES) .numRows(NUM_ROWS) .selfLink(SELF_LINK) - .streamingBuffer(STREAMING_BUFFER) .build(); @Test public void testToBuilder() { compareTableInfo(TABLE_INFO, TABLE_INFO.toBuilder().build()); - compareTableInfo(VIEW_INFO, VIEW_INFO.toBuilder().build()); - compareTableInfo(EXTERNAL_TABLE_INFO, EXTERNAL_TABLE_INFO.toBuilder().build()); - TableInfo tableInfo = TABLE_INFO.toBuilder() + compareViewInfo(VIEW_INFO, VIEW_INFO.toBuilder().build()); + compareExternalTableInfo(EXTERNAL_TABLE_INFO, EXTERNAL_TABLE_INFO.toBuilder().build()); + BaseTableInfo tableInfo = TABLE_INFO.toBuilder() .description("newDescription") .build(); assertEquals("newDescription", tableInfo.description()); tableInfo = tableInfo.toBuilder() .description("description") .build(); - compareTableInfo(TABLE_INFO, tableInfo); + compareBaseTableInfo(TABLE_INFO, tableInfo); } @Test public void testToBuilderIncomplete() { - TableInfo tableInfo = TableInfo.of(TABLE_ID, TableType.view(VIEW_QUERY)); + BaseTableInfo tableInfo = TableInfo.of(TABLE_ID, TABLE_SCHEMA); + assertEquals(tableInfo, tableInfo.toBuilder().build()); + tableInfo = ViewInfo.of(TABLE_ID, VIEW_QUERY); + assertEquals(tableInfo, tableInfo.toBuilder().build()); + tableInfo = ExternalTableInfo.of(TABLE_ID, CONFIGURATION); assertEquals(tableInfo, tableInfo.toBuilder().build()); } + @Test public void testBuilder() { assertEquals(TABLE_ID, TABLE_INFO.tableId()); - assertTrue(VIEW_INFO.type() instanceof TableType); assertEquals(TABLE_SCHEMA, TABLE_INFO.schema()); assertEquals(CREATION_TIME, TABLE_INFO.creationTime()); assertEquals(DESCRIPTION, TABLE_INFO.description()); @@ -154,11 +155,11 @@ public void testBuilder() { assertEquals(NUM_ROWS, TABLE_INFO.numRows()); assertEquals(SELF_LINK, TABLE_INFO.selfLink()); assertEquals(STREAMING_BUFFER, TABLE_INFO.streamingBuffer()); - assertEquals(TableType.Type.TABLE, TABLE_INFO.type().type()); + assertEquals(BaseTableInfo.Type.TABLE, TABLE_INFO.type()); assertEquals(TABLE_ID, VIEW_INFO.tableId()); assertEquals(null, VIEW_INFO.schema()); - assertTrue(VIEW_INFO.type() instanceof TableType.View); - assertEquals(VIEW_QUERY, ((TableType.View) VIEW_INFO.type()).query()); + assertEquals(VIEW_QUERY, VIEW_INFO.query()); + assertEquals(BaseTableInfo.Type.VIEW, VIEW_INFO.type()); assertEquals(CREATION_TIME, VIEW_INFO.creationTime()); assertEquals(DESCRIPTION, VIEW_INFO.description()); assertEquals(ETAG, VIEW_INFO.etag()); @@ -166,17 +167,13 @@ public void testBuilder() { assertEquals(FRIENDLY_NAME, VIEW_INFO.friendlyName()); assertEquals(ID, VIEW_INFO.id()); assertEquals(LAST_MODIFIED_TIME, VIEW_INFO.lastModifiedTime()); - assertEquals(LOCATION, VIEW_INFO.location()); assertEquals(NUM_BYTES, VIEW_INFO.numBytes()); assertEquals(NUM_ROWS, VIEW_INFO.numRows()); assertEquals(SELF_LINK, VIEW_INFO.selfLink()); - assertEquals(STREAMING_BUFFER, VIEW_INFO.streamingBuffer()); - assertEquals(TableType.Type.VIEW, VIEW_INFO.type().type()); + assertEquals(BaseTableInfo.Type.VIEW, VIEW_INFO.type()); assertEquals(TABLE_ID, EXTERNAL_TABLE_INFO.tableId()); assertEquals(null, EXTERNAL_TABLE_INFO.schema()); - assertTrue(EXTERNAL_TABLE_INFO.type() instanceof TableType.ExternalTable); - assertEquals(CONFIGURATION, - ((TableType.ExternalTable) EXTERNAL_TABLE_INFO.type()).configuration()); + assertEquals(CONFIGURATION, EXTERNAL_TABLE_INFO.configuration()); assertEquals(CREATION_TIME, EXTERNAL_TABLE_INFO.creationTime()); assertEquals(DESCRIPTION, EXTERNAL_TABLE_INFO.description()); assertEquals(ETAG, EXTERNAL_TABLE_INFO.etag()); @@ -184,22 +181,25 @@ public void testBuilder() { assertEquals(FRIENDLY_NAME, EXTERNAL_TABLE_INFO.friendlyName()); assertEquals(ID, EXTERNAL_TABLE_INFO.id()); assertEquals(LAST_MODIFIED_TIME, EXTERNAL_TABLE_INFO.lastModifiedTime()); - assertEquals(LOCATION, EXTERNAL_TABLE_INFO.location()); assertEquals(NUM_BYTES, EXTERNAL_TABLE_INFO.numBytes()); assertEquals(NUM_ROWS, EXTERNAL_TABLE_INFO.numRows()); assertEquals(SELF_LINK, EXTERNAL_TABLE_INFO.selfLink()); assertEquals(STREAMING_BUFFER, EXTERNAL_TABLE_INFO.streamingBuffer()); - assertEquals(TableType.Type.EXTERNAL, EXTERNAL_TABLE_INFO.type().type()); + assertEquals(BaseTableInfo.Type.EXTERNAL, EXTERNAL_TABLE_INFO.type()); } @Test public void testToAndFromPb() { - compareTableInfo(TABLE_INFO, TableInfo.fromPb(TABLE_INFO.toPb())); - compareTableInfo(VIEW_INFO, TableInfo.fromPb(VIEW_INFO.toPb())); - compareTableInfo(EXTERNAL_TABLE_INFO, TableInfo.fromPb(EXTERNAL_TABLE_INFO.toPb())); + assertTrue(BaseTableInfo.fromPb(TABLE_INFO.toPb()) instanceof BaseTableInfo); + compareTableInfo(TABLE_INFO, BaseTableInfo.fromPb(TABLE_INFO.toPb())); + assertTrue(BaseTableInfo.fromPb(VIEW_INFO.toPb()) instanceof ViewInfo); + compareViewInfo(VIEW_INFO, BaseTableInfo.fromPb(VIEW_INFO.toPb())); + assertTrue(BaseTableInfo.fromPb(EXTERNAL_TABLE_INFO.toPb()) instanceof ExternalTableInfo); + compareExternalTableInfo(EXTERNAL_TABLE_INFO, + BaseTableInfo.fromPb(EXTERNAL_TABLE_INFO.toPb())); } - private void compareTableInfo(TableInfo expected, TableInfo value) { + private void compareBaseTableInfo(BaseTableInfo expected, BaseTableInfo value) { assertEquals(expected, value); assertEquals(expected.tableId(), value.tableId()); assertEquals(expected.schema(), value.schema()); @@ -211,11 +211,30 @@ private void compareTableInfo(TableInfo expected, TableInfo value) { assertEquals(expected.friendlyName(), value.friendlyName()); assertEquals(expected.id(), value.id()); assertEquals(expected.lastModifiedTime(), value.lastModifiedTime()); - assertEquals(expected.location(), value.location()); assertEquals(expected.numBytes(), value.numBytes()); assertEquals(expected.numRows(), value.numRows()); assertEquals(expected.selfLink(), value.selfLink()); - assertEquals(expected.streamingBuffer(), value.streamingBuffer()); assertEquals(expected.type(), value.type()); } + + private void compareTableInfo(TableInfo expected, TableInfo value) { + compareBaseTableInfo(expected, value); + assertEquals(expected, value); + assertEquals(expected.location(), value.location()); + assertEquals(expected.streamingBuffer(), value.streamingBuffer()); + } + + private void compareViewInfo(ViewInfo expected, ViewInfo value) { + compareBaseTableInfo(expected, value); + assertEquals(expected, value); + assertEquals(expected.query(), value.query()); + assertEquals(expected.userDefinedFunctions(), value.userDefinedFunctions()); + } + + private void compareExternalTableInfo(ExternalTableInfo expected, ExternalTableInfo value) { + compareBaseTableInfo(expected, value); + assertEquals(expected, value); + assertEquals(expected.configuration(), value.configuration()); + assertEquals(expected.streamingBuffer(), value.streamingBuffer()); + } } diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java deleted file mode 100644 index 6b67e8968e03..000000000000 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTypeTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import com.google.common.collect.ImmutableList; - -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertEquals; - -public class TableTypeTest { - - private static final Field FIELD_SCHEMA1 = - Field.builder("StringField", Field.Type.string()) - .mode(Field.Mode.NULLABLE) - .description("FieldDescription1") - .build(); - private static final Field FIELD_SCHEMA2 = - Field.builder("IntegerField", Field.Type.integer()) - .mode(Field.Mode.REPEATED) - .description("FieldDescription2") - .build(); - private static final Field FIELD_SCHEMA3 = - Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) - .mode(Field.Mode.REQUIRED) - .description("FieldDescription3") - .build(); - private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); - private static final String VIEW_QUERY = "VIEW QUERY"; - private static final ExternalDataConfiguration CONFIGURATION = ExternalDataConfiguration - .builder(ImmutableList.of("URI"), TABLE_SCHEMA, FormatOptions.datastoreBackup()) - .compression("ZIP") - .ignoreUnknownValues(true) - .maxBadRecords(42) - .build(); - private static List FUNCTIONS = ImmutableList.of( - UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); - private static final TableType TABLE = TableType.table(TABLE_SCHEMA); - private static final TableType.View VIEW = TableType.view(VIEW_QUERY); - private static final TableType.View VIEW_WITH_FUNCTIONS = TableType.view(VIEW_QUERY, FUNCTIONS); - private static final TableType.ExternalTable EXTERNAL = TableType.externalTable(CONFIGURATION); - - @Test - public void testConstructor() { - TableType table = new TableType(TableType.Type.TABLE, TABLE_SCHEMA); - assertEquals(TableType.Type.TABLE, table.type()); - assertEquals(TABLE_SCHEMA, table.schema()); - TableType.View view = new TableType.View(VIEW_QUERY); - assertEquals(TableType.Type.VIEW, view.type()); - assertEquals(null, view.schema()); - assertEquals(VIEW_QUERY, view.query()); - assertEquals(null, view.userDefinedFunctions()); - view = new TableType.View(VIEW_QUERY, FUNCTIONS); - assertEquals(TableType.Type.VIEW, view.type()); - assertEquals(null, view.schema()); - assertEquals(VIEW_QUERY, view.query()); - assertEquals(FUNCTIONS, view.userDefinedFunctions()); - view = new TableType.View(TABLE_SCHEMA, VIEW_QUERY, FUNCTIONS); - assertEquals(TableType.Type.VIEW, view.type()); - assertEquals(TABLE_SCHEMA, view.schema()); - assertEquals(VIEW_QUERY, view.query()); - assertEquals(FUNCTIONS, view.userDefinedFunctions()); - TableType.ExternalTable extern = new TableType.ExternalTable(CONFIGURATION); - assertEquals(TableType.Type.EXTERNAL, extern.type()); - assertEquals(null, extern.schema()); - assertEquals(CONFIGURATION, extern.configuration()); - extern = new TableType.ExternalTable(TABLE_SCHEMA, CONFIGURATION); - assertEquals(TableType.Type.EXTERNAL, extern.type()); - assertEquals(TABLE_SCHEMA, extern.schema()); - assertEquals(CONFIGURATION, extern.configuration()); - } - - @Test - public void testFactoryMethods() { - TableType table = TableType.table(TABLE_SCHEMA); - assertEquals(TableType.Type.TABLE, table.type()); - assertEquals(TABLE_SCHEMA, table.schema()); - TableType.View view = TableType.view(VIEW_QUERY); - assertEquals(TableType.Type.VIEW, view.type()); - assertEquals(null, view.schema()); - assertEquals(VIEW_QUERY, view.query()); - assertEquals(null, view.userDefinedFunctions()); - view = TableType.view(VIEW_QUERY, FUNCTIONS); - assertEquals(TableType.Type.VIEW, view.type()); - assertEquals(null, view.schema()); - assertEquals(VIEW_QUERY, view.query()); - assertEquals(FUNCTIONS, view.userDefinedFunctions()); - TableType.ExternalTable extern = TableType.externalTable(CONFIGURATION); - assertEquals(TableType.Type.EXTERNAL, extern.type()); - assertEquals(null, extern.schema()); - assertEquals(CONFIGURATION, extern.configuration()); - } - - @Test - public void testEquals() { - assertEquals(new TableType(TableType.Type.TABLE, TABLE_SCHEMA), TableType.table(TABLE_SCHEMA)); - assertEquals(new TableType.View(VIEW_QUERY), TableType.view(VIEW_QUERY)); - assertEquals(new TableType.View(VIEW_QUERY, FUNCTIONS), TableType.view(VIEW_QUERY, FUNCTIONS)); - assertEquals(new TableType.ExternalTable(CONFIGURATION), - TableType.externalTable(CONFIGURATION)); - assertEquals(new TableType(TableType.Type.TABLE, TABLE_SCHEMA).hashCode(), - TableType.table(TABLE_SCHEMA).hashCode()); - assertEquals(new TableType.View(VIEW_QUERY).hashCode(), - TableType.view(VIEW_QUERY).hashCode()); - assertEquals(new TableType.View(VIEW_QUERY, FUNCTIONS).hashCode(), - TableType.view(VIEW_QUERY, FUNCTIONS).hashCode()); - assertEquals(new TableType.ExternalTable(CONFIGURATION).hashCode(), - TableType.externalTable(CONFIGURATION).hashCode()); - } -} From 338ede110628da0e980bcae8c7ec499b89c12dc1 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 4 Dec 2015 15:33:57 +0100 Subject: [PATCH 09/10] Fix mode's nullability by internally storing a string --- .../gcloud/bigquery/ExternalDataConfiguration.java | 2 +- .../main/java/com/google/gcloud/bigquery/Field.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java index 9e049c4f13d6..4344aeba186b 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java @@ -283,7 +283,7 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() { if (sourceUris != null) { externalConfigurationPb.setSourceUris(sourceUris); } - if (formatOptions instanceof CsvOptions) { + if (formatOptions != null && FormatOptions.CSV.equals(formatOptions.type())) { externalConfigurationPb.setCsvOptions(((CsvOptions) formatOptions).toPb()); } return externalConfigurationPb; diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java index d28e5eb9fed7..54fdb9f50329 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Field.java @@ -182,19 +182,19 @@ public boolean equals(Object obj) { * than one value. */ public enum Mode { - NULLABLE, REQUIRED, REPEATED, NOT_SET + NULLABLE, REQUIRED, REPEATED } private final String name; private final Type type; - private final Mode mode; + private final String mode; private final String description; public static final class Builder { private String name; private Type type; - private Mode mode; + private String mode; private String description; private Builder() {} @@ -231,7 +231,7 @@ public Builder type(Type type) { * Sets the mode of the field. When not specified {@link Mode#NULLABLE} is used. */ public Builder mode(Mode mode) { - this.mode = firstNonNull(mode, Mode.NOT_SET); + this.mode = mode != null ? mode.name() : Data.nullOf(String.class); return this; } @@ -279,7 +279,7 @@ public Type type() { * Returns the field mode. By default {@link Mode#NULLABLE} is used. */ public Mode mode() { - return mode == Mode.NOT_SET ? null : mode; + return mode != null ? Mode.valueOf(mode) : null; } /** @@ -329,7 +329,7 @@ TableFieldSchema toPb() { fieldSchemaPb.setName(name); fieldSchemaPb.setType(type.value().name()); if (mode != null) { - fieldSchemaPb.setMode(mode == Mode.NOT_SET ? Data.nullOf(String.class) : mode.name()); + fieldSchemaPb.setMode(mode); } if (description != null) { fieldSchemaPb.setDescription(description); From 9df4005c0a9c7e782a51bbd842420f8d241904aa Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Fri, 4 Dec 2015 23:07:02 +0100 Subject: [PATCH 10/10] Make BaseTableInfo abstract and other minor fixes --- .../google/gcloud/bigquery/BaseTableInfo.java | 117 ++++++------------ .../gcloud/bigquery/ExternalTableInfo.java | 27 ++-- .../com/google/gcloud/bigquery/TableInfo.java | 24 ++-- .../com/google/gcloud/bigquery/ViewInfo.java | 29 +++-- 4 files changed, 83 insertions(+), 114 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java index d409e440aabd..8845179b31c5 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java @@ -22,10 +22,8 @@ import com.google.api.client.util.Data; import com.google.api.services.bigquery.model.Streamingbuffer; import com.google.api.services.bigquery.model.Table; -import com.google.api.services.bigquery.model.ViewDefinition; import com.google.common.base.Function; import com.google.common.base.MoreObjects; -import com.google.common.collect.Lists; import java.io.Serializable; import java.math.BigInteger; @@ -38,7 +36,7 @@ * * @see Managing Tables */ -public class BaseTableInfo implements Serializable { +public abstract class BaseTableInfo implements Serializable { static final Function FROM_PB_FUNCTION = new Function() { @@ -167,7 +165,7 @@ static StreamingBuffer fromPb(Streamingbuffer streamingBufferPb) { private final Long expirationTime; private final Long lastModifiedTime; - public static class Builder> { + public static abstract class Builder> { private String etag; private String id; @@ -201,6 +199,28 @@ protected Builder(BaseTableInfo tableInfo) { this.lastModifiedTime = tableInfo.lastModifiedTime; } + protected Builder(Table tablePb) { + this.type = Type.valueOf(tablePb.getType()); + this.tableId = TableId.fromPb(tablePb.getTableReference()); + if (tablePb.getSchema() != null) { + this.schema(Schema.fromPb(tablePb.getSchema())); + } + if (tablePb.getLastModifiedTime() != null) { + this.lastModifiedTime(tablePb.getLastModifiedTime().longValue()); + } + if (tablePb.getNumRows() != null) { + this.numRows(tablePb.getNumRows().longValue()); + } + this.description = tablePb.getDescription(); + this.expirationTime = tablePb.getExpirationTime(); + this.friendlyName = tablePb.getFriendlyName(); + this.creationTime = tablePb.getCreationTime(); + this.etag = tablePb.getEtag(); + this.id = tablePb.getId(); + this.numBytes = tablePb.getNumBytes(); + this.selfLink = tablePb.getSelfLink(); + } + @SuppressWarnings("unchecked") protected B self() { return (B) this; @@ -288,12 +308,9 @@ public B schema(Schema schema) { } /** - * Creates a {@code BaseTableInfo} object. + * Creates an object. */ - @SuppressWarnings("unchecked") - public T build() { - return (T) new BaseTableInfo(this); - } + public abstract T build(); } protected BaseTableInfo(Builder builder) { @@ -408,11 +425,9 @@ public Long lastModifiedTime() { } /** - * Returns a builder for the {@code BaseTableInfo} object. + * Returns a builder for the object. */ - public Builder toBuilder() { - return new Builder(this); - } + public abstract Builder toBuilder(); protected MoreObjects.ToStringHelper toStringHelper() { return MoreObjects.toStringHelper(this) @@ -458,6 +473,7 @@ Table toPb() { if (schema != null) { tablePb.setSchema(schema.toPb()); } + tablePb.setType(type.name()); tablePb.setCreationTime(creationTime); tablePb.setDescription(description); tablePb.setEtag(etag); @@ -471,71 +487,16 @@ Table toPb() { @SuppressWarnings("unchecked") static T fromPb(Table tablePb) { - Builder builder; - TableId tableId = TableId.fromPb(tablePb.getTableReference()); - Schema schema = tablePb.getSchema() != null ? Schema.fromPb(tablePb.getSchema()) : null; - if (Objects.equals(tablePb.getType(), Type.VIEW.name()) || tablePb.getView() != null) { - ViewDefinition viewPb = tablePb.getView(); - ViewInfo.Builder viewBuilder = ViewInfo.builder(tableId, viewPb.getQuery()); - if (tablePb.getView().getUserDefinedFunctionResources() != null) { - viewBuilder.userDefinedFunctions(Lists.transform(viewPb.getUserDefinedFunctionResources(), - UserDefinedFunction.FROM_PB_FUNCTION)); - } - builder = viewBuilder; - } else if (Objects.equals(tablePb.getType(), Type.EXTERNAL.name()) - || tablePb.getExternalDataConfiguration() != null) { - ExternalTableInfo.Builder externalBuilder = ExternalTableInfo.builder(tableId, - ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration())); - if (tablePb.getStreamingBuffer() != null) { - externalBuilder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); - } - builder = externalBuilder; - } else if (Objects.equals(tablePb.getType(), Type.TABLE.name()) || schema != null) { - TableInfo.Builder tableBuilder = TableInfo.builder(tableId, schema); - if (tablePb.getLocation() != null) { - tableBuilder.location(tablePb.getLocation()); - } - if (tablePb.getStreamingBuffer() != null) { - tableBuilder.streamingBuffer(StreamingBuffer.fromPb(tablePb.getStreamingBuffer())); - } - builder = tableBuilder; - } else { - // This is for incomplete tables returned by bigquery.listTables - builder = new Builder().tableId(tableId); - } - if (schema != null) { - builder.schema(schema); - } - if (tablePb.getDescription() != null) { - builder.description(tablePb.getDescription()); - } - if (tablePb.getExpirationTime() != null) { - builder.expirationTime(tablePb.getExpirationTime()); - } - if (tablePb.getFriendlyName() != null) { - builder.friendlyName(tablePb.getFriendlyName()); - } - if (tablePb.getLastModifiedTime() != null) { - builder.lastModifiedTime(tablePb.getLastModifiedTime().longValue()); - } - if (tablePb.getNumRows() != null) { - builder.numRows(tablePb.getNumRows().longValue()); - } - if (tablePb.getCreationTime() != null) { - builder.creationTime(tablePb.getCreationTime()); - } - if (tablePb.getEtag() != null) { - builder.etag(tablePb.getEtag()); - } - if (tablePb.getId() != null) { - builder.id(tablePb.getId()); - } - if (tablePb.getNumBytes() != null) { - builder.numBytes(tablePb.getNumBytes()); - } - if (tablePb.getSelfLink() != null) { - builder.selfLink(tablePb.getSelfLink()); + switch (Type.valueOf(tablePb.getType())) { + case TABLE: + return (T) TableInfo.fromPb(tablePb); + case VIEW: + return (T) ViewInfo.fromPb(tablePb); + case EXTERNAL: + return (T) ExternalTableInfo.fromPb(tablePb); + default: + // never reached + throw new IllegalArgumentException("Format " + tablePb.getType() + " is not supported"); } - return (T) builder.build(); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java index a885eeb77d1b..775a0294db77 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalTableInfo.java @@ -21,8 +21,6 @@ import com.google.api.services.bigquery.model.Table; import com.google.common.base.MoreObjects; -import java.util.Objects; - /** * Google BigQuery External Table information. BigQuery's external tables are tables whose data * reside outside of BigQuery but can be queried as normal BigQuery tables. External tables are @@ -51,9 +49,15 @@ private Builder(ExternalTableInfo tableInfo) { this.streamingBuffer = tableInfo.streamingBuffer; } - @Override - protected Builder self() { - return this; + protected Builder(Table tablePb) { + super(tablePb); + if (tablePb.getExternalDataConfiguration() != null) { + this.configuration = + ExternalDataConfiguration.fromPb(tablePb.getExternalDataConfiguration()); + } + if (tablePb.getStreamingBuffer() != null) { + this.streamingBuffer = StreamingBuffer.fromPb(tablePb.getStreamingBuffer()); + } } /** @@ -83,7 +87,7 @@ public ExternalTableInfo build() { private ExternalTableInfo(Builder builder) { super(builder); - this.configuration = checkNotNull(builder.configuration); + this.configuration = builder.configuration; this.streamingBuffer = builder.streamingBuffer; } @@ -121,12 +125,6 @@ protected MoreObjects.ToStringHelper toStringHelper() { .add("streamingBuffer", streamingBuffer); } - @Override - public boolean equals(Object obj) { - return obj instanceof ExternalTableInfo - && Objects.equals(toPb(), ((ExternalTableInfo) obj).toPb()); - } - @Override Table toPb() { Table tablePb = super.toPb(); @@ -156,4 +154,9 @@ public static Builder builder(TableId tableId, ExternalDataConfiguration configu public static ExternalTableInfo of(TableId table, ExternalDataConfiguration configuration) { return builder(table, configuration).build(); } + + @SuppressWarnings("unchecked") + static ExternalTableInfo fromPb(Table tablePb) { + return new Builder(tablePb).build(); + } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java index 732444ebdf26..6594a3f25a67 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableInfo.java @@ -19,8 +19,6 @@ import com.google.api.services.bigquery.model.Table; import com.google.common.base.MoreObjects; -import java.util.Objects; - /** * A Google BigQuery Table information. A BigQuery table is a standard, two-dimensional table with * individual records organized in rows, and a data type assigned to each column (also called a @@ -49,9 +47,12 @@ private Builder(TableInfo tableInfo) { this.streamingBuffer = tableInfo.streamingBuffer; } - @Override - protected Builder self() { - return this; + protected Builder(Table tablePb) { + super(tablePb); + this.location = tablePb.getLocation(); + if (tablePb.getStreamingBuffer() != null) { + this.streamingBuffer = StreamingBuffer.fromPb(tablePb.getStreamingBuffer()); + } } Builder location(String location) { @@ -119,7 +120,7 @@ public static BaseTableInfo of(TableId tableId, Schema schema) { } /** - * Returns a builder for the {@code ExternalTableInfo} object. + * Returns a builder for the {@code TableInfo} object. */ @Override public Builder toBuilder() { @@ -133,12 +134,6 @@ protected MoreObjects.ToStringHelper toStringHelper() { .add("streamingBuffer", streamingBuffer); } - @Override - public boolean equals(Object obj) { - return obj instanceof TableInfo - && Objects.equals(toPb(), ((TableInfo) obj).toPb()); - } - @Override Table toPb() { Table tablePb = super.toPb(); @@ -148,4 +143,9 @@ Table toPb() { } return tablePb; } + + @SuppressWarnings("unchecked") + static TableInfo fromPb(Table tablePb) { + return new Builder(tablePb).build(); + } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java index 6a98a446e294..01e07663b363 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ViewInfo.java @@ -25,7 +25,6 @@ import com.google.common.collect.Lists; import java.util.List; -import java.util.Objects; /** * Google BigQuery View Table information. BigQuery's views are logical views, not materialized @@ -54,9 +53,16 @@ private Builder(ViewInfo viewInfo) { this.userDefinedFunctions = viewInfo.userDefinedFunctions; } - @Override - protected Builder self() { - return this; + protected Builder(Table tablePb) { + super(tablePb); + ViewDefinition viewPb = tablePb.getView(); + if (viewPb != null) { + this.query = viewPb.getQuery(); + if (viewPb.getUserDefinedFunctionResources() != null) { + this.userDefinedFunctions = Lists.transform(viewPb.getUserDefinedFunctionResources(), + UserDefinedFunction.FROM_PB_FUNCTION); + } + } } /** @@ -100,7 +106,7 @@ public ViewInfo build() { private ViewInfo(Builder builder) { super(builder); - this.query = checkNotNull(builder.query); + this.query = builder.query; this.userDefinedFunctions = builder.userDefinedFunctions; } @@ -137,16 +143,10 @@ protected MoreObjects.ToStringHelper toStringHelper() { .add("userDefinedFunctions", userDefinedFunctions); } - @Override - public boolean equals(Object obj) { - return obj instanceof ViewInfo && Objects.equals(toPb(), ((ViewInfo) obj).toPb()); - } - @Override Table toPb() { Table tablePb = super.toPb(); - ViewDefinition viewDefinition = new ViewDefinition() - .setQuery(query); + ViewDefinition viewDefinition = new ViewDefinition().setQuery(query); if (userDefinedFunctions != null) { viewDefinition.setUserDefinedFunctionResources(Lists.transform(userDefinedFunctions, UserDefinedFunction.TO_PB_FUNCTION)); @@ -226,4 +226,9 @@ public static ViewInfo of(TableId table, String query, List public static ViewInfo of(TableId table, String query, UserDefinedFunction... functions) { return builder(table, query, functions).build(); } + + @SuppressWarnings("unchecked") + static ViewInfo fromPb(Table tablePb) { + return new Builder(tablePb).build(); + } }