diff --git a/README.md b/README.md index 520bf8b62c55..a544653e90da 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ import com.google.gcloud.bigquery.BigQuery; import com.google.gcloud.bigquery.BigQueryOptions; import com.google.gcloud.bigquery.Field; import com.google.gcloud.bigquery.JobStatus; -import com.google.gcloud.bigquery.LoadJobInfo; +import com.google.gcloud.bigquery.JobInfo; import com.google.gcloud.bigquery.Schema; import com.google.gcloud.bigquery.TableId; import com.google.gcloud.bigquery.TableInfo; @@ -144,7 +144,8 @@ if (info == null) { bigquery.create(TableInfo.of(tableId, Schema.of(integerField))); } else { System.out.println("Loading data into table " + tableId); - LoadJobInfo loadJob = LoadJobInfo.of(tableId, "gs://bucket/path"); + LoadJobConfiguration configuration = LoadJobConfiguration.of(tableId, "gs://bucket/path"); + JobInfo loadJob = JobInfo.of(configuration); loadJob = bigquery.create(loadJob); while (loadJob.status().state() != JobStatus.State.DONE) { Thread.sleep(1000L); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java index 977da0981d2a..8bb30f025c06 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BaseTableInfo.java @@ -394,6 +394,10 @@ protected final boolean baseEquals(BaseTableInfo tableInfo) { return Objects.equals(toPb(), tableInfo.toPb()); } + BaseTableInfo setProjectId(String projectId) { + return toBuilder().tableId(tableId().setProjectId(projectId)).build(); + } + Table toPb() { Table tablePb = new Table(); tablePb.setTableReference(tableId.toPb()); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java index d3c712229348..6bc6a2ebabb5 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQuery.java @@ -371,8 +371,9 @@ public static JobListOption startPageToken(String pageToken) { * is not provided all job's fields are returned. {@code JobOption.fields()} can be used to * specify only the fields of interest. {@link JobInfo#jobId()}, {@link JobStatus#state()}, * {@link JobStatus#error()} as well as type-specific configuration (e.g. - * {@link QueryJobInfo#query()} for Query Jobs) are always returned, even if not specified. - * {@link JobField#SELF_LINK} and {@link JobField#ETAG} can not be selected when listing jobs. + * {@link QueryJobConfiguration#query()} for Query Jobs) are always returned, even if not + * specified. {@link JobField#SELF_LINK} and {@link JobField#ETAG} can not be selected when + * listing jobs. */ public static JobListOption fields(JobField... fields) { String selector = JobField.selector(fields); @@ -397,8 +398,8 @@ private JobOption(BigQueryRpc.Option option, Object value) { * Returns an option to specify the job's fields to be returned by the RPC call. If this option * is not provided all job's fields are returned. {@code JobOption.fields()} can be used to * specify only the fields of interest. {@link JobInfo#jobId()} as well as type-specific - * configuration (e.g. {@link QueryJobInfo#query()} for Query Jobs) are always returned, even if - * not specified. + * configuration (e.g. {@link QueryJobConfiguration#query()} for Query Jobs) are always + * returned, even if not specified. */ public static JobOption fields(JobField... fields) { return new JobOption(BigQueryRpc.Option.FIELDS, JobField.selector(fields)); @@ -470,7 +471,7 @@ public static QueryResultsOption maxWaitTime(long maxWaitTime) { * * @throws BigQueryException upon failure */ - T create(T job, JobOption... options) throws BigQueryException; + JobInfo create(JobInfo job, JobOption... options) throws BigQueryException; /** * Returns the requested dataset or {@code null} if not found. @@ -611,14 +612,14 @@ Page> listTableData(TableId tableId, TableDataListOption... opt * * @throws BigQueryException upon failure */ - T getJob(String jobId, JobOption... options) throws BigQueryException; + JobInfo getJob(String jobId, JobOption... options) throws BigQueryException; /** * Returns the requested job or {@code null} if not found. * * @throws BigQueryException upon failure */ - T getJob(JobId jobId, JobOption... options) throws BigQueryException; + JobInfo getJob(JobId jobId, JobOption... options) throws BigQueryException; /** * Lists the jobs. @@ -665,9 +666,9 @@ Page> listTableData(TableId tableId, TableDataListOption... opt /** * Returns a channel to write data to be inserted into a BigQuery table. Data format and other - * options can be configured using the {@link LoadConfiguration} parameter. + * options can be configured using the {@link WriteChannelConfiguration} parameter. * * @throws BigQueryException upon failure */ - TableDataWriteChannel writer(LoadConfiguration loadConfiguration); + TableDataWriteChannel writer(WriteChannelConfiguration writeChannelConfiguration); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryException.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryException.java index c42bbd96db66..a157afd25db2 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryException.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryException.java @@ -16,8 +16,6 @@ package com.google.gcloud.bigquery; -import com.google.api.client.googleapis.json.GoogleJsonError; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.common.collect.ImmutableSet; import com.google.gcloud.BaseServiceException; import com.google.gcloud.RetryHelper.RetryHelperException; diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java index dde918d1f516..e521228d73bb 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java @@ -25,7 +25,6 @@ import com.google.api.services.bigquery.model.Table; import com.google.api.services.bigquery.model.TableDataInsertAllRequest; import com.google.api.services.bigquery.model.TableDataInsertAllRequest.Rows; -import com.google.api.services.bigquery.model.TableReference; import com.google.api.services.bigquery.model.TableRow; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -159,7 +158,7 @@ public QueryResult nextPage() { @Override public DatasetInfo create(DatasetInfo dataset, DatasetOption... options) throws BigQueryException { - final Dataset datasetPb = setProjectId(dataset).toPb(); + final Dataset datasetPb = dataset.setProjectId(options().projectId()).toPb(); final Map optionsMap = optionMap(options); try { return DatasetInfo.fromPb(runWithRetries(new Callable() { @@ -176,7 +175,7 @@ public Dataset call() { @Override public T create(T table, TableOption... options) throws BigQueryException { - final Table tablePb = setProjectId(table).toPb(); + final Table tablePb = table.setProjectId(options().projectId()).toPb(); final Map optionsMap = optionMap(options); try { return BaseTableInfo.fromPb(runWithRetries(new Callable() { @@ -191,8 +190,8 @@ public Table call() { } @Override - public T create(T job, JobOption... options) throws BigQueryException { - final Job jobPb = setProjectId(job).toPb(); + public JobInfo create(JobInfo job, JobOption... options) throws BigQueryException { + final Job jobPb = job.setProjectId(options().projectId()).toPb(); final Map optionsMap = optionMap(options); try { return JobInfo.fromPb(runWithRetries(new Callable() { @@ -295,7 +294,7 @@ public Boolean call() { @Override public DatasetInfo update(DatasetInfo dataset, DatasetOption... options) throws BigQueryException { - final Dataset datasetPb = setProjectId(dataset).toPb(); + final Dataset datasetPb = dataset.setProjectId(options().projectId()).toPb(); final Map optionsMap = optionMap(options); try { return DatasetInfo.fromPb(runWithRetries(new Callable() { @@ -312,7 +311,7 @@ public Dataset call() { @Override public T update(T table, TableOption... options) throws BigQueryException { - final Table tablePb = setProjectId(table).toPb(); + final Table tablePb = table.setProjectId(options().projectId()).toPb(); final Map optionsMap = optionMap(options); try { return BaseTableInfo.fromPb(runWithRetries(new Callable
() { @@ -442,12 +441,12 @@ public List apply(TableRow rowPb) { } @Override - public T getJob(String jobId, JobOption... options) throws BigQueryException { + public JobInfo getJob(String jobId, JobOption... options) throws BigQueryException { return getJob(JobId.of(jobId), options); } @Override - public T getJob(final JobId jobId, JobOption... options) + public JobInfo getJob(final JobId jobId, JobOption... options) throws BigQueryException { final Map optionsMap = optionMap(options); try { @@ -457,7 +456,7 @@ public Job call() { return bigQueryRpc.getJob(jobId.job(), optionsMap); } }, options().retryParams(), EXCEPTION_HANDLER); - return answer == null ? null : JobInfo.fromPb(answer); + return answer == null ? null : JobInfo.fromPb(answer); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } @@ -508,7 +507,7 @@ public QueryResponse query(final QueryRequest request) throws BigQueryException runWithRetries(new Callable() { @Override public com.google.api.services.bigquery.model.QueryResponse call() { - return bigQueryRpc.query(setProjectId(request).toPb()); + return bigQueryRpc.query(request.setProjectId(options().projectId()).toPb()); } }, options().retryParams(), EXCEPTION_HANDLER); QueryResponse.Builder builder = QueryResponse.builder(); @@ -596,8 +595,9 @@ private static QueryResult.Builder transformQueryResults(JobId jobId, List
optionMap(Option... options) { @@ -608,93 +608,4 @@ public TableDataWriteChannel writer(LoadConfiguration loadConfiguration) { } return optionMap; } - - private DatasetInfo setProjectId(DatasetInfo dataset) { - DatasetInfo.Builder datasetBuilder = dataset.toBuilder(); - datasetBuilder.datasetId(setProjectId(dataset.datasetId())); - if (dataset.acl() != null) { - List acls = Lists.newArrayListWithCapacity(dataset.acl().size()); - for (Acl acl : dataset.acl()) { - if (acl.entity().type() == Acl.Entity.Type.VIEW) { - Dataset.Access accessPb = acl.toPb(); - TableReference viewReferencePb = accessPb.getView(); - if (viewReferencePb.getProjectId() == null) { - viewReferencePb.setProjectId(options().projectId()); - } - acls.add(Acl.of(new Acl.View(TableId.fromPb(viewReferencePb)))); - } else { - acls.add(acl); - } - } - datasetBuilder.acl(acls); - } - return datasetBuilder.build(); - } - - private DatasetId setProjectId(DatasetId dataset) { - return dataset.project() != null ? dataset - : DatasetId.of(options().projectId(), dataset.dataset()); - } - - private BaseTableInfo setProjectId(BaseTableInfo table) { - return table.toBuilder().tableId(setProjectId(table.tableId())).build(); - } - - private TableId setProjectId(TableId table) { - return table.project() != null ? table - : TableId.of(options().projectId(), table.dataset(), table.table()); - } - - private JobInfo setProjectId(JobInfo job) { - if (job instanceof CopyJobInfo) { - CopyJobInfo copyJob = (CopyJobInfo) job; - CopyJobInfo.Builder copyBuilder = copyJob.toBuilder(); - copyBuilder.destinationTable(setProjectId(copyJob.destinationTable())); - copyBuilder.sourceTables( - Lists.transform(copyJob.sourceTables(), new Function() { - @Override - public TableId apply(TableId tableId) { - return setProjectId(tableId); - } - })); - return copyBuilder.build(); - } - if (job instanceof QueryJobInfo) { - QueryJobInfo queryJob = (QueryJobInfo) job; - QueryJobInfo.Builder queryBuilder = queryJob.toBuilder(); - if (queryJob.destinationTable() != null) { - queryBuilder.destinationTable(setProjectId(queryJob.destinationTable())); - } - if (queryJob.defaultDataset() != null) { - queryBuilder.defaultDataset(setProjectId(queryJob.defaultDataset())); - } - return queryBuilder.build(); - } - if (job instanceof ExtractJobInfo) { - ExtractJobInfo extractJob = (ExtractJobInfo) job; - ExtractJobInfo.Builder extractBuilder = extractJob.toBuilder(); - extractBuilder.sourceTable(setProjectId(extractJob.sourceTable())); - return extractBuilder.build(); - } - if (job instanceof LoadJobInfo) { - LoadJobInfo loadJob = (LoadJobInfo) job; - LoadJobInfo.Builder loadBuilder = loadJob.toBuilder(); - return loadBuilder.configuration(setProjectId(loadJob.configuration())).build(); - } - return job; - } - - private QueryRequest setProjectId(QueryRequest request) { - QueryRequest.Builder builder = request.toBuilder(); - if (request.defaultDataset() != null) { - builder.defaultDataset(setProjectId(request.defaultDataset())); - } - return builder.build(); - } - - private LoadConfiguration setProjectId(LoadConfiguration configuration) { - LoadConfiguration.Builder builder = configuration.toBuilder(); - builder.destinationTable(setProjectId(configuration.destinationTable())); - return builder.build(); - } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java similarity index 57% rename from gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobInfo.java rename to gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java index d42c90455e50..1da4fdbe3cdd 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CopyJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,9 +18,8 @@ import static com.google.common.base.Preconditions.checkNotNull; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; import com.google.api.services.bigquery.model.JobConfigurationTableCopy; +import com.google.common.base.Function; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -29,37 +28,41 @@ import java.util.Objects; /** - * Google BigQuery Copy Job. A Copy Job copies an existing table to another new or existing table. + * Google BigQuery copy job configuration. A copy job copies an existing table to another new or + * existing table. Copy job configurations have {@link JobConfiguration.Type#COPY} type. */ -public class CopyJobInfo extends JobInfo { +public final class CopyJobConfiguration extends JobConfiguration { - private static final long serialVersionUID = 7830335512951916299L; + private static final long serialVersionUID = 1140509641399762967L; private final List sourceTables; private final TableId destinationTable; - private final CreateDisposition createDisposition; - private final WriteDisposition writeDisposition; + private final JobInfo.CreateDisposition createDisposition; + private final JobInfo.WriteDisposition writeDisposition; - public static final class Builder extends JobInfo.Builder { + public static final class Builder + extends JobConfiguration.Builder { private List sourceTables; private TableId destinationTable; - private CreateDisposition createDisposition; - private WriteDisposition writeDisposition; + private JobInfo.CreateDisposition createDisposition; + private JobInfo.WriteDisposition writeDisposition; - private Builder() {} + private Builder() { + super(Type.COPY); + } - private Builder(CopyJobInfo jobInfo) { - super(jobInfo); - this.sourceTables = jobInfo.sourceTables; - this.destinationTable = jobInfo.destinationTable; - this.createDisposition = jobInfo.createDisposition; - this.writeDisposition = jobInfo.writeDisposition; + private Builder(CopyJobConfiguration jobConfiguration) { + this(); + this.sourceTables = jobConfiguration.sourceTables; + this.destinationTable = jobConfiguration.destinationTable; + this.createDisposition = jobConfiguration.createDisposition; + this.writeDisposition = jobConfiguration.writeDisposition; } - private Builder(Job jobPb) { - super(jobPb); - JobConfigurationTableCopy copyConfigurationPb = jobPb.getConfiguration().getCopy(); + private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + this(); + JobConfigurationTableCopy copyConfigurationPb = configurationPb.getCopy(); this.destinationTable = TableId.fromPb(copyConfigurationPb.getDestinationTable()); if (copyConfigurationPb.getSourceTables() != null) { this.sourceTables = @@ -69,10 +72,11 @@ private Builder(Job jobPb) { } if (copyConfigurationPb.getCreateDisposition() != null) { this.createDisposition = - CreateDisposition.valueOf(copyConfigurationPb.getCreateDisposition()); + JobInfo.CreateDisposition.valueOf(copyConfigurationPb.getCreateDisposition()); } if (copyConfigurationPb.getWriteDisposition() != null) { - this.writeDisposition = WriteDisposition.valueOf(copyConfigurationPb.getWriteDisposition()); + this.writeDisposition = JobInfo.WriteDisposition.valueOf( + copyConfigurationPb.getWriteDisposition()); } } @@ -81,7 +85,7 @@ private Builder(Job jobPb) { */ public Builder sourceTables(List sourceTables) { this.sourceTables = sourceTables != null ? ImmutableList.copyOf(sourceTables) : null; - return self(); + return this; } /** @@ -89,7 +93,7 @@ public Builder sourceTables(List sourceTables) { */ public Builder destinationTable(TableId destinationTable) { this.destinationTable = destinationTable; - return self(); + return this; } /** @@ -98,9 +102,9 @@ public Builder destinationTable(TableId destinationTable) { * @see * Create Disposition */ - public Builder createDisposition(CreateDisposition createDisposition) { + public Builder createDisposition(JobInfo.CreateDisposition createDisposition) { this.createDisposition = createDisposition; - return self(); + return this; } /** @@ -109,18 +113,17 @@ public Builder createDisposition(CreateDisposition createDisposition) { * @see * Write Disposition */ - public Builder writeDisposition(WriteDisposition writeDisposition) { + public Builder writeDisposition(JobInfo.WriteDisposition writeDisposition) { this.writeDisposition = writeDisposition; - return self(); + return this; } - @Override - public CopyJobInfo build() { - return new CopyJobInfo(this); + public CopyJobConfiguration build() { + return new CopyJobConfiguration(this); } } - private CopyJobInfo(Builder builder) { + private CopyJobConfiguration(Builder builder) { super(builder); this.sourceTables = checkNotNull(builder.sourceTables); this.destinationTable = checkNotNull(builder.destinationTable); @@ -148,7 +151,7 @@ public TableId destinationTable() { * @see * Create Disposition */ - public CreateDisposition createDisposition() { + public JobInfo.CreateDisposition createDisposition() { return this.createDisposition; } @@ -158,7 +161,7 @@ public CreateDisposition createDisposition() { * @see * Write Disposition */ - public WriteDisposition writeDisposition() { + public JobInfo.WriteDisposition writeDisposition() { return writeDisposition; } @@ -178,7 +181,7 @@ ToStringHelper toStringHelper() { @Override public boolean equals(Object obj) { - return obj instanceof CopyJobInfo && baseEquals((CopyJobInfo) obj); + return obj instanceof CopyJobConfiguration && baseEquals((CopyJobConfiguration) obj); } @Override @@ -188,71 +191,67 @@ public int hashCode() { } @Override - Job toPb() { - JobConfigurationTableCopy copyConfigurationPb = new JobConfigurationTableCopy(); - copyConfigurationPb.setDestinationTable(destinationTable.toPb()); + CopyJobConfiguration setProjectId(final String projectId) { + Builder builder = toBuilder(); + builder.sourceTables( + Lists.transform(sourceTables(), new Function() { + @Override + public TableId apply(TableId tableId) { + return tableId.setProjectId(projectId); + } + })); + builder.destinationTable(destinationTable().setProjectId(projectId)); + return builder.build(); + } + + com.google.api.services.bigquery.model.JobConfiguration toPb() { + JobConfigurationTableCopy configurationPb = new JobConfigurationTableCopy(); + configurationPb.setDestinationTable(destinationTable.toPb()); if (sourceTables.size() == 1) { - copyConfigurationPb.setSourceTable(sourceTables.get(0).toPb()); + configurationPb.setSourceTable(sourceTables.get(0).toPb()); } else { - copyConfigurationPb.setSourceTables(Lists.transform(sourceTables, TableId.TO_PB_FUNCTION)); + configurationPb.setSourceTables(Lists.transform(sourceTables, TableId.TO_PB_FUNCTION)); } if (createDisposition != null) { - copyConfigurationPb.setCreateDisposition(createDisposition.toString()); + configurationPb.setCreateDisposition(createDisposition.toString()); } if (writeDisposition != null) { - copyConfigurationPb.setWriteDisposition(writeDisposition.toString()); + configurationPb.setWriteDisposition(writeDisposition.toString()); } - return super.toPb().setConfiguration(new JobConfiguration().setCopy(copyConfigurationPb)); + return new com.google.api.services.bigquery.model.JobConfiguration().setCopy(configurationPb); } /** - * Creates a builder for a BigQuery Copy Job given destination and source table. + * Creates a builder for a BigQuery Copy Job configuration given destination and source table. */ public static Builder builder(TableId destinationTable, TableId sourceTable) { return builder(destinationTable, ImmutableList.of(checkNotNull(sourceTable))); } /** - * Creates a builder for a BigQuery Copy Job given destination and source tables. + * Creates a builder for a BigQuery Copy Job configuration given destination and source tables. */ public static Builder builder(TableId destinationTable, List sourceTables) { return new Builder().destinationTable(destinationTable).sourceTables(sourceTables); } /** - * Returns a BigQuery Copy Job for the given destination and source table. Job's id is chosen by - * the service. + * Returns a BigQuery Copy Job configuration for the given destination and source table. */ - public static CopyJobInfo of(TableId destinationTable, TableId sourceTable) { + public static CopyJobConfiguration of(TableId destinationTable, TableId sourceTable) { return builder(destinationTable, sourceTable).build(); } /** - * Returns a BigQuery Copy Job for the given destination and source tables. Job's id is chosen by - * the service. + * Returns a BigQuery Copy Job configuration for the given destination and source tables. */ - public static CopyJobInfo of(TableId destinationTable, List sourceTables) { + public static CopyJobConfiguration of(TableId destinationTable, List sourceTables) { return builder(destinationTable, sourceTables).build(); } - /** - * Returns a BigQuery Copy Job for the given destination and source table. Job's id is set to the - * provided value. - */ - public static CopyJobInfo of(JobId jobId, TableId destinationTable, TableId sourceTable) { - return builder(destinationTable, sourceTable).jobId(jobId).build(); - } - - /** - * Returns a BigQuery Copy Job for the given destination and source tables. Job's id is set to the - * provided value. - */ - public static CopyJobInfo of(JobId jobId, TableId destinationTable, List sourceTables) { - return builder(destinationTable, sourceTables).jobId(jobId).build(); - } - @SuppressWarnings("unchecked") - static CopyJobInfo fromPb(Job jobPb) { + static CopyJobConfiguration fromPb( + com.google.api.services.bigquery.model.JobConfiguration jobPb) { return new Builder(jobPb).build(); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetId.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetId.java index 942322ea51d3..006c089f8d63 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetId.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetId.java @@ -81,6 +81,10 @@ public String toString() { return toPb().toString(); } + DatasetId setProjectId(String projectId) { + return project() != null ? this : DatasetId.of(projectId, dataset()); + } + DatasetReference toPb() { return new DatasetReference().setProjectId(project).setDatasetId(dataset); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetInfo.java index 95897ba3a801..c6330308c8ce 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/DatasetInfo.java @@ -21,6 +21,7 @@ import com.google.api.client.util.Data; import com.google.api.services.bigquery.model.Dataset; +import com.google.api.services.bigquery.model.TableReference; import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; @@ -333,6 +334,28 @@ public boolean equals(Object obj) { return obj instanceof DatasetInfo && Objects.equals(toPb(), ((DatasetInfo) obj).toPb()); } + DatasetInfo setProjectId(String projectId) { + Builder builder = toBuilder(); + builder.datasetId(datasetId().setProjectId(projectId)); + if (acl() != null) { + List acls = Lists.newArrayListWithCapacity(acl().size()); + for (Acl acl : acl()) { + if (acl.entity().type() == Acl.Entity.Type.VIEW) { + Dataset.Access accessPb = acl.toPb(); + TableReference viewReferencePb = accessPb.getView(); + if (viewReferencePb.getProjectId() == null) { + viewReferencePb.setProjectId(projectId); + } + acls.add(Acl.of(new Acl.View(TableId.fromPb(viewReferencePb)))); + } else { + acls.add(acl); + } + } + builder.acl(acls); + } + return builder.build(); + } + Dataset toPb() { Dataset datasetPb = new Dataset(); datasetPb.setDatasetReference(datasetId.toPb()); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java similarity index 63% rename from gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobInfo.java rename to gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java index 67f643ce4e19..d8e57bd17254 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExtractJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,23 +18,21 @@ import static com.google.common.base.Preconditions.checkNotNull; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; import com.google.api.services.bigquery.model.JobConfigurationExtract; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; -import com.google.gcloud.bigquery.JobStatistics.ExtractStatistics; import java.util.List; import java.util.Objects; /** - * Google BigQuery Extract Jobs. An Extract Job exports a BigQuery table to Google Cloud Storage. - * The extract destination provided as URIs that point to objects in Google Cloud Storage. + * Google BigQuery extract job configuration. An extract job exports a BigQuery table to Google + * Cloud Storage. The extract destination provided as URIs that point to objects in Google Cloud + * Storage. Extract job configurations have {@link JobConfiguration.Type#EXTRACT} type. */ -public class ExtractJobInfo extends JobInfo { +public final class ExtractJobConfiguration extends JobConfiguration { - private static final long serialVersionUID = -9126951217071361576L; + private static final long serialVersionUID = 4147749733166593761L; private final TableId sourceTable; private final List destinationUris; @@ -43,8 +41,8 @@ public class ExtractJobInfo extends JobInfo { private final String format; private final String compression; - public static final class Builder extends JobInfo.Builder { + public static final class Builder + extends JobConfiguration.Builder { private TableId sourceTable; private List destinationUris; @@ -53,10 +51,12 @@ public static final class Builder extends JobInfo.Builder destinationUris) { this.destinationUris = destinationUris != null ? ImmutableList.copyOf(destinationUris) : null; - return self(); + return this; } /** @@ -98,7 +98,7 @@ public Builder destinationUris(List destinationUris) { */ public Builder printHeader(Boolean printHeader) { this.printHeader = printHeader; - return self(); + return this; } /** @@ -106,7 +106,7 @@ public Builder printHeader(Boolean printHeader) { */ public Builder fieldDelimiter(String fieldDelimiter) { this.fieldDelimiter = fieldDelimiter; - return self(); + return this; } /** @@ -117,7 +117,7 @@ public Builder fieldDelimiter(String fieldDelimiter) { */ public Builder format(String format) { this.format = format; - return self(); + return this; } /** @@ -129,16 +129,15 @@ public Builder format(String format) { */ public Builder compression(String compression) { this.compression = compression; - return self(); + return this; } - @Override - public ExtractJobInfo build() { - return new ExtractJobInfo(this); + public ExtractJobConfiguration build() { + return new ExtractJobConfiguration(this); } } - private ExtractJobInfo(Builder builder) { + private ExtractJobConfiguration(Builder builder) { super(builder); this.sourceTable = checkNotNull(builder.sourceTable); this.destinationUris = checkNotNull(builder.destinationUris); @@ -212,7 +211,7 @@ ToStringHelper toStringHelper() { @Override public boolean equals(Object obj) { - return obj instanceof ExtractJobInfo && baseEquals((ExtractJobInfo) obj); + return obj instanceof ExtractJobConfiguration && baseEquals((ExtractJobConfiguration) obj); } @Override @@ -222,7 +221,11 @@ public int hashCode() { } @Override - Job toPb() { + ExtractJobConfiguration setProjectId(String projectId) { + return toBuilder().sourceTable(sourceTable().setProjectId(projectId)).build(); + } + + com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); @@ -230,92 +233,61 @@ Job toPb() { extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); - return super.toPb().setConfiguration(new JobConfiguration().setExtract(extractConfigurationPb)); + return new com.google.api.services.bigquery.model.JobConfiguration() + .setExtract(extractConfigurationPb); } /** - * Creates a builder for a BigQuery Extract Job given source table and destination URI. + * Creates a builder for a BigQuery Extract Job configuration given source table and destination + * URI. */ public static Builder builder(TableId sourceTable, String destinationUri) { return builder(sourceTable, ImmutableList.of(checkNotNull(destinationUri))); } /** - * Creates a builder for a BigQuery Extract Job given source table and destination URIs. + * Creates a builder for a BigQuery Extract Job configuration given source table and destination + * URIs. */ public static Builder builder(TableId sourceTable, List destinationUris) { return new Builder().sourceTable(sourceTable).destinationUris(destinationUris); } /** - * Returns a BigQuery Extract Job for the given source table and destination URI. Job's id is - * chosen by the service. + * Returns a BigQuery Extract Job configuration for the given source table and destination URI. */ - public static ExtractJobInfo of(TableId sourceTable, String destinationUri) { + public static ExtractJobConfiguration of(TableId sourceTable, String destinationUri) { return builder(sourceTable, destinationUri).build(); } /** - * Returns a BigQuery Extract Job for the given source table and destination URIs. Job's id is - * chosen by the service. + * Returns a BigQuery Extract Job configuration for the given source table and destination URIs. */ - public static ExtractJobInfo of(TableId sourceTable, List destinationUris) { + public static ExtractJobConfiguration of(TableId sourceTable, List destinationUris) { return builder(sourceTable, destinationUris).build(); } /** - * Returns a BigQuery Extract Job for the given source table, format and destination URI. Job's id - * is chosen by the service. + * Returns a BigQuery Extract Job configuration for the given source table, format and destination + * URI. */ - public static ExtractJobInfo of(TableId sourceTable, String format, String destinationUri) { + public static ExtractJobConfiguration of(TableId sourceTable, String destinationUri, + String format) { return builder(sourceTable, destinationUri).format(format).build(); } /** - * Returns a BigQuery Extract Job for the given source table, format and destination URIs. Job's - * id is chosen by the service. + * Returns a BigQuery Extract Job configuration for the given source table, format and destination + * URIs. */ - public static ExtractJobInfo of(TableId sourceTable, String format, - List destinationUris) { + public static ExtractJobConfiguration of(TableId sourceTable, List destinationUris, + String format) { return builder(sourceTable, destinationUris).format(format).build(); } - /** - * Returns a BigQuery Extract Job for the given source table and destination URI. Job's id is set - * to the provided value. - */ - public static ExtractJobInfo of(JobId jobId, TableId sourceTable, String destinationUri) { - return builder(sourceTable, destinationUri).jobId(jobId).build(); - } - - /** - * Returns a BigQuery Extract Job for the given source table and destination URIs. Job's id is set - * to the provided value. - */ - public static ExtractJobInfo of(JobId jobId, TableId sourceTable, List destinationUris) { - return builder(sourceTable, destinationUris).jobId(jobId).build(); - } - - /** - * Returns a BigQuery Extract Job for the given source table, format and destination URI. Job's id - * is set to the provided value. - */ - public static ExtractJobInfo of(JobId jobId, TableId sourceTable, String format, - String destinationUri) { - return builder(sourceTable, destinationUri).format(format).jobId(jobId).build(); - } - - /** - * Returns a BigQuery Extract Job for the given source table, format and destination URIs. Job's - * id is set to the provided value. - */ - public static ExtractJobInfo of(JobId jobId, TableId sourceTable, String format, - List destinationUris) { - return builder(sourceTable, destinationUris).format(format).jobId(jobId).build(); - } - @SuppressWarnings("unchecked") - static ExtractJobInfo fromPb(Job jobPb) { - return new Builder(jobPb).build(); + static ExtractJobConfiguration fromPb( + com.google.api.services.bigquery.model.JobConfiguration confPb) { + return new Builder(confPb).build(); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java new file mode 100644 index 000000000000..2244969567ef --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobConfiguration.java @@ -0,0 +1,145 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Base class for a BigQuery job configuration. + */ +public abstract class JobConfiguration implements Serializable { + + private static final long serialVersionUID = -548132177415406526L; + + private final Type type; + + /** + * Type of a BigQuery Job. + */ + enum Type { + /** + * A Copy Job copies an existing table to another new or existing table. Instances of + * {@code JobConfiguration} for this type are implemented by {@link CopyJobConfiguration}. + */ + COPY, + /** + * An Extract Job exports a BigQuery table to Google Cloud Storage. Instances of + * {@code JobConfiguration} for this type are implemented by {@link ExtractJobConfiguration}. + */ + EXTRACT, + /** + * A Load Job loads data from one of several formats into a table. Instances of + * {@code JobConfiguration} for this type are implemented by {@link LoadJobConfiguration}. + */ + LOAD, + /** + * A Query Job runs a query against BigQuery data. Instances of + * {@code JobConfiguration} for this type are implemented by {@link QueryJobConfiguration}. + */ + QUERY + } + + /** + * Base builder for job configurations. + * + * @param the job configuration type + * @param the job configuration builder + */ + public abstract static class Builder> { + + private Type type; + + Builder(Type type) { + this.type = checkNotNull(type); + } + + @SuppressWarnings("unchecked") + B self() { + return (B) this; + } + + B type(Type type) { + this.type = checkNotNull(type); + return self(); + } + + /** + * Creates an object. + */ + public abstract T build(); + } + + JobConfiguration(Builder builder) { + this.type = builder.type; + } + + /** + * Returns the type of the job configuration. + */ + public Type type() { + return type; + } + + /** + * Returns a builder for the object. + */ + public abstract Builder toBuilder(); + + ToStringHelper toStringHelper() { + return MoreObjects.toStringHelper(this).add("type", type); + } + + @Override + public String toString() { + return toStringHelper().toString(); + } + + final int baseHashCode() { + return Objects.hash(type); + } + + final boolean baseEquals(JobConfiguration jobConfiguration) { + return Objects.equals(toPb(), jobConfiguration.toPb()); + } + + abstract JobConfiguration setProjectId(String projectId); + + abstract com.google.api.services.bigquery.model.JobConfiguration toPb(); + + @SuppressWarnings("unchecked") + static T fromPb( + com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + if (configurationPb.getCopy() != null) { + return (T) CopyJobConfiguration.fromPb(configurationPb); + } else if (configurationPb.getExtract() != null) { + return (T) ExtractJobConfiguration.fromPb(configurationPb); + } else if (configurationPb.getLoad() != null) { + return (T) LoadJobConfiguration.fromPb(configurationPb); + } else if (configurationPb.getQuery() != null) { + return (T) QueryJobConfiguration.fromPb(configurationPb); + } else { + // never reached + throw new IllegalArgumentException("Job configuration is not supported"); + } + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java index e623aec26dd2..47135b6d97d0 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobInfo.java @@ -19,23 +19,20 @@ import com.google.api.services.bigquery.model.Job; import com.google.common.base.Function; import com.google.common.base.MoreObjects; -import com.google.common.base.MoreObjects.ToStringHelper; import java.io.Serializable; import java.util.Objects; /** - * Base class for Google BigQuery Job information. Jobs are objects that manage asynchronous tasks - * such as running queries, loading data, and exporting data. Use {@link CopyJobInfo} for a job that - * copies an existing table. Use {@link ExtractJobInfo} for a job that exports a table to Google - * Cloud Storage. Use {@link LoadJobInfo} for a job that loads data from Google Cloud Storage into - * a table. Use {@link QueryJobInfo} for a job that runs a query. + * Google BigQuery Job information. Jobs are objects that manage asynchronous tasks such as running + * queries, loading data, and exporting data. Use {@link CopyJobConfiguration} for a job that + * copies an existing table. Use {@link ExtractJobConfiguration} for a job that exports a table to + * Google Cloud Storage. Use {@link LoadJobConfiguration} for a job that loads data from Google + * Cloud Storage into a table. Use {@link QueryJobConfiguration} for a job that runs a query. * * @see Jobs - * - * @param the statistics type */ -public abstract class JobInfo implements Serializable { +public final class JobInfo implements Serializable { static final Function FROM_PB_FUNCTION = new Function() { @@ -44,7 +41,7 @@ public JobInfo apply(Job pb) { return JobInfo.fromPb(pb); } }; - private static final long serialVersionUID = -7086529810736715842L; + private static final long serialVersionUID = -3272941007234620265L; /** * Specifies whether the job is allowed to create new tables. @@ -86,30 +83,24 @@ public enum WriteDisposition { private final JobId jobId; private final String selfLink; private final JobStatus status; - private final S statistics; + private final JobStatistics statistics; private final String userEmail; + private final JobConfiguration configuration; - /** - * Base builder for jobs. - * - * @param the job type - * @param the job statistics type - * @param the job builder - */ - public abstract static class Builder> { + public static final class Builder { private String etag; private String id; private JobId jobId; private String selfLink; private JobStatus status; - private S statistics; + private JobStatistics statistics; private String userEmail; + private JobConfiguration configuration; - protected Builder() {} + private Builder() {} - protected Builder(JobInfo jobInfo) { + private Builder(JobInfo jobInfo) { this.etag = jobInfo.etag; this.id = jobInfo.id; this.jobId = jobInfo.jobId; @@ -117,7 +108,7 @@ protected Builder(JobInfo jobInfo) { this.status = jobInfo.status; this.statistics = jobInfo.statistics; this.userEmail = jobInfo.userEmail; - + this.configuration = jobInfo.configuration; } protected Builder(Job jobPb) { @@ -134,55 +125,58 @@ protected Builder(Job jobPb) { this.statistics = JobStatistics.fromPb(jobPb.getStatistics()); } this.userEmail = jobPb.getUserEmail(); + this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); } - @SuppressWarnings("unchecked") - protected B self() { - return (B) this; - } - - B etag(String etag) { + Builder etag(String etag) { this.etag = etag; - return self(); + return this; } - B id(String id) { + Builder id(String id) { this.id = id; - return self(); + return this; } /** * Sets the job identity. */ - public B jobId(JobId jobId) { + public Builder jobId(JobId jobId) { this.jobId = jobId; - return self(); + return this; } - B selfLink(String selfLink) { + Builder selfLink(String selfLink) { this.selfLink = selfLink; - return self(); + return this; } - B status(JobStatus status) { + Builder status(JobStatus status) { this.status = status; - return self(); + return this; } - B statistics(S statistics) { + Builder statistics(JobStatistics statistics) { this.statistics = statistics; - return self(); + return this; } - B userEmail(String userEmail) { + Builder userEmail(String userEmail) { this.userEmail = userEmail; - return self(); + return this; + } + + public Builder configuration(JobConfiguration configuration) { + this.configuration = configuration; + return this; } - public abstract T build(); + public JobInfo build() { + return new JobInfo(this); + } } - protected JobInfo(Builder builder) { + private JobInfo(Builder builder) { this.jobId = builder.jobId; this.etag = builder.etag; this.id = builder.id; @@ -190,6 +184,7 @@ protected JobInfo(Builder builder) { this.status = builder.status; this.statistics = builder.statistics; this.userEmail = builder.userEmail; + this.configuration = builder.configuration; } /** @@ -232,8 +227,9 @@ public JobStatus status() { /** * Returns information about the job, including starting time and ending time of the job. */ - public S statistics() { - return statistics; + @SuppressWarnings("unchecked") + public S statistics() { + return (S) statistics; } /** @@ -243,12 +239,23 @@ public String userEmail() { return userEmail; } + /** + * Returns the job's configuration. + */ + @SuppressWarnings("unchecked") + public C configuration() { + return (C) configuration; + } + /** * Returns a builder for the job. */ - public abstract Builder toBuilder(); + public Builder toBuilder() { + return new Builder(this); + } - ToStringHelper toStringHelper() { + @Override + public String toString() { return MoreObjects.toStringHelper(this) .add("job", jobId) .add("status", status) @@ -256,20 +263,23 @@ ToStringHelper toStringHelper() { .add("userEmail", userEmail) .add("etag", etag) .add("id", id) - .add("selfLink", selfLink); + .add("selfLink", selfLink) + .add("configuration", configuration) + .toString(); } @Override - public String toString() { - return toStringHelper().toString(); + public int hashCode() { + return Objects.hash(jobId); } - protected final int baseHashCode() { - return Objects.hash(jobId); + @Override + public boolean equals(Object obj) { + return obj instanceof JobInfo && Objects.equals(toPb(), ((JobInfo) obj).toPb()); } - protected final boolean baseEquals(JobInfo jobInfo) { - return Objects.equals(toPb(), jobInfo.toPb()); + JobInfo setProjectId(String projectId) { + return toBuilder().configuration(configuration.setProjectId(projectId)).build(); } Job toPb() { @@ -287,22 +297,23 @@ Job toPb() { if (statistics != null) { jobPb.setStatistics(statistics.toPb()); } + jobPb.setConfiguration(configuration.toPb()); return jobPb; } - @SuppressWarnings("unchecked") - static T fromPb(Job jobPb) { - if (jobPb.getConfiguration().getLoad() != null) { - return (T) LoadJobInfo.fromPb(jobPb); - } else if (jobPb.getConfiguration().getCopy() != null) { - return (T) CopyJobInfo.fromPb(jobPb); - } else if (jobPb.getConfiguration().getExtract() != null) { - return (T) ExtractJobInfo.fromPb(jobPb); - } else if (jobPb.getConfiguration().getQuery() != null) { - return (T) QueryJobInfo.fromPb(jobPb); - } else { - // never reached - throw new IllegalArgumentException("Job configuration is not supported"); - } + public static Builder builder(JobConfiguration configuration) { + return new Builder().configuration(configuration); + } + + public static JobInfo of(JobConfiguration configuration) { + return builder(configuration).build(); + } + + public static JobInfo of(JobId jobId, JobConfiguration configuration) { + return builder(configuration).jobId(jobId).build(); + } + + static JobInfo fromPb(Job jobPb) { + return new Builder(jobPb).build(); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobStatistics.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobStatistics.java index b2d50882aabb..34e4917921ba 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobStatistics.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/JobStatistics.java @@ -61,7 +61,7 @@ private ExtractStatistics(Builder builder) { /** * Returns the number of files per destination URI or URI pattern specified in the extract job. * These values will be in the same order as the URIs specified by - * {@link ExtractJobInfo#destinationUris()}. + * {@link ExtractJobConfiguration#destinationUris()}. */ public List destinationUriFileCounts() { return destinationUriFileCounts; diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java index 18cb8ae6bedb..223a25a478e0 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadConfiguration.java @@ -16,103 +16,26 @@ package com.google.gcloud.bigquery; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; import com.google.gcloud.bigquery.JobInfo.CreateDisposition; import com.google.gcloud.bigquery.JobInfo.WriteDisposition; -import java.io.Serializable; -import java.nio.channels.SeekableByteChannel; import java.util.List; -import java.util.Objects; /** - * Google BigQuery Configuration for a load operation. A load configuration can be used to build a - * {@link LoadJobInfo} or to load data into a table with a {@link com.google.gcloud.WriteChannel} - * ({@link BigQuery#writer(LoadConfiguration)}). + * Common interface for a load configuration. A load configuration + * ({@link WriteChannelConfiguration}) can be used to load data into a table with a + * {@link com.google.gcloud.WriteChannel} ({@link BigQuery#writer(WriteChannelConfiguration)}). + * A load configuration ({@link LoadJobConfiguration}) can also be used to create a load job + * ({@link JobInfo#of(JobConfiguration)}). */ -public class LoadConfiguration implements Serializable { - - private static final long serialVersionUID = 470267591917413578L; - - private final TableId destinationTable; - private final CreateDisposition createDisposition; - private final WriteDisposition writeDisposition; - private final FormatOptions formatOptions; - private final Integer maxBadRecords; - private final Schema schema; - private final Boolean ignoreUnknownValues; - private final List projectionFields; - - public static final class Builder { - - private TableId destinationTable; - private CreateDisposition createDisposition; - private WriteDisposition writeDisposition; - private FormatOptions formatOptions; - private Integer maxBadRecords; - private Schema schema; - private Boolean ignoreUnknownValues; - private List projectionFields; - - private Builder() {} +public interface LoadConfiguration { - private Builder(LoadConfiguration loadConfiguration) { - this.destinationTable = loadConfiguration.destinationTable; - this.createDisposition = loadConfiguration.createDisposition; - this.writeDisposition = loadConfiguration.writeDisposition; - this.formatOptions = loadConfiguration.formatOptions; - this.maxBadRecords = loadConfiguration.maxBadRecords; - this.schema = loadConfiguration.schema; - this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues; - this.projectionFields = loadConfiguration.projectionFields; - } - - private Builder(JobConfigurationLoad loadConfigurationPb) { - this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); - if (loadConfigurationPb.getCreateDisposition() != null) { - this.createDisposition = - CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); - } - if (loadConfigurationPb.getWriteDisposition() != null) { - this.writeDisposition = WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); - } - if (loadConfigurationPb.getSourceFormat() != null) { - this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); - } - if (loadConfigurationPb.getAllowJaggedRows() != null - || loadConfigurationPb.getAllowQuotedNewlines() != null - || loadConfigurationPb.getEncoding() != null - || loadConfigurationPb.getFieldDelimiter() != null - || loadConfigurationPb.getQuote() != null - || loadConfigurationPb.getSkipLeadingRows() != null) { - CsvOptions.Builder builder = CsvOptions.builder() - .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) - .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) - .encoding(loadConfigurationPb.getEncoding()) - .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) - .quote(loadConfigurationPb.getQuote()) - .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); - this.formatOptions = builder.build(); - } - this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); - if (loadConfigurationPb.getSchema() != null) { - this.schema = Schema.fromPb(loadConfigurationPb.getSchema()); - } - this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues(); - this.projectionFields = loadConfigurationPb.getProjectionFields(); - } + interface Builder { /** * Sets the destination table to load the data into. */ - public Builder destinationTable(TableId destinationTable) { - this.destinationTable = destinationTable; - return this; - } + Builder destinationTable(TableId destinationTable); /** * Sets whether the job is allowed to create new tables. @@ -120,10 +43,7 @@ public Builder destinationTable(TableId destinationTable) { * @see * Create Disposition */ - public Builder createDisposition(CreateDisposition createDisposition) { - this.createDisposition = createDisposition; - return this; - } + Builder createDisposition(CreateDisposition createDisposition); /** * Sets the action that should occur if the destination table already exists. @@ -131,10 +51,7 @@ public Builder createDisposition(CreateDisposition createDisposition) { * @see * Write Disposition */ - public Builder writeDisposition(WriteDisposition writeDisposition) { - this.writeDisposition = writeDisposition; - return this; - } + Builder writeDisposition(WriteDisposition writeDisposition); /** * Sets the source format, and possibly some parsing options, of the external data. Supported @@ -144,30 +61,21 @@ public Builder writeDisposition(WriteDisposition writeDisposition) { * * Source Format */ - public Builder formatOptions(FormatOptions formatOptions) { - this.formatOptions = formatOptions; - return this; - } + Builder formatOptions(FormatOptions formatOptions); /** * Sets the maximum number of bad records that BigQuery can ignore when running the job. If the * number of bad records exceeds this value, an invalid error is returned in the job result. * By default no bad record is ignored. */ - public Builder maxBadRecords(Integer maxBadRecords) { - this.maxBadRecords = maxBadRecords; - return this; - } + Builder maxBadRecords(Integer maxBadRecords); /** * Sets the schema for the destination table. The schema can be omitted if the destination table * already exists, or if you're loading data from a Google Cloud Datastore backup (i.e. * {@code DATASTORE_BACKUP} format option). */ - public Builder schema(Schema schema) { - this.schema = schema; - return this; - } + Builder schema(Schema schema); /** * Sets whether BigQuery should allow extra values that are not represented in the table schema. @@ -175,10 +83,7 @@ public Builder schema(Schema schema) { * are treated as bad records, and if there are too many bad records, an invalid error is * returned in the job result. By default unknown values are not allowed. */ - public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { - this.ignoreUnknownValues = ignoreUnknownValues; - return this; - } + Builder ignoreUnknownValues(Boolean ignoreUnknownValues); /** * Sets which entity properties to load into BigQuery from a Cloud Datastore backup. This field @@ -187,34 +92,15 @@ public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { * all properties. If any named property isn't found in the Cloud Datastore backup, an invalid * error is returned in the job result. */ - public Builder projectionFields(List projectionFields) { - this.projectionFields = - projectionFields != null ? ImmutableList.copyOf(projectionFields) : null; - return this; - } - - public LoadConfiguration build() { - return new LoadConfiguration(this); - } - } + Builder projectionFields(List projectionFields); - private LoadConfiguration(Builder builder) { - this.destinationTable = checkNotNull(builder.destinationTable); - this.createDisposition = builder.createDisposition; - this.writeDisposition = builder.writeDisposition; - this.formatOptions = builder.formatOptions; - this.maxBadRecords = builder.maxBadRecords; - this.schema = builder.schema; - this.ignoreUnknownValues = builder.ignoreUnknownValues; - this.projectionFields = builder.projectionFields; + LoadConfiguration build(); } /** * Returns the destination table to load the data into. */ - public TableId destinationTable() { - return destinationTable; - } + TableId destinationTable(); /** * Returns whether the job is allowed to create new tables. @@ -222,9 +108,7 @@ public TableId destinationTable() { * @see * Create Disposition */ - public CreateDisposition createDisposition() { - return this.createDisposition; - } + CreateDisposition createDisposition(); /** * Returns the action that should occur if the destination table already exists. @@ -232,40 +116,30 @@ public CreateDisposition createDisposition() { * @see * Write Disposition */ - public WriteDisposition writeDisposition() { - return writeDisposition; - } + WriteDisposition writeDisposition(); /** * Returns additional properties used to parse CSV data (used when {@link #format()} is set * to CSV). Returns {@code null} if not set. */ - public CsvOptions csvOptions() { - return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; - } + CsvOptions csvOptions(); /** * Returns the maximum number of bad records that BigQuery can ignore when running the job. If the * number of bad records exceeds this value, an invalid error is returned in the job result. * By default no bad record is ignored. */ - public Integer maxBadRecords() { - return maxBadRecords; - } + Integer maxBadRecords(); /** * Returns the schema for the destination table, if set. Returns {@code null} otherwise. */ - public Schema schema() { - return schema; - } + Schema schema(); /** * Returns the format of the data files. */ - public String format() { - return formatOptions != null ? formatOptions.type() : null; - } + String format(); /** * Returns whether BigQuery should allow extra values that are not represented in the table @@ -273,9 +147,7 @@ public String format() { * columns are treated as bad records, and if there are too many bad records, an invalid error is * returned in the job result. By default unknown values are not allowed. */ - public Boolean ignoreUnknownValues() { - return ignoreUnknownValues; - } + Boolean ignoreUnknownValues(); /** * Returns which entity properties to load into BigQuery from a Cloud Datastore backup. This field @@ -284,99 +156,10 @@ public Boolean ignoreUnknownValues() { * all properties. If any named property isn't found in the Cloud Datastore backup, an invalid * error is returned in the job result. */ - public List projectionFields() { - return projectionFields; - } - - public Builder toBuilder() { - return new Builder(this); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("destinationTable", destinationTable) - .add("createDisposition", createDisposition) - .add("writeDisposition", writeDisposition) - .add("formatOptions", formatOptions) - .add("maxBadRecords", maxBadRecords) - .add("schema", schema) - .add("ignoreUnknownValue", ignoreUnknownValues) - .add("projectionFields", projectionFields) - .toString(); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof LoadConfiguration - && Objects.equals(toPb(), ((LoadConfiguration) obj).toPb()); - } - - @Override - public int hashCode() { - return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions, - maxBadRecords, schema, ignoreUnknownValues, projectionFields); - } - - JobConfigurationLoad toPb() { - JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); - loadConfigurationPb.setDestinationTable(destinationTable.toPb()); - if (createDisposition != null) { - loadConfigurationPb.setCreateDisposition(createDisposition.toString()); - } - if (writeDisposition != null) { - loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); - } - if (csvOptions() != null) { - CsvOptions csvOptions = csvOptions(); - loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) - .setAllowJaggedRows(csvOptions.allowJaggedRows()) - .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) - .setEncoding(csvOptions.encoding()) - .setQuote(csvOptions.quote()) - .setSkipLeadingRows(csvOptions.skipLeadingRows()); - } - if (schema != null) { - loadConfigurationPb.setSchema(schema.toPb()); - } - if (formatOptions != null) { - loadConfigurationPb.setSourceFormat(formatOptions.type()); - } - loadConfigurationPb.setMaxBadRecords(maxBadRecords); - loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); - loadConfigurationPb.setProjectionFields(projectionFields); - return loadConfigurationPb; - } - - static LoadConfiguration fromPb(JobConfigurationLoad configurationPb) { - return new Builder(configurationPb).build(); - } + List projectionFields(); /** - * Creates a builder for a BigQuery Load Configuration given the destination table. + * Returns a builder for the load configuration object. */ - public static Builder builder(TableId destinationTable) { - return new Builder().destinationTable(destinationTable); - } - - /** - * Creates a builder for a BigQuery Load Configuration given the destination table and format. - */ - public static Builder builder(TableId destinationTable, FormatOptions format) { - return new Builder().destinationTable(destinationTable).formatOptions(format); - } - - /** - * Returns a BigQuery Load Configuration for the given destination table. - */ - public static LoadConfiguration of(TableId destinationTable) { - return builder(destinationTable).build(); - } - - /** - * Returns a BigQuery Load Configuration for the given destination table and format. - */ - public static LoadConfiguration of(TableId destinationTable, FormatOptions format) { - return builder(destinationTable).formatOptions(format).build(); - } + Builder toBuilder(); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java new file mode 100644 index 000000000000..1f98a3dfaca1 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobConfiguration.java @@ -0,0 +1,389 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.JobConfigurationLoad; +import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery load job configuration. A load job loads data from one of several formats into a + * table. Data is provided as URIs that point to objects in Google Cloud Storage. Load job + * configurations have {@link JobConfiguration.Type#LOAD} type. + */ +public final class LoadJobConfiguration extends JobConfiguration implements LoadConfiguration { + + private static final long serialVersionUID = -2673554846792429829L; + + private final List sourceUris; + private final TableId destinationTable; + private final JobInfo.CreateDisposition createDisposition; + private final JobInfo.WriteDisposition writeDisposition; + private final FormatOptions formatOptions; + private final Integer maxBadRecords; + private final Schema schema; + private final Boolean ignoreUnknownValues; + private final List projectionFields; + + public static final class Builder + extends JobConfiguration.Builder + implements LoadConfiguration.Builder { + + private List sourceUris; + private TableId destinationTable; + private JobInfo.CreateDisposition createDisposition; + private JobInfo.WriteDisposition writeDisposition; + private FormatOptions formatOptions; + private Integer maxBadRecords; + private Schema schema; + private Boolean ignoreUnknownValues; + private List projectionFields; + + private Builder() { + super(Type.LOAD); + } + + private Builder(LoadJobConfiguration loadConfiguration) { + this(); + this.destinationTable = loadConfiguration.destinationTable; + this.createDisposition = loadConfiguration.createDisposition; + this.writeDisposition = loadConfiguration.writeDisposition; + this.formatOptions = loadConfiguration.formatOptions; + this.maxBadRecords = loadConfiguration.maxBadRecords; + this.schema = loadConfiguration.schema; + this.ignoreUnknownValues = loadConfiguration.ignoreUnknownValues; + this.projectionFields = loadConfiguration.projectionFields; + this.sourceUris = loadConfiguration.sourceUris; + } + + private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + this(); + JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); + this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); + if (loadConfigurationPb.getCreateDisposition() != null) { + this.createDisposition = + JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); + } + if (loadConfigurationPb.getWriteDisposition() != null) { + this.writeDisposition = + JobInfo.WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); + } + if (loadConfigurationPb.getSourceFormat() != null) { + this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); + } + if (loadConfigurationPb.getAllowJaggedRows() != null + || loadConfigurationPb.getAllowQuotedNewlines() != null + || loadConfigurationPb.getEncoding() != null + || loadConfigurationPb.getFieldDelimiter() != null + || loadConfigurationPb.getQuote() != null + || loadConfigurationPb.getSkipLeadingRows() != null) { + CsvOptions.Builder builder = CsvOptions.builder() + .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) + .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) + .encoding(loadConfigurationPb.getEncoding()) + .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) + .quote(loadConfigurationPb.getQuote()) + .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + this.formatOptions = builder.build(); + } + this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); + if (loadConfigurationPb.getSchema() != null) { + this.schema = Schema.fromPb(loadConfigurationPb.getSchema()); + } + this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues(); + this.projectionFields = loadConfigurationPb.getProjectionFields(); + if (loadConfigurationPb.getSourceUris() != null) { + this.sourceUris = ImmutableList.copyOf(configurationPb.getLoad().getSourceUris()); + } + } + + @Override + public Builder destinationTable(TableId destinationTable) { + this.destinationTable = destinationTable; + return this; + } + + @Override + public Builder createDisposition(JobInfo.CreateDisposition createDisposition) { + this.createDisposition = createDisposition; + return this; + } + + @Override + public Builder writeDisposition(JobInfo.WriteDisposition writeDisposition) { + this.writeDisposition = writeDisposition; + return this; + } + + @Override + public Builder formatOptions(FormatOptions formatOptions) { + this.formatOptions = formatOptions; + return this; + } + + @Override + public Builder maxBadRecords(Integer maxBadRecords) { + this.maxBadRecords = maxBadRecords; + return this; + } + + @Override + public Builder schema(Schema schema) { + this.schema = schema; + return this; + } + + @Override + public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { + this.ignoreUnknownValues = ignoreUnknownValues; + return this; + } + + @Override + public Builder projectionFields(List projectionFields) { + this.projectionFields = + projectionFields != null ? ImmutableList.copyOf(projectionFields) : null; + return this; + } + + /** + * Sets the fully-qualified URIs that point to source data in Google Cloud Storage (e.g. + * gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the + * 'bucket' name. + */ + public Builder sourceUris(List sourceUris) { + this.sourceUris = ImmutableList.copyOf(checkNotNull(sourceUris)); + return this; + } + + @Override + public LoadJobConfiguration build() { + return new LoadJobConfiguration(this); + } + } + + private LoadJobConfiguration(Builder builder) { + super(builder); + this.sourceUris = builder.sourceUris; + this.destinationTable = builder.destinationTable; + this.createDisposition = builder.createDisposition; + this.writeDisposition = builder.writeDisposition; + this.formatOptions = builder.formatOptions; + this.maxBadRecords = builder.maxBadRecords; + this.schema = builder.schema; + this.ignoreUnknownValues = builder.ignoreUnknownValues; + this.projectionFields = builder.projectionFields; + } + + @Override + public TableId destinationTable() { + return destinationTable; + } + + @Override + public JobInfo.CreateDisposition createDisposition() { + return this.createDisposition; + } + + @Override + public JobInfo.WriteDisposition writeDisposition() { + return writeDisposition; + } + + @Override + public CsvOptions csvOptions() { + return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; + } + + @Override + public Integer maxBadRecords() { + return maxBadRecords; + } + + @Override + public Schema schema() { + return schema; + } + + @Override + public String format() { + return formatOptions != null ? formatOptions.type() : null; + } + + @Override + public Boolean ignoreUnknownValues() { + return ignoreUnknownValues; + } + + @Override + public List projectionFields() { + return projectionFields; + } + + /** + * Returns the fully-qualified URIs that point to source data in Google Cloud Storage (e.g. + * gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the + * 'bucket' name. + */ + public List sourceUris() { + return sourceUris; + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + @Override + ToStringHelper toStringHelper() { + return super.toStringHelper() + .add("destinationTable", destinationTable) + .add("createDisposition", createDisposition) + .add("writeDisposition", writeDisposition) + .add("formatOptions", formatOptions) + .add("maxBadRecords", maxBadRecords) + .add("schema", schema) + .add("ignoreUnknownValue", ignoreUnknownValues) + .add("projectionFields", projectionFields) + .add("sourceUris", sourceUris); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof LoadJobConfiguration && baseEquals((LoadJobConfiguration) obj); + } + + @Override + public int hashCode() { + return Objects.hash(baseHashCode(), sourceUris); + } + + @Override + LoadJobConfiguration setProjectId(String projectId) { + return toBuilder().destinationTable(destinationTable().setProjectId(projectId)).build(); + } + + com.google.api.services.bigquery.model.JobConfiguration toPb() { + JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); + loadConfigurationPb.setDestinationTable(destinationTable.toPb()); + if (createDisposition != null) { + loadConfigurationPb.setCreateDisposition(createDisposition.toString()); + } + if (writeDisposition != null) { + loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); + } + if (csvOptions() != null) { + CsvOptions csvOptions = csvOptions(); + loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) + .setAllowJaggedRows(csvOptions.allowJaggedRows()) + .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) + .setEncoding(csvOptions.encoding()) + .setQuote(csvOptions.quote()) + .setSkipLeadingRows(csvOptions.skipLeadingRows()); + } + if (schema != null) { + loadConfigurationPb.setSchema(schema.toPb()); + } + if (formatOptions != null) { + loadConfigurationPb.setSourceFormat(formatOptions.type()); + } + loadConfigurationPb.setMaxBadRecords(maxBadRecords); + loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); + loadConfigurationPb.setProjectionFields(projectionFields); + if (sourceUris != null) { + loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris)); + } + return new com.google.api.services.bigquery.model.JobConfiguration() + .setLoad(loadConfigurationPb); + } + + /** + * Creates a builder for a BigQuery Load Job configuration given the destination table and source + * URIs. + */ + public static Builder builder(TableId destinationTable, List sourceUris) { + return new Builder().destinationTable(destinationTable).sourceUris(sourceUris); + } + + /** + * Creates a builder for a BigQuery Load Job configuration given the destination table and source + * URI. + */ + public static Builder builder(TableId destinationTable, String sourceUri) { + return builder(destinationTable, ImmutableList.of(sourceUri)); + } + + /** + * Creates a builder for a BigQuery Load Job configuration given the destination table, format and + * source URIs. + */ + public static Builder builder(TableId destinationTable, List sourceUris, + FormatOptions format) { + return builder(destinationTable, sourceUris).formatOptions(format); + } + + /** + * Creates a builder for a BigQuery Load Job configuration given the destination table, format and + * source URI. + */ + public static Builder builder(TableId destinationTable, String sourceUri, FormatOptions format) { + return builder(destinationTable, ImmutableList.of(sourceUri), format); + } + + /** + * Returns a BigQuery Load Job Configuration for the given destination table and source URIs. + */ + public static LoadJobConfiguration of(TableId destinationTable, List sourceUris) { + return builder(destinationTable, sourceUris).build(); + } + + /** + * Returns a BigQuery Load Job Configuration for the given destination table and source URI. + */ + public static LoadJobConfiguration of(TableId destinationTable, String sourceUri) { + return of(destinationTable, ImmutableList.of(sourceUri)); + } + + /** + * Returns a BigQuery Load Job Configuration for the given destination table, format and source + * URI. + */ + public static LoadJobConfiguration of(TableId destinationTable, List sourceUris, + FormatOptions format) { + return builder(destinationTable, sourceUris, format).build(); + } + + /** + * Returns a BigQuery Load Job Configuration for the given destination table, format and source + * URI. + */ + public static LoadJobConfiguration of(TableId destinationTable, String sourceUri, + FormatOptions format) { + return of(destinationTable, ImmutableList.of(sourceUri), format); + } + + @SuppressWarnings("unchecked") + static LoadJobConfiguration fromPb( + com.google.api.services.bigquery.model.JobConfiguration confPb) { + return new Builder(confPb).build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobInfo.java deleted file mode 100644 index 21fe51baa4ae..000000000000 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/LoadJobInfo.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.common.base.MoreObjects.ToStringHelper; -import com.google.common.collect.ImmutableList; -import com.google.gcloud.bigquery.JobStatistics.LoadStatistics; - -import java.util.List; -import java.util.Objects; - -/** - * Google BigQuery Load Job. A Load Job loads data from one of several formats into a table. Data is - * provided as URIs that point to objects in Google Cloud Storage. - */ -public class LoadJobInfo extends JobInfo { - - private static final long serialVersionUID = 6349304826867750535L; - - private final List sourceUris; - private final LoadConfiguration configuration; - - public static final class Builder extends JobInfo.Builder { - - private List sourceUris; - private LoadConfiguration configuration; - - private Builder() {} - - private Builder(LoadJobInfo jobInfo) { - super(jobInfo); - this.sourceUris = jobInfo.sourceUris; - this.configuration = jobInfo.configuration; - } - - private Builder(Job jobPb) { - super(jobPb); - JobConfigurationLoad loadConfigurationPb = jobPb.getConfiguration().getLoad(); - this.configuration = LoadConfiguration.fromPb(loadConfigurationPb); - this.sourceUris = loadConfigurationPb.getSourceUris(); - } - - /** - * Sets the fully-qualified URIs that point to source data in Google Cloud Storage (e.g. - * gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the - * 'bucket' name. - */ - public Builder sourceUris(List sourceUris) { - this.sourceUris = sourceUris != null ? ImmutableList.copyOf(sourceUris) : null; - return this; - } - - /** - * Sets the configuration for the BigQuery Load Job. - */ - public Builder configuration(LoadConfiguration configuration) { - this.configuration = configuration; - return this; - } - - @Override - public LoadJobInfo build() { - return new LoadJobInfo(this); - } - } - - private LoadJobInfo(Builder builder) { - super(builder); - this.sourceUris = builder.sourceUris; - this.configuration = builder.configuration; - } - - /** - * Returns the fully-qualified URIs that point to source data in Google Cloud Storage (e.g. - * gs://bucket/path). Each URI can contain one '*' wildcard character and it must come after the - * 'bucket' name. - */ - public List sourceUris() { - return sourceUris; - } - - /** - * Returns the configuration for the BigQuery Load Job. - */ - public LoadConfiguration configuration() { - return configuration; - } - - @Override - public Builder toBuilder() { - return new Builder(this); - } - - @Override - ToStringHelper toStringHelper() { - return super.toStringHelper().add("sourceUris", sourceUris).add("configuration", configuration); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof LoadJobInfo && baseEquals((LoadJobInfo) obj); - } - - @Override - public int hashCode() { - return Objects.hash(baseHashCode(), sourceUris, configuration); - } - - @Override - Job toPb() { - JobConfigurationLoad loadConfigurationPb = configuration.toPb(); - loadConfigurationPb.setSourceUris(sourceUris); - return super.toPb().setConfiguration(new JobConfiguration().setLoad(loadConfigurationPb)); - } - - /** - * Creates a builder for a BigQuery Load Job given the load configuration and source URI. - */ - public static Builder builder(LoadConfiguration configuration, String sourceUri) { - return builder(configuration, ImmutableList.of(checkNotNull(sourceUri))); - } - - /** - * Creates a builder for a BigQuery Load Job given the load configuration and source URIs. - */ - public static Builder builder(LoadConfiguration configuration, List sourceUris) { - return new Builder().configuration(configuration).sourceUris(sourceUris); - } - - /** - * Returns a BigQuery Load Job for the given load configuration and source URI. Job's id is chosen - * by the service. - */ - public static LoadJobInfo of(LoadConfiguration configuration, String sourceUri) { - return builder(configuration, sourceUri).build(); - } - - /** - * Returns a BigQuery Load Job for the given load configuration and source URIs. Job's id is - * chosen by the service. - */ - public static LoadJobInfo of(LoadConfiguration configuration, List sourceUris) { - return builder(configuration, sourceUris).build(); - } - - /** - * Returns a BigQuery Load Job for the given load configuration and source URI. Job's id is set to - * the provided value. - */ - public static LoadJobInfo of(JobId jobId, LoadConfiguration configuration, String sourceUri) { - return builder(configuration, sourceUri).jobId(jobId).build(); - } - - /** - * Returns a BigQuery Load Job for the given load configuration and source URIs. Job's id is set - * to the provided value. - */ - public static LoadJobInfo of(JobId jobId, LoadConfiguration configuration, - List sourceUris) { - return builder(configuration, sourceUris).jobId(jobId).build(); - } - - @SuppressWarnings("unchecked") - static LoadJobInfo fromPb(Job jobPb) { - return new Builder(jobPb).build(); - } -} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobInfo.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java similarity index 84% rename from gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobInfo.java rename to gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java index e2a505ffc4dd..630a3d5b9088 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobInfo.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,26 +18,26 @@ import static com.google.common.base.Preconditions.checkNotNull; -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfiguration; import com.google.api.services.bigquery.model.JobConfigurationQuery; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.gcloud.bigquery.JobStatistics.QueryStatistics; +import com.google.gcloud.bigquery.JobInfo.CreateDisposition; +import com.google.gcloud.bigquery.JobInfo.WriteDisposition; import java.util.List; import java.util.Map; import java.util.Objects; /** - * Google BigQuery Query Job. A Query Job runs a query against BigQuery data. + * Google BigQuery Query Job configuration. A Query Job runs a query against BigQuery data. Query + * job configurations have {@link JobConfiguration.Type#QUERY} type. */ -public class QueryJobInfo extends JobInfo { +public final class QueryJobConfiguration extends JobConfiguration { - private static final long serialVersionUID = -8708709356039780158L; + private static final long serialVersionUID = -1108948249081804890L; /** * Priority levels for a query. If not specified the priority is assumed to be @@ -72,8 +72,8 @@ public enum Priority { private final Boolean flattenResults; private final Boolean dryRun; - public static final class Builder extends JobInfo.Builder { + public static final class Builder + extends JobConfiguration.Builder { private String query; private TableId destinationTable; @@ -88,32 +88,34 @@ public static final class Builder extends JobInfo.Builder tableDefinitions) { this.tableDefinitions = tableDefinitions != null ? Maps.newHashMap(tableDefinitions) : null; - return self(); + return this; } /** @@ -180,7 +184,7 @@ public Builder addTableDefinition(String tableName, ExternalDataConfiguration ta this.tableDefinitions = Maps.newHashMap(); } this.tableDefinitions.put(checkNotNull(tableName), checkNotNull(tableDefinition)); - return self(); + return this; } /** @@ -191,7 +195,7 @@ public Builder addTableDefinition(String tableName, ExternalDataConfiguration ta public Builder userDefinedFunctions(List userDefinedFunctions) { this.userDefinedFunctions = userDefinedFunctions != null ? ImmutableList.copyOf(userDefinedFunctions) : null; - return self(); + return this; } /** @@ -202,7 +206,7 @@ public Builder userDefinedFunctions(List userDefinedFunctio */ public Builder createDisposition(CreateDisposition createDisposition) { this.createDisposition = createDisposition; - return self(); + return this; } /** @@ -213,7 +217,7 @@ public Builder createDisposition(CreateDisposition createDisposition) { */ public Builder writeDisposition(WriteDisposition writeDisposition) { this.writeDisposition = writeDisposition; - return self(); + return this; } /** @@ -222,7 +226,7 @@ public Builder writeDisposition(WriteDisposition writeDisposition) { */ public Builder defaultDataset(DatasetId defaultDataset) { this.defaultDataset = defaultDataset; - return self(); + return this; } /** @@ -239,7 +243,7 @@ public Builder defaultDataset(String defaultDataset) { */ public Builder priority(Priority priority) { this.priority = priority; - return self(); + return this; } /** @@ -252,7 +256,7 @@ public Builder priority(Priority priority) { */ public Builder allowLargeResults(Boolean allowLargeResults) { this.allowLargeResults = allowLargeResults; - return self(); + return this; } /** @@ -264,7 +268,7 @@ public Builder allowLargeResults(Boolean allowLargeResults) { */ public Builder useQueryCache(Boolean useQueryCache) { this.useQueryCache = useQueryCache; - return self(); + return this; } /** @@ -276,7 +280,7 @@ public Builder useQueryCache(Boolean useQueryCache) { */ public Builder flattenResults(Boolean flattenResults) { this.flattenResults = flattenResults; - return self(); + return this; } /** @@ -286,16 +290,15 @@ public Builder flattenResults(Boolean flattenResults) { */ public Builder dryRun(Boolean dryRun) { this.dryRun = dryRun; - return self(); + return this; } - @Override - public QueryJobInfo build() { - return new QueryJobInfo(this); + public QueryJobConfiguration build() { + return new QueryJobConfiguration(this); } } - private QueryJobInfo(Builder builder) { + private QueryJobConfiguration(Builder builder) { super(builder); this.query = checkNotNull(builder.query); this.allowLargeResults = builder.allowLargeResults; @@ -447,7 +450,7 @@ ToStringHelper toStringHelper() { @Override public boolean equals(Object obj) { - return obj instanceof QueryJobInfo && baseEquals((QueryJobInfo) obj); + return obj instanceof QueryJobConfiguration && baseEquals((QueryJobConfiguration) obj); } @Override @@ -458,8 +461,20 @@ public int hashCode() { } @Override - Job toPb() { - JobConfiguration configurationPb = new JobConfiguration(); + QueryJobConfiguration setProjectId(String projectId) { + Builder builder = toBuilder(); + if (destinationTable() != null) { + builder.destinationTable(destinationTable().setProjectId(projectId)); + } + if (defaultDataset() != null) { + builder.defaultDataset(defaultDataset().setProjectId(projectId)); + } + return builder.build(); + } + + com.google.api.services.bigquery.model.JobConfiguration toPb() { + com.google.api.services.bigquery.model.JobConfiguration configurationPb = + new com.google.api.services.bigquery.model.JobConfiguration(); JobConfigurationQuery queryConfigurationPb = new JobConfigurationQuery(); queryConfigurationPb.setQuery(query); configurationPb.setDryRun(dryRun()); @@ -495,7 +510,7 @@ Job toPb() { if (writeDisposition != null) { queryConfigurationPb.setWriteDisposition(writeDisposition.toString()); } - return super.toPb().setConfiguration(configurationPb.setQuery(queryConfigurationPb)); + return configurationPb.setQuery(queryConfigurationPb); } /** @@ -509,20 +524,13 @@ public static Builder builder(String query) { * Returns a BigQuery Copy Job for the given the query to be run. Job's id is chosen by the * service. */ - public static QueryJobInfo of(String query) { + public static QueryJobConfiguration of(String query) { return builder(query).build(); } - /** - * Returns a BigQuery Copy Job for the given the query to be run. Job's id is set to the provided - * value. - */ - public static QueryJobInfo of(JobId jobId, String query) { - return builder(query).jobId(jobId).build(); - } - @SuppressWarnings("unchecked") - static QueryJobInfo fromPb(Job jobPb) { + static QueryJobConfiguration fromPb( + com.google.api.services.bigquery.model.JobConfiguration jobPb) { return new Builder(jobPb).build(); } } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryRequest.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryRequest.java index 64fbb3e931fc..0bcfb3d4a9ae 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryRequest.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/QueryRequest.java @@ -120,8 +120,8 @@ public Builder defaultDataset(String defaultDataset) { * Sets how long to wait for the query to complete, in milliseconds, before the request times * out and returns. Note that this is only a timeout for the request, not the query. If the * query takes longer to run than the timeout value, the call returns without any results and - * with the {@link QueryResponse#jobCompleted()} set to {@code false}. If not set, a wait time of - * 10000 milliseconds (10 seconds) is used. + * with the {@link QueryResponse#jobCompleted()} set to {@code false}. If not set, a wait time + * of 10000 milliseconds (10 seconds) is used. */ public Builder maxWaitTime(Long maxWaitTime) { this.maxWaitTime = maxWaitTime; @@ -253,6 +253,14 @@ public boolean equals(Object obj) { return obj instanceof QueryRequest && Objects.equals(toPb(), ((QueryRequest) obj).toPb()); } + QueryRequest setProjectId(String projectId) { + Builder builder = toBuilder(); + if (defaultDataset() != null) { + builder.defaultDataset(defaultDataset().setProjectId(projectId)); + } + return builder.build(); + } + com.google.api.services.bigquery.model.QueryRequest toPb() { com.google.api.services.bigquery.model.QueryRequest queryRequestPb = new com.google.api.services.bigquery.model.QueryRequest().setQuery(query); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Table.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Table.java index b4cc1df1d997..1344b31c9b68 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Table.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/Table.java @@ -200,8 +200,8 @@ Job copy(String destinationDataset, String destinationTable, BigQuery.JobOption. * @throws BigQueryException upon failure */ Job copy(TableId destinationTable, BigQuery.JobOption... options) throws BigQueryException { - JobInfo job = bigquery.create(CopyJobInfo.of(destinationTable, info.tableId()), options); - return new Job(bigquery, job); + CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, info.tableId()); + return new Job(bigquery, bigquery.create(JobInfo.of(configuration), options)); } /** @@ -231,8 +231,9 @@ Job extract(String format, String destinationUri, BigQuery.JobOption... options) */ Job extract(String format, List destinationUris, BigQuery.JobOption... options) throws BigQueryException { - return new Job(bigquery, - bigquery.create(ExtractJobInfo.of(info.tableId(), format, destinationUris), options)); + ExtractJobConfiguration extractConfiguration = + ExtractJobConfiguration.of(info.tableId(), destinationUris, format); + return new Job(bigquery, bigquery.create(JobInfo.of(extractConfiguration), options)); } /** @@ -262,8 +263,8 @@ Job load(FormatOptions format, String sourceUri, BigQuery.JobOption... options) */ Job load(FormatOptions format, List sourceUris, BigQuery.JobOption... options) throws BigQueryException { - LoadConfiguration configuration = LoadConfiguration.of(info.tableId(), format); - return new Job(bigquery, bigquery.create(LoadJobInfo.of(configuration, sourceUris), options)); + LoadJobConfiguration loadConfig = LoadJobConfiguration.of(info.tableId(), sourceUris, format); + return new Job(bigquery, bigquery.create(JobInfo.of(loadConfig), options)); } /** diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java index c4cee5a9a303..bee0340a29a8 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableDataWriteChannel.java @@ -24,18 +24,18 @@ import com.google.gcloud.RetryHelper; import com.google.gcloud.WriteChannel; -import java.util.Arrays; - /** * WriteChannel implementation to stream data into a BigQuery table. */ -class TableDataWriteChannel extends BaseWriteChannel { +class TableDataWriteChannel extends BaseWriteChannel { - TableDataWriteChannel(BigQueryOptions options, LoadConfiguration loadConfiguration) { - this(options, loadConfiguration, options.rpc().open(loadConfiguration.toPb())); + TableDataWriteChannel(BigQueryOptions options, + WriteChannelConfiguration writeChannelConfiguration) { + this(options, writeChannelConfiguration, options.rpc().open(writeChannelConfiguration.toPb())); } - TableDataWriteChannel(BigQueryOptions options, LoadConfiguration config, String uploadId) { + TableDataWriteChannel(BigQueryOptions options, WriteChannelConfiguration config, + String uploadId) { super(options, config, uploadId); } @@ -57,7 +57,8 @@ protected StateImpl.Builder stateBuilder() { return StateImpl.builder(options(), entity(), uploadId()); } - static class StateImpl extends BaseWriteChannel.BaseState { + static class StateImpl + extends BaseWriteChannel.BaseState { private static final long serialVersionUID = -787362105981823738L; @@ -66,9 +67,10 @@ static class StateImpl extends BaseWriteChannel.BaseState { + extends BaseWriteChannel.BaseState.Builder { - private Builder(BigQueryOptions options, LoadConfiguration configuration, String uploadId) { + private Builder(BigQueryOptions options, WriteChannelConfiguration configuration, + String uploadId) { super(options, configuration, uploadId); } @@ -77,7 +79,8 @@ public RestorableState build() { } } - static Builder builder(BigQueryOptions options, LoadConfiguration config, String uploadId) { + static Builder builder(BigQueryOptions options, WriteChannelConfiguration config, + String uploadId) { return new Builder(options, config, uploadId); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableId.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableId.java index 7a4e0bbb38b4..20ed53cc1a5d 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableId.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/TableId.java @@ -105,6 +105,10 @@ public String toString() { return toPb().toString(); } + TableId setProjectId(String projectId) { + return project() != null ? this : TableId.of(projectId, dataset(), table()); + } + TableReference toPb() { return new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table); } diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java new file mode 100644 index 000000000000..18342bac1bff --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/WriteChannelConfiguration.java @@ -0,0 +1,320 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.bigquery.model.JobConfigurationLoad; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.gcloud.bigquery.JobInfo.CreateDisposition; +import com.google.gcloud.bigquery.JobInfo.WriteDisposition; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +/** + * Google BigQuery Configuration for a load operation. A load configuration can be used to load data + * into a table with a {@link com.google.gcloud.WriteChannel} + * ({@link BigQuery#writer(WriteChannelConfiguration)}). + */ +public class WriteChannelConfiguration implements LoadConfiguration, Serializable { + + private static final long serialVersionUID = 470267591917413578L; + + private final TableId destinationTable; + private final CreateDisposition createDisposition; + private final WriteDisposition writeDisposition; + private final FormatOptions formatOptions; + private final Integer maxBadRecords; + private final Schema schema; + private final Boolean ignoreUnknownValues; + private final List projectionFields; + + public static final class Builder implements LoadConfiguration.Builder { + + private TableId destinationTable; + private CreateDisposition createDisposition; + private WriteDisposition writeDisposition; + private FormatOptions formatOptions; + private Integer maxBadRecords; + private Schema schema; + private Boolean ignoreUnknownValues; + private List projectionFields; + + private Builder() {} + + private Builder(WriteChannelConfiguration writeChannelConfiguration) { + this.destinationTable = writeChannelConfiguration.destinationTable; + this.createDisposition = writeChannelConfiguration.createDisposition; + this.writeDisposition = writeChannelConfiguration.writeDisposition; + this.formatOptions = writeChannelConfiguration.formatOptions; + this.maxBadRecords = writeChannelConfiguration.maxBadRecords; + this.schema = writeChannelConfiguration.schema; + this.ignoreUnknownValues = writeChannelConfiguration.ignoreUnknownValues; + this.projectionFields = writeChannelConfiguration.projectionFields; + } + + private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); + this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); + if (loadConfigurationPb.getCreateDisposition() != null) { + this.createDisposition = + CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); + } + if (loadConfigurationPb.getWriteDisposition() != null) { + this.writeDisposition = WriteDisposition.valueOf(loadConfigurationPb.getWriteDisposition()); + } + if (loadConfigurationPb.getSourceFormat() != null) { + this.formatOptions = FormatOptions.of(loadConfigurationPb.getSourceFormat()); + } + if (loadConfigurationPb.getAllowJaggedRows() != null + || loadConfigurationPb.getAllowQuotedNewlines() != null + || loadConfigurationPb.getEncoding() != null + || loadConfigurationPb.getFieldDelimiter() != null + || loadConfigurationPb.getQuote() != null + || loadConfigurationPb.getSkipLeadingRows() != null) { + CsvOptions.Builder builder = CsvOptions.builder() + .allowJaggedRows(loadConfigurationPb.getAllowJaggedRows()) + .allowQuotedNewLines(loadConfigurationPb.getAllowQuotedNewlines()) + .encoding(loadConfigurationPb.getEncoding()) + .fieldDelimiter(loadConfigurationPb.getFieldDelimiter()) + .quote(loadConfigurationPb.getQuote()) + .skipLeadingRows(loadConfigurationPb.getSkipLeadingRows()); + this.formatOptions = builder.build(); + } + this.maxBadRecords = loadConfigurationPb.getMaxBadRecords(); + if (loadConfigurationPb.getSchema() != null) { + this.schema = Schema.fromPb(loadConfigurationPb.getSchema()); + } + this.ignoreUnknownValues = loadConfigurationPb.getIgnoreUnknownValues(); + this.projectionFields = loadConfigurationPb.getProjectionFields(); + } + + @Override + public Builder destinationTable(TableId destinationTable) { + this.destinationTable = destinationTable; + return this; + } + + @Override + public Builder createDisposition(CreateDisposition createDisposition) { + this.createDisposition = createDisposition; + return this; + } + + @Override + public Builder writeDisposition(WriteDisposition writeDisposition) { + this.writeDisposition = writeDisposition; + return this; + } + + @Override + public Builder formatOptions(FormatOptions formatOptions) { + this.formatOptions = formatOptions; + return this; + } + + @Override + public Builder maxBadRecords(Integer maxBadRecords) { + this.maxBadRecords = maxBadRecords; + return this; + } + + @Override + public Builder schema(Schema schema) { + this.schema = schema; + return this; + } + + @Override + public Builder ignoreUnknownValues(Boolean ignoreUnknownValues) { + this.ignoreUnknownValues = ignoreUnknownValues; + return this; + } + + @Override + public Builder projectionFields(List projectionFields) { + this.projectionFields = + projectionFields != null ? ImmutableList.copyOf(projectionFields) : null; + return this; + } + + public WriteChannelConfiguration build() { + return new WriteChannelConfiguration(this); + } + } + + protected WriteChannelConfiguration(Builder builder) { + this.destinationTable = checkNotNull(builder.destinationTable); + this.createDisposition = builder.createDisposition; + this.writeDisposition = builder.writeDisposition; + this.formatOptions = builder.formatOptions; + this.maxBadRecords = builder.maxBadRecords; + this.schema = builder.schema; + this.ignoreUnknownValues = builder.ignoreUnknownValues; + this.projectionFields = builder.projectionFields; + } + + @Override + public TableId destinationTable() { + return destinationTable; + } + + @Override + public CreateDisposition createDisposition() { + return this.createDisposition; + } + + @Override + public WriteDisposition writeDisposition() { + return writeDisposition; + } + + @Override + public CsvOptions csvOptions() { + return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null; + } + + @Override + public Integer maxBadRecords() { + return maxBadRecords; + } + + @Override + public Schema schema() { + return schema; + } + + @Override + public String format() { + return formatOptions != null ? formatOptions.type() : null; + } + + @Override + public Boolean ignoreUnknownValues() { + return ignoreUnknownValues; + } + + @Override + public List projectionFields() { + return projectionFields; + } + + @Override + public Builder toBuilder() { + return new Builder(this); + } + + MoreObjects.ToStringHelper toStringHelper() { + return MoreObjects.toStringHelper(this) + .add("destinationTable", destinationTable) + .add("createDisposition", createDisposition) + .add("writeDisposition", writeDisposition) + .add("formatOptions", formatOptions) + .add("maxBadRecords", maxBadRecords) + .add("schema", schema) + .add("ignoreUnknownValue", ignoreUnknownValues) + .add("projectionFields", projectionFields); + } + + @Override + public String toString() { + return toStringHelper().toString(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof WriteChannelConfiguration + && Objects.equals(toPb(), ((WriteChannelConfiguration) obj).toPb()); + } + + @Override + public int hashCode() { + return Objects.hash(destinationTable, createDisposition, writeDisposition, formatOptions, + maxBadRecords, schema, ignoreUnknownValues, projectionFields); + } + + WriteChannelConfiguration setProjectId(String projectId) { + return toBuilder().destinationTable(destinationTable().setProjectId(projectId)).build(); + } + + com.google.api.services.bigquery.model.JobConfiguration toPb() { + JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); + loadConfigurationPb.setDestinationTable(destinationTable.toPb()); + if (createDisposition != null) { + loadConfigurationPb.setCreateDisposition(createDisposition.toString()); + } + if (writeDisposition != null) { + loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); + } + if (csvOptions() != null) { + CsvOptions csvOptions = csvOptions(); + loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) + .setAllowJaggedRows(csvOptions.allowJaggedRows()) + .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) + .setEncoding(csvOptions.encoding()) + .setQuote(csvOptions.quote()) + .setSkipLeadingRows(csvOptions.skipLeadingRows()); + } + if (schema != null) { + loadConfigurationPb.setSchema(schema.toPb()); + } + if (formatOptions != null) { + loadConfigurationPb.setSourceFormat(formatOptions.type()); + } + loadConfigurationPb.setMaxBadRecords(maxBadRecords); + loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); + loadConfigurationPb.setProjectionFields(projectionFields); + return new com.google.api.services.bigquery.model.JobConfiguration() + .setLoad(loadConfigurationPb); + } + + static WriteChannelConfiguration fromPb( + com.google.api.services.bigquery.model.JobConfiguration configurationPb) { + return new Builder(configurationPb).build(); + } + + /** + * Creates a builder for a BigQuery Load Configuration given the destination table. + */ + public static Builder builder(TableId destinationTable) { + return new Builder().destinationTable(destinationTable); + } + + /** + * Creates a builder for a BigQuery Load Configuration given the destination table and format. + */ + public static Builder builder(TableId destinationTable, FormatOptions format) { + return builder(destinationTable).formatOptions(format); + } + + /** + * Returns a BigQuery Load Configuration for the given destination table. + */ + public static WriteChannelConfiguration of(TableId destinationTable) { + return builder(destinationTable).build(); + } + + /** + * Returns a BigQuery Load Configuration for the given destination table and format. + */ + public static WriteChannelConfiguration of(TableId destinationTable, FormatOptions format) { + return builder(destinationTable).formatOptions(format).build(); + } +} diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/package-info.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/package-info.java index 8553de221fdc..dd57da2b606a 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/package-info.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/package-info.java @@ -28,7 +28,8 @@ * bigquery.create(TableInfo.of(tableId, Schema.of(integerField))); * } else { * System.out.println("Loading data into table " + tableId); - * LoadJobInfo loadJob = LoadJobInfo.of(tableId, "gs://bucket/path"); + * LoadJobConfiguration configuration = LoadJobConfiguration.of(tableId, "gs://bucket/path"); + * JobInfo loadJob = JobInfo.of(configuration); * loadJob = bigquery.create(loadJob); * while (loadJob.status().state() != JobStatus.State.DONE) { * Thread.sleep(1000L); diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java index 5f17f60f2bb5..6062e19950e0 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java @@ -19,7 +19,7 @@ import com.google.api.services.bigquery.model.Dataset; import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfigurationLoad; +import com.google.api.services.bigquery.model.JobConfiguration; import com.google.api.services.bigquery.model.QueryRequest; import com.google.api.services.bigquery.model.QueryResponse; import com.google.api.services.bigquery.model.Table; @@ -193,7 +193,7 @@ GetQueryResultsResponse getQueryResults(String jobId, Map options) * @param configuration load configuration * @throws BigQueryException upon failure */ - String open(JobConfigurationLoad configuration) throws BigQueryException; + String open(JobConfiguration configuration) throws BigQueryException; /** * Uploads the provided data to the resumable upload session at the specified position. diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java index 0a1dc046bf74..b57f1dc8a128 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java @@ -24,7 +24,6 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; -import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.http.ByteArrayContent; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; @@ -43,7 +42,6 @@ import com.google.api.services.bigquery.model.GetQueryResultsResponse; import com.google.api.services.bigquery.model.Job; import com.google.api.services.bigquery.model.JobConfiguration; -import com.google.api.services.bigquery.model.JobConfigurationLoad; import com.google.api.services.bigquery.model.JobList; import com.google.api.services.bigquery.model.JobStatus; import com.google.api.services.bigquery.model.QueryRequest; @@ -408,9 +406,9 @@ public QueryResponse query(QueryRequest request) throws BigQueryException { } @Override - public String open(JobConfigurationLoad configuration) throws BigQueryException { + public String open(JobConfiguration configuration) throws BigQueryException { try { - Job loadJob = new Job().setConfiguration(new JobConfiguration().setLoad(configuration)); + Job loadJob = new Job().setConfiguration(configuration); StringBuilder builder = new StringBuilder() .append(BASE_RESUMABLE_URI) .append(options.projectId()) diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java index 3fe1824fd3c2..8af8c700cd8c 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java @@ -108,42 +108,42 @@ public class BigQueryImplTest { private static final TableInfo OTHER_TABLE_INFO = TableInfo.of(OTHER_TABLE_ID, TABLE_SCHEMA); private static final TableInfo TABLE_INFO_WITH_PROJECT = TableInfo.of(TABLE_ID_WITH_PROJECT, TABLE_SCHEMA); - private static final LoadJobInfo LOAD_JOB = LoadJobInfo.of(LoadConfiguration.of(TABLE_ID), "URI"); - private static final LoadJobInfo LOAD_JOB_WITH_PROJECT = - LoadJobInfo.of(LoadConfiguration.of(TABLE_ID_WITH_PROJECT), "URI"); - private static final LoadJobInfo COMPLETE_LOAD_JOB = - LoadJobInfo.builder(LoadConfiguration.of(TABLE_ID_WITH_PROJECT), "URI") - .jobId(JobId.of(PROJECT, JOB)) - .build(); - private static final CopyJobInfo COPY_JOB = - CopyJobInfo.of(TABLE_ID, ImmutableList.of(TABLE_ID, TABLE_ID)); - private static final CopyJobInfo COPY_JOB_WITH_PROJECT = - CopyJobInfo.of(TABLE_ID_WITH_PROJECT, ImmutableList.of(TABLE_ID_WITH_PROJECT, + private static final LoadJobConfiguration LOAD_JOB_CONFIGURATION = + LoadJobConfiguration.of(TABLE_ID, "URI"); + private static final LoadJobConfiguration LOAD_JOB_CONFIGURATION_WITH_PROJECT = + LoadJobConfiguration.of(TABLE_ID_WITH_PROJECT, "URI"); + private static final JobInfo LOAD_JOB = + JobInfo.of(LOAD_JOB_CONFIGURATION); + private static final JobInfo COMPLETE_LOAD_JOB = + JobInfo.of(JobId.of(PROJECT, JOB), LOAD_JOB_CONFIGURATION_WITH_PROJECT); + private static final CopyJobConfiguration COPY_JOB_CONFIGURATION = + CopyJobConfiguration.of(TABLE_ID, ImmutableList.of(TABLE_ID, TABLE_ID)); + private static final CopyJobConfiguration COPY_JOB_CONFIGURATION_WITH_PROJECT = + CopyJobConfiguration.of(TABLE_ID_WITH_PROJECT, ImmutableList.of(TABLE_ID_WITH_PROJECT, TABLE_ID_WITH_PROJECT)); - private static final CopyJobInfo COMPLETE_COPY_JOB = - CopyJobInfo.builder(TABLE_ID_WITH_PROJECT, ImmutableList.of(TABLE_ID_WITH_PROJECT, - TABLE_ID_WITH_PROJECT)) - .jobId(JobId.of(PROJECT, JOB)) + private static final JobInfo COPY_JOB = JobInfo.of(COPY_JOB_CONFIGURATION); + private static final JobInfo COMPLETE_COPY_JOB = + JobInfo.of(JobId.of(PROJECT, JOB), COPY_JOB_CONFIGURATION_WITH_PROJECT); + private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = + QueryJobConfiguration.builder("SQL") + .defaultDataset(DatasetId.of(DATASET)) + .destinationTable(TABLE_ID) .build(); - private static final QueryJobInfo QUERY_JOB = QueryJobInfo.builder("SQL") - .defaultDataset(DatasetId.of(DATASET)) - .destinationTable(TABLE_ID) - .build(); - private static final QueryJobInfo QUERY_JOB_WITH_PROJECT = QueryJobInfo.builder("SQL") - .defaultDataset(DatasetId.of(PROJECT, DATASET)) - .destinationTable(TABLE_ID_WITH_PROJECT) - .build(); - private static final QueryJobInfo COMPLETE_QUERY_JOB = QueryJobInfo.builder("SQL") - .defaultDataset(DatasetId.of(PROJECT, DATASET)).destinationTable(TABLE_ID_WITH_PROJECT) - .jobId(JobId.of(PROJECT, JOB)) - .build(); - private static final ExtractJobInfo EXTRACT_JOB = ExtractJobInfo.of(TABLE_ID, "URI"); - private static final ExtractJobInfo EXTRACT_JOB_WITH_PROJECT = - ExtractJobInfo.of(TABLE_ID_WITH_PROJECT, "URI"); - private static final ExtractJobInfo COMPLETE_EXTRACT_JOB = - ExtractJobInfo.builder(TABLE_ID_WITH_PROJECT, "URI") - .jobId(JobId.of(PROJECT, JOB)) + private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION_WITH_PROJECT = + QueryJobConfiguration.builder("SQL") + .defaultDataset(DatasetId.of(PROJECT, DATASET)) + .destinationTable(TABLE_ID_WITH_PROJECT) .build(); + private static final JobInfo QUERY_JOB = JobInfo.of(QUERY_JOB_CONFIGURATION); + private static final JobInfo COMPLETE_QUERY_JOB = + JobInfo.of(JobId.of(PROJECT, JOB), QUERY_JOB_CONFIGURATION_WITH_PROJECT); + private static final ExtractJobConfiguration EXTRACT_JOB_CONFIGURATION = + ExtractJobConfiguration.of(TABLE_ID, "URI"); + private static final ExtractJobConfiguration EXTRACT_JOB_CONFIGURATION_WITH_PROJECT = + ExtractJobConfiguration.of(TABLE_ID_WITH_PROJECT, "URI"); + private static final JobInfo EXTRACT_JOB = JobInfo.of(EXTRACT_JOB_CONFIGURATION); + private static final JobInfo COMPLETE_EXTRACT_JOB = + JobInfo.of(JobId.of(PROJECT, JOB), EXTRACT_JOB_CONFIGURATION_WITH_PROJECT); private static final TableCell BOOLEAN_FIELD = new TableCell().setV("false"); private static final TableCell INTEGER_FIELD = new TableCell().setV("1"); private static final TableRow TABLE_ROW = @@ -728,53 +728,57 @@ public void testListTableDataWithOptions() { @Test public void testCreateQueryJob() { - EasyMock.expect(bigqueryRpcMock.create(QUERY_JOB_WITH_PROJECT.toPb(), EMPTY_RPC_OPTIONS)) + EasyMock.expect(bigqueryRpcMock.create( + JobInfo.of(QUERY_JOB_CONFIGURATION_WITH_PROJECT).toPb(), EMPTY_RPC_OPTIONS)) .andReturn(COMPLETE_QUERY_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - QueryJobInfo job = bigquery.create(QUERY_JOB); + JobInfo job = bigquery.create(QUERY_JOB); assertEquals(COMPLETE_QUERY_JOB, job); } @Test public void testCreateLoadJob() { - EasyMock.expect(bigqueryRpcMock.create(LOAD_JOB_WITH_PROJECT.toPb(), EMPTY_RPC_OPTIONS)) + EasyMock.expect(bigqueryRpcMock.create( + JobInfo.of(LOAD_JOB_CONFIGURATION_WITH_PROJECT).toPb(), EMPTY_RPC_OPTIONS)) .andReturn(COMPLETE_LOAD_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - LoadJobInfo job = bigquery.create(LOAD_JOB); + JobInfo job = bigquery.create(LOAD_JOB); assertEquals(COMPLETE_LOAD_JOB, job); } @Test public void testCreateCopyJob() { - EasyMock.expect(bigqueryRpcMock.create(COPY_JOB_WITH_PROJECT.toPb(), EMPTY_RPC_OPTIONS)) + EasyMock.expect(bigqueryRpcMock.create( + JobInfo.of(COPY_JOB_CONFIGURATION_WITH_PROJECT).toPb(), EMPTY_RPC_OPTIONS)) .andReturn(COMPLETE_COPY_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - CopyJobInfo job = bigquery.create(COPY_JOB); + JobInfo job = bigquery.create(COPY_JOB); assertEquals(COMPLETE_COPY_JOB, job); } @Test public void testCreateExtractJob() { - EasyMock.expect(bigqueryRpcMock.create(EXTRACT_JOB_WITH_PROJECT.toPb(), EMPTY_RPC_OPTIONS)) + EasyMock.expect(bigqueryRpcMock.create( + JobInfo.of(EXTRACT_JOB_CONFIGURATION_WITH_PROJECT).toPb(), EMPTY_RPC_OPTIONS)) .andReturn(COMPLETE_EXTRACT_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - ExtractJobInfo job = bigquery.create(EXTRACT_JOB); + JobInfo job = bigquery.create(EXTRACT_JOB); assertEquals(COMPLETE_EXTRACT_JOB, job); } @Test public void testCreateJobWithSelectedFields() { Capture> capturedOptions = Capture.newInstance(); - EasyMock.expect( - bigqueryRpcMock.create(eq(QUERY_JOB_WITH_PROJECT.toPb()), capture(capturedOptions))) + EasyMock.expect(bigqueryRpcMock.create( + eq(JobInfo.of(QUERY_JOB_CONFIGURATION_WITH_PROJECT).toPb()), capture(capturedOptions))) .andReturn(COMPLETE_QUERY_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - QueryJobInfo job = bigquery.create(QUERY_JOB, JOB_OPTION_FIELDS); + JobInfo job = bigquery.create(QUERY_JOB, JOB_OPTION_FIELDS); assertEquals(COMPLETE_QUERY_JOB, job); String selector = (String) capturedOptions.getValue().get(JOB_OPTION_FIELDS.rpcOption()); assertTrue(selector.contains("jobReference")); @@ -789,7 +793,7 @@ public void testGetJob() { .andReturn(COMPLETE_COPY_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - CopyJobInfo job = bigquery.getJob(JOB); + JobInfo job = bigquery.getJob(JOB); assertEquals(COMPLETE_COPY_JOB, job); } @@ -799,15 +803,14 @@ public void testGetJobFromJobId() { .andReturn(COMPLETE_COPY_JOB.toPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - CopyJobInfo job = bigquery.getJob(JobId.of(PROJECT, JOB)); + JobInfo job = bigquery.getJob(JobId.of(PROJECT, JOB)); assertEquals(COMPLETE_COPY_JOB, job); } @Test public void testListJobs() { String cursor = "cursor"; - ImmutableList jobList = - ImmutableList.of(QUERY_JOB_WITH_PROJECT, LOAD_JOB_WITH_PROJECT); + ImmutableList jobList = ImmutableList.of(COMPLETE_QUERY_JOB, COMPLETE_LOAD_JOB); Tuple> result = Tuple.of(cursor, Iterables.transform(jobList, new Function() { @Override @@ -826,8 +829,7 @@ public Job apply(JobInfo jobInfo) { @Test public void testListJobsWithOptions() { String cursor = "cursor"; - ImmutableList jobList = - ImmutableList.of(QUERY_JOB_WITH_PROJECT, LOAD_JOB_WITH_PROJECT); + ImmutableList jobList = ImmutableList.of(COMPLETE_QUERY_JOB, COMPLETE_LOAD_JOB); Tuple> result = Tuple.of(cursor, Iterables.transform(jobList, new Function() { @Override @@ -848,8 +850,7 @@ public Job apply(JobInfo jobInfo) { public void testListJobsWithSelectedFields() { String cursor = "cursor"; Capture> capturedOptions = Capture.newInstance(); - ImmutableList jobList = - ImmutableList.of(QUERY_JOB_WITH_PROJECT, LOAD_JOB_WITH_PROJECT); + ImmutableList jobList = ImmutableList.of(COMPLETE_QUERY_JOB, COMPLETE_LOAD_JOB); Tuple> result = Tuple.of(cursor, Iterables.transform(jobList, new Function() { @Override @@ -1009,12 +1010,13 @@ public void testGetQueryResultsWithOptions() { @Test public void testWriter() { - LoadConfiguration loadConfiguration = LoadConfiguration.of(TABLE_ID); - EasyMock.expect(bigqueryRpcMock.open(LoadConfiguration.of(TABLE_ID_WITH_PROJECT).toPb())) - .andReturn("upload-id"); + WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.of(TABLE_ID); + EasyMock.expect( + bigqueryRpcMock.open(WriteChannelConfiguration.of(TABLE_ID_WITH_PROJECT).toPb())) + .andReturn("upload-id"); EasyMock.replay(bigqueryRpcMock); bigquery = options.service(); - WriteChannel channel = bigquery.writer(loadConfiguration); + WriteChannel channel = bigquery.writer(writeChannelConfiguration); assertNotNull(channel); assertTrue(channel.isOpen()); } diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java new file mode 100644 index 000000000000..3f3f6f0fd15c --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobConfigurationTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.google.common.collect.ImmutableList; +import com.google.gcloud.bigquery.JobInfo.CreateDisposition; +import com.google.gcloud.bigquery.JobInfo.WriteDisposition; + +import org.junit.Test; + +import java.util.List; + +public class CopyJobConfigurationTest { + + private static final TableId SOURCE_TABLE = TableId.of("dataset", "sourceTable"); + private static final List SOURCE_TABLES = ImmutableList.of( + TableId.of("dataset", "sourceTable1"), + TableId.of("dataset", "sourceTable2")); + private static final TableId DESTINATION_TABLE = TableId.of("dataset", "destinationTable"); + private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED; + private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; + private static final CopyJobConfiguration COPY_JOB_CONFIGURATION = + CopyJobConfiguration.builder(DESTINATION_TABLE, SOURCE_TABLE) + .createDisposition(CREATE_DISPOSITION) + .writeDisposition(WRITE_DISPOSITION) + .build(); + private static final CopyJobConfiguration COPY_JOB_CONFIGURATION_MULTIPLE_TABLES = + CopyJobConfiguration.builder(DESTINATION_TABLE, SOURCE_TABLES) + .createDisposition(CREATE_DISPOSITION) + .writeDisposition(WRITE_DISPOSITION) + .build(); + + @Test + public void testToBuilder() { + compareCopyJobConfiguration(COPY_JOB_CONFIGURATION, COPY_JOB_CONFIGURATION.toBuilder().build()); + compareCopyJobConfiguration(COPY_JOB_CONFIGURATION_MULTIPLE_TABLES, + COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.toBuilder().build()); + CopyJobConfiguration jobConfiguration = COPY_JOB_CONFIGURATION.toBuilder() + .destinationTable(TableId.of("dataset", "newTable")) + .build(); + assertEquals("newTable", jobConfiguration.destinationTable().table()); + jobConfiguration = jobConfiguration.toBuilder().destinationTable(DESTINATION_TABLE).build(); + compareCopyJobConfiguration(COPY_JOB_CONFIGURATION, jobConfiguration); + } + + @Test + public void testOf() { + CopyJobConfiguration job = CopyJobConfiguration.of(DESTINATION_TABLE, SOURCE_TABLES); + assertEquals(DESTINATION_TABLE, job.destinationTable()); + assertEquals(SOURCE_TABLES, job.sourceTables()); + job = CopyJobConfiguration.of(DESTINATION_TABLE, SOURCE_TABLE); + assertEquals(DESTINATION_TABLE, job.destinationTable()); + assertEquals(ImmutableList.of(SOURCE_TABLE), job.sourceTables()); + } + + @Test + public void testToBuilderIncomplete() { + CopyJobConfiguration jobConfiguration = + CopyJobConfiguration.of(DESTINATION_TABLE, SOURCE_TABLES); + compareCopyJobConfiguration(jobConfiguration, jobConfiguration.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(DESTINATION_TABLE, COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.destinationTable()); + assertEquals(SOURCE_TABLES, COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.sourceTables()); + assertEquals(CREATE_DISPOSITION, COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.createDisposition()); + assertEquals(WRITE_DISPOSITION, COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.writeDisposition()); + assertEquals(DESTINATION_TABLE, COPY_JOB_CONFIGURATION.destinationTable()); + assertEquals(ImmutableList.of(SOURCE_TABLE), COPY_JOB_CONFIGURATION.sourceTables()); + assertEquals(CREATE_DISPOSITION, COPY_JOB_CONFIGURATION.createDisposition()); + assertEquals(WRITE_DISPOSITION, COPY_JOB_CONFIGURATION.writeDisposition()); + } + + @Test + public void testToPbAndFromPb() { + assertNotNull(COPY_JOB_CONFIGURATION.toPb().getCopy()); + assertNull(COPY_JOB_CONFIGURATION.toPb().getExtract()); + assertNull(COPY_JOB_CONFIGURATION.toPb().getLoad()); + assertNull(COPY_JOB_CONFIGURATION.toPb().getQuery()); + assertNull(COPY_JOB_CONFIGURATION.toPb().getCopy().getSourceTables()); + assertNull(COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.toPb().getCopy().getSourceTable()); + compareCopyJobConfiguration(COPY_JOB_CONFIGURATION, + CopyJobConfiguration.fromPb(COPY_JOB_CONFIGURATION.toPb())); + compareCopyJobConfiguration(COPY_JOB_CONFIGURATION_MULTIPLE_TABLES, + CopyJobConfiguration.fromPb(COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.toPb())); + CopyJobConfiguration jobConfiguration = + CopyJobConfiguration.of(DESTINATION_TABLE, SOURCE_TABLES); + compareCopyJobConfiguration( + jobConfiguration, CopyJobConfiguration.fromPb(jobConfiguration.toPb())); + } + + @Test + public void testSetProjectId() { + CopyJobConfiguration configuration = COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId("p"); + assertEquals("p", configuration.destinationTable().project()); + for (TableId sourceTable : configuration.sourceTables()) { + assertEquals("p", sourceTable.project()); + } + } + + private void compareCopyJobConfiguration(CopyJobConfiguration expected, + CopyJobConfiguration value) { + assertEquals(expected, value); + assertEquals(expected.hashCode(), value.hashCode()); + assertEquals(expected.toString(), value.toString()); + assertEquals(expected.destinationTable(), value.destinationTable()); + assertEquals(expected.sourceTables(), value.sourceTables()); + assertEquals(expected.createDisposition(), value.createDisposition()); + assertEquals(expected.writeDisposition(), value.writeDisposition()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobInfoTest.java deleted file mode 100644 index 81da59644cf0..000000000000 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/CopyJobInfoTest.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.google.common.collect.ImmutableList; -import com.google.gcloud.bigquery.JobInfo.CreateDisposition; -import com.google.gcloud.bigquery.JobInfo.WriteDisposition; - -import org.junit.Test; - -import java.util.List; - -public class CopyJobInfoTest { - - private static final String ETAG = "etag"; - private static final String ID = "id"; - private static final String SELF_LINK = "selfLink"; - private static final String EMAIL = "email"; - private static final TableId SOURCE_TABLE = TableId.of("dataset", "sourceTable"); - private static final List SOURCE_TABLES = ImmutableList.of( - TableId.of("dataset", "sourceTable1"), - TableId.of("dataset", "sourceTable2") - ); - private static final TableId DESTINATION_TABLE = TableId.of("dataset", "destinationTable"); - private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED; - private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; - private static final JobId JOB_ID = JobId.of("job"); - private static final JobStatus JOB_STATUS = new JobStatus(JobStatus.State.DONE); - private static final JobStatistics JOB_STATISTICS = JobStatistics.builder() - .creationTime(1L) - .endTime(3L) - .startTime(2L) - .build(); - private static final CopyJobInfo COPY_JOB = - CopyJobInfo.builder(DESTINATION_TABLE, SOURCE_TABLE) - .etag(ETAG) - .id(ID) - .selfLink(SELF_LINK) - .userEmail(EMAIL) - .jobId(JOB_ID) - .status(JOB_STATUS) - .createDisposition(CREATE_DISPOSITION) - .writeDisposition(WRITE_DISPOSITION) - .statistics(JOB_STATISTICS) - .build(); - private static final CopyJobInfo COPY_JOB_INFO_MULTIPLE_TABLES = - CopyJobInfo.builder(DESTINATION_TABLE, SOURCE_TABLES) - .etag(ETAG) - .id(ID) - .selfLink(SELF_LINK) - .userEmail(EMAIL) - .jobId(JOB_ID) - .status(JOB_STATUS) - .createDisposition(CREATE_DISPOSITION) - .writeDisposition(WRITE_DISPOSITION) - .build(); - - @Test - public void testToBuilder() { - compareCopyJobInfo(COPY_JOB, COPY_JOB.toBuilder().build()); - compareCopyJobInfo(COPY_JOB_INFO_MULTIPLE_TABLES, - COPY_JOB_INFO_MULTIPLE_TABLES.toBuilder().build()); - CopyJobInfo job = COPY_JOB.toBuilder() - .destinationTable(TableId.of("dataset", "newTable")) - .build(); - assertEquals("newTable", job.destinationTable().table()); - job = job.toBuilder().destinationTable(DESTINATION_TABLE).build(); - compareCopyJobInfo(COPY_JOB, job); - } - - @Test - public void testOf() { - CopyJobInfo job = CopyJobInfo.of(DESTINATION_TABLE, SOURCE_TABLES); - assertEquals(DESTINATION_TABLE, job.destinationTable()); - assertEquals(SOURCE_TABLES, job.sourceTables()); - job = CopyJobInfo.of(DESTINATION_TABLE, SOURCE_TABLE); - assertEquals(DESTINATION_TABLE, job.destinationTable()); - assertEquals(ImmutableList.of(SOURCE_TABLE), job.sourceTables()); - job = CopyJobInfo.of(JOB_ID, DESTINATION_TABLE, SOURCE_TABLES); - assertEquals(JOB_ID, job.jobId()); - assertEquals(DESTINATION_TABLE, job.destinationTable()); - assertEquals(SOURCE_TABLES, job.sourceTables()); - job = CopyJobInfo.of(JOB_ID, DESTINATION_TABLE, SOURCE_TABLE); - assertEquals(JOB_ID, job.jobId()); - assertEquals(DESTINATION_TABLE, job.destinationTable()); - assertEquals(ImmutableList.of(SOURCE_TABLE), job.sourceTables()); - } - - @Test - public void testToBuilderIncomplete() { - CopyJobInfo job = CopyJobInfo.of(DESTINATION_TABLE, SOURCE_TABLES); - compareCopyJobInfo(job, job.toBuilder().build()); - } - - @Test - public void testBuilder() { - assertEquals(ETAG, COPY_JOB_INFO_MULTIPLE_TABLES.etag()); - assertEquals(ID, COPY_JOB_INFO_MULTIPLE_TABLES.id()); - assertEquals(SELF_LINK, COPY_JOB_INFO_MULTIPLE_TABLES.selfLink()); - assertEquals(EMAIL, COPY_JOB_INFO_MULTIPLE_TABLES.userEmail()); - assertEquals(JOB_ID, COPY_JOB_INFO_MULTIPLE_TABLES.jobId()); - assertEquals(JOB_STATUS, COPY_JOB_INFO_MULTIPLE_TABLES.status()); - assertEquals(DESTINATION_TABLE, COPY_JOB_INFO_MULTIPLE_TABLES.destinationTable()); - assertEquals(SOURCE_TABLES, COPY_JOB_INFO_MULTIPLE_TABLES.sourceTables()); - assertEquals(CREATE_DISPOSITION, COPY_JOB_INFO_MULTIPLE_TABLES.createDisposition()); - assertEquals(WRITE_DISPOSITION, COPY_JOB_INFO_MULTIPLE_TABLES.writeDisposition()); - assertEquals(ETAG, COPY_JOB.etag()); - assertEquals(ID, COPY_JOB.id()); - assertEquals(SELF_LINK, COPY_JOB.selfLink()); - assertEquals(EMAIL, COPY_JOB.userEmail()); - assertEquals(JOB_ID, COPY_JOB.jobId()); - assertEquals(JOB_STATUS, COPY_JOB.status()); - assertEquals(DESTINATION_TABLE, COPY_JOB.destinationTable()); - assertEquals(ImmutableList.of(SOURCE_TABLE), COPY_JOB.sourceTables()); - assertEquals(CREATE_DISPOSITION, COPY_JOB.createDisposition()); - assertEquals(WRITE_DISPOSITION, COPY_JOB.writeDisposition()); - assertEquals(JOB_STATISTICS, COPY_JOB.statistics()); - } - - @Test - public void testToPbAndFromPb() { - assertNotNull(COPY_JOB.toPb().getConfiguration().getCopy()); - assertNull(COPY_JOB.toPb().getConfiguration().getExtract()); - assertNull(COPY_JOB.toPb().getConfiguration().getLoad()); - assertNull(COPY_JOB.toPb().getConfiguration().getQuery()); - assertNull(COPY_JOB.toPb().getConfiguration().getCopy().getSourceTables()); - assertEquals(JOB_STATISTICS, JobStatistics.fromPb(COPY_JOB.statistics().toPb())); - assertNull(COPY_JOB_INFO_MULTIPLE_TABLES.toPb().getConfiguration().getCopy().getSourceTable()); - compareCopyJobInfo(COPY_JOB, CopyJobInfo.fromPb(COPY_JOB.toPb())); - compareCopyJobInfo(COPY_JOB, (CopyJobInfo) JobInfo.fromPb(COPY_JOB.toPb())); - compareCopyJobInfo(COPY_JOB_INFO_MULTIPLE_TABLES, - CopyJobInfo.fromPb(COPY_JOB_INFO_MULTIPLE_TABLES.toPb())); - compareCopyJobInfo(COPY_JOB_INFO_MULTIPLE_TABLES, - (CopyJobInfo) JobInfo.fromPb(COPY_JOB_INFO_MULTIPLE_TABLES.toPb())); - CopyJobInfo job = CopyJobInfo.of(DESTINATION_TABLE, SOURCE_TABLES); - compareCopyJobInfo(job, CopyJobInfo.fromPb(job.toPb())); - compareCopyJobInfo(job, (CopyJobInfo) JobInfo.fromPb(job.toPb())); - } - - private void compareCopyJobInfo(CopyJobInfo expected, CopyJobInfo value) { - assertEquals(expected, value); - assertEquals(expected.hashCode(), value.hashCode()); - assertEquals(expected.toString(), value.toString()); - assertEquals(expected.etag(), value.etag()); - assertEquals(expected.id(), value.id()); - assertEquals(expected.jobId(), value.jobId()); - assertEquals(expected.selfLink(), value.selfLink()); - assertEquals(expected.status(), value.status()); - assertEquals(expected.statistics(), value.statistics()); - assertEquals(expected.userEmail(), value.userEmail()); - assertEquals(expected.destinationTable(), value.destinationTable()); - assertEquals(expected.sourceTables(), value.sourceTables()); - assertEquals(expected.createDisposition(), value.createDisposition()); - assertEquals(expected.writeDisposition(), value.writeDisposition()); - } -} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetIdTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetIdTest.java index 0af665895d71..ec645d71c96f 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetIdTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetIdTest.java @@ -45,6 +45,11 @@ public void testToPbAndFromPb() { compareDatasetIds(DATASET_COMPLETE, DatasetId.fromPb(DATASET_COMPLETE.toPb())); } + @Test + public void testSetProjectId() { + assertEquals(DATASET_COMPLETE, DATASET.setProjectId("project")); + } + private void compareDatasetIds(DatasetId expected, DatasetId value) { assertEquals(expected, value); assertEquals(expected.project(), value.project()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetInfoTest.java index 733253a2d790..20875c0fc853 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/DatasetInfoTest.java @@ -29,7 +29,10 @@ public class DatasetInfoTest { private static final List ACCESS_RULES = ImmutableList.of( Acl.of(Acl.Group.ofAllAuthenticatedUsers(), Acl.Role.READER), - Acl.of(new Acl.View(TableId.of("project", "dataset", "table")), Acl.Role.WRITER)); + Acl.of(new Acl.View(TableId.of("dataset", "table")))); + private static final List ACCESS_RULES_COMPLETE = ImmutableList.of( + Acl.of(Acl.Group.ofAllAuthenticatedUsers(), Acl.Role.READER), + Acl.of(new Acl.View(TableId.of("project", "dataset", "table")))); private static final Long CREATION_TIME = System.currentTimeMillis(); private static final Long DEFAULT_TABLE_EXPIRATION = CREATION_TIME + 100; private static final String DESCRIPTION = "description"; @@ -55,6 +58,7 @@ public class DatasetInfoTest { .build(); private static final DatasetInfo DATASET_INFO_COMPLETE = DATASET_INFO.toBuilder() .datasetId(DATASET_ID_COMPLETE) + .acl(ACCESS_RULES_COMPLETE) .build(); @Test @@ -91,7 +95,7 @@ public void testBuilder() { assertEquals(LOCATION, DATASET_INFO.location()); assertEquals(SELF_LINK, DATASET_INFO.selfLink()); assertEquals(DATASET_ID_COMPLETE, DATASET_INFO_COMPLETE.datasetId()); - assertEquals(ACCESS_RULES, DATASET_INFO_COMPLETE.acl()); + assertEquals(ACCESS_RULES_COMPLETE, DATASET_INFO_COMPLETE.acl()); assertEquals(CREATION_TIME, DATASET_INFO_COMPLETE.creationTime()); assertEquals(DEFAULT_TABLE_EXPIRATION, DATASET_INFO_COMPLETE.defaultTableLifetime()); assertEquals(DESCRIPTION, DATASET_INFO_COMPLETE.description()); @@ -110,6 +114,11 @@ public void testToPbAndFromPb() { compareDatasets(datasetInfo, DatasetInfo.fromPb(datasetInfo.toPb())); } + @Test + public void testSetProjectId() { + assertEquals(DATASET_INFO_COMPLETE, DATASET_INFO.setProjectId("project")); + } + private void compareDatasets(DatasetInfo expected, DatasetInfo value) { assertEquals(expected, value); assertEquals(expected.datasetId(), value.datasetId()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java new file mode 100644 index 000000000000..7ac67f41b1f8 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobConfigurationTest.java @@ -0,0 +1,139 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; + +import java.util.List; + +public class ExtractJobConfigurationTest { + + private static final List DESTINATION_URIS = ImmutableList.of("uri1", "uri2"); + private static final String DESTINATION_URI = "uri1"; + private static final TableId TABLE_ID = TableId.of("dataset", "table"); + private static final String FIELD_DELIMITER = ","; + private static final String FORMAT = "CSV"; + private static final String JSON_FORMAT = "NEWLINE_DELIMITED_JSON"; + private static final Boolean PRINT_HEADER = true; + private static final String COMPRESSION = "GZIP"; + private static final ExtractJobConfiguration EXTRACT_CONFIGURATION = + ExtractJobConfiguration.builder(TABLE_ID, DESTINATION_URIS) + .printHeader(PRINT_HEADER) + .fieldDelimiter(FIELD_DELIMITER) + .compression(COMPRESSION) + .format(FORMAT) + .build(); + private static final ExtractJobConfiguration EXTRACT_CONFIGURATION_ONE_URI = + ExtractJobConfiguration.builder(TABLE_ID, DESTINATION_URI) + .printHeader(PRINT_HEADER) + .fieldDelimiter(FIELD_DELIMITER) + .compression(COMPRESSION) + .format(FORMAT) + .build(); + + @Test + public void testToBuilder() { + compareExtractJobConfiguration( + EXTRACT_CONFIGURATION, EXTRACT_CONFIGURATION.toBuilder().build()); + ExtractJobConfiguration job = EXTRACT_CONFIGURATION.toBuilder() + .sourceTable(TableId.of("dataset", "newTable")) + .build(); + assertEquals("newTable", job.sourceTable().table()); + job = job.toBuilder().sourceTable(TABLE_ID).build(); + compareExtractJobConfiguration(EXTRACT_CONFIGURATION, job); + } + + @Test + public void testOf() { + ExtractJobConfiguration job = ExtractJobConfiguration.of(TABLE_ID, DESTINATION_URIS); + assertEquals(TABLE_ID, job.sourceTable()); + assertEquals(DESTINATION_URIS, job.destinationUris()); + job = ExtractJobConfiguration.of(TABLE_ID, DESTINATION_URI); + assertEquals(TABLE_ID, job.sourceTable()); + assertEquals(ImmutableList.of(DESTINATION_URI), job.destinationUris()); + job = ExtractJobConfiguration.of(TABLE_ID, DESTINATION_URIS, JSON_FORMAT); + assertEquals(TABLE_ID, job.sourceTable()); + assertEquals(DESTINATION_URIS, job.destinationUris()); + assertEquals(JSON_FORMAT, job.format()); + job = ExtractJobConfiguration.of(TABLE_ID, DESTINATION_URI, JSON_FORMAT); + assertEquals(TABLE_ID, job.sourceTable()); + assertEquals(ImmutableList.of(DESTINATION_URI), job.destinationUris()); + assertEquals(JSON_FORMAT, job.format()); + } + + @Test + public void testToBuilderIncomplete() { + ExtractJobConfiguration job = ExtractJobConfiguration.of(TABLE_ID, DESTINATION_URIS); + compareExtractJobConfiguration(job, job.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(TABLE_ID, EXTRACT_CONFIGURATION.sourceTable()); + assertEquals(DESTINATION_URIS, EXTRACT_CONFIGURATION.destinationUris()); + assertEquals(FIELD_DELIMITER, EXTRACT_CONFIGURATION.fieldDelimiter()); + assertEquals(COMPRESSION, EXTRACT_CONFIGURATION.compression()); + assertEquals(PRINT_HEADER, EXTRACT_CONFIGURATION.printHeader()); + assertEquals(FORMAT, EXTRACT_CONFIGURATION.format()); + assertEquals(TABLE_ID, EXTRACT_CONFIGURATION_ONE_URI.sourceTable()); + assertEquals(ImmutableList.of(DESTINATION_URI), + EXTRACT_CONFIGURATION_ONE_URI.destinationUris()); + assertEquals(FIELD_DELIMITER, EXTRACT_CONFIGURATION_ONE_URI.fieldDelimiter()); + assertEquals(COMPRESSION, EXTRACT_CONFIGURATION_ONE_URI.compression()); + assertEquals(PRINT_HEADER, EXTRACT_CONFIGURATION_ONE_URI.printHeader()); + assertEquals(FORMAT, EXTRACT_CONFIGURATION_ONE_URI.format()); + } + + @Test + public void testToPbAndFromPb() { + assertNotNull(EXTRACT_CONFIGURATION.toPb().getExtract()); + assertNull(EXTRACT_CONFIGURATION.toPb().getCopy()); + assertNull(EXTRACT_CONFIGURATION.toPb().getLoad()); + assertNull(EXTRACT_CONFIGURATION.toPb().getQuery()); + compareExtractJobConfiguration(EXTRACT_CONFIGURATION, + ExtractJobConfiguration.fromPb(EXTRACT_CONFIGURATION.toPb())); + compareExtractJobConfiguration(EXTRACT_CONFIGURATION_ONE_URI, + ExtractJobConfiguration.fromPb(EXTRACT_CONFIGURATION_ONE_URI.toPb())); + ExtractJobConfiguration job = ExtractJobConfiguration.of(TABLE_ID, DESTINATION_URIS); + compareExtractJobConfiguration(job, ExtractJobConfiguration.fromPb(job.toPb())); + } + + @Test + public void testSetProjectId() { + ExtractJobConfiguration configuration = EXTRACT_CONFIGURATION.setProjectId("p"); + assertEquals("p", configuration.sourceTable().project()); + } + + private void compareExtractJobConfiguration(ExtractJobConfiguration expected, + ExtractJobConfiguration value) { + assertEquals(expected, value); + assertEquals(expected.hashCode(), value.hashCode()); + assertEquals(expected.toString(), value.toString()); + assertEquals(expected.sourceTable(), value.sourceTable()); + assertEquals(expected.destinationUris(), value.destinationUris()); + assertEquals(expected.compression(), value.compression()); + assertEquals(expected.printHeader(), value.printHeader()); + assertEquals(expected.fieldDelimiter(), value.fieldDelimiter()); + assertEquals(expected.format(), value.format()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobInfoTest.java deleted file mode 100644 index bb47112b6410..000000000000 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ExtractJobInfoTest.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.google.common.collect.ImmutableList; -import com.google.gcloud.bigquery.JobStatistics.ExtractStatistics; - -import org.junit.Test; - -import java.util.List; - -public class ExtractJobInfoTest { - - private static final String ETAG = "etag"; - private static final String ID = "id"; - private static final String SELF_LINK = "selfLink"; - private static final String EMAIL = "email"; - private static final List DESTINATION_URIS = ImmutableList.of("uri1", "uri2"); - private static final String DESTINATION_URI = "uri1"; - private static final TableId TABLE_ID = TableId.of("dataset", "table"); - private static final String FIELD_DELIMITER = ","; - private static final String FORMAT = "CSV"; - private static final String JSON_FORMAT = "NEWLINE_DELIMITED_JSON"; - private static final Boolean PRINT_HEADER = true; - private static final String COMPRESSION = "GZIP"; - private static final JobId JOB_ID = JobId.of("job"); - private static final JobStatus JOB_STATUS = new JobStatus(JobStatus.State.DONE); - private static final ExtractStatistics JOB_STATISTICS = ExtractStatistics.builder() - .creationTime(1L) - .endTime(3L) - .startTime(2L) - .destinationUriFileCounts(ImmutableList.of(42L)) - .build(); - private static final ExtractJobInfo EXTRACT_JOB = - ExtractJobInfo.builder(TABLE_ID, DESTINATION_URIS) - .etag(ETAG) - .id(ID) - .selfLink(SELF_LINK) - .userEmail(EMAIL) - .jobId(JOB_ID) - .status(JOB_STATUS) - .printHeader(PRINT_HEADER) - .fieldDelimiter(FIELD_DELIMITER) - .compression(COMPRESSION) - .format(FORMAT) - .statistics(JOB_STATISTICS) - .build(); - private static final ExtractJobInfo EXTRACT_JOB_ONE_URI = - ExtractJobInfo.builder(TABLE_ID, DESTINATION_URI) - .etag(ETAG) - .id(ID) - .selfLink(SELF_LINK) - .userEmail(EMAIL) - .jobId(JOB_ID) - .status(JOB_STATUS) - .printHeader(PRINT_HEADER) - .fieldDelimiter(FIELD_DELIMITER) - .compression(COMPRESSION) - .format(FORMAT) - .build(); - - @Test - public void testToBuilder() { - compareExtractJobInfo(EXTRACT_JOB, EXTRACT_JOB.toBuilder().build()); - ExtractJobInfo job = EXTRACT_JOB.toBuilder() - .sourceTable(TableId.of("dataset", "newTable")) - .build(); - assertEquals("newTable", job.sourceTable().table()); - job = job.toBuilder().sourceTable(TABLE_ID).build(); - compareExtractJobInfo(EXTRACT_JOB, job); - } - - @Test - public void testOf() { - ExtractJobInfo job = ExtractJobInfo.of(TABLE_ID, DESTINATION_URIS); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(DESTINATION_URIS, job.destinationUris()); - job = ExtractJobInfo.of(TABLE_ID, DESTINATION_URI); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(ImmutableList.of(DESTINATION_URI), job.destinationUris()); - job = ExtractJobInfo.of(TABLE_ID, JSON_FORMAT, DESTINATION_URIS); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(DESTINATION_URIS, job.destinationUris()); - assertEquals(JSON_FORMAT, job.format()); - job = ExtractJobInfo.of(TABLE_ID, JSON_FORMAT, DESTINATION_URI); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(ImmutableList.of(DESTINATION_URI), job.destinationUris()); - assertEquals(JSON_FORMAT, job.format()); - job = ExtractJobInfo.of(JOB_ID, TABLE_ID, DESTINATION_URIS); - assertEquals(JOB_ID, job.jobId()); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(DESTINATION_URIS, job.destinationUris()); - job = ExtractJobInfo.of(JOB_ID, TABLE_ID, DESTINATION_URI); - assertEquals(JOB_ID, job.jobId()); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(ImmutableList.of(DESTINATION_URI), job.destinationUris()); - job = ExtractJobInfo.of(JOB_ID, TABLE_ID, JSON_FORMAT, DESTINATION_URIS); - assertEquals(JOB_ID, job.jobId()); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(DESTINATION_URIS, job.destinationUris()); - assertEquals(JSON_FORMAT, job.format()); - job = ExtractJobInfo.of(JOB_ID, TABLE_ID, JSON_FORMAT, DESTINATION_URI); - assertEquals(JOB_ID, job.jobId()); - assertEquals(TABLE_ID, job.sourceTable()); - assertEquals(ImmutableList.of(DESTINATION_URI), job.destinationUris()); - assertEquals(JSON_FORMAT, job.format()); - } - - @Test - public void testToBuilderIncomplete() { - ExtractJobInfo job = ExtractJobInfo.of(TABLE_ID, DESTINATION_URIS); - compareExtractJobInfo(job, job.toBuilder().build()); - } - - @Test - public void testBuilder() { - assertEquals(ETAG, EXTRACT_JOB.etag()); - assertEquals(ID, EXTRACT_JOB.id()); - assertEquals(SELF_LINK, EXTRACT_JOB.selfLink()); - assertEquals(EMAIL, EXTRACT_JOB.userEmail()); - assertEquals(JOB_ID, EXTRACT_JOB.jobId()); - assertEquals(JOB_STATUS, EXTRACT_JOB.status()); - assertEquals(TABLE_ID, EXTRACT_JOB.sourceTable()); - assertEquals(DESTINATION_URIS, EXTRACT_JOB.destinationUris()); - assertEquals(FIELD_DELIMITER, EXTRACT_JOB.fieldDelimiter()); - assertEquals(COMPRESSION, EXTRACT_JOB.compression()); - assertEquals(PRINT_HEADER, EXTRACT_JOB.printHeader()); - assertEquals(FORMAT, EXTRACT_JOB.format()); - assertEquals(JOB_STATISTICS, EXTRACT_JOB.statistics()); - assertEquals(ETAG, EXTRACT_JOB_ONE_URI.etag()); - assertEquals(ID, EXTRACT_JOB_ONE_URI.id()); - assertEquals(SELF_LINK, EXTRACT_JOB_ONE_URI.selfLink()); - assertEquals(EMAIL, EXTRACT_JOB_ONE_URI.userEmail()); - assertEquals(JOB_ID, EXTRACT_JOB_ONE_URI.jobId()); - assertEquals(JOB_STATUS, EXTRACT_JOB_ONE_URI.status()); - assertEquals(TABLE_ID, EXTRACT_JOB_ONE_URI.sourceTable()); - assertEquals(ImmutableList.of(DESTINATION_URI), - EXTRACT_JOB_ONE_URI.destinationUris()); - assertEquals(FIELD_DELIMITER, EXTRACT_JOB_ONE_URI.fieldDelimiter()); - assertEquals(COMPRESSION, EXTRACT_JOB_ONE_URI.compression()); - assertEquals(PRINT_HEADER, EXTRACT_JOB_ONE_URI.printHeader()); - assertEquals(FORMAT, EXTRACT_JOB_ONE_URI.format()); - } - - @Test - public void testToPbAndFromPb() { - assertNotNull(EXTRACT_JOB.toPb().getConfiguration().getExtract()); - assertNull(EXTRACT_JOB.toPb().getConfiguration().getCopy()); - assertNull(EXTRACT_JOB.toPb().getConfiguration().getLoad()); - assertNull(EXTRACT_JOB.toPb().getConfiguration().getQuery()); - assertEquals(JOB_STATISTICS, JobStatistics.fromPb(EXTRACT_JOB.toPb().getStatistics())); - compareExtractJobInfo(EXTRACT_JOB, - ExtractJobInfo.fromPb(EXTRACT_JOB.toPb())); - compareExtractJobInfo(EXTRACT_JOB, - (ExtractJobInfo) JobInfo.fromPb(EXTRACT_JOB.toPb())); - compareExtractJobInfo(EXTRACT_JOB_ONE_URI, - ExtractJobInfo.fromPb(EXTRACT_JOB_ONE_URI.toPb())); - compareExtractJobInfo(EXTRACT_JOB_ONE_URI, - (ExtractJobInfo) JobInfo.fromPb(EXTRACT_JOB_ONE_URI.toPb())); - ExtractJobInfo job = ExtractJobInfo.of(TABLE_ID, DESTINATION_URIS); - compareExtractJobInfo(job, ExtractJobInfo.fromPb(job.toPb())); - compareExtractJobInfo(job, (ExtractJobInfo) JobInfo.fromPb(job.toPb())); - } - - private void compareExtractJobInfo(ExtractJobInfo expected, ExtractJobInfo value) { - assertEquals(expected, value); - assertEquals(expected.hashCode(), value.hashCode()); - assertEquals(expected.toString(), value.toString()); - assertEquals(expected.etag(), value.etag()); - assertEquals(expected.id(), value.id()); - assertEquals(expected.jobId(), value.jobId()); - assertEquals(expected.selfLink(), value.selfLink()); - assertEquals(expected.status(), value.status()); - assertEquals(expected.statistics(), value.statistics()); - assertEquals(expected.userEmail(), value.userEmail()); - assertEquals(expected.sourceTable(), value.sourceTable()); - assertEquals(expected.destinationUris(), value.destinationUris()); - assertEquals(expected.compression(), value.compression()); - assertEquals(expected.printHeader(), value.printHeader()); - assertEquals(expected.fieldDelimiter(), value.fieldDelimiter()); - assertEquals(expected.format(), value.format()); - } -} \ No newline at end of file diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java index f672815bcb7a..e083d3682d8c 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java @@ -152,12 +152,12 @@ public static void beforeClass() throws IOException, InterruptedException { JSON_CONTENT.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.builder(DATASET).description(DESCRIPTION).build(); bigquery.create(info); - LoadConfiguration configuration = LoadConfiguration.builder(TABLE_ID, FormatOptions.json()) + LoadJobConfiguration configuration = LoadJobConfiguration.builder( + TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .schema(TABLE_SCHEMA) .build(); - LoadJobInfo job = LoadJobInfo.of(configuration, "gs://" + BUCKET + "/" + JSON_LOAD_FILE); - job = bigquery.create(job); + JobInfo job = bigquery.create(JobInfo.of(configuration)); while (job.status().state() != JobStatus.State.DONE) { Thread.sleep(1000); job = bigquery.getJob(job.jobId()); @@ -646,8 +646,9 @@ public void testQuery() throws InterruptedException { rowCount++; } assertEquals(2, rowCount); - QueryJobInfo queryJob = bigquery.getJob(response.jobId()); - assertNotNull(queryJob.statistics().queryPlan()); + JobInfo queryJob = bigquery.getJob(response.jobId()); + JobStatistics.QueryStatistics statistics = queryJob.statistics(); + assertNotNull(statistics.queryPlan()); } @Test @@ -685,14 +686,18 @@ public void testCreateAndGetJob() throws InterruptedException { assertEquals(DATASET, createdTableInfo.tableId().dataset()); assertEquals(sourceTableName, createdTableInfo.tableId().table()); TableId destinationTable = TableId.of(DATASET, destinationTableName); - CopyJobInfo job = CopyJobInfo.of(destinationTable, sourceTable); - CopyJobInfo createdJob = bigquery.create(job); - CopyJobInfo remoteJob = bigquery.getJob(createdJob.jobId()); + CopyJobConfiguration copyJobConfiguration = + CopyJobConfiguration.of(destinationTable, sourceTable); + JobInfo job = JobInfo.of(copyJobConfiguration); + JobInfo createdJob = bigquery.create(job); + JobInfo remoteJob = bigquery.getJob(createdJob.jobId()); assertEquals(createdJob.jobId(), remoteJob.jobId()); - assertEquals(createdJob.sourceTables(), remoteJob.sourceTables()); - assertEquals(createdJob.destinationTable(), remoteJob.destinationTable()); - assertEquals(createdJob.createDisposition(), remoteJob.createDisposition()); - assertEquals(createdJob.writeDisposition(), remoteJob.writeDisposition()); + CopyJobConfiguration createdConfiguration = createdJob.configuration(); + CopyJobConfiguration remoteConfiguration = remoteJob.configuration(); + assertEquals(createdConfiguration.sourceTables(), remoteConfiguration.sourceTables()); + assertEquals(createdConfiguration.destinationTable(), remoteConfiguration.destinationTable()); + assertEquals(createdConfiguration.createDisposition(), remoteConfiguration.createDisposition()); + assertEquals(createdConfiguration.writeDisposition(), remoteConfiguration.writeDisposition()); assertNotNull(remoteJob.etag()); assertNotNull(remoteJob.statistics()); assertNotNull(remoteJob.status()); @@ -713,23 +718,25 @@ public void testCreateAndGetJobWithSelectedFields() throws InterruptedException assertEquals(DATASET, createdTableInfo.tableId().dataset()); assertEquals(sourceTableName, createdTableInfo.tableId().table()); TableId destinationTable = TableId.of(DATASET, destinationTableName); - CopyJobInfo job = CopyJobInfo.of(destinationTable, sourceTable); - CopyJobInfo createdJob = bigquery.create(job, JobOption.fields(JobField.ETAG)); + CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable); + JobInfo createdJob = + bigquery.create(JobInfo.of(configuration), JobOption.fields(JobField.ETAG)); + CopyJobConfiguration createdConfiguration = createdJob.configuration(); assertNotNull(createdJob.jobId()); - assertNotNull(createdJob.sourceTables()); - assertNotNull(createdJob.destinationTable()); + assertNotNull(createdConfiguration.sourceTables()); + assertNotNull(createdConfiguration.destinationTable()); assertNotNull(createdJob.etag()); assertNull(createdJob.statistics()); assertNull(createdJob.status()); assertNull(createdJob.selfLink()); assertNull(createdJob.userEmail()); - CopyJobInfo remoteJob = bigquery.getJob(createdJob.jobId(), - JobOption.fields(JobField.ETAG)); + JobInfo remoteJob = bigquery.getJob(createdJob.jobId(), JobOption.fields(JobField.ETAG)); + CopyJobConfiguration remoteConfiguration = remoteJob.configuration(); assertEquals(createdJob.jobId(), remoteJob.jobId()); - assertEquals(createdJob.sourceTables(), remoteJob.sourceTables()); - assertEquals(createdJob.destinationTable(), remoteJob.destinationTable()); - assertEquals(createdJob.createDisposition(), remoteJob.createDisposition()); - assertEquals(createdJob.writeDisposition(), remoteJob.writeDisposition()); + assertEquals(createdConfiguration.sourceTables(), remoteConfiguration.sourceTables()); + assertEquals(createdConfiguration.destinationTable(), remoteConfiguration.destinationTable()); + assertEquals(createdConfiguration.createDisposition(), remoteConfiguration.createDisposition()); + assertEquals(createdConfiguration.writeDisposition(), remoteConfiguration.writeDisposition()); assertNotNull(remoteJob.etag()); assertNull(remoteJob.statistics()); assertNull(remoteJob.status()); @@ -750,8 +757,8 @@ public void testCopyJob() throws InterruptedException { assertEquals(DATASET, createdTableInfo.tableId().dataset()); assertEquals(sourceTableName, createdTableInfo.tableId().table()); TableId destinationTable = TableId.of(DATASET, destinationTableName); - CopyJobInfo job = CopyJobInfo.of(destinationTable, sourceTable); - CopyJobInfo remoteJob = bigquery.create(job); + CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable); + JobInfo remoteJob = bigquery.create(JobInfo.of(configuration)); while (remoteJob.status().state() != JobStatus.State.DONE) { Thread.sleep(1000); remoteJob = bigquery.getJob(remoteJob.jobId()); @@ -774,11 +781,11 @@ public void testQueryJob() throws InterruptedException { .append(TABLE_ID.table()) .toString(); TableId destinationTable = TableId.of(DATASET, tableName); - QueryJobInfo job = QueryJobInfo.builder(query) + QueryJobConfiguration configuration = QueryJobConfiguration.builder(query) .defaultDataset(DatasetId.of(DATASET)) .destinationTable(destinationTable) .build(); - QueryJobInfo remoteJob = bigquery.create(job); + JobInfo remoteJob = bigquery.create(JobInfo.of(configuration)); while (remoteJob.status().state() != JobStatus.State.DONE) { Thread.sleep(1000); remoteJob = bigquery.getJob(remoteJob.jobId()); @@ -807,30 +814,33 @@ public void testQueryJob() throws InterruptedException { } assertEquals(2, rowCount); assertTrue(bigquery.delete(DATASET, tableName)); - QueryJobInfo queryJob = bigquery.getJob(remoteJob.jobId()); - assertNotNull(queryJob.statistics().queryPlan()); + JobInfo queryJob = bigquery.getJob(remoteJob.jobId()); + JobStatistics.QueryStatistics statistics = queryJob.statistics(); + assertNotNull(statistics.queryPlan()); } @Test public void testExtractJob() throws InterruptedException { String tableName = "test_export_job_table"; TableId destinationTable = TableId.of(DATASET, tableName); - LoadConfiguration configuration = LoadConfiguration.builder(destinationTable) - .schema(SIMPLE_SCHEMA) - .build(); - LoadJobInfo remoteLoadJob = - bigquery.create(LoadJobInfo.of(configuration, "gs://" + BUCKET + "/" + LOAD_FILE)); + LoadJobConfiguration configuration = + LoadJobConfiguration.builder(destinationTable, "gs://" + BUCKET + "/" + LOAD_FILE) + .schema(SIMPLE_SCHEMA) + .build(); + JobInfo remoteLoadJob = + bigquery.create(JobInfo.of(configuration)); while (remoteLoadJob.status().state() != JobStatus.State.DONE) { Thread.sleep(1000); remoteLoadJob = bigquery.getJob(remoteLoadJob.jobId()); } assertNull(remoteLoadJob.status().error()); - ExtractJobInfo extractJob = - ExtractJobInfo.builder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE) - .printHeader(false) - .build(); - ExtractJobInfo remoteExtractJob = bigquery.create(extractJob); + ExtractJobConfiguration extractConfiguration = + ExtractJobConfiguration.builder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE) + .printHeader(false) + .build(); + JobInfo extractJob = JobInfo.of(extractConfiguration); + JobInfo remoteExtractJob = bigquery.create(extractJob); while (remoteExtractJob.status().state() != JobStatus.State.DONE) { Thread.sleep(1000); remoteExtractJob = bigquery.getJob(remoteExtractJob.jobId()); @@ -846,11 +856,11 @@ public void testCancelJob() throws InterruptedException { String destinationTableName = "test_cancel_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.table(); TableId destinationTable = TableId.of(DATASET, destinationTableName); - QueryJobInfo job = QueryJobInfo.builder(query) + QueryJobConfiguration configuration = QueryJobConfiguration.builder(query) .defaultDataset(DatasetId.of(DATASET)) .destinationTable(destinationTable) .build(); - JobInfo remoteJob = bigquery.create(job); + JobInfo remoteJob = bigquery.create(JobInfo.of(configuration)); assertTrue(bigquery.cancel(remoteJob.jobId())); while (remoteJob.status().state() != JobStatus.State.DONE) { Thread.sleep(1000); @@ -868,7 +878,7 @@ public void testCancelNonExistingJob() throws InterruptedException { public void testInsertFromFile() throws InterruptedException, FileNotFoundException { String destinationTableName = "test_insert_from_file_table"; TableId tableId = TableId.of(DATASET, destinationTableName); - LoadConfiguration configuration = LoadConfiguration.builder(tableId) + WriteChannelConfiguration configuration = WriteChannelConfiguration.builder(tableId) .formatOptions(FormatOptions.json()) .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .schema(TABLE_SCHEMA) diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java new file mode 100644 index 000000000000..96bf8d1838c4 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobInfoTest.java @@ -0,0 +1,370 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gcloud.bigquery.JobInfo.CreateDisposition; +import com.google.gcloud.bigquery.JobInfo.WriteDisposition; +import com.google.gcloud.bigquery.JobStatistics.ExtractStatistics; +import com.google.gcloud.bigquery.JobStatistics.LoadStatistics; +import com.google.gcloud.bigquery.JobStatistics.QueryStatistics; + +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +public class JobInfoTest { + + private static final String ETAG = "etag"; + private static final String ID = "id"; + private static final String SELF_LINK = "selfLink"; + private static final String EMAIL = "email"; + private static final JobId JOB_ID = JobId.of("job"); + private static final JobStatus JOB_STATUS = new JobStatus(JobStatus.State.DONE); + private static final JobStatistics COPY_JOB_STATISTICS = JobStatistics.builder() + .creationTime(1L) + .endTime(3L) + .startTime(2L) + .build(); + private static final ExtractStatistics EXTRACT_JOB_STATISTICS = + ExtractStatistics.builder() + .creationTime(1L) + .endTime(3L) + .startTime(2L) + .destinationUriFileCounts(ImmutableList.of(42L)) + .build(); + private static final LoadStatistics LOAD_JOB_STATISTICS = + LoadStatistics.builder() + .creationTime(1L) + .endTime(3L) + .startTime(2L) + .inputFiles(42L) + .outputBytes(1024L) + .inputBytes(2048L) + .outputRows(24L) + .build(); + private static final QueryStatistics QUERY_JOB_STATISTICS = + QueryStatistics.builder() + .creationTime(1L) + .endTime(3L) + .startTime(2L) + .totalBytesProcessed(2048L) + .totalBytesBilled(1024L) + .cacheHit(false) + .billingTier(42) + .build(); + private static final TableId SOURCE_TABLE = TableId.of("dataset", "sourceTable"); + private static final TableId DESTINATION_TABLE = TableId.of("dataset", "destinationTable"); + private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED; + private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; + private static final CopyJobConfiguration COPY_CONFIGURATION = + CopyJobConfiguration.builder(DESTINATION_TABLE, SOURCE_TABLE) + .createDisposition(CREATE_DISPOSITION) + .writeDisposition(WRITE_DISPOSITION) + .build(); + private static final List DESTINATION_URIS = ImmutableList.of("uri1", "uri2"); + private static final TableId TABLE_ID = TableId.of("dataset", "table"); + private static final DatasetId DATASET_ID = DatasetId.of("dataset"); + private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final Field FIELD_SCHEMA1 = + Field.builder("StringField", Field.Type.string()) + .mode(Field.Mode.NULLABLE) + .description("FieldDescription1") + .build(); + private static final Field FIELD_SCHEMA2 = + Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REPEATED) + .description("FieldDescription2") + .build(); + private static final Field FIELD_SCHEMA3 = + Field.builder("RecordField", Field.Type.record(FIELD_SCHEMA1, FIELD_SCHEMA2)) + .mode(Field.Mode.REQUIRED) + .description("FieldDescription3") + .build(); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA1, FIELD_SCHEMA2, FIELD_SCHEMA3); + private static final String FIELD_DELIMITER = ","; + private static final String FORMAT = "CSV"; + private static final Boolean PRINT_HEADER = true; + private static final String COMPRESSION = "GZIP"; + private static final ExtractJobConfiguration EXTRACT_CONFIGURATION = + ExtractJobConfiguration.builder(TABLE_ID, DESTINATION_URIS) + .printHeader(PRINT_HEADER) + .fieldDelimiter(FIELD_DELIMITER) + .compression(COMPRESSION) + .format(FORMAT) + .build(); + private static final List PROJECTION_FIELDS = ImmutableList.of("field1", "field2"); + private static final Integer MAX_BAD_RECORDS = 42; + private static final Boolean IGNORE_UNKNOWN_VALUES = true; + private static final CsvOptions CSV_OPTIONS = CsvOptions.builder().build(); + private static final ExternalDataConfiguration TABLE_CONFIGURATION = ExternalDataConfiguration + .builder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) + .compression(COMPRESSION) + .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .maxBadRecords(MAX_BAD_RECORDS) + .build(); + private static final LoadJobConfiguration LOAD_CONFIGURATION = + LoadJobConfiguration.builder(TABLE_ID, SOURCE_URIS) + .createDisposition(CREATE_DISPOSITION) + .writeDisposition(WRITE_DISPOSITION) + .formatOptions(CSV_OPTIONS) + .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .maxBadRecords(MAX_BAD_RECORDS) + .projectionFields(PROJECTION_FIELDS) + .schema(TABLE_SCHEMA) + .build(); + private static final String QUERY = "BigQuery SQL"; + private static final Map TABLE_DEFINITIONS = + ImmutableMap.of("tableName", TABLE_CONFIGURATION); + private static final QueryJobConfiguration.Priority PRIORITY = + QueryJobConfiguration.Priority.BATCH; + private static final boolean ALLOW_LARGE_RESULTS = true; + private static final boolean USE_QUERY_CACHE = false; + private static final boolean FLATTEN_RESULTS = true; + private static final List USER_DEFINED_FUNCTIONS = ImmutableList.of( + UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); + private static final QueryJobConfiguration QUERY_CONFIGURATION = + QueryJobConfiguration.builder(QUERY) + .useQueryCache(USE_QUERY_CACHE) + .tableDefinitions(TABLE_DEFINITIONS) + .allowLargeResults(ALLOW_LARGE_RESULTS) + .createDisposition(CREATE_DISPOSITION) + .defaultDataset(DATASET_ID) + .destinationTable(TABLE_ID) + .writeDisposition(WRITE_DISPOSITION) + .priority(PRIORITY) + .flattenResults(FLATTEN_RESULTS) + .userDefinedFunctions(USER_DEFINED_FUNCTIONS) + .dryRun(true) + .build(); + private static final JobInfo COPY_JOB = JobInfo.builder(COPY_CONFIGURATION) + .jobId(JOB_ID) + .statistics(COPY_JOB_STATISTICS) + .jobId(JOB_ID) + .etag(ETAG) + .id(ID) + .selfLink(SELF_LINK) + .userEmail(EMAIL) + .status(JOB_STATUS) + .build(); + private static final JobInfo EXTRACT_JOB = JobInfo.builder(EXTRACT_CONFIGURATION) + .jobId(JOB_ID) + .statistics(EXTRACT_JOB_STATISTICS) + .jobId(JOB_ID) + .etag(ETAG) + .id(ID) + .selfLink(SELF_LINK) + .userEmail(EMAIL) + .status(JOB_STATUS) + .build(); + private static final JobInfo LOAD_JOB = JobInfo.builder(LOAD_CONFIGURATION) + .jobId(JOB_ID) + .statistics(LOAD_JOB_STATISTICS) + .jobId(JOB_ID) + .etag(ETAG) + .id(ID) + .selfLink(SELF_LINK) + .userEmail(EMAIL) + .status(JOB_STATUS) + .build(); + private static final JobInfo QUERY_JOB = JobInfo.builder(QUERY_CONFIGURATION) + .jobId(JOB_ID) + .statistics(QUERY_JOB_STATISTICS) + .jobId(JOB_ID) + .etag(ETAG) + .id(ID) + .selfLink(SELF_LINK) + .userEmail(EMAIL) + .status(JOB_STATUS) + .build(); + + + @Test + public void testToBuilder() { + compareJobInfo(COPY_JOB, COPY_JOB.toBuilder().build()); + compareJobInfo(EXTRACT_JOB, EXTRACT_JOB.toBuilder().build()); + compareJobInfo(LOAD_JOB, LOAD_JOB.toBuilder().build()); + compareJobInfo(QUERY_JOB, QUERY_JOB.toBuilder().build()); + JobInfo job = COPY_JOB.toBuilder() + .userEmail("newEmail") + .build(); + assertEquals("newEmail", job.userEmail()); + job = job.toBuilder().userEmail(EMAIL).build(); + compareJobInfo(COPY_JOB, job); + job = EXTRACT_JOB.toBuilder() + .userEmail("newEmail") + .build(); + assertEquals("newEmail", job.userEmail()); + job = job.toBuilder().userEmail(EMAIL).build(); + compareJobInfo(EXTRACT_JOB, job); + job = LOAD_JOB.toBuilder() + .userEmail("newEmail") + .build(); + assertEquals("newEmail", job.userEmail()); + job = job.toBuilder().userEmail(EMAIL).build(); + compareJobInfo(LOAD_JOB, job); + job = QUERY_JOB.toBuilder() + .userEmail("newEmail") + .build(); + assertEquals("newEmail", job.userEmail()); + job = job.toBuilder().userEmail(EMAIL).build(); + compareJobInfo(QUERY_JOB, job); + } + + @Test + public void testOf() { + JobInfo job = JobInfo.of(COPY_CONFIGURATION); + assertEquals(COPY_CONFIGURATION, job.configuration()); + job = JobInfo.of(EXTRACT_CONFIGURATION); + assertEquals(EXTRACT_CONFIGURATION, job.configuration()); + job = JobInfo.of(LOAD_CONFIGURATION); + assertEquals(LOAD_CONFIGURATION, job.configuration()); + job = JobInfo.of(QUERY_CONFIGURATION); + assertEquals(QUERY_CONFIGURATION, job.configuration()); + job = JobInfo.of(JOB_ID, COPY_CONFIGURATION); + assertEquals(JOB_ID, job.jobId()); + assertEquals(COPY_CONFIGURATION, job.configuration()); + job = JobInfo.of(JOB_ID, EXTRACT_CONFIGURATION); + assertEquals(JOB_ID, job.jobId()); + assertEquals(EXTRACT_CONFIGURATION, job.configuration()); + job = JobInfo.of(JOB_ID, LOAD_CONFIGURATION); + assertEquals(JOB_ID, job.jobId()); + assertEquals(LOAD_CONFIGURATION, job.configuration()); + job = JobInfo.of(JOB_ID, QUERY_CONFIGURATION); + assertEquals(JOB_ID, job.jobId()); + assertEquals(QUERY_CONFIGURATION, job.configuration()); + + } + + @Test + public void testToBuilderIncomplete() { + JobInfo job = JobInfo.of(COPY_CONFIGURATION); + compareJobInfo(job, job.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(ETAG, COPY_JOB.etag()); + assertEquals(ID, COPY_JOB.id()); + assertEquals(SELF_LINK, COPY_JOB.selfLink()); + assertEquals(EMAIL, COPY_JOB.userEmail()); + assertEquals(JOB_ID, COPY_JOB.jobId()); + assertEquals(JOB_STATUS, COPY_JOB.status()); + assertEquals(COPY_CONFIGURATION, COPY_JOB.configuration()); + assertEquals(COPY_JOB_STATISTICS, COPY_JOB.statistics()); + + assertEquals(ETAG, EXTRACT_JOB.etag()); + assertEquals(ID, EXTRACT_JOB.id()); + assertEquals(SELF_LINK, EXTRACT_JOB.selfLink()); + assertEquals(EMAIL, EXTRACT_JOB.userEmail()); + assertEquals(JOB_ID, EXTRACT_JOB.jobId()); + assertEquals(JOB_STATUS, EXTRACT_JOB.status()); + assertEquals(EXTRACT_CONFIGURATION, EXTRACT_JOB.configuration()); + assertEquals(EXTRACT_JOB_STATISTICS, EXTRACT_JOB.statistics()); + + assertEquals(ETAG, LOAD_JOB.etag()); + assertEquals(ID, LOAD_JOB.id()); + assertEquals(SELF_LINK, LOAD_JOB.selfLink()); + assertEquals(EMAIL, LOAD_JOB.userEmail()); + assertEquals(JOB_ID, LOAD_JOB.jobId()); + assertEquals(JOB_STATUS, LOAD_JOB.status()); + assertEquals(LOAD_CONFIGURATION, LOAD_JOB.configuration()); + assertEquals(LOAD_JOB_STATISTICS, LOAD_JOB.statistics()); + + assertEquals(ETAG, QUERY_JOB.etag()); + assertEquals(ID, QUERY_JOB.id()); + assertEquals(SELF_LINK, QUERY_JOB.selfLink()); + assertEquals(EMAIL, QUERY_JOB.userEmail()); + assertEquals(JOB_ID, QUERY_JOB.jobId()); + assertEquals(JOB_STATUS, QUERY_JOB.status()); + assertEquals(QUERY_CONFIGURATION, QUERY_JOB.configuration()); + assertEquals(QUERY_JOB_STATISTICS, QUERY_JOB.statistics()); + } + + @Test + public void testToPbAndFromPb() { + assertNotNull(COPY_JOB.toPb().getConfiguration().getCopy()); + assertNull(COPY_JOB.toPb().getConfiguration().getExtract()); + assertNull(COPY_JOB.toPb().getConfiguration().getLoad()); + assertNull(COPY_JOB.toPb().getConfiguration().getQuery()); + assertEquals(COPY_JOB_STATISTICS, JobStatistics.fromPb(COPY_JOB.statistics().toPb())); + compareJobInfo(COPY_JOB, JobInfo.fromPb(COPY_JOB.toPb())); + assertTrue(JobInfo.fromPb(COPY_JOB.toPb()).configuration() instanceof CopyJobConfiguration); + assertNull(EXTRACT_JOB.toPb().getConfiguration().getCopy()); + assertNotNull(EXTRACT_JOB.toPb().getConfiguration().getExtract()); + assertNull(EXTRACT_JOB.toPb().getConfiguration().getLoad()); + assertNull(EXTRACT_JOB.toPb().getConfiguration().getQuery()); + assertEquals(EXTRACT_JOB_STATISTICS, JobStatistics.fromPb(EXTRACT_JOB.statistics().toPb())); + compareJobInfo(EXTRACT_JOB, JobInfo.fromPb(EXTRACT_JOB.toPb())); + assertTrue( + JobInfo.fromPb(EXTRACT_JOB.toPb()).configuration() instanceof ExtractJobConfiguration); + assertTrue(JobInfo.fromPb(EXTRACT_JOB.toPb()).statistics() instanceof ExtractStatistics); + assertNull(LOAD_JOB.toPb().getConfiguration().getCopy()); + assertNull(LOAD_JOB.toPb().getConfiguration().getExtract()); + assertNotNull(LOAD_JOB.toPb().getConfiguration().getLoad()); + assertNull(LOAD_JOB.toPb().getConfiguration().getQuery()); + assertEquals(LOAD_JOB_STATISTICS, JobStatistics.fromPb(LOAD_JOB.statistics().toPb())); + compareJobInfo(LOAD_JOB, JobInfo.fromPb(LOAD_JOB.toPb())); + assertTrue(JobInfo.fromPb(LOAD_JOB.toPb()).configuration() instanceof LoadJobConfiguration); + assertTrue(JobInfo.fromPb(LOAD_JOB.toPb()).statistics() instanceof LoadStatistics); + assertNull(QUERY_JOB.toPb().getConfiguration().getCopy()); + assertNull(QUERY_JOB.toPb().getConfiguration().getExtract()); + assertNull(QUERY_JOB.toPb().getConfiguration().getLoad()); + assertNotNull(QUERY_JOB.toPb().getConfiguration().getQuery()); + assertEquals(QUERY_JOB_STATISTICS, JobStatistics.fromPb(QUERY_JOB.statistics().toPb())); + compareJobInfo(QUERY_JOB, JobInfo.fromPb(QUERY_JOB.toPb())); + assertTrue(JobInfo.fromPb(QUERY_JOB.toPb()).configuration() instanceof QueryJobConfiguration); + assertTrue(JobInfo.fromPb(QUERY_JOB.toPb()).statistics() instanceof QueryStatistics); + } + + @Test + public void testSetProjectId() { + CopyJobConfiguration copyConfiguration = COPY_JOB.setProjectId("p").configuration(); + assertEquals("p", copyConfiguration.destinationTable().project()); + for (TableId sourceTable : copyConfiguration.sourceTables()) { + assertEquals("p", sourceTable.project()); + } + ExtractJobConfiguration extractConfiguration = EXTRACT_JOB.setProjectId("p").configuration(); + assertEquals("p", extractConfiguration.sourceTable().project()); + LoadJobConfiguration loadConfiguration = LOAD_JOB.setProjectId("p").configuration(); + assertEquals("p", loadConfiguration.destinationTable().project()); + QueryJobConfiguration queryConfiguration = QUERY_JOB.setProjectId("p").configuration(); + assertEquals("p", queryConfiguration.defaultDataset().project()); + assertEquals("p", queryConfiguration.destinationTable().project()); + } + + private void compareJobInfo(JobInfo expected, JobInfo value) { + assertEquals(expected, value); + assertEquals(expected.hashCode(), value.hashCode()); + assertEquals(expected.toString(), value.toString()); + assertEquals(expected.etag(), value.etag()); + assertEquals(expected.id(), value.id()); + assertEquals(expected.jobId(), value.jobId()); + assertEquals(expected.selfLink(), value.selfLink()); + assertEquals(expected.status(), value.status()); + assertEquals(expected.statistics(), value.statistics()); + assertEquals(expected.userEmail(), value.userEmail()); + assertEquals(expected.configuration(), value.configuration()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobTest.java index 6a5cef6508cc..90b602d978e0 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/JobTest.java @@ -36,7 +36,8 @@ public class JobTest { private static final JobId JOB_ID = JobId.of("dataset", "job"); private static final TableId TABLE_ID1 = TableId.of("dataset", "table1"); private static final TableId TABLE_ID2 = TableId.of("dataset", "table2"); - private static final JobInfo JOB_INFO = CopyJobInfo.of(JOB_ID, TABLE_ID1, TABLE_ID2); + private static final JobInfo JOB_INFO = + JobInfo.of(JOB_ID, CopyJobConfiguration.of(TABLE_ID1, TABLE_ID2)); private BigQuery bigquery; private Job job; diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java new file mode 100644 index 000000000000..88ae6a4fc1b8 --- /dev/null +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobConfigurationTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.google.gcloud.bigquery.JobInfo.CreateDisposition; +import com.google.gcloud.bigquery.JobInfo.WriteDisposition; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +public class LoadJobConfigurationTest { + + private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() + .allowJaggedRows(true) + .allowQuotedNewLines(false) + .encoding(StandardCharsets.UTF_8) + .build(); + private static final TableId TABLE_ID = TableId.of("dataset", "table"); + private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED; + private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; + private static final Integer MAX_BAD_RECORDS = 42; + private static final String FORMAT = "CSV"; + private static final Boolean IGNORE_UNKNOWN_VALUES = true; + private static final List PROJECTION_FIELDS = ImmutableList.of("field1", "field2"); + private static final Field FIELD_SCHEMA = Field.builder("IntegerField", Field.Type.integer()) + .mode(Field.Mode.REQUIRED) + .description("FieldDescription") + .build(); + private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); + private static final LoadJobConfiguration LOAD_CONFIGURATION = + LoadJobConfiguration.builder(TABLE_ID, SOURCE_URIS) + .createDisposition(CREATE_DISPOSITION) + .writeDisposition(WRITE_DISPOSITION) + .formatOptions(CSV_OPTIONS) + .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) + .maxBadRecords(MAX_BAD_RECORDS) + .projectionFields(PROJECTION_FIELDS) + .schema(TABLE_SCHEMA) + .build(); + + @Test + public void testToBuilder() { + compareLoadJobConfiguration(LOAD_CONFIGURATION, LOAD_CONFIGURATION.toBuilder().build()); + LoadJobConfiguration configuration = LOAD_CONFIGURATION.toBuilder() + .destinationTable(TableId.of("dataset", "newTable")) + .build(); + assertEquals("newTable", configuration.destinationTable().table()); + configuration = configuration.toBuilder().destinationTable(TABLE_ID).build(); + compareLoadJobConfiguration(LOAD_CONFIGURATION, configuration); + } + + @Test + public void testOf() { + LoadJobConfiguration configuration = LoadJobConfiguration.of(TABLE_ID, SOURCE_URIS); + assertEquals(TABLE_ID, configuration.destinationTable()); + assertEquals(SOURCE_URIS, configuration.sourceUris()); + configuration = LoadJobConfiguration.of(TABLE_ID, SOURCE_URIS, CSV_OPTIONS); + assertEquals(TABLE_ID, configuration.destinationTable()); + assertEquals(FORMAT, configuration.format()); + assertEquals(CSV_OPTIONS, configuration.csvOptions()); + assertEquals(SOURCE_URIS, configuration.sourceUris()); + configuration = LoadJobConfiguration.of(TABLE_ID, "uri1"); + assertEquals(TABLE_ID, configuration.destinationTable()); + assertEquals(ImmutableList.of("uri1"), configuration.sourceUris()); + configuration = LoadJobConfiguration.of(TABLE_ID, "uri1", CSV_OPTIONS); + assertEquals(TABLE_ID, configuration.destinationTable()); + assertEquals(FORMAT, configuration.format()); + assertEquals(CSV_OPTIONS, configuration.csvOptions()); + assertEquals(ImmutableList.of("uri1"), configuration.sourceUris()); + } + + @Test + public void testToBuilderIncomplete() { + LoadJobConfiguration configuration = LoadJobConfiguration.of(TABLE_ID, SOURCE_URIS); + compareLoadJobConfiguration(configuration, configuration.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(TABLE_ID, LOAD_CONFIGURATION.destinationTable()); + assertEquals(CREATE_DISPOSITION, LOAD_CONFIGURATION.createDisposition()); + assertEquals(WRITE_DISPOSITION, LOAD_CONFIGURATION.writeDisposition()); + assertEquals(CSV_OPTIONS, LOAD_CONFIGURATION.csvOptions()); + assertEquals(FORMAT, LOAD_CONFIGURATION.format()); + assertEquals(IGNORE_UNKNOWN_VALUES, LOAD_CONFIGURATION.ignoreUnknownValues()); + assertEquals(MAX_BAD_RECORDS, LOAD_CONFIGURATION.maxBadRecords()); + assertEquals(PROJECTION_FIELDS, LOAD_CONFIGURATION.projectionFields()); + assertEquals(TABLE_SCHEMA, LOAD_CONFIGURATION.schema()); + } + + @Test + public void testToPbAndFromPb() { + compareLoadJobConfiguration(LOAD_CONFIGURATION, + LoadJobConfiguration.fromPb(LOAD_CONFIGURATION.toPb())); + LoadJobConfiguration configuration = LoadJobConfiguration.of(TABLE_ID, SOURCE_URIS); + compareLoadJobConfiguration(configuration, LoadJobConfiguration.fromPb(configuration.toPb())); + } + + @Test + public void testSetProjectId() { + LoadConfiguration configuration = LOAD_CONFIGURATION.setProjectId("p"); + assertEquals("p", configuration.destinationTable().project()); + } + + private void compareLoadJobConfiguration(LoadJobConfiguration expected, + LoadJobConfiguration value) { + assertEquals(expected, value); + assertEquals(expected.hashCode(), value.hashCode()); + assertEquals(expected.toString(), value.toString()); + assertEquals(expected.destinationTable(), value.destinationTable()); + assertEquals(expected.createDisposition(), value.createDisposition()); + assertEquals(expected.writeDisposition(), value.writeDisposition()); + assertEquals(expected.csvOptions(), value.csvOptions()); + assertEquals(expected.format(), value.format()); + assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues()); + assertEquals(expected.maxBadRecords(), value.maxBadRecords()); + assertEquals(expected.projectionFields(), value.projectionFields()); + assertEquals(expected.schema(), value.schema()); + } +} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobInfoTest.java deleted file mode 100644 index 499d0d939698..000000000000 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadJobInfoTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.bigquery; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.google.common.collect.ImmutableList; -import com.google.gcloud.bigquery.JobInfo.CreateDisposition; -import com.google.gcloud.bigquery.JobInfo.WriteDisposition; -import com.google.gcloud.bigquery.JobStatistics.LoadStatistics; - -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.util.List; - -public class LoadJobInfoTest { - - private static final String ETAG = "etag"; - private static final String ID = "id"; - private static final String SELF_LINK = "selfLink"; - private static final String EMAIL = "email"; - private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() - .allowJaggedRows(true) - .allowQuotedNewLines(false) - .encoding(StandardCharsets.UTF_8) - .build(); - private static final String SOURCE_URI = "uri"; - private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); - private static final TableId TABLE_ID = TableId.of("dataset", "table"); - private static final CreateDisposition CREATE_DISPOSITION = CreateDisposition.CREATE_IF_NEEDED; - private static final WriteDisposition WRITE_DISPOSITION = WriteDisposition.WRITE_APPEND; - private static final Integer MAX_BAD_RECORDS = 42; - private static final Boolean IGNORE_UNKNOWN_VALUES = true; - private static final List PROJECTION_FIELDS = ImmutableList.of("field1", "field2"); - private static final JobId JOB_ID = JobId.of("job"); - private static final JobStatus JOB_STATUS = new JobStatus(JobStatus.State.DONE); - private static final Field FIELD_SCHEMA = Field.builder("IntegerField", Field.Type.integer()) - .mode(Field.Mode.REQUIRED) - .description("FieldDescription") - .build(); - private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); - private static final LoadStatistics JOB_STATISTICS = LoadStatistics.builder() - .creationTime(1L) - .endTime(3L) - .startTime(2L) - .inputFiles(42L) - .outputBytes(1024L) - .inputBytes(2048L) - .outputRows(24L) - .build(); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) - .createDisposition(CREATE_DISPOSITION) - .writeDisposition(WRITE_DISPOSITION) - .formatOptions(CSV_OPTIONS) - .ignoreUnknownValues(IGNORE_UNKNOWN_VALUES) - .maxBadRecords(MAX_BAD_RECORDS) - .projectionFields(PROJECTION_FIELDS) - .schema(TABLE_SCHEMA) - .build(); - private static final LoadJobInfo LOAD_JOB = LoadJobInfo.builder(LOAD_CONFIGURATION, SOURCE_URIS) - .etag(ETAG) - .id(ID) - .selfLink(SELF_LINK) - .userEmail(EMAIL) - .jobId(JOB_ID) - .status(JOB_STATUS) - .statistics(JOB_STATISTICS) - .build(); - - @Test - public void testToBuilder() { - compareLoadJobInfo(LOAD_JOB, LOAD_JOB.toBuilder().build()); - LoadJobInfo job = LOAD_JOB.toBuilder().etag("newEtag").build(); - assertEquals("newEtag", job.etag()); - job = job.toBuilder().etag(ETAG).build(); - compareLoadJobInfo(LOAD_JOB, job); - } - - @Test - public void testOf() { - LoadJobInfo job = LoadJobInfo.of(LOAD_CONFIGURATION, SOURCE_URIS); - assertEquals(LOAD_CONFIGURATION, job.configuration()); - assertEquals(SOURCE_URIS, job.sourceUris()); - job = LoadJobInfo.of(LOAD_CONFIGURATION, SOURCE_URI); - assertEquals(LOAD_CONFIGURATION, job.configuration()); - assertEquals(ImmutableList.of(SOURCE_URI), job.sourceUris()); - job = LoadJobInfo.of(JOB_ID, LOAD_CONFIGURATION, SOURCE_URIS); - assertEquals(JOB_ID, job.jobId()); - assertEquals(LOAD_CONFIGURATION, job.configuration()); - assertEquals(SOURCE_URIS, job.sourceUris()); - job = LoadJobInfo.of(JOB_ID, LOAD_CONFIGURATION, SOURCE_URI); - assertEquals(JOB_ID, job.jobId()); - assertEquals(LOAD_CONFIGURATION, job.configuration()); - assertEquals(ImmutableList.of(SOURCE_URI), job.sourceUris()); - } - - @Test - public void testToBuilderIncomplete() { - LoadJobInfo job = LoadJobInfo.of(LOAD_CONFIGURATION, SOURCE_URIS); - compareLoadJobInfo(job, job.toBuilder().build()); - } - - @Test - public void testBuilder() { - assertEquals(ETAG, LOAD_JOB.etag()); - assertEquals(ID, LOAD_JOB.id()); - assertEquals(SELF_LINK, LOAD_JOB.selfLink()); - assertEquals(EMAIL, LOAD_JOB.userEmail()); - assertEquals(JOB_ID, LOAD_JOB.jobId()); - assertEquals(JOB_STATUS, LOAD_JOB.status()); - assertEquals(LOAD_CONFIGURATION, LOAD_JOB.configuration()); - assertEquals(SOURCE_URIS, LOAD_JOB.sourceUris()); - assertEquals(JOB_STATISTICS, LOAD_JOB.statistics()); - } - - @Test - public void testToPbAndFromPb() { - assertNotNull(LOAD_JOB.toPb().getConfiguration().getLoad()); - assertNull(LOAD_JOB.toPb().getConfiguration().getExtract()); - assertNull(LOAD_JOB.toPb().getConfiguration().getCopy()); - assertNull(LOAD_JOB.toPb().getConfiguration().getQuery()); - assertEquals(JOB_STATISTICS, JobStatistics.fromPb(LOAD_JOB.toPb().getStatistics())); - compareLoadJobInfo(LOAD_JOB, LoadJobInfo.fromPb(LOAD_JOB.toPb())); - compareLoadJobInfo(LOAD_JOB, (LoadJobInfo) JobInfo.fromPb(LOAD_JOB.toPb())); - LoadJobInfo job = LoadJobInfo.of(LOAD_CONFIGURATION, SOURCE_URIS); - compareLoadJobInfo(job, LoadJobInfo.fromPb(job.toPb())); - compareLoadJobInfo(job, (LoadJobInfo) JobInfo.fromPb(job.toPb())); - } - - private void compareLoadJobInfo(LoadJobInfo expected, LoadJobInfo value) { - assertEquals(expected, value); - assertEquals(expected.hashCode(), value.hashCode()); - assertEquals(expected.toString(), value.toString()); - assertEquals(expected.etag(), value.etag()); - assertEquals(expected.id(), value.id()); - assertEquals(expected.jobId(), value.jobId()); - assertEquals(expected.selfLink(), value.selfLink()); - assertEquals(expected.status(), value.status()); - assertEquals(expected.statistics(), value.statistics()); - assertEquals(expected.userEmail(), value.userEmail()); - assertEquals(expected.configuration(), value.configuration()); - assertEquals(expected.sourceUris(), value.sourceUris()); - } -} diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java similarity index 51% rename from gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobInfoTest.java rename to gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java index f99bec19efd9..69b2f992fe22 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryJobConfigurationTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 Google Inc. All Rights Reserved. + * Copyright 2016 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,23 +25,18 @@ import com.google.common.collect.ImmutableMap; import com.google.gcloud.bigquery.JobInfo.CreateDisposition; import com.google.gcloud.bigquery.JobInfo.WriteDisposition; -import com.google.gcloud.bigquery.JobStatistics.QueryStatistics; -import com.google.gcloud.bigquery.QueryJobInfo.Priority; +import com.google.gcloud.bigquery.QueryJobConfiguration.Priority; import org.junit.Test; import java.util.List; import java.util.Map; -public class QueryJobInfoTest { +public class QueryJobConfigurationTest { - private static final String ETAG = "etag"; - private static final String ID = "id"; - private static final String SELF_LINK = "selfLink"; - private static final String EMAIL = "email"; private static final String QUERY = "BigQuery SQL"; - private static final DatasetId DATASET_ID = DatasetId.of("project", "dataset"); - private static final TableId TABLE_ID = TableId.of("project", "dataset", "table"); + private static final DatasetId DATASET_ID = DatasetId.of("dataset"); + private static final TableId TABLE_ID = TableId.of("dataset", "table"); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); private static final Field FIELD_SCHEMA1 = Field.builder("StringField", Field.Type.string()) @@ -79,114 +74,86 @@ public class QueryJobInfoTest { private static final boolean FLATTEN_RESULTS = true; private static final List USER_DEFINED_FUNCTIONS = ImmutableList.of( UserDefinedFunction.inline("Function"), UserDefinedFunction.fromUri("URI")); - private static final JobId JOB_ID = JobId.of("job"); - private static final JobStatus JOB_STATUS = new JobStatus(JobStatus.State.DONE); - private static final QueryStatistics JOB_STATISTICS = QueryStatistics.builder() - .creationTime(1L) - .endTime(3L) - .startTime(2L) - .totalBytesProcessed(2048L) - .totalBytesBilled(1024L) - .cacheHit(false) - .billingTier(42) - .build(); - private static final QueryJobInfo QUERY_JOB = QueryJobInfo.builder(QUERY) - .etag(ETAG) - .id(ID) - .selfLink(SELF_LINK) - .userEmail(EMAIL) - .jobId(JOB_ID) - .status(JOB_STATUS) - .useQueryCache(USE_QUERY_CACHE) - .tableDefinitions(TABLE_DEFINITIONS) - .allowLargeResults(ALLOW_LARGE_RESULTS) - .createDisposition(CREATE_DISPOSITION) - .defaultDataset(DATASET_ID) - .destinationTable(TABLE_ID) - .writeDisposition(WRITE_DISPOSITION) - .priority(PRIORITY) - .flattenResults(FLATTEN_RESULTS) - .userDefinedFunctions(USER_DEFINED_FUNCTIONS) - .dryRun(true) - .statistics(JOB_STATISTICS) - .build(); + private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = + QueryJobConfiguration.builder(QUERY) + .useQueryCache(USE_QUERY_CACHE) + .tableDefinitions(TABLE_DEFINITIONS) + .allowLargeResults(ALLOW_LARGE_RESULTS) + .createDisposition(CREATE_DISPOSITION) + .defaultDataset(DATASET_ID) + .destinationTable(TABLE_ID) + .writeDisposition(WRITE_DISPOSITION) + .priority(PRIORITY) + .flattenResults(FLATTEN_RESULTS) + .userDefinedFunctions(USER_DEFINED_FUNCTIONS) + .dryRun(true) + .build(); @Test public void testToBuilder() { - compareQueryJobInfo(QUERY_JOB, QUERY_JOB.toBuilder().build()); - QueryJobInfo job = QUERY_JOB.toBuilder() + compareQueryJobConfiguration(QUERY_JOB_CONFIGURATION, + QUERY_JOB_CONFIGURATION.toBuilder().build()); + QueryJobConfiguration job = QUERY_JOB_CONFIGURATION.toBuilder() .query("New BigQuery SQL") .build(); assertEquals("New BigQuery SQL", job.query()); job = job.toBuilder().query(QUERY).build(); - compareQueryJobInfo(QUERY_JOB, job); + compareQueryJobConfiguration(QUERY_JOB_CONFIGURATION, job); } @Test public void testOf() { - QueryJobInfo job = QueryJobInfo.of(QUERY); - assertEquals(QUERY, job.query()); - job = QueryJobInfo.of(JOB_ID, QUERY); - assertEquals(JOB_ID, job.jobId()); + QueryJobConfiguration job = QueryJobConfiguration.of(QUERY); assertEquals(QUERY, job.query()); } @Test public void testToBuilderIncomplete() { - QueryJobInfo job = QueryJobInfo.of(QUERY); - compareQueryJobInfo(job, job.toBuilder().build()); + QueryJobConfiguration job = QueryJobConfiguration.of(QUERY); + compareQueryJobConfiguration(job, job.toBuilder().build()); } @Test public void testBuilder() { - assertEquals(ETAG, QUERY_JOB.etag()); - assertEquals(ID, QUERY_JOB.id()); - assertEquals(SELF_LINK, QUERY_JOB.selfLink()); - assertEquals(EMAIL, QUERY_JOB.userEmail()); - assertEquals(JOB_ID, QUERY_JOB.jobId()); - assertEquals(JOB_STATUS, QUERY_JOB.status()); - assertEquals(ALLOW_LARGE_RESULTS, QUERY_JOB.allowLargeResults()); - assertEquals(CREATE_DISPOSITION, QUERY_JOB.createDisposition()); - assertEquals(DATASET_ID, QUERY_JOB.defaultDataset()); - assertEquals(TABLE_ID, QUERY_JOB.destinationTable()); - assertEquals(FLATTEN_RESULTS, QUERY_JOB.flattenResults()); - assertEquals(PRIORITY, QUERY_JOB.priority()); - assertEquals(QUERY, QUERY_JOB.query()); - assertEquals(TABLE_DEFINITIONS, QUERY_JOB.tableDefinitions()); - assertEquals(USE_QUERY_CACHE, QUERY_JOB.useQueryCache()); - assertEquals(USER_DEFINED_FUNCTIONS, QUERY_JOB.userDefinedFunctions()); - assertEquals(WRITE_DISPOSITION, QUERY_JOB.writeDisposition()); - assertTrue(QUERY_JOB.dryRun()); - assertEquals(JOB_STATISTICS, QUERY_JOB.statistics()); + assertEquals(ALLOW_LARGE_RESULTS, QUERY_JOB_CONFIGURATION.allowLargeResults()); + assertEquals(CREATE_DISPOSITION, QUERY_JOB_CONFIGURATION.createDisposition()); + assertEquals(DATASET_ID, QUERY_JOB_CONFIGURATION.defaultDataset()); + assertEquals(TABLE_ID, QUERY_JOB_CONFIGURATION.destinationTable()); + assertEquals(FLATTEN_RESULTS, QUERY_JOB_CONFIGURATION.flattenResults()); + assertEquals(PRIORITY, QUERY_JOB_CONFIGURATION.priority()); + assertEquals(QUERY, QUERY_JOB_CONFIGURATION.query()); + assertEquals(TABLE_DEFINITIONS, QUERY_JOB_CONFIGURATION.tableDefinitions()); + assertEquals(USE_QUERY_CACHE, QUERY_JOB_CONFIGURATION.useQueryCache()); + assertEquals(USER_DEFINED_FUNCTIONS, QUERY_JOB_CONFIGURATION.userDefinedFunctions()); + assertEquals(WRITE_DISPOSITION, QUERY_JOB_CONFIGURATION.writeDisposition()); + assertTrue(QUERY_JOB_CONFIGURATION.dryRun()); } @Test public void testToPbAndFromPb() { - assertNotNull(QUERY_JOB.toPb().getConfiguration().getQuery()); - assertNull(QUERY_JOB.toPb().getConfiguration().getExtract()); - assertNull(QUERY_JOB.toPb().getConfiguration().getCopy()); - assertNull(QUERY_JOB.toPb().getConfiguration().getLoad()); - assertEquals(JOB_STATISTICS, JobStatistics.fromPb(QUERY_JOB.statistics().toPb())); - compareQueryJobInfo(QUERY_JOB, QueryJobInfo.fromPb(QUERY_JOB.toPb())); - compareQueryJobInfo(QUERY_JOB, - (QueryJobInfo) JobInfo.fromPb(QUERY_JOB.toPb())); - QueryJobInfo job = QueryJobInfo.of(QUERY); - compareQueryJobInfo(job, QueryJobInfo.fromPb(job.toPb())); - compareQueryJobInfo(job, (QueryJobInfo) JobInfo.fromPb(job.toPb())); + assertNotNull(QUERY_JOB_CONFIGURATION.toPb().getQuery()); + assertNull(QUERY_JOB_CONFIGURATION.toPb().getExtract()); + assertNull(QUERY_JOB_CONFIGURATION.toPb().getCopy()); + assertNull(QUERY_JOB_CONFIGURATION.toPb().getLoad()); + compareQueryJobConfiguration(QUERY_JOB_CONFIGURATION, + QueryJobConfiguration.fromPb(QUERY_JOB_CONFIGURATION.toPb())); + QueryJobConfiguration job = QueryJobConfiguration.of(QUERY); + compareQueryJobConfiguration(job, QueryJobConfiguration.fromPb(job.toPb())); + } + + @Test + public void testSetProjectId() { + QueryJobConfiguration configuration = QUERY_JOB_CONFIGURATION.setProjectId("p"); + assertEquals("p", configuration.defaultDataset().project()); + assertEquals("p", configuration.destinationTable().project()); } - private void compareQueryJobInfo(QueryJobInfo expected, QueryJobInfo value) { + private void compareQueryJobConfiguration(QueryJobConfiguration expected, + QueryJobConfiguration value) { assertEquals(expected, value); assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); - assertEquals(expected.etag(), value.etag()); - assertEquals(expected.id(), value.id()); - assertEquals(expected.jobId(), value.jobId()); - assertEquals(expected.selfLink(), value.selfLink()); - assertEquals(expected.status(), value.status()); - assertEquals(expected.statistics(), value.statistics()); assertEquals(expected.dryRun(), value.dryRun()); - assertEquals(expected.userEmail(), value.userEmail()); assertEquals(expected.allowLargeResults(), value.allowLargeResults()); assertEquals(expected.createDisposition(), value.createDisposition()); assertEquals(expected.defaultDataset(), value.defaultDataset()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryRequestTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryRequestTest.java index 276e4f6792b3..370b4d614cbf 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryRequestTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/QueryRequestTest.java @@ -26,7 +26,7 @@ public class QueryRequestTest { private static final String QUERY = "BigQuery SQL"; - private static final DatasetId DATASET_ID = DatasetId.of("project", "dataset"); + private static final DatasetId DATASET_ID = DatasetId.of("dataset"); private static final Boolean USE_QUERY_CACHE = true; private static final Boolean DRY_RUN = false; private static final Long MAX_RESULTS = 42L; @@ -91,6 +91,11 @@ public void testToPbAndFromPb() { compareQueryRequest(queryRequest, QueryRequest.fromPb(queryRequest.toPb())); } + @Test + public void testSetProjectId() { + assertEquals("p", QUERY_REQUEST.setProjectId("p").defaultDataset().project()); + } + private void compareQueryRequest(QueryRequest expected, QueryRequest value) { assertEquals(expected, value); assertEquals(expected.query(), value.query()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java index 1a20682aa447..19b281f073b3 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/SerializationTest.java @@ -168,18 +168,24 @@ public class SerializationTest { private static final JobStatus JOB_STATUS = new JobStatus(JobStatus.State.DONE, BIGQUERY_ERROR, ImmutableList.of(BIGQUERY_ERROR)); private static final JobId JOB_ID = JobId.of("project", "job"); - private static final CopyJobInfo COPY_JOB = CopyJobInfo.of(TABLE_ID, TABLE_ID); - private static final ExtractJobInfo EXTRACT_JOB = ExtractJobInfo.of(TABLE_ID, SOURCE_URIS); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) - .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) - .formatOptions(CSV_OPTIONS) - .ignoreUnknownValues(true) - .maxBadRecords(10) - .schema(TABLE_SCHEMA) - .build(); - private static final LoadJobInfo LOAD_JOB = LoadJobInfo.of(LOAD_CONFIGURATION, SOURCE_URIS); - private static final QueryJobInfo QUERY_JOB = QueryJobInfo.of("query"); + private static final CopyJobConfiguration COPY_JOB_CONFIGURATION = + CopyJobConfiguration.of(TABLE_ID, TABLE_ID); + private static final ExtractJobConfiguration EXTRACT_JOB_CONFIGURATION = + ExtractJobConfiguration.of(TABLE_ID, SOURCE_URIS); + private static final WriteChannelConfiguration LOAD_CONFIGURATION = + WriteChannelConfiguration.builder(TABLE_ID) + .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) + .formatOptions(CSV_OPTIONS) + .ignoreUnknownValues(true) + .maxBadRecords(10) + .schema(TABLE_SCHEMA) + .build(); + private static final LoadJobConfiguration LOAD_JOB_CONFIGURATION = + LoadJobConfiguration.of(TABLE_ID, SOURCE_URIS); + private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION = + QueryJobConfiguration.of("query"); + private static final JobInfo JOB_INFO = JobInfo.of(COPY_JOB_CONFIGURATION); private static final Map CONTENT1 = ImmutableMap.of("key", "val1"); private static final Map CONTENT2 = @@ -241,8 +247,9 @@ public void testModelAndRequests() throws Exception { DATASET_INFO, TABLE_ID, CSV_OPTIONS, STREAMING_BUFFER, EXTERNAL_DATA_CONFIGURATION, TABLE_SCHEMA, TABLE_INFO, VIEW_INFO, EXTERNAL_TABLE_INFO, INLINE_FUNCTION, URI_FUNCTION, JOB_STATISTICS, EXTRACT_STATISTICS, LOAD_STATISTICS, QUERY_STATISTICS, BIGQUERY_ERROR, - JOB_STATUS, JOB_ID, COPY_JOB, EXTRACT_JOB, LOAD_CONFIGURATION, LOAD_JOB, QUERY_JOB, - INSERT_ALL_REQUEST, INSERT_ALL_RESPONSE, FIELD_VALUE, QUERY_REQUEST, QUERY_RESPONSE, + JOB_STATUS, JOB_ID, COPY_JOB_CONFIGURATION, EXTRACT_JOB_CONFIGURATION, LOAD_CONFIGURATION, + LOAD_JOB_CONFIGURATION, QUERY_JOB_CONFIGURATION, JOB_INFO, INSERT_ALL_REQUEST, + INSERT_ALL_RESPONSE, FIELD_VALUE, QUERY_REQUEST, QUERY_RESPONSE, BigQuery.DatasetOption.fields(), BigQuery.DatasetDeleteOption.deleteContents(), BigQuery.DatasetListOption.all(), BigQuery.TableOption.fields(), BigQuery.TableListOption.maxResults(42L), BigQuery.JobOption.fields(), diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java index 67933407e377..6b7edcd76db1 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableDataWriteChannelTest.java @@ -50,13 +50,14 @@ public class TableDataWriteChannelTest { private static final String UPLOAD_ID = "uploadid"; private static final TableId TABLE_ID = TableId.of("dataset", "table"); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) - .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) - .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) - .formatOptions(FormatOptions.json()) - .ignoreUnknownValues(true) - .maxBadRecords(10) - .build(); + private static final WriteChannelConfiguration LOAD_CONFIGURATION = + WriteChannelConfiguration.builder(TABLE_ID) + .createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .writeDisposition(JobInfo.WriteDisposition.WRITE_APPEND) + .formatOptions(FormatOptions.json()) + .ignoreUnknownValues(true) + .maxBadRecords(10) + .build(); private static final int MIN_CHUNK_SIZE = 256 * 1024; private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE; private static final int CUSTOM_CHUNK_SIZE = 4 * MIN_CHUNK_SIZE; diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableIdTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableIdTest.java index 9da050bf5951..bc013bfa5c31 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableIdTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableIdTest.java @@ -47,6 +47,11 @@ public void testToPbAndFromPb() { compareTableIds(TABLE_COMPLETE, TableId.fromPb(TABLE_COMPLETE.toPb())); } + @Test + public void testSetProjectId() { + assertEquals(TABLE_COMPLETE, TABLE.setProjectId("project")); + } + private void compareTableIds(TableId expected, TableId value) { assertEquals(expected, value); assertEquals(expected.project(), value.project()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java index c636a31ad1ff..7326f6c51b95 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableInfoTest.java @@ -196,6 +196,13 @@ public void testToAndFromPb() { BaseTableInfo.fromPb(EXTERNAL_TABLE_INFO.toPb())); } + @Test + public void testSetProjectId() { + assertEquals("project", TABLE_INFO.setProjectId("project").tableId().project()); + assertEquals("project", EXTERNAL_TABLE_INFO.setProjectId("project").tableId().project()); + assertEquals("project", VIEW_INFO.setProjectId("project").tableId().project()); + } + private void compareBaseTableInfo(BaseTableInfo expected, BaseTableInfo value) { assertEquals(expected, value); assertEquals(expected.tableId(), value.tableId()); diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTest.java index c931d768def1..2d0b7e528750 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/TableTest.java @@ -47,15 +47,13 @@ public class TableTest { private static final TableId TABLE_ID1 = TableId.of("dataset", "table1"); private static final TableId TABLE_ID2 = TableId.of("dataset", "table2"); - private static final JobInfo COPY_JOB_INFO = CopyJobInfo.of(TABLE_ID2, TABLE_ID1); - private static final JobInfo LOAD_JOB_INFO = LoadJobInfo.builder( - LoadConfiguration.builder(TABLE_ID1).formatOptions(FormatOptions.json()).build(), - ImmutableList.of("URI")) - .build(); + private static final CopyJobConfiguration COPY_JOB_CONFIGURATION = + CopyJobConfiguration.of(TABLE_ID2, TABLE_ID1); + private static final JobInfo COPY_JOB_INFO = JobInfo.of(COPY_JOB_CONFIGURATION); + private static final JobInfo LOAD_JOB_INFO = + JobInfo.of(LoadJobConfiguration.of(TABLE_ID1, ImmutableList.of("URI"), FormatOptions.json())); private static final JobInfo EXTRACT_JOB_INFO = - ExtractJobInfo.builder(TABLE_ID1, ImmutableList.of("URI")) - .format("CSV") - .build(); + JobInfo.of(ExtractJobConfiguration.of(TABLE_ID1, ImmutableList.of("URI"), "CSV")); private static final Field FIELD = Field.of("FieldName", Field.Type.integer()); private static final TableInfo TABLE_INFO = TableInfo.of(TABLE_ID1, Schema.of(FIELD)); private static final List ROWS_TO_INSERT = ImmutableList.of( diff --git a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadConfigurationTest.java b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/WriteChannelConfigurationTest.java similarity index 83% rename from gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadConfigurationTest.java rename to gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/WriteChannelConfigurationTest.java index e72101829cdf..17fa8446d097 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/LoadConfigurationTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/WriteChannelConfigurationTest.java @@ -28,7 +28,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; -public class LoadConfigurationTest { +public class WriteChannelConfigurationTest { private static final CsvOptions CSV_OPTIONS = CsvOptions.builder() .allowJaggedRows(true) @@ -47,7 +47,7 @@ public class LoadConfigurationTest { .description("FieldDescription") .build(); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); - private static final LoadConfiguration LOAD_CONFIGURATION = LoadConfiguration.builder(TABLE_ID) + private static final WriteChannelConfiguration LOAD_CONFIGURATION = WriteChannelConfiguration.builder(TABLE_ID) .createDisposition(CREATE_DISPOSITION) .writeDisposition(WRITE_DISPOSITION) .formatOptions(CSV_OPTIONS) @@ -60,7 +60,7 @@ public class LoadConfigurationTest { @Test public void testToBuilder() { compareLoadConfiguration(LOAD_CONFIGURATION, LOAD_CONFIGURATION.toBuilder().build()); - LoadConfiguration configuration = LOAD_CONFIGURATION.toBuilder() + WriteChannelConfiguration configuration = LOAD_CONFIGURATION.toBuilder() .destinationTable(TableId.of("dataset", "newTable")) .build(); assertEquals("newTable", configuration.destinationTable().table()); @@ -70,9 +70,9 @@ public void testToBuilder() { @Test public void testOf() { - LoadConfiguration configuration = LoadConfiguration.of(TABLE_ID); + WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); assertEquals(TABLE_ID, configuration.destinationTable()); - configuration = LoadConfiguration.of(TABLE_ID, CSV_OPTIONS); + configuration = WriteChannelConfiguration.of(TABLE_ID, CSV_OPTIONS); assertEquals(TABLE_ID, configuration.destinationTable()); assertEquals(FORMAT, configuration.format()); assertEquals(CSV_OPTIONS, configuration.csvOptions()); @@ -80,7 +80,7 @@ public void testOf() { @Test public void testToBuilderIncomplete() { - LoadConfiguration configuration = LoadConfiguration.of(TABLE_ID); + WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); compareLoadConfiguration(configuration, configuration.toBuilder().build()); } @@ -99,14 +99,14 @@ public void testBuilder() { @Test public void testToPbAndFromPb() { - assertNull(LOAD_CONFIGURATION.toPb().getSourceUris()); + assertNull(LOAD_CONFIGURATION.toPb().getLoad().getSourceUris()); compareLoadConfiguration(LOAD_CONFIGURATION, - LoadConfiguration.fromPb(LOAD_CONFIGURATION.toPb())); - LoadConfiguration configuration = LoadConfiguration.of(TABLE_ID); - compareLoadConfiguration(configuration, LoadConfiguration.fromPb(configuration.toPb())); + WriteChannelConfiguration.fromPb(LOAD_CONFIGURATION.toPb())); + WriteChannelConfiguration configuration = WriteChannelConfiguration.of(TABLE_ID); + compareLoadConfiguration(configuration, WriteChannelConfiguration.fromPb(configuration.toPb())); } - private void compareLoadConfiguration(LoadConfiguration expected, LoadConfiguration value) { + private void compareLoadConfiguration(WriteChannelConfiguration expected, WriteChannelConfiguration value) { assertEquals(expected, value); assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java index 895502e3d6d5..8fe78cbd50ad 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/BigQueryExample.java @@ -22,20 +22,20 @@ import com.google.gcloud.bigquery.BigQuery; import com.google.gcloud.bigquery.BigQueryError; import com.google.gcloud.bigquery.BigQueryOptions; -import com.google.gcloud.bigquery.CopyJobInfo; +import com.google.gcloud.bigquery.CopyJobConfiguration; import com.google.gcloud.bigquery.DatasetId; import com.google.gcloud.bigquery.DatasetInfo; import com.google.gcloud.bigquery.ExternalDataConfiguration; import com.google.gcloud.bigquery.ExternalTableInfo; -import com.google.gcloud.bigquery.ExtractJobInfo; +import com.google.gcloud.bigquery.ExtractJobConfiguration; import com.google.gcloud.bigquery.Field; import com.google.gcloud.bigquery.FieldValue; import com.google.gcloud.bigquery.FormatOptions; import com.google.gcloud.bigquery.JobId; import com.google.gcloud.bigquery.JobInfo; import com.google.gcloud.bigquery.JobStatus; -import com.google.gcloud.bigquery.LoadConfiguration; -import com.google.gcloud.bigquery.LoadJobInfo; +import com.google.gcloud.bigquery.WriteChannelConfiguration; +import com.google.gcloud.bigquery.LoadJobConfiguration; import com.google.gcloud.bigquery.QueryRequest; import com.google.gcloud.bigquery.QueryResponse; import com.google.gcloud.bigquery.Schema; @@ -544,15 +544,15 @@ void run(BigQuery bigquery, JobInfo job) throws Exception { */ private static class LoadAction extends JobRunAction { @Override - LoadJobInfo parse(String... args) throws Exception { + JobInfo parse(String... args) throws Exception { if (args.length >= 4) { String dataset = args[0]; String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); - LoadConfiguration configuration = LoadConfiguration.of(tableId, FormatOptions.of(format)); - return LoadJobInfo.builder(configuration, Arrays.asList(args).subList(3, args.length)) - .build(); + LoadJobConfiguration configuration = LoadJobConfiguration.of( + tableId, Arrays.asList(args).subList(3, args.length), FormatOptions.of(format)); + return JobInfo.of(configuration); } throw new IllegalArgumentException("Missing required arguments."); } @@ -570,15 +570,15 @@ protected String params() { */ private static class ExtractAction extends JobRunAction { @Override - ExtractJobInfo parse(String... args) throws Exception { + JobInfo parse(String... args) throws Exception { if (args.length >= 4) { String dataset = args[0]; String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); - return ExtractJobInfo.builder(tableId, Arrays.asList(args).subList(3, args.length)) - .format(format) - .build(); + ExtractJobConfiguration configuration = ExtractJobConfiguration.of( + tableId, Arrays.asList(args).subList(3, args.length), format); + return JobInfo.of(configuration); } throw new IllegalArgumentException("Missing required arguments."); } @@ -596,12 +596,12 @@ protected String params() { */ private static class CopyAction extends JobRunAction { @Override - CopyJobInfo parse(String... args) throws Exception { + JobInfo parse(String... args) throws Exception { String message; if (args.length == 4) { TableId sourceTableId = TableId.of(args[0], args[1]); TableId destinationTableId = TableId.of(args[2], args[3]); - return CopyJobInfo.of(destinationTableId, sourceTableId); + return JobInfo.of(CopyJobConfiguration.of(destinationTableId, sourceTableId)); } else if (args.length < 3) { message = "Missing required source or destination table."; } else { @@ -671,9 +671,11 @@ protected String params() { * @see Resumable * Upload */ - private static class LoadFileAction extends BigQueryAction> { + private static class LoadFileAction + extends BigQueryAction> { @Override - void run(BigQuery bigquery, Tuple configuration) throws Exception { + void run(BigQuery bigquery, Tuple configuration) + throws Exception { System.out.println("Running insert"); try (FileChannel fileChannel = FileChannel.open(Paths.get(configuration.y()))) { WriteChannel writeChannel = bigquery.writer(configuration.x()); @@ -688,13 +690,14 @@ void run(BigQuery bigquery, Tuple configuration) thro } @Override - Tuple parse(String... args) throws Exception { + Tuple parse(String... args) throws Exception { if (args.length == 4) { String dataset = args[0]; String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); - LoadConfiguration configuration = LoadConfiguration.of(tableId, FormatOptions.of(format)); + WriteChannelConfiguration configuration = + WriteChannelConfiguration.of(tableId, FormatOptions.of(format)); return Tuple.of(configuration, args[3]); } throw new IllegalArgumentException("Missing required arguments.");