From 381d0563f1035395a4dbad759bef3c8bfc9166a6 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 12 Nov 2015 19:01:23 +0100 Subject: [PATCH 1/4] Add BigQueryRpc default implementation --- .../google/gcloud/spi/DefaultBigQueryRpc.java | 421 ++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java new file mode 100644 index 000000000000..fc9760bfa794 --- /dev/null +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java @@ -0,0 +1,421 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.gcloud.spi; + +import static com.google.gcloud.spi.BigQueryRpc.Option.DELETE_CONTENTS; +import static com.google.gcloud.spi.BigQueryRpc.Option.FIELDS; +import static com.google.gcloud.spi.BigQueryRpc.Option.QUOTA_USER; +import static com.google.gcloud.spi.BigQueryRpc.Option.START_INDEX; +import static com.google.gcloud.spi.BigQueryRpc.Option.TIMEOUT; +import static com.google.gcloud.spi.BigQueryRpc.Option.USER_IP; + +import com.google.api.client.googleapis.json.GoogleJsonError; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.model.Dataset; +import com.google.api.services.bigquery.model.DatasetList; +import com.google.api.services.bigquery.model.DatasetReference; +import com.google.api.services.bigquery.model.GetQueryResultsResponse; +import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.JobList; +import com.google.api.services.bigquery.model.JobReference; +import com.google.api.services.bigquery.model.QueryRequest; +import com.google.api.services.bigquery.model.QueryResponse; +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.TableDataInsertAllRequest; +import com.google.api.services.bigquery.model.TableDataInsertAllResponse; +import com.google.api.services.bigquery.model.TableDataList; +import com.google.api.services.bigquery.model.TableList; +import com.google.api.services.bigquery.model.TableReference; +import com.google.api.services.bigquery.model.TableRow; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; + +import static com.google.gcloud.spi.BigQueryRpc.Option.MAX_RESULTS; +import static com.google.gcloud.spi.BigQueryRpc.Option.PAGE_TOKEN; + +import com.google.gcloud.bigquery.BigQueryException; +import com.google.gcloud.bigquery.BigQueryOptions; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class DefaultBigQueryRpc implements BigQueryRpc { + + public static final String DEFAULT_PROJECTION = "full"; + // see: https://cloud.google.com/bigquery/troubleshooting-errors + private static final Set RETRYABLE_CODES = ImmutableSet.of(503); + private final BigQueryOptions options; + private final Bigquery bigquery; + + public DefaultBigQueryRpc(BigQueryOptions options) { + HttpTransport transport = options.httpTransportFactory().create(); + HttpRequestInitializer initializer = options.httpRequestInitializer(); + this.options = options; + bigquery = new Bigquery.Builder(transport, new JacksonFactory(), initializer) + .setRootUrl(options.host()) + .setApplicationName(options.applicationName()) + .build(); + } + + private static BigQueryException translate(IOException exception) { + BigQueryException translated; + if (exception instanceof GoogleJsonResponseException) { + translated = translate(((GoogleJsonResponseException) exception).getDetails()); + } else { + translated = + new BigQueryException(BigQueryException.UNKNOWN_CODE, exception.getMessage(), false); + } + translated.initCause(exception); + return translated; + } + + private static BigQueryException translate(GoogleJsonError exception) { + boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) + || "InternalError".equals(exception.getMessage()); + return new BigQueryException(exception.getCode(), exception.getMessage(), retryable); + } + + @Override + public Dataset getDataset(String datasetId, Map options) throws BigQueryException { + try { + return bigquery.datasets() + .get(this.options.projectId(), datasetId) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Tuple> listDatasets(Map options) + throws BigQueryException { + try { + DatasetList datasetsList = bigquery.datasets() + .list(this.options.projectId()) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .setAll(Option.ALL_DATASETS.getBoolean(options)) + .setMaxResults(MAX_RESULTS.getLong(options)) + .setPageToken(PAGE_TOKEN.getString(options)) + .execute(); + Iterable datasets = datasetsList.getDatasets(); + return Tuple.of(datasetsList.getNextPageToken(), + Iterables.transform(datasets != null ? datasets : ImmutableList.of(), + new Function() { + @Override + public Dataset apply(DatasetList.Datasets f) { + return new Dataset() + .setDatasetReference(f.getDatasetReference()) + .setFriendlyName(f.getFriendlyName()) + .setId(f.getId()) + .setKind(f.getKind()); + } + })); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Dataset create(Dataset dataset, Map options) throws BigQueryException { + try { + return bigquery.datasets().insert(this.options.projectId(), dataset) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public boolean deleteDataset(String datasetId, Map options) throws BigQueryException { + try { + bigquery.datasets().delete(this.options.projectId(), datasetId) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .setDeleteContents(DELETE_CONTENTS.getBoolean(options)) + .execute(); + return true; + } catch (IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == 404) { + return false; + } + throw serviceException; + } + } + + @Override + public Dataset patch(Dataset dataset, Map options) throws BigQueryException { + try { + DatasetReference reference = dataset.getDatasetReference(); + return bigquery.datasets().patch(reference.getProjectId(), reference.getDatasetId(), dataset) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Table getTable(String datasetId, String tableId, Map options) + throws BigQueryException { + try { + return bigquery.tables() + .get(this.options.projectId(), datasetId, tableId) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Tuple> listTables(String datasetId, Map options) + throws BigQueryException { + try { + TableList tableList = bigquery.tables() + .list(this.options.projectId(), datasetId) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .setMaxResults(MAX_RESULTS.getLong(options)) + .setPageToken(PAGE_TOKEN.getString(options)) + .execute(); + return Tuple.of(tableList.getNextPageToken(), + Iterables.transform(tableList.getTables(), + new Function() { + @Override + public Table apply(TableList.Tables f) { + return new Table() + .setFriendlyName(f.getFriendlyName()) + .setId(f.getId()) + .setKind(f.getKind()) + .setTableReference(f.getTableReference()) + .setType(f.getType()); + } + })); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Table create(Table table, Map options) + throws BigQueryException { + try { + return bigquery.tables() + .insert(this.options.projectId(), table.getTableReference().getDatasetId(), table) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public boolean deleteTable(String datasetId, String tableId, Map options) + throws BigQueryException { + try { + bigquery.tables().delete(this.options.projectId(), datasetId, tableId) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + return true; + } catch (IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == 404) { + return false; + } + throw serviceException; + } + } + + @Override + public Table patch(Table table, Map options) throws BigQueryException { + try { + TableReference reference = table.getTableReference(); + return bigquery.tables() + .patch(reference.getProjectId(), reference.getDatasetId(), reference.getTableId(), table) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public TableDataInsertAllResponse insertAll(TableReference table, + TableDataInsertAllRequest request, Map options) throws BigQueryException { + try { + return bigquery.tabledata() + .insertAll(this.options.projectId(), table.getDatasetId(), table.getTableId(), request) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Tuple> listTableData(String datasetId, String tableId, + Map options) throws BigQueryException { + try { + TableDataList tableDataList = bigquery.tabledata() + .list(this.options.projectId(), datasetId, tableId) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .setMaxResults(MAX_RESULTS.getLong(options)) + .setPageToken(PAGE_TOKEN.getString(options)) + .setStartIndex(START_INDEX.getLong(options) != null ? + BigInteger.valueOf(START_INDEX.getLong(options)) : null) + .execute(); + return Tuple.>of(tableDataList.getPageToken(), + tableDataList.getRows()); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Job getJob(String jobId, Map options) throws BigQueryException { + try { + return bigquery.jobs() + .get(this.options.projectId(), jobId) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Tuple> listJobs(Map options) throws BigQueryException { + try { + JobList jobsList = bigquery.jobs() + .list(this.options.projectId()) + .setAllUsers(Option.ALL_USERS.getBoolean(options)) + .setFields(Option.FIELDS.getString(options)) + .setStateFilter(Option.STATE_FILTER.>get(options)) + .setMaxResults(MAX_RESULTS.getLong(options)) + .setPageToken(PAGE_TOKEN.getString(options)) + .setProjection(DEFAULT_PROJECTION) + .execute(); + return Tuple.of(jobsList.getNextPageToken(), + Iterables.transform(jobsList.getJobs(), + new Function() { + @Override + public Job apply(JobList.Jobs f) { + return new Job() + .setConfiguration(f.getConfiguration()) + .setId(f.getId()) + .setJobReference(f.getJobReference()) + .setKind(f.getKind()) + .setStatistics(f.getStatistics()) + .setStatus(f.getStatus()) + .setUserEmail(f.getUserEmail()); + } + })); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public Job create(Job job, Map options) throws BigQueryException { + try { + return bigquery.jobs() + .insert(this.options.projectId(), job) + .setFields(FIELDS.getString(options)) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public boolean cancel(String jobId, Map options) throws BigQueryException { + try { + bigquery.jobs().cancel(this.options.projectId(), jobId) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + return true; + } catch (IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == 404) { + return false; + } + throw serviceException; + } + } + + @Override + public GetQueryResultsResponse getQueryResults(JobReference job, Map options) + throws BigQueryException { + try { + return bigquery.jobs().getQueryResults(this.options.projectId(), job.getProjectId()) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .setMaxResults(MAX_RESULTS.getLong(options)) + .setPageToken(PAGE_TOKEN.getString(options)) + .setStartIndex(START_INDEX.getLong(options) != null ? + BigInteger.valueOf(START_INDEX.getLong(options)) : null) + .setTimeoutMs(TIMEOUT.getLong(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } + + @Override + public QueryResponse query(QueryRequest request, Map options) + throws BigQueryException { + try { + return bigquery.jobs().query(this.options.projectId(), request) + .setQuotaUser(QUOTA_USER.getString(options)) + .setUserIp(USER_IP.getString(options)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } +} From e3a99c2f9314f938bafd9fd5d19a98a43335bd98 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 19 Nov 2015 09:41:42 +0100 Subject: [PATCH 2/4] Add error codes, handle NOT_FOUND in BigQuery RPC, add javadoc --- .../com/google/gcloud/spi/BigQueryRpc.java | 44 ++++++++++++++++++ .../google/gcloud/spi/DefaultBigQueryRpc.java | 45 +++++++++++++------ 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java index 7cce35ab3eb9..a16fed1eb1ab 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java @@ -98,23 +98,57 @@ public Y y() { } } + /** + * Returns the requested dataset or {@code null} if not found. + * + * @throws BigQueryException upon failure + */ Dataset getDataset(String datasetId, Map options) throws BigQueryException; + /** + * Lists the project's datasets. Partial information is returned on a dataset (datasetReference, + * friendlyName and id). To get full information use {@link #getDataset(String, Map)}. + * + * @throws BigQueryException upon failure + */ Tuple> listDatasets(Map options) throws BigQueryException; Dataset create(Dataset dataset, Map options) throws BigQueryException; + /** + * Delete the requested dataset. + * + * @return {@code true} if dataset was deleted, {@code false} if it was not found + * @throws BigQueryException upon failure + */ boolean deleteDataset(String datasetId, Map options) throws BigQueryException; Dataset patch(Dataset dataset, Map options) throws BigQueryException; + /** + * Returns the requested table or {@code null} if not found. + * + * @throws BigQueryException upon failure + */ Table getTable(String datasetId, String tableId, Map options) throws BigQueryException; + /** + * Lists the dataset's tables. Partial information is returned on a table (tableReference, + * friendlyName, id and type). To get full information use {@link #getTable(String, String, Map)}. + * + * @throws BigQueryException upon failure + */ Tuple> listTables(String dataset, Map options) throws BigQueryException; Table create(Table table, Map options) throws BigQueryException; + /** + * Delete the requested table. + * + * @return {@code true} if table was deleted, {@code false} if it was not found + * @throws BigQueryException upon failure + */ boolean deleteTable(String datasetId, String tableId, Map options) throws BigQueryException; @@ -126,8 +160,18 @@ TableDataInsertAllResponse insertAll(TableReference table, TableDataInsertAllReq Tuple> listTableData(String datasetId, String tableId, Map options) throws BigQueryException; + /** + * Returns the requested job or {@code null} if not found. + * + * @throws BigQueryException upon failure + */ Job getJob(String jobId, Map options) throws BigQueryException; + /** + * Lists the project's jobs. + * + * @throws BigQueryException upon failure + */ Tuple> listJobs(Map options) throws BigQueryException; Job create(Job job, Map options) throws BigQueryException; diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java index fc9760bfa794..a77417e27fb9 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java @@ -20,6 +20,7 @@ import static com.google.gcloud.spi.BigQueryRpc.Option.START_INDEX; import static com.google.gcloud.spi.BigQueryRpc.Option.TIMEOUT; import static com.google.gcloud.spi.BigQueryRpc.Option.USER_IP; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; @@ -64,7 +65,7 @@ public class DefaultBigQueryRpc implements BigQueryRpc { public static final String DEFAULT_PROJECTION = "full"; // see: https://cloud.google.com/bigquery/troubleshooting-errors - private static final Set RETRYABLE_CODES = ImmutableSet.of(503); + private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 502, 503, 504); private final BigQueryOptions options; private final Bigquery bigquery; @@ -80,7 +81,8 @@ public DefaultBigQueryRpc(BigQueryOptions options) { private static BigQueryException translate(IOException exception) { BigQueryException translated; - if (exception instanceof GoogleJsonResponseException) { + if (exception instanceof GoogleJsonResponseException + && ((GoogleJsonResponseException) exception).getDetails() != null) { translated = translate(((GoogleJsonResponseException) exception).getDetails()); } else { translated = @@ -91,8 +93,7 @@ private static BigQueryException translate(IOException exception) { } private static BigQueryException translate(GoogleJsonError exception) { - boolean retryable = RETRYABLE_CODES.contains(exception.getCode()) - || "InternalError".equals(exception.getMessage()); + boolean retryable = RETRYABLE_CODES.contains(exception.getCode()); return new BigQueryException(exception.getCode(), exception.getMessage(), retryable); } @@ -105,8 +106,12 @@ public Dataset getDataset(String datasetId, Map options) throws BigQu .setQuotaUser(QUOTA_USER.getString(options)) .setUserIp(USER_IP.getString(options)) .execute(); - } catch (IOException ex) { - throw translate(ex); + } catch(IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == HTTP_NOT_FOUND) { + return null; + } + throw serviceException; } } @@ -164,7 +169,7 @@ public boolean deleteDataset(String datasetId, Map options) throws Bi return true; } catch (IOException ex) { BigQueryException serviceException = translate(ex); - if (serviceException.code() == 404) { + if (serviceException.code() == HTTP_NOT_FOUND) { return false; } throw serviceException; @@ -195,8 +200,12 @@ public Table getTable(String datasetId, String tableId, Map options) .setQuotaUser(QUOTA_USER.getString(options)) .setUserIp(USER_IP.getString(options)) .execute(); - } catch (IOException ex) { - throw translate(ex); + } catch(IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == HTTP_NOT_FOUND) { + return null; + } + throw serviceException; } } @@ -320,8 +329,12 @@ public Job getJob(String jobId, Map options) throws BigQueryException .setQuotaUser(QUOTA_USER.getString(options)) .setUserIp(USER_IP.getString(options)) .execute(); - } catch (IOException ex) { - throw translate(ex); + } catch(IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == HTTP_NOT_FOUND) { + return null; + } + throw serviceException; } } @@ -381,7 +394,7 @@ public boolean cancel(String jobId, Map options) throws BigQueryExcep return true; } catch (IOException ex) { BigQueryException serviceException = translate(ex); - if (serviceException.code() == 404) { + if (serviceException.code() == HTTP_NOT_FOUND) { return false; } throw serviceException; @@ -401,8 +414,12 @@ public GetQueryResultsResponse getQueryResults(JobReference job, Map BigInteger.valueOf(START_INDEX.getLong(options)) : null) .setTimeoutMs(TIMEOUT.getLong(options)) .execute(); - } catch (IOException ex) { - throw translate(ex); + } catch(IOException ex) { + BigQueryException serviceException = translate(ex); + if (serviceException.code() == HTTP_NOT_FOUND) { + return null; + } + throw serviceException; } } From 12aa55591cde5f0f5b3bbf6e4d29c7d049925c5f Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 19 Nov 2015 19:11:07 +0100 Subject: [PATCH 3/4] Fix empty list bug, handle job state and error in listJobs --- .../com/google/gcloud/spi/DefaultBigQueryRpc.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java index a77417e27fb9..68607e50685f 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java @@ -35,6 +35,7 @@ import com.google.api.services.bigquery.model.Job; import com.google.api.services.bigquery.model.JobList; import com.google.api.services.bigquery.model.JobReference; +import com.google.api.services.bigquery.model.JobStatus; import com.google.api.services.bigquery.model.QueryRequest; import com.google.api.services.bigquery.model.QueryResponse; import com.google.api.services.bigquery.model.Table; @@ -220,8 +221,9 @@ public Tuple> listTables(String datasetId, Map tables = tableList.getTables(); return Tuple.of(tableList.getNextPageToken(), - Iterables.transform(tableList.getTables(), + Iterables.transform(tables != null ? tables : ImmutableList.of(), new Function() { @Override public Table apply(TableList.Tables f) { @@ -350,11 +352,19 @@ public Tuple> listJobs(Map options) throws BigQ .setPageToken(PAGE_TOKEN.getString(options)) .setProjection(DEFAULT_PROJECTION) .execute(); + Iterable jobs = jobsList.getJobs(); return Tuple.of(jobsList.getNextPageToken(), - Iterables.transform(jobsList.getJobs(), + Iterables.transform(jobs != null ? jobs : ImmutableList.of(), new Function() { @Override public Job apply(JobList.Jobs f) { + JobStatus statusPb = f.getStatus() != null ? f.getStatus() : new JobStatus(); + if (statusPb.getState() == null) { + statusPb.setState(f.getState()); + } + if (statusPb.getErrorResult() == null) { + statusPb.setErrorResult(f.getErrorResult()); + } return new Job() .setConfiguration(f.getConfiguration()) .setId(f.getId()) From 120fe1228f8df9e770980be6c576cffab7e48f2f Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 24 Nov 2015 18:47:01 +0100 Subject: [PATCH 4/4] Remove quota options, always use options.projectId() --- .../com/google/gcloud/spi/BigQueryRpc.java | 2 - .../google/gcloud/spi/DefaultBigQueryRpc.java | 54 +++---------------- 2 files changed, 8 insertions(+), 48 deletions(-) diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java index a16fed1eb1ab..c45cb847eecb 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/BigQueryRpc.java @@ -35,8 +35,6 @@ public interface BigQueryRpc { // These options are part of the Google Cloud BigQuery query parameters enum Option { - QUOTA_USER("quotaUser"), - USER_IP("userIp"), FIELDS("fields"), DELETE_CONTENTS("deleteContents"), ALL_DATASETS("all"), diff --git a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java index 68607e50685f..d2555d50f478 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java +++ b/gcloud-java-bigquery/src/main/java/com/google/gcloud/spi/DefaultBigQueryRpc.java @@ -16,10 +16,8 @@ import static com.google.gcloud.spi.BigQueryRpc.Option.DELETE_CONTENTS; import static com.google.gcloud.spi.BigQueryRpc.Option.FIELDS; -import static com.google.gcloud.spi.BigQueryRpc.Option.QUOTA_USER; import static com.google.gcloud.spi.BigQueryRpc.Option.START_INDEX; import static com.google.gcloud.spi.BigQueryRpc.Option.TIMEOUT; -import static com.google.gcloud.spi.BigQueryRpc.Option.USER_IP; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import com.google.api.client.googleapis.json.GoogleJsonError; @@ -104,8 +102,6 @@ public Dataset getDataset(String datasetId, Map options) throws BigQu return bigquery.datasets() .get(this.options.projectId(), datasetId) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch(IOException ex) { BigQueryException serviceException = translate(ex); @@ -122,8 +118,6 @@ public Tuple> listDatasets(Map options) try { DatasetList datasetsList = bigquery.datasets() .list(this.options.projectId()) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .setAll(Option.ALL_DATASETS.getBoolean(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) @@ -151,8 +145,6 @@ public Dataset create(Dataset dataset, Map options) throws BigQueryEx try { return bigquery.datasets().insert(this.options.projectId(), dataset) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); @@ -163,8 +155,6 @@ public Dataset create(Dataset dataset, Map options) throws BigQueryEx public boolean deleteDataset(String datasetId, Map options) throws BigQueryException { try { bigquery.datasets().delete(this.options.projectId(), datasetId) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .setDeleteContents(DELETE_CONTENTS.getBoolean(options)) .execute(); return true; @@ -181,10 +171,9 @@ public boolean deleteDataset(String datasetId, Map options) throws Bi public Dataset patch(Dataset dataset, Map options) throws BigQueryException { try { DatasetReference reference = dataset.getDatasetReference(); - return bigquery.datasets().patch(reference.getProjectId(), reference.getDatasetId(), dataset) + return bigquery.datasets() + .patch(this.options.projectId(), reference.getDatasetId(), dataset) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); @@ -198,8 +187,6 @@ public Table getTable(String datasetId, String tableId, Map options) return bigquery.tables() .get(this.options.projectId(), datasetId, tableId) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch(IOException ex) { BigQueryException serviceException = translate(ex); @@ -216,8 +203,6 @@ public Tuple> listTables(String datasetId, Map options) return bigquery.tables() .insert(this.options.projectId(), table.getTableReference().getDatasetId(), table) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); @@ -259,14 +242,11 @@ public Table create(Table table, Map options) public boolean deleteTable(String datasetId, String tableId, Map options) throws BigQueryException { try { - bigquery.tables().delete(this.options.projectId(), datasetId, tableId) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) - .execute(); + bigquery.tables().delete(this.options.projectId(), datasetId, tableId).execute(); return true; } catch (IOException ex) { BigQueryException serviceException = translate(ex); - if (serviceException.code() == 404) { + if (serviceException.code() == HTTP_NOT_FOUND) { return false; } throw serviceException; @@ -278,10 +258,8 @@ public Table patch(Table table, Map options) throws BigQueryException try { TableReference reference = table.getTableReference(); return bigquery.tables() - .patch(reference.getProjectId(), reference.getDatasetId(), reference.getTableId(), table) + .patch(this.options.projectId(), reference.getDatasetId(), reference.getTableId(), table) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); @@ -294,8 +272,6 @@ public TableDataInsertAllResponse insertAll(TableReference table, try { return bigquery.tabledata() .insertAll(this.options.projectId(), table.getDatasetId(), table.getTableId(), request) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); @@ -308,8 +284,6 @@ public Tuple> listTableData(String datasetId, String try { TableDataList tableDataList = bigquery.tabledata() .list(this.options.projectId(), datasetId, tableId) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setStartIndex(START_INDEX.getLong(options) != null ? @@ -328,8 +302,6 @@ public Job getJob(String jobId, Map options) throws BigQueryException return bigquery.jobs() .get(this.options.projectId(), jobId) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch(IOException ex) { BigQueryException serviceException = translate(ex); @@ -386,8 +358,6 @@ public Job create(Job job, Map options) throws BigQueryException { return bigquery.jobs() .insert(this.options.projectId(), job) .setFields(FIELDS.getString(options)) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); @@ -397,10 +367,7 @@ public Job create(Job job, Map options) throws BigQueryException { @Override public boolean cancel(String jobId, Map options) throws BigQueryException { try { - bigquery.jobs().cancel(this.options.projectId(), jobId) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) - .execute(); + bigquery.jobs().cancel(this.options.projectId(), jobId).execute(); return true; } catch (IOException ex) { BigQueryException serviceException = translate(ex); @@ -415,9 +382,7 @@ public boolean cancel(String jobId, Map options) throws BigQueryExcep public GetQueryResultsResponse getQueryResults(JobReference job, Map options) throws BigQueryException { try { - return bigquery.jobs().getQueryResults(this.options.projectId(), job.getProjectId()) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) + return bigquery.jobs().getQueryResults(this.options.projectId(), job.getJobId()) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setStartIndex(START_INDEX.getLong(options) != null ? @@ -437,10 +402,7 @@ public GetQueryResultsResponse getQueryResults(JobReference job, Map public QueryResponse query(QueryRequest request, Map options) throws BigQueryException { try { - return bigquery.jobs().query(this.options.projectId(), request) - .setQuotaUser(QUOTA_USER.getString(options)) - .setUserIp(USER_IP.getString(options)) - .execute(); + return bigquery.jobs().query(this.options.projectId(), request).execute(); } catch (IOException ex) { throw translate(ex); }