Skip to content

Commit

Permalink
Reconcile RetrySettings & Clock (google-cloud-java part) (#1738)
Browse files Browse the repository at this point in the history 
Reconcile RetrySettings & Clock (google-cloud-java part)

Reconcile RetrySettings in GAX with RetryParams in google-cloud-java
#1574
Reconcile NanoClock in GAX with Clock in google-cloud-java
#1575
  • Loading branch information
@vam-google
vam-google authored Mar 21, 2017
1 parent 6c76a82 commit 95b70eb
Show file tree
Hide file tree
Showing 71 changed files with 1,624 additions and 2,107 deletions.
4 changes: 1 addition & 3 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryException.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import com.google.cloud.BaseServiceException;
import com.google.cloud.RetryHelper.RetryHelperException;
import com.google.cloud.RetryHelper.RetryInterruptedException;
import com.google.common.collect.ImmutableSet;

import java.io.IOException;
Expand Down Expand Up @@ -102,10 +101,9 @@ public int hashCode() {
* always throw an exception.
*
* @throws BigQueryException when {@code ex} was caused by a {@code BigQueryException}
* @throws RetryInterruptedException when {@code ex} is a {@code RetryInterruptedException}
*/
static BaseServiceException translateAndThrow(RetryHelperException ex) {
BaseServiceException.translateAndPropagateIfPossible(ex);
BaseServiceException.translate(ex);
throw new BigQueryException(UNKNOWN_CODE, ex.getMessage(), ex.getCause());
}
}
34 changes: 17 additions & 17 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public Dataset create(DatasetInfo datasetInfo, DatasetOption... options) {
public com.google.api.services.bigquery.model.Dataset call() {
return bigQueryRpc.create(datasetPb, optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock()));
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()));
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -189,7 +189,7 @@ public Table create(TableInfo tableInfo, TableOption... options) {
public com.google.api.services.bigquery.model.Table call() {
return bigQueryRpc.create(tablePb, optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock()));
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()));
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -207,7 +207,7 @@ public Job create(JobInfo jobInfo, JobOption... options) {
public com.google.api.services.bigquery.model.Job call() {
return bigQueryRpc.create(jobPb, optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock()));
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()));
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -230,7 +230,7 @@ public com.google.api.services.bigquery.model.Dataset call() {
return bigQueryRpc.getDataset(
completeDatasetId.getProject(), completeDatasetId.getDataset(), optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
return answer == null ? null : Dataset.fromPb(this, answer);
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
Expand Down Expand Up @@ -258,7 +258,7 @@ private static Page<Dataset> listDatasets(final String projectId,
Iterable<com.google.api.services.bigquery.model.Dataset>> call() {
return serviceOptions.getBigQueryRpcV2().listDatasets(projectId, optionsMap);
}
}, serviceOptions.getRetryParams(), EXCEPTION_HANDLER, serviceOptions.getClock());
}, serviceOptions.getRetrySettings(), EXCEPTION_HANDLER, serviceOptions.getClock());
String cursor = result.x();
return new PageImpl<>(new DatasetPageFetcher(projectId, serviceOptions, cursor, optionsMap),
cursor, Iterables.transform(result.y(),
Expand Down Expand Up @@ -289,7 +289,7 @@ public Boolean call() {
return bigQueryRpc.deleteDataset(
completeDatasetId.getProject(), completeDatasetId.getDataset(), optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -310,7 +310,7 @@ public Boolean call() {
return bigQueryRpc.deleteTable(completeTableId.getProject(), completeTableId.getDataset(),
completeTableId.getTable());
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -328,7 +328,7 @@ public Dataset update(DatasetInfo datasetInfo, DatasetOption... options) {
public com.google.api.services.bigquery.model.Dataset call() {
return bigQueryRpc.patch(datasetPb, optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock()));
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()));
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -346,7 +346,7 @@ public Table update(TableInfo tableInfo, TableOption... options) {
public com.google.api.services.bigquery.model.Table call() {
return bigQueryRpc.patch(tablePb, optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock()));
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()));
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -369,7 +369,7 @@ public com.google.api.services.bigquery.model.Table call() {
return bigQueryRpc.getTable(completeTableId.getProject(),
completeTableId.getDataset(), completeTableId.getTable(), optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
return answer == null ? null : Table.fromPb(this, answer);
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
Expand Down Expand Up @@ -400,7 +400,7 @@ private static Page<Table> listTables(final DatasetId datasetId,
return serviceOptions.getBigQueryRpcV2().listTables(
datasetId.getProject(), datasetId.getDataset(), optionsMap);
}
}, serviceOptions.getRetryParams(), EXCEPTION_HANDLER, serviceOptions.getClock());
}, serviceOptions.getRetrySettings(), EXCEPTION_HANDLER, serviceOptions.getClock());
String cursor = result.x();
Iterable<Table> tables = Iterables.transform(result.y(),
new Function<com.google.api.services.bigquery.model.Table, Table>() {
Expand Down Expand Up @@ -458,7 +458,7 @@ public BigQueryRpc.Tuple<String, Iterable<TableRow>> call() {
.listTableData(completeTableId.getProject(), completeTableId.getDataset(),
completeTableId.getTable(), optionsMap);
}
}, serviceOptions.getRetryParams(), EXCEPTION_HANDLER, serviceOptions.getClock());
}, serviceOptions.getRetrySettings(), EXCEPTION_HANDLER, serviceOptions.getClock());
String cursor = result.x();
return new PageImpl<>(new TableDataPageFetcher(tableId, serviceOptions, cursor, optionsMap),
cursor, transformTableData(result.y()));
Expand Down Expand Up @@ -495,7 +495,7 @@ public com.google.api.services.bigquery.model.Job call() {
return bigQueryRpc.getJob(completeJobId.getProject(), completeJobId.getJob(),
optionsMap);
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
return answer == null ? null : Job.fromPb(this, answer);
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
Expand All @@ -517,7 +517,7 @@ private static Page<Job> listJobs(final BigQueryOptions serviceOptions,
call() {
return serviceOptions.getBigQueryRpcV2().listJobs(serviceOptions.getProjectId(), optionsMap);
}
}, serviceOptions.getRetryParams(), EXCEPTION_HANDLER, serviceOptions.getClock());
}, serviceOptions.getRetrySettings(), EXCEPTION_HANDLER, serviceOptions.getClock());
String cursor = result.x();
Iterable<Job> jobs = Iterables.transform(result.y(),
new Function<com.google.api.services.bigquery.model.Job, Job>() {
Expand All @@ -543,7 +543,7 @@ public boolean cancel(JobId jobId) {
public Boolean call() {
return bigQueryRpc.cancel(completeJobId.getProject(), completeJobId.getJob());
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand All @@ -558,7 +558,7 @@ public QueryResponse query(final QueryRequest request) {
public com.google.api.services.bigquery.model.QueryResponse call() {
return bigQueryRpc.query(request.setProjectId(getOptions().getProjectId()).toPb());
}
}, getOptions().getRetryParams(), EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock());
QueryResponse.Builder builder = QueryResponse.newBuilder();
JobId completeJobId = JobId.fromPb(results.getJobReference());
builder.setJobId(completeJobId);
Expand Down Expand Up @@ -605,7 +605,7 @@ public GetQueryResultsResponse call() {
return serviceOptions.getBigQueryRpcV2().getQueryResults(
completeJobId.getProject(), completeJobId.getJob(), optionsMap);
}
}, serviceOptions.getRetryParams(), EXCEPTION_HANDLER, serviceOptions.getClock());
}, serviceOptions.getRetrySettings(), EXCEPTION_HANDLER, serviceOptions.getClock());
QueryResponse.Builder builder = QueryResponse.newBuilder();
builder.setJobId(JobId.fromPb(results.getJobReference()));
builder.setEtag(results.getEtag());
Expand Down
8 changes: 4 additions & 4 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Job.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.cloud.Clock;
import com.google.api.gax.core.ApiClock;
import com.google.cloud.WaitForOption;
import com.google.cloud.WaitForOption.CheckingPeriod;
import com.google.cloud.WaitForOption.Timeout;
Expand Down Expand Up @@ -209,10 +209,10 @@ public Job waitFor(WaitForOption... waitOptions) throws InterruptedException, Ti
Timeout timeout = Timeout.getOrDefault(waitOptions);
CheckingPeriod checkingPeriod = CheckingPeriod.getOrDefault(waitOptions);
long timeoutMillis = timeout.getTimeoutMillis();
Clock clock = options.getClock();
long startTime = clock.millis();
ApiClock clock = options.getClock();
long startTime = clock.millisTime();
while (!isDone()) {
if (timeoutMillis != -1 && (clock.millis() - startTime) >= timeoutMillis) {
if (timeoutMillis != -1 && (clock.millisTime() - startTime) >= timeoutMillis) {
throw new TimeoutException();
}
checkingPeriod.sleep();
Expand Down
4 changes: 2 additions & 2 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDataWriteChannel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public com.google.api.services.bigquery.model.Job call() {
return getOptions().getBigQueryRpcV2().write(
getUploadId(), getBuffer(), 0, getPosition(), length, last);
}
}, getOptions().getRetryParams(), BigQueryImpl.EXCEPTION_HANDLER, getOptions().getClock());
}, getOptions().getRetrySettings(), BigQueryImpl.EXCEPTION_HANDLER, getOptions().getClock());
job = jobPb != null ? Job.fromPb(getOptions().getService(), jobPb) : null;
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
Expand All @@ -77,7 +77,7 @@ private static String open(final BigQueryOptions options,
public String call() {
return options.getBigQueryRpcV2().open(writeChannelConfiguration.toPb());
}
}, options.getRetryParams(), BigQueryImpl.EXCEPTION_HANDLER, options.getClock());
}, options.getRetrySettings(), BigQueryImpl.EXCEPTION_HANDLER, options.getClock());
} catch (RetryHelper.RetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
}
Expand Down
33 changes: 18 additions & 15 deletions ...-cloud-bigquery/src/main/java/com/google/cloud/bigquery/testing/RemoteBigQueryHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.HttpTransportOptions;
import com.google.cloud.RetryParams;
import com.google.api.gax.core.RetrySettings;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
Expand All @@ -28,15 +28,16 @@
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.joda.time.Duration;

/**
* Utility to create a remote BigQuery configuration for testing. BigQuery options can be obtained
* via the {@link #getOptions()} method. Returned options have custom
* {@link BigQueryOptions#getRetryParams()}: {@link RetryParams#getRetryMaxAttempts()} is
* {@code 10}, {@link RetryParams#getRetryMinAttempts()} is {@code 6},
* {@link RetryParams#getMaxRetryDelayMillis()} is {@code 30000},
* {@link RetryParams#getTotalRetryPeriodMillis()} is {@code 120000} and
* {@link RetryParams#getInitialRetryDelayMillis()} is {@code 250}.
* {@link BigQueryOptions#getRetrySettings()}: {@link RetrySettings#getMaxAttempts()} is
* {@code 10},
* {@link RetrySettings#getMaxRetryDelay()} is {@code 30000},
* {@link RetrySettings#getTotalTimeout()} is {@code 120000} and
* {@link RetrySettings#getInitialRetryDelay()} is {@code 250}.
* {@link HttpTransportOptions#getConnectTimeout()} and
* {@link HttpTransportOptions#getReadTimeout()} are both
* set to {@code 60000}.
Expand Down Expand Up @@ -96,7 +97,7 @@ public static RemoteBigQueryHelper create(String projectId, InputStream keyStrea
BigQueryOptions bigqueryOptions = BigQueryOptions.newBuilder()
.setCredentials(ServiceAccountCredentials.fromStream(keyStream))
.setProjectId(projectId)
.setRetryParams(retryParams())
.setRetrySettings(retrySettings())
.setTransportOptions(transportOptions)
.build();
return new RemoteBigQueryHelper(bigqueryOptions);
Expand All @@ -117,19 +118,21 @@ public static RemoteBigQueryHelper create() {
transportOptions = transportOptions.toBuilder().setConnectTimeout(60000).setReadTimeout(60000)
.build();
BigQueryOptions bigqueryOptions = BigQueryOptions.newBuilder()
.setRetryParams(retryParams())
.setRetrySettings(retrySettings())
.setTransportOptions(transportOptions)
.build();
return new RemoteBigQueryHelper(bigqueryOptions);
}

private static RetryParams retryParams() {
return RetryParams.newBuilder()
.setRetryMaxAttempts(10)
.setRetryMinAttempts(6)
.setMaxRetryDelayMillis(30000)
.setTotalRetryPeriodMillis(120000)
.setInitialRetryDelayMillis(250)
private static RetrySettings retrySettings() {
return RetrySettings.newBuilder().setMaxAttempts(10)
.setMaxRetryDelay(Duration.millis(30000L))
.setTotalTimeout(Duration.millis(120000L))
.setInitialRetryDelay(Duration.millis(250L))
.setRetryDelayMultiplier(1.0)
.setInitialRpcTimeout(Duration.millis(120000L))
.setRpcTimeoutMultiplier(1.0)
.setMaxRpcTimeout(Duration.millis(120000L))
.build();
}

Expand Down
Loading

0 comments on commit 95b70eb

Please sign in to comment.