Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BigQuery query samples: command line parameters. #305

Merged
merged 1 commit into from
Aug 18, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions bigquery/README.md
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
# Getting Started with BigQuery and the Google Java API Client library

Google's BigQuery Service features a REST-based API that allows developers to create applications to run ad-hoc queries on massive datasets. These sample Java applications demonstrate how to access the BigQuery API using the Google Java API Client Libraries. For more information, read the [Getting Started with BigQuery and the Google Java API Client library][1] codelab.
Google's BigQuery Service features a REST-based API that allows developers to create applications to run ad-hoc queries
on massive datasets. These sample Java applications demonstrate how to access the BigQuery API using the Google Java API
Client Libraries.

For more information, read the [Getting Started with BigQuery and the Google Java API Client
library][1] codelab.

## Quickstart

Install [Maven](http://maven.apache.org/).

Build your project with:

mvn package -DskipTests
mvn clean package -DskipTests

You can then run a given `ClassName` via:

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@

import java.io.IOException;
import java.util.Iterator;
import java.util.Scanner;

/**
* Example of authorizing with BigQuery and reading from a public dataset.
*/
public class AsyncQuerySample {
private static final String DEFAULT_QUERY =
"SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;";

// [START main]
/**
* Prompts for all the parameters required to make a query.
Expand All @@ -39,17 +41,42 @@ public class AsyncQuerySample {
* @throws InterruptedException InterruptedException
*/
public static void main(final String[] args) throws IOException, InterruptedException {
Scanner scanner = new Scanner(System.in);
System.out.println("Enter your project id: ");
String projectId = scanner.nextLine();
System.out.println("Enter your query string: ");
String queryString = scanner.nextLine();
System.out.println("Run query in batch mode? [true|false] ");
boolean batch = Boolean.valueOf(scanner.nextLine());
System.out.println("Enter how often to check if your job is complete " + "(milliseconds): ");
long waitTime = scanner.nextLong();
scanner.close();
Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, batch, waitTime);
String projectId = System.getProperty("projectId");
if (projectId == null || projectId.isEmpty()) {
System.err.println("The projectId property must be set.");
System.exit(1);
}
System.out.printf("projectId: %s\n", projectId);

String queryString = System.getProperty("query");
if (queryString == null || queryString.isEmpty()) {
System.out.println("The query property was not set, using default.");
queryString = DEFAULT_QUERY;
}
System.out.printf("query: %s\n", queryString);

String useBatchString = System.getProperty("useBatchMode");
if (useBatchString == null || useBatchString.isEmpty()) {
useBatchString = "false";
}
boolean useBatchMode = Boolean.parseBoolean(useBatchString);
System.out.printf("useBatchMode: %b\n", useBatchMode);

String waitTimeString = System.getProperty("waitTime");
if (waitTimeString == null || waitTimeString.isEmpty()) {
waitTimeString = "1000";
}
long waitTime = Long.parseLong(waitTimeString);
System.out.printf("waitTime: %d (milliseconds)\n", waitTime);

String useLegacySqlString = System.getProperty("useLegacySql");
if (useLegacySqlString == null || useLegacySqlString.isEmpty()) {
useLegacySqlString = "false";
}
boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString);

Iterator<GetQueryResultsResponse> pages =
run(projectId, queryString, useBatchMode, waitTime, useLegacySql);
while (pages.hasNext()) {
BigQueryUtils.printRows(pages.next().getRows(), System.out);
}
Expand All @@ -62,19 +89,24 @@ public static void main(final String[] args) throws IOException, InterruptedExce
*
* @param projectId Get this from Google Developers console
* @param queryString Query we want to run against BigQuery
* @param batch True if you want to batch the queries
* @param useBatchMode True if you want to batch the queries
* @param waitTime How long to wait before retries
* @param useLegacySql Boolean that is false if using standard SQL syntax.
* @return An iterator to the result of your pages
* @throws IOException Thrown if there's an IOException
* @throws InterruptedException Thrown if there's an Interrupted Exception
*/
public static Iterator<GetQueryResultsResponse> run(
final String projectId, final String queryString, final boolean batch, final long waitTime)
final String projectId,
final String queryString,
final boolean useBatchMode,
final long waitTime,
final boolean useLegacySql)
throws IOException, InterruptedException {

Bigquery bigquery = BigQueryServiceFactory.getService();

Job query = asyncQuery(bigquery, projectId, queryString, batch);
Job query = asyncQuery(bigquery, projectId, queryString, useBatchMode, useLegacySql);
Bigquery.Jobs.Get getRequest =
bigquery.jobs().get(projectId, query.getJobReference().getJobId());

Expand All @@ -96,17 +128,27 @@ public static Iterator<GetQueryResultsResponse> run(
* @param bigquery an authorized BigQuery client
* @param projectId a String containing the project ID
* @param querySql the actual query string
* @param batch True if you want to run the query as BATCH
* @param useBatchMode True if you want to run the query as BATCH
* @param useLegacySql Boolean that is false if using standard SQL syntax.
* @return a reference to the inserted query job
* @throws IOException Thrown if there's a network exception
*/
public static Job asyncQuery(
final Bigquery bigquery, final String projectId, final String querySql, final boolean batch)
final Bigquery bigquery,
final String projectId,
final String querySql,
final boolean useBatchMode,
final boolean useLegacySql)
throws IOException {

JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(querySql);
JobConfigurationQuery queryConfig =
new JobConfigurationQuery()
.setQuery(querySql)
// Set the useLegacySql parameter to false to use standard SQL syntax. See:
// https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql
.setUseLegacySql(useLegacySql);

if (batch) {
if (useBatchMode) {
queryConfig.setPriority("BATCH");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@

import java.io.IOException;
import java.util.Iterator;
import java.util.Scanner;

/**
* Runs a synchronous query against Bigtable.
*/
public class SyncQuerySample {
private static final String DEFAULT_QUERY =
"SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;";
private static final long TEN_SECONDS_MILLIS = 10000;

/**
* Protected because this is a collection of static methods.
Expand All @@ -41,18 +44,39 @@ protected SyncQuerySample() {}
* @throws IOException ioexceptino
*/
public static void main(final String[] args) throws IOException {
Scanner scanner = new Scanner(System.in);
System.out.println("Enter your project id: ");
String projectId = scanner.nextLine();
System.out.println("Enter your query string: ");
String queryString = scanner.nextLine();
System.out.println(
"Enter how long to wait for the query to complete"
+ " (in milliseconds):\n "
+ "(if longer than 10 seconds, use an asynchronous query)");
long waitTime = scanner.nextLong();
scanner.close();
Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, waitTime);
String projectId = System.getProperty("projectId");
if (projectId == null || projectId.isEmpty()) {
System.err.println("The projectId property must be set.");
System.exit(1);
}
System.out.printf("projectId: %s\n", projectId);

String queryString = System.getProperty("query");
if (queryString == null || queryString.isEmpty()) {
System.out.println("The query property was not set, using default.");
queryString = DEFAULT_QUERY;
}
System.out.printf("query: %s\n", queryString);

String waitTimeString = System.getProperty("waitTime");
if (waitTimeString == null || waitTimeString.isEmpty()) {
waitTimeString = "1000";
}
long waitTime = Long.parseLong(waitTimeString);
System.out.printf("waitTime: %d (milliseconds)\n", waitTime);
if (waitTime > TEN_SECONDS_MILLIS) {
System.out.println(
"WARNING: If the query is going to take longer than 10 seconds to complete, use an"
+ " asynchronous query.");
}

String useLegacySqlString = System.getProperty("useLegacySql");
if (useLegacySqlString == null || useLegacySqlString.isEmpty()) {
useLegacySqlString = "false";
}
boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString);

Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, waitTime, useLegacySql);
while (pages.hasNext()) {
BigQueryUtils.printRows(pages.next().getRows(), System.out);
}
Expand All @@ -65,23 +89,33 @@ public static void main(final String[] args) throws IOException {
* @param projectId project id from developer console
* @param queryString query to run
* @param waitTime Timeout in milliseconds before we abort
* @param useLegacySql Boolean that is false if using standard SQL syntax.
* @return Iterator that pages through the results of the query
* @throws IOException ioexception
*/
// [START run]
public static Iterator<GetQueryResultsResponse> run(
final String projectId, final String queryString, final long waitTime) throws IOException {
final String projectId,
final String queryString,
final long waitTime,
final boolean useLegacySql) throws IOException {
Bigquery bigquery = BigQueryServiceFactory.getService();
//Wait until query is done with 10 second timeout, at most 5 retries on error

// Wait until query is done with `waitTime` millisecond timeout, at most 5 retries on error.
QueryResponse query =
bigquery
.jobs()
.query(projectId, new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString))
.query(
projectId,
new QueryRequest()
.setTimeoutMs(waitTime)
.setQuery(queryString)
// Set the useLegacySql parameter to false to use standard SQL syntax. See:
// https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql
.setUseLegacySql(useLegacySql))
.execute();

//Make a request to get the results of the query
//(timeout is zero since job should be complete)

// Make a request to get the results of the query.
GetQueryResults getRequest =
bigquery
.jobs()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,41 @@ public class AsyncQuerySampleTest {
@Test
public void testInteractive() throws IOException, InterruptedException {
Iterator<GetQueryResultsResponse> pages =
AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, false, 5000);
AsyncQuerySample.run(
Constants.PROJECT_ID,
"SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;",
false /* useBatchMode */,
5000,
false /* useLegacySql */);
while (pages.hasNext()) {
assertThat(pages.next().getRows()).isNotEmpty();
}
}

@Test
@Ignore // Batches can take up to 3 hours to run, probably shouldn't use this
public void testInteractiveLegacySql() throws IOException, InterruptedException {
Iterator<GetQueryResultsResponse> pages =
AsyncQuerySample.run(
Constants.PROJECT_ID,
Constants.QUERY,
false /* useBatchMode */,
5000,
true /* useLegacySql */);
while (pages.hasNext()) {
assertThat(pages.next().getRows()).isNotEmpty();
}
}

@Test
@Ignore // Batches can take up to 3 hours to run, don't run during the system tests.
public void testBatch() throws IOException, InterruptedException {
Iterator<GetQueryResultsResponse> pages =
AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, true, 5000);
AsyncQuerySample.run(
Constants.PROJECT_ID,
Constants.QUERY,
true /* useBatchMode */,
5000,
true /* useLegacySql */);
while (pages.hasNext()) {
assertThat(pages.next().getRows()).isNotEmpty();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,20 @@ public class SyncQuerySampleTest {
@Test
public void testSyncQuery() throws IOException {
Iterator<GetQueryResultsResponse> pages =
SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000);
SyncQuerySample.run(
Constants.PROJECT_ID,
"SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;",
10000,
false /* useLegacySql */);
while (pages.hasNext()) {
assertThat(pages.next().getRows()).isNotEmpty();
}
}

@Test
public void testSyncQueryLegacySql() throws IOException {
Iterator<GetQueryResultsResponse> pages =
SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000, true /* useLegacySql */);
while (pages.hasNext()) {
assertThat(pages.next().getRows()).isNotEmpty();
}
Expand Down