BQ: remove API client lib samples. Use a job for queries.

bigquery/README.md

@@ -18,8 +18,6 @@ recommended way to access the API.
     way to interact with BigQuery.
 - rest
   - This uses BigQuery's RESTful API directly. Not recommended.
-- src
-  - This uses [Google API Client Libraries](https://developers.google.com/api-client-library/java/). Not recommended.
 
 ## Quickstart
 

bigquery/cloud-client/README.md

@@ -26,11 +26,12 @@ You can then run a given `ClassName` via:
 
     mvn exec:java -Dexec.mainClass=com.example.bigquery.QuickstartSample
 
-### Running a synchronous query
+### Running a query using standard SQL syntax
 
-    mvn exec:java -Dexec.mainClass=com.example.bigquery.SyncQuerySample \
-        -Dquery='SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;' \
-        -DuseLegacySql=false
+    mvn exec:java -Dexec.mainClass=com.example.bigquery.QuerySample \
+        -Dexec.args=' \
+            --query="SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;" \
+            --runStandardSqlQuery'
 
 ### Running the simple app example
 

bigquery/cloud-client/pom.xml

@@ -39,6 +39,11 @@
       <artifactId>google-cloud-bigquery</artifactId>
       <version>0.12.0-beta</version>
     </dependency>
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.3.1</version>
+    </dependency>
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>

bigquery/cloud-client/src/main/java/com/example/bigquery/QueryParametersSample.java

@@ -33,46 +33,38 @@
 import java.util.Iterator;
 import java.util.List;
 
-/**
- * A sample that demonstrates use of query parameters.
- */
+/** A sample that demonstrates use of query parameters. */
 public class QueryParametersSample {
   private static final int ERROR_CODE = 1;
 
   private static void printUsage() {
     System.err.println("Usage:");
     System.err.printf(
         "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n",
-        QueryParametersSample.class.getCanonicalName(),
-        "${sample}");
+        QueryParametersSample.class.getCanonicalName(), "${sample}");
     System.err.println();
     System.err.println("${sample} can be one of: named, array, timestamp");
     System.err.println();
     System.err.println("Usage for ${sample}=named:");
     System.err.printf(
         "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n",
-        QueryParametersSample.class.getCanonicalName(),
-        "named ${corpus} ${minWordCount}");
+        QueryParametersSample.class.getCanonicalName(), "named ${corpus} ${minWordCount}");
     System.err.println();
     System.err.println("Usage for sample=array:");
     System.err.printf(
         "\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n",
-        QueryParametersSample.class.getCanonicalName(),
-        "array ${gender} ${states...}");
+        QueryParametersSample.class.getCanonicalName(), "array ${gender} ${states...}");
     System.err.println();
     System.err.println("\twhere ${gender} can be on of: M, F");
     System.err.println(
         "\tand ${states} is any upper-case 2-letter code for U.S. a state, e.g. CA.");
     System.err.println();
     System.err.printf(
         "\t\tmvn exec:java -Dexec.mainClass=%s -Dexec.args='%s'\n",
-        QueryParametersSample.class.getCanonicalName(),
-        "array F MD WA");
+        QueryParametersSample.class.getCanonicalName(), "array F MD WA");
   }
 
-  /**
-   * Prompts the user for the required parameters to perform a query.
-   */
+  /** Prompts the user for the required parameters to perform a query. */
   public static void main(final String[] args) throws IOException, InterruptedException {
     if (args.length < 1) {
       System.err.println("Expected first argument 'sample'");
@@ -125,11 +117,12 @@ private static void runNamed(final String corpus, final long minWordCount)
     BigQuery bigquery =
         new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance());
 
-    String queryString = "SELECT word, word_count\n"
-        + "FROM `bigquery-public-data.samples.shakespeare`\n"
-        + "WHERE corpus = @corpus\n"
-        + "AND word_count >= @min_word_count\n"
-        + "ORDER BY word_count DESC";
+    String queryString =
+        "SELECT word, word_count\n"
+            + "FROM `bigquery-public-data.samples.shakespeare`\n"
+            + "WHERE corpus = @corpus\n"
+            + "AND word_count >= @min_word_count\n"
+            + "ORDER BY word_count DESC";
     QueryRequest queryRequest =
         QueryRequest.newBuilder(queryString)
             .addNamedParameter("corpus", QueryParameterValue.string(corpus))
@@ -161,10 +154,7 @@ private static void runNamed(final String corpus, final long minWordCount)
 
     while (iter.hasNext()) {
       List<FieldValue> row = iter.next();
-      System.out.printf(
-          "%s: %d\n",
-          row.get(0).getStringValue(),
-          row.get(1).getLongValue());
+      System.out.printf("%s: %d\n", row.get(0).getStringValue(), row.get(1).getLongValue());
     }
   }
   // [END bigquery_query_params]
@@ -173,24 +163,22 @@ private static void runNamed(final String corpus, final long minWordCount)
    * Query the baby names database to find the most popular names for a gender in a list of states.
    */
   // [START bigquery_query_params_arrays]
-  private static void runArray(String gender, String[] states)
-      throws InterruptedException {
+  private static void runArray(String gender, String[] states) throws InterruptedException {
     BigQuery bigquery =
         new BigQueryOptions.DefaultBigqueryFactory().create(BigQueryOptions.getDefaultInstance());
 
-    String queryString = "SELECT name, sum(number) as count\n"
-        + "FROM `bigquery-public-data.usa_names.usa_1910_2013`\n"
-        + "WHERE gender = @gender\n"
-        + "AND state IN UNNEST(@states)\n"
-        + "GROUP BY name\n"
-        + "ORDER BY count DESC\n"
-        + "LIMIT 10;";
+    String queryString =
+        "SELECT name, sum(number) as count\n"
+            + "FROM `bigquery-public-data.usa_names.usa_1910_2013`\n"
+            + "WHERE gender = @gender\n"
+            + "AND state IN UNNEST(@states)\n"
+            + "GROUP BY name\n"
+            + "ORDER BY count DESC\n"
+            + "LIMIT 10;";
     QueryRequest queryRequest =
         QueryRequest.newBuilder(queryString)
             .addNamedParameter("gender", QueryParameterValue.string(gender))
-            .addNamedParameter(
-                "states",
-                QueryParameterValue.array(states, String.class))
+            .addNamedParameter("states", QueryParameterValue.array(states, String.class))
             // Standard SQL syntax is required for parameterized queries.
             // See: https://cloud.google.com/bigquery/sql-reference/
             .setUseLegacySql(false)
@@ -272,8 +260,7 @@ private static void runTimestamp() throws InterruptedException {
               new DateTime(
                   // Timestamp values are returned in microseconds since 1970-01-01T00:00:00 UTC,
                   // but org.joda.time.DateTime constructor accepts times in milliseconds.
-                  row.get(0).getTimestampValue() / 1000,
-                  DateTimeZone.UTC)));
+                  row.get(0).getTimestampValue() / 1000, DateTimeZone.UTC)));
     }
   }
   // [END bigquery_query_params_timestamps]

bigquery/cloud-client/src/main/java/com/example/bigquery/QuerySample.java

@@ -0,0 +1,208 @@
+/*
+  Copyright 2016, Google, Inc.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+package com.example.bigquery;
+
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.FieldValue;
+import com.google.cloud.bigquery.Job;
+import com.google.cloud.bigquery.JobId;
+import com.google.cloud.bigquery.JobInfo;
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.QueryResponse;
+import com.google.cloud.bigquery.QueryResult;
+import com.google.cloud.bigquery.TableId;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+/** Runs a query against BigQuery. */
+public class QuerySample {
+  // [START query_config_simple]
+  public static void runSimpleQuery(String queryString)
+      throws TimeoutException, InterruptedException {
+    QueryJobConfiguration queryConfig =
+        QueryJobConfiguration.newBuilder(queryString).build();
+
+    runQuery(queryConfig);
+  }
+  // [END query_config_simple]
+
+  // [START query_config_standard_sql]
+  public static void runStandardSqlQuery(String queryString)
+      throws TimeoutException, InterruptedException {
+    QueryJobConfiguration queryConfig =
+        QueryJobConfiguration.newBuilder(queryString)
+            // To use standard SQL syntax, set useLegacySql to false.
+            // See: https://cloud.google.com/bigquery/sql-reference/
+            .setUseLegacySql(false)
+            .build();
+
+    runQuery(queryConfig);
+  }
+  // [END query_config_standard_sql]
+
+  // [START query_config_permanent_table]
+  public static void runQueryPermanentTable(
+      String queryString,
+      String destinationDataset,
+      String destinationTable,
+      boolean allowLargeResults) throws TimeoutException, InterruptedException {
+    QueryJobConfiguration queryConfig =
+        QueryJobConfiguration.newBuilder(queryString)
+            // Save the results of the query to a permanent table.
+            // See: https://cloud.google.com/bigquery/querying-data#permanent-table
+            .setDestinationTable(TableId.of(destinationDataset, destinationTable))
+            // Allow results larger than the maximum response size.
+            // If true, a destination table must be set.
+            // See: https://cloud.google.com/bigquery/querying-data#large-results
+            .setAllowLargeResults(allowLargeResults)
+            .build();
+
+    runQuery(queryConfig);
+  }
+  // [END query_config_permanent_table]
+
+  // [START query_config_cache]
+  public static void runUncachedQuery(String queryString)
+      throws TimeoutException, InterruptedException {
+    QueryJobConfiguration queryConfig =
+        QueryJobConfiguration.newBuilder(queryString)
+            // Do not use the query cache. Force live query evaluation.
+            // See: https://cloud.google.com/bigquery/querying-data#query-caching
+            .setUseQueryCache(false)
+            .build();
+
+    runQuery(queryConfig);
+  }
+  // [END query_config_cache]
+
+  // [START query_config_batch]
+  public static void runBatchQuery(String queryString)
+      throws TimeoutException, InterruptedException {
+    QueryJobConfiguration queryConfig =
+        QueryJobConfiguration.newBuilder(queryString)
+            // Run at batch priority, which won't count toward concurrent rate
+            // limit.
+            // See: https://cloud.google.com/bigquery/querying-data#interactive-batch
+            .setPriority(QueryJobConfiguration.Priority.BATCH)
+            .build();
+
+    runQuery(queryConfig);
+  }
+  // [END query_config_batch]
+
+
+  // [START run_query]
+  public static void runQuery(QueryJobConfiguration queryConfig)
+      throws TimeoutException, InterruptedException {
+    BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
+
+    // Create a job ID so that we can safely retry.
+    JobId jobId = JobId.of(UUID.randomUUID().toString());
+    Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());
+
+    // Wait for the query to complete.
+    queryJob = queryJob.waitFor();
+
+    // Check for errors
+    if (queryJob == null) {
+      throw new RuntimeException("Job no longer exists");
+    } else if (queryJob.getStatus().getError() != null) {
+      // You can also look at queryJob.getStatus().getExecutionErrors() for all
+      // errors, not just the latest one.
+      throw new RuntimeException(queryJob.getStatus().getError().toString());
+    }
+
+    // Get the results.
+    QueryResponse response = bigquery.getQueryResults(jobId);
+    QueryResult result = response.getResult();
+
+    // Print all pages of the results.
+    while (result != null) {
+      if (response.hasErrors()) {
+        String firstError = "";
+        if (response.getExecutionErrors().size() != 0) {
+          firstError = response.getExecutionErrors().get(0).getMessage();
+        }
+        throw new RuntimeException(firstError);
+      }
+
+      Iterator<List<FieldValue>> iter = result.iterateAll();
+      while (iter.hasNext()) {
+        List<FieldValue> row = iter.next();
+        for (FieldValue val : row) {
+          System.out.printf("%s,", val.toString());
+        }
+        System.out.printf("\n");
+      }
+
+      result = result.getNextPage();
+    }
+  }
+  // [END run_query]
+
+  /** Prompts the user for the required parameters to perform a query. */
+  public static void main(final String[] args)
+      throws IOException, InterruptedException, TimeoutException, ParseException {
+    Options options = new Options();
+
+    // Use an OptionsGroup to choose which sample to run.
+    OptionGroup samples = new OptionGroup();
+    samples.addOption(Option.builder().longOpt("runSimpleQuery").build());
+    samples.addOption(Option.builder().longOpt("runStandardSqlQuery").build());
+    samples.addOption(Option.builder().longOpt("runPermanentTableQuery").build());
+    samples.addOption(Option.builder().longOpt("runUncachedQuery").build());
+    samples.addOption(Option.builder().longOpt("runBatchQuery").build());
+    samples.isRequired();
+    options.addOptionGroup(samples);
+
+    options.addOption(Option.builder().longOpt("query").hasArg().required().build());
+    options.addOption(Option.builder().longOpt("destDataset").hasArg().build());
+    options.addOption(Option.builder().longOpt("destTable").hasArg().build());
+    options.addOption(Option.builder().longOpt("allowLargeResults").build());
+
+    CommandLineParser parser = new DefaultParser();
+    CommandLine cmd = parser.parse(options, args);
+
+    String query = cmd.getOptionValue("query");
+    if (cmd.hasOption("runSimpleQuery")) {
+      runSimpleQuery(query);
+    } else if (cmd.hasOption("runStandardSqlQuery")) {
+      runStandardSqlQuery(query);
+    } else if (cmd.hasOption("runPermanentTableQuery")) {
+      String destDataset = cmd.getOptionValue("destDataset");
+      String destTable = cmd.getOptionValue("destTable");
+      boolean allowLargeResults = cmd.hasOption("allowLargeResults");
+      runQueryPermanentTable(query, destDataset, destTable, allowLargeResults);
+    } else if (cmd.hasOption("runUncachedQuery")) {
+      runUncachedQuery(query);
+    } else if (cmd.hasOption("runBatchQuery")) {
+      runBatchQuery(query);
+    }
+  }
+}

bigquery/cloud-client/src/main/java/com/example/bigquery/SimpleApp.java

@@ -41,10 +41,10 @@ public static void main(String... args) throws Exception {
     // [START run_query]
     QueryJobConfiguration queryConfig =
         QueryJobConfiguration.newBuilder(
-            "SELECT "
-                + "APPROX_TOP_COUNT(corpus, 10) as title, "
-                + "COUNT(*) as unique_words "
-                + "FROM `publicdata.samples.shakespeare`;")
+                "SELECT "
+                    + "APPROX_TOP_COUNT(corpus, 10) as title, "
+                    + "COUNT(*) as unique_words "
+                    + "FROM `publicdata.samples.shakespeare`;")
             // Use standard SQL syntax for queries.
             // See: https://cloud.google.com/bigquery/sql-reference/
             .setUseLegacySql(false)
@@ -98,4 +98,3 @@ public static void main(String... args) throws Exception {
   }
 }
 // [END all]
-