From 830d87ce8376f4641ad6abe236836004597ab1f8 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 25 Jan 2016 16:23:17 -0800 Subject: [PATCH 1/2] Use pagination in DatastoreExample and fix readme bug and formatting --- gcloud-java-examples/README.md | 94 +++++++++---------- .../gcloud/examples/DatastoreExample.java | 38 +++++--- 2 files changed, 71 insertions(+), 61 deletions(-) diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md index 8d64c8ea202f..3d9baadf0b6b 100644 --- a/gcloud-java-examples/README.md +++ b/gcloud-java-examples/README.md @@ -39,54 +39,54 @@ To run examples from your command line: 4. Run an example using Maven from command line. - Here's an example run of `BigQueryExample`. - - Before running the example, go to the [Google Developers Console][developers-console] to ensure - that BigQuery API is enabled. You can upload a CSV file `my_csv_file` to the `my_bucket` bucket - (replace `my_csv_file` and `my_bucket` with actual file and bucket names) using the GCS - [web browser](https://console.developers.google.com/storage/browser). The CSV file will be used to - load data into a BigQuery table and should look something like: - ```csv - value1 - value2 - value3 - ``` - Then you are ready to run the following example: - ``` - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="create dataset new_dataset_id" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="create table new_dataset_id new_table_id field_name:string" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="list tables new_dataset_id" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="load new_dataset_id new_table_id CSV gs://my_bucket/my_csv_file" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="query 'select * from new_dataset_id.new_table_id'" - ``` - - Here's an example run of `DatastoreExample`. + * Here's an example run of `BigQueryExample`. + + Before running the example, go to the [Google Developers Console][developers-console] to ensure + that BigQuery API is enabled. You can upload a CSV file `my_csv_file` to the `my_bucket` bucket + (replace `my_csv_file` and `my_bucket` with actual file and bucket names) using the GCS + [web browser](https://console.developers.google.com/storage/browser). The CSV file will be used to + load data into a BigQuery table and should look something like: + ```csv + value1 + value2 + value3 + ``` + Then you are ready to run the following example: + ``` + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="create dataset new_dataset_id" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="create table new_dataset_id new_table_id field_name:string" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="list tables new_dataset_id" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="load new_dataset_id new_table_id CSV gs://my_bucket/my_csv_file" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.BigQueryExample" -Dexec.args="query 'select * from new_dataset_id.new_table_id'" + ``` + + * Here's an example run of `DatastoreExample`. - Note that you have to enable the Google Cloud Datastore API on the [Google Developers Console][developers-console] before running the following commands. - ``` - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.args="my_name add my\ comment" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.args="my_name display" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.args="my_name delete" - ``` - - Here's an example run of `ResourceManagerExample`. - - Be sure to change the placeholder project ID "my-project-id" with your own globally unique project ID. - ``` - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.ResourceManagerExample" -Dexec.args="create my-project-id" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.ResourceManagerExample" -Dexec.args="list" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.ResourceManagerExample" -Dexec.args="get my-project-id" - ``` - - Here's an example run of `StorageExample`. - - Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled and that you have a bucket. Also ensure that you have a test file (`test.txt` is chosen here) to upload to Cloud Storage stored locally on your machine. - ``` - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="upload /path/to/test.txt " - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="list " - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="download test.txt" - $mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="delete test.txt" - ``` + Be sure to change the placeholder project ID "your-project-id" with your own project ID. Also note that you have to enable the Google Cloud Datastore API on the [Google Developers Console][developers-console] before running the following commands. + ``` + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.args="your-project-id my_name add my\ comment" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.args="your-project-id my_name display" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.args="your-project-id my_name delete" + ``` + + * Here's an example run of `ResourceManagerExample`. + + Be sure to change the placeholder project ID "your-project-id" with your own globally unique project ID. + ``` + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.ResourceManagerExample" -Dexec.args="create your-project-id" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.ResourceManagerExample" -Dexec.args="list" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.ResourceManagerExample" -Dexec.args="get your-project-id" + ``` + + * Here's an example run of `StorageExample`. + + Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled and that you have a bucket. Also ensure that you have a test file (`test.txt` is chosen here) to upload to Cloud Storage stored locally on your machine. + ``` + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="upload /path/to/test.txt " + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="list " + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="download test.txt" + mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample" -Dexec.args="delete test.txt" + ``` Troubleshooting --------------- diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java index e408bab1338e..62d39c32204d 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java @@ -25,8 +25,8 @@ import com.google.gcloud.datastore.Key; import com.google.gcloud.datastore.KeyFactory; import com.google.gcloud.datastore.Query; -import com.google.gcloud.datastore.Query.ResultType; import com.google.gcloud.datastore.QueryResults; +import com.google.gcloud.datastore.StructuredQuery; import com.google.gcloud.datastore.StructuredQuery.PropertyFilter; import com.google.gcloud.datastore.Transaction; @@ -100,21 +100,31 @@ public void run(Transaction tx, Key userKey, String... args) { return; } System.out.printf("User '%s' has %d comment[s].%n", userKey.name(), user.getLong("count")); - // ORDER BY timestamp"; - String gql = "SELECT * FROM " + COMMENT_KIND + " WHERE __key__ HAS ANCESTOR @1"; - Query query = Query.gqlQueryBuilder(ResultType.ENTITY, gql) - .namespace(NAMESPACE) - .addBinding(userKey) - .build(); - QueryResults results = tx.run(query); - // We could have added "ORDER BY timestamp" to the query to avoid the sorting bellow - // but that would require adding an ancestor index for timestamp - // see: https://cloud.google.com/datastore/docs/tools/indexconfig + int limit = 10; Map sortedComments = new TreeMap<>(); - while (results.hasNext()) { - Entity result = results.next(); - sortedComments.put(result.getDateTime("timestamp"), result.getString("content")); + StructuredQuery query = + Query.entityQueryBuilder() + .namespace(NAMESPACE) + .kind(COMMENT_KIND) + .filter(PropertyFilter.hasAncestor(userKey)) + .limit(limit) + .build(); + while (true) { + QueryResults results = tx.run(query); + int resultCount = 0; + while (results.hasNext()) { + Entity result = results.next(); + sortedComments.put(result.getDateTime("timestamp"), result.getString("content")); + resultCount++; + } + if (resultCount < limit) { + break; + } + query = query.toBuilder().startCursor(results.cursorAfter()).build(); } + // We could have added "ORDER BY timestamp" to the query to avoid sorting, but that would + // require adding an ancestor index for timestamp. + // See: https://cloud.google.com/datastore/docs/tools/indexconfig for (Map.Entry entry : sortedComments.entrySet()) { System.out.printf("\t%s: %s%n", entry.getKey(), entry.getValue()); } From 360e048824a322a0b065e71380be9a28caa81788 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 25 Jan 2016 16:51:13 -0800 Subject: [PATCH 2/2] Increase limit on query size --- .../main/java/com/google/gcloud/examples/DatastoreExample.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java index 62d39c32204d..1e65a018a1fb 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java @@ -100,7 +100,7 @@ public void run(Transaction tx, Key userKey, String... args) { return; } System.out.printf("User '%s' has %d comment[s].%n", userKey.name(), user.getLong("count")); - int limit = 10; + int limit = 200; Map sortedComments = new TreeMap<>(); StructuredQuery query = Query.entityQueryBuilder()