Merge pull request #502 from mziccard/bigquery

Merge master into bigquery and add BigQuery to README and TESTING

README.md

@@ -14,6 +14,8 @@ This client supports the following Google Cloud Platform services:
 
 -  [Google Cloud Datastore] (#google-cloud-datastore)
 -  [Google Cloud Storage] (#google-cloud-storage)
+-  [Google Cloud Resource Manager] (#google-cloud-resource-manager)
+-  [Google Cloud BigQuery] (#google-cloud-bigquery)
 
 > Note: This client is a work-in-progress, and may occasionally
 > make backwards-incompatible changes.
@@ -182,6 +184,82 @@ if (blob == null) {
 }
 ```
 
+Google Cloud Resource Manager
+----------------------
+
+- [API Documentation][resourcemanager-api]
+- [Official Documentation][cloud-resourcemanager-docs]
+
+#### Preview
+
+Here is a code snippet showing a simple usage example. Note that you must supply Google SDK credentials for this service, not other forms of authentication listed in the [Authentication section](#authentication).
+
+```java
+import com.google.gcloud.resourcemanager.ProjectInfo;
+import com.google.gcloud.resourcemanager.ResourceManager;
+import com.google.gcloud.resourcemanager.ResourceManagerOptions;
+
+import java.util.Iterator;
+
+ResourceManager resourceManager = ResourceManagerOptions.defaultInstance().service();
+ProjectInfo myProject = resourceManager.get("some-project-id"); // Use an existing project's ID
+ProjectInfo newProjectInfo = resourceManager.replace(myProject.toBuilder()
+    .addLabel("launch-status", "in-development").build());
+System.out.println("Updated the labels of project " + newProjectInfo.projectId()
+    + " to be " + newProjectInfo.labels());
+// List all the projects you have permission to view.
+Iterator<ProjectInfo> projectIterator = resourceManager.list().iterateAll();
+System.out.println("Projects I can view:");
+while (projectIterator.hasNext()) {
+  System.out.println(projectIterator.next().projectId());
+}
+```
+
+Google Cloud BigQuery
+----------------------
+
+- [API Documentation][bigquery-api]
+- [Official Documentation][cloud-bigquery-docs]
+
+#### Preview
+
+Here is a code snippet showing a simple usage example from within Compute/App Engine. Note that you
+must [supply credentials](#authentication) and a project ID if running this snippet elsewhere.
+
+```java
+import com.google.gcloud.bigquery.BaseTableInfo;
+import com.google.gcloud.bigquery.BigQuery;
+import com.google.gcloud.bigquery.BigQueryOptions;
+import com.google.gcloud.bigquery.Field;
+import com.google.gcloud.bigquery.JobStatus;
+import com.google.gcloud.bigquery.LoadJobInfo;
+import com.google.gcloud.bigquery.Schema;
+import com.google.gcloud.bigquery.TableId;
+import com.google.gcloud.bigquery.TableInfo;
+
+BigQuery bigquery = BigQueryOptions.defaultInstance().service();
+TableId tableId = TableId.of("dataset", "table");
+BaseTableInfo info = bigquery.getTable(tableId);
+if (info == null) {
+  System.out.println("Creating table " + tableId);
+  Field integerField = Field.of("fieldName", Field.Type.integer());
+  bigquery.create(TableInfo.of(tableId, Schema.of(integerField)));
+} else {
+  System.out.println("Loading data into table " + tableId);
+  LoadJobInfo loadJob = LoadJobInfo.of(tableId, "gs://bucket/path");
+  loadJob = bigquery.create(loadJob);
+  while (loadJob.status().state() != JobStatus.State.DONE) {
+    Thread.sleep(1000L);
+    loadJob = bigquery.getJob(loadJob.jobId());
+  }
+  if (loadJob.status().error() != null) {
+    System.out.println("Job completed with errors");
+  } else {
+    System.out.println("Job succeeded");
+  }
+}
+```
+
 Troubleshooting
 ---------------
 
@@ -241,3 +319,10 @@ Apache 2.0 - See [LICENSE] for more information.
 [cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
 [cloud-storage-activation]: https://cloud.google.com/storage/docs/signup
 [storage-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/storage/package-summary.html
+
+[resourcemanager-api]:http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/resourcemanager/package-summary.html
+[cloud-resourcemanager-docs]:https://cloud.google.com/resource-manager/
+
+[cloud-bigquery]: https://cloud.google.com/bigquery/
+[cloud-bigquery-docs]: https://cloud.google.com/bigquery/docs/overview
+[bigquery-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/bigquery/package-summary.html

TESTING.md

@@ -1,6 +1,11 @@
 ## gcloud-java tools for testing
 
-This library provides tools to help write tests for code that uses gcloud-java services.
+This library provides tools to help write tests for code that uses the following gcloud-java services:
+
+-  [Datastore] (#testing-code-that-uses-datastore)
+-  [Storage] (#testing-code-that-uses-storage)
+-  [Resource Manager] (#testing-code-that-uses-resource-manager)
+-  [BigQuery] (#testing-code-that-uses-bigquery)
 
 ### Testing code that uses Datastore
 
@@ -51,7 +56,8 @@ Currently, there isn't an emulator for Google Cloud Storage, so an alternative i
 3. Create a `RemoteGcsHelper` object using your project ID and JSON key.
 Here is an example that uses the `RemoteGcsHelper` to create a bucket.
   ```java
-  RemoteGcsHelper gcsHelper = RemoteGcsHelper.create(PROJECT_ID, "/path/to/my/JSON/key.json");
+  RemoteGcsHelper gcsHelper =
+      RemoteGcsHelper.create(PROJECT_ID, new FileInputStream("/path/to/my/JSON/key.json"));
   Storage storage = gcsHelper.options().service();
   String bucket = RemoteGcsHelper.generateBucketName();
   storage.create(BucketInfo.of(bucket));
@@ -65,5 +71,67 @@ Here is an example that clears the bucket created in Step 3 with a timeout of 5
   RemoteGcsHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS);
   ```
 
+### Testing code that uses Resource Manager
+
+#### On your machine
+
+You can test against a temporary local Resource Manager by following these steps:
+
+1. Before running your testing code, start the Resource Manager emulator `LocalResourceManagerHelper`. This can be done as follows:
+
+  ```java
+  import com.google.gcloud.resourcemanager.testing.LocalResourceManagerHelper;
+
+  LocalResourceManagerHelper helper = LocalResourceManagerHelper.create();
+  helper.start();
+  ```
+
+  This will spawn a server thread that listens to `localhost` at an ephemeral port for Resource Manager requests.
+
+2. In your program, create and use a Resource Manager service object whose host is set to `localhost` at the appropriate port.  For example:
+
+  ```java
+  ResourceManager resourceManager = LocalResourceManagerHelper.options().service();
+  ```
+
+3. Run your tests.
+
+4. Stop the Resource Manager emulator.
+
+  ```java
+  helper.stop();
+  ```
+
+  This method will block until the server thread has been terminated.
+
+### Testing code that uses BigQuery
+
+Currently, there isn't an emulator for Google BigQuery, so an alternative is to create a test
+project. `RemoteBigQueryHelper` contains convenience methods to make setting up and cleaning up the
+test project easier. To use this class, follow the steps below:
+
+1. Create a test Google Cloud project.
+
+2. Download a [JSON service account credentials file][create-service-account] from the Google
+Developer's Console.
+
+3. Create a `RemoteBigQueryHelper` object using your project ID and JSON key.
+Here is an example that uses the `RemoteBigQueryHelper` to create a dataset.
+  ```java
+  RemoteBigQueryHelper bigqueryHelper =
+      RemoteBigQueryHelper.create(PROJECT_ID, new FileInputStream("/path/to/my/JSON/key.json"));
+  BigQuery bigquery = bigqueryHelper.options().service();
+  String dataset = RemoteBigQueryHelper.generateDatasetName();
+  bigquery.create(DatasetInfo.builder(dataset).build());
+  ```
+
+4. Run your tests.
+
+5. Clean up the test project by using `forceDelete` to clear any datasets used.
+Here is an example that clears the dataset created in Step 3.
+  ```java
+  RemoteBigQueryHelper.forceDelete(bigquery, dataset);
+  ```
 
 [cloud-platform-storage-authentication]:https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts
+[create-service-account]:https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java

@@ -199,7 +199,7 @@ protected DatastoreException newInvalidRequest(String msg, Object... params) {
     return DatastoreException.throwInvalidRequest(String.format(msg, params));
   }
 
-  protected DatastoreV1.Mutation.Builder toMutationPb() {
+  DatastoreV1.Mutation.Builder toMutationPb() {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
     for (FullEntity<IncompleteKey> entity : toAddAutoId()) {
       mutationPb.addInsertAutoId(entity.toPb());

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java

@@ -90,7 +90,7 @@ private B self() {
     }
 
     @SuppressWarnings("unchecked")
-    protected B fill(DatastoreV1.Entity entityPb) {
+    B fill(DatastoreV1.Entity entityPb) {
       Map<String, Value<?>> copiedProperties = Maps.newHashMap();
       for (DatastoreV1.Property property : entityPb.getPropertyList()) {
         copiedProperties.put(property.getName(), Value.fromPb(property.getValue()));
@@ -375,7 +375,7 @@ ImmutableSortedMap<String, Value<?>> properties() {
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     Builder<?, ?> builder = emptyBuilder();
     builder.fill(DatastoreV1.Entity.parseFrom(bytesPb));
     return builder.build();
@@ -384,7 +384,7 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   protected abstract Builder<?, ?> emptyBuilder();
 
   @Override
-  protected final DatastoreV1.Entity toPb() {
+  final DatastoreV1.Entity toPb() {
     DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder();
     for (Map.Entry<String, Value<?>> entry : properties.entrySet()) {
       DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder();

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseKey.java

@@ -172,7 +172,7 @@ public boolean equals(Object obj) {
   }
 
   @Override
-  protected DatastoreV1.Key toPb() {
+  DatastoreV1.Key toPb() {
     DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder();
     DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder();
     if (projectId != null) {

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Blob.java

@@ -147,12 +147,12 @@ public static Blob copyFrom(InputStream input) throws IOException {
   }
 
   @Override
-  protected Value toPb() {
+  Value toPb() {
     return DatastoreV1.Value.newBuilder().setBlobValue(byteString).build();
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return new Blob(DatastoreV1.Value.parseFrom(bytesPb).getBlobValue());
   }
 }

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java

@@ -102,12 +102,12 @@ public static Cursor copyFrom(byte[] bytes) {
   }
 
   @Override
-  protected Value toPb() {
+  Value toPb() {
     return DatastoreV1.Value.newBuilder().setBlobValue(byteString).build();
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Value.parseFrom(bytesPb));
   }
 

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java

@@ -98,12 +98,12 @@ public static DateTime copyFrom(Calendar calendar) {
   }
 
   @Override
-  protected Value toPb() {
+  Value toPb() {
     return DatastoreV1.Value.newBuilder().setIntegerValue(timestampMicroseconds).build();
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return new DateTime(DatastoreV1.Value.parseFrom(bytesPb).getIntegerValue());
   }
 }

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java

@@ -126,7 +126,7 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    protected DatastoreV1.GqlQueryArg toPb() {
+    DatastoreV1.GqlQueryArg toPb() {
       DatastoreV1.GqlQueryArg.Builder argPb = DatastoreV1.GqlQueryArg.newBuilder();
       if (name != null) {
         argPb.setName(name);
@@ -141,7 +141,7 @@ protected DatastoreV1.GqlQueryArg toPb() {
     }
 
     @Override
-    protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+    Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
       return fromPb(DatastoreV1.GqlQueryArg.parseFrom(bytesPb));
     }
 
@@ -370,7 +370,7 @@ public boolean equals(Object obj) {
   }
 
   @Override
-  protected DatastoreV1.GqlQuery toPb() {
+  DatastoreV1.GqlQuery toPb() {
     DatastoreV1.GqlQuery.Builder queryPb = DatastoreV1.GqlQuery.newBuilder();
     queryPb.setQueryString(queryString);
     queryPb.setAllowLiteral(allowLiteral);
@@ -384,18 +384,18 @@ protected DatastoreV1.GqlQuery toPb() {
   }
 
   @Override
-  protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
+  void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
     requestPb.setGqlQuery(toPb());
   }
 
   @Override
-  protected GqlQuery<V> nextQuery(DatastoreV1.QueryResultBatch responsePb) {
+  GqlQuery<V> nextQuery(DatastoreV1.QueryResultBatch responsePb) {
     // See issue #17
     throw new UnsupportedOperationException("paging for this query is not implemented yet");
   }
 
   @Override
-  protected Object fromPb(ResultType<V> resultType, String namespace, byte[] bytesPb)
+  Object fromPb(ResultType<V> resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
     return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/IncompleteKey.java

@@ -54,7 +54,7 @@ public IncompleteKey build() {
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Key.parseFrom(bytesPb));
   }
 

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java

@@ -164,7 +164,7 @@ public static Key fromUrlSafe(String urlSafe) {
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Key.parseFrom(bytesPb));
   }
 

gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/PathElement.java

@@ -86,7 +86,7 @@ public boolean equals(Object obj) {
   }
 
   @Override
-  protected DatastoreV1.Key.PathElement toPb() {
+  DatastoreV1.Key.PathElement toPb() {
     DatastoreV1.Key.PathElement.Builder pathElementPb = DatastoreV1.Key.PathElement.newBuilder();
     pathElementPb.setKind(kind);
     if (id != null) {
@@ -98,7 +98,7 @@ protected DatastoreV1.Key.PathElement toPb() {
   }
 
   @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+  Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Key.PathElement.parseFrom(bytesPb));
   }